#!/usr/bin/env python
"""
    %prog serun

Check the outputs for the input run.  You really want to use parallel 
instead if hdfs is used.
"""

import sys, os
from sys import stderr
import copy
import deswl
import esutil as eu
from esutil import json_util
from esutil.io import fexists

from optparse import OptionParser
parser = OptionParser(__doc__)
parser.add_option("--fs",default='nfs', help="file system, nfs or hdfs")

def main():
    options, args = parser.parse_args(sys.argv[1:])
    if len(args) < 1:
        parser.print_help()
        sys.exit(45)

    serun=args[0]

    sej = deswl.files.ShearFiles(serun, fs=options.fs)
    flist = sej.get_flist()

    n=len(flist)
    goodlist=[]
    badlist=[]
    print >>stderr,'checking',n
    for i,fdict in enumerate(flist):
        if ((i+1) % 100) == 0:
            print >>stderr,'%05d/%05d: %s-%02d' % (i+1,n,fdict['expname'],fdict['ccd'])

        info = copy.deepcopy(fdict)
        problem_found=False

        missing=[]
        #for f in ['stat','stars','fitpsf','psf','shear','qa']:
        for f in ['stat']:
            if not fexists(fdict[f]):
                # only print first missing one
                if len(missing) == 0:
                    print >>stderr,'file missing:',fdict[f]
                missing.append(f)

        if len(missing) > 0:
            problem_found=True
            fmiss='\n'.join(missing)
            info['failtype'] = 'files missing:\n%s' % fmiss
            badlist.append(info)
        else:
            status_read = False
            try:
                # this will deal properly with hdfs
                stat=eu.io.read(fdict['stat'])
                status_read = True
            except ValueError as e:
                problem_found=True
                print >>stderr,"Error reading file:",fdict['stat']
                info['failtype'] = 'Error reading stat file'
                badlist.append(info)

            if status_read:
                exit_status= stat['exit_status']
                if exit_status != 0:
                    problem_found=True
                    print >>stderr,"Found non-zero exit status %s in stat file %s" % \
                            (exit_status,fdict['stat'])
                    info['failtype'] = 'Processing error'
                    badlist.append(info)

        if not problem_found:
            goodlist.append(info)

    print >>stderr,'Found %s/%s problems' % (len(badlist),n)

    goodfile=deswl.files.collated_path(serun, 'goodlist')
    badfile=deswl.files.collated_path(serun, 'badlist')

    eu.ostools.makedirs_fromfile(goodfile)

    print >>stderr,"Writing goodlist: %s",goodfile
    json_util.write(goodlist, goodfile)
    print >>stderr,"Writing badlist: %s",badfile
    json_util.write(badlist, badfile)
 

main()
