#!/usr/bin/env python
"""
    %prog [options] serun

Generate all the wq job files for the given serun. By default it works
by exposure name, generating a wq script for each exposure.  It also
generates a wq script for checking the result of each exposure.

Finally, generate a script to reduce the results of the parallelized check
scripts

"""

import os,sys
from sys import stderr
import deswl
from deswl.files import ShearFiles,SEWQJob
import desdb

from optparse import OptionParser
parser = OptionParser(__doc__)
parser.add_option("-g","--groups",default='new,new2', help="groups for queue")
parser.add_option("--fs",default='hdfs', help="file system, nfs or hdfs")
parser.add_option('--byccd',action='store_true', help="file system, nfs or hdfs, default %default")


def write_reducer(serun):
    text="""
from sys import stderr
import glob
import deswl
import esutil as eu

files=glob.glob('byccd/*-check.json')
goodlist=[]
badlist=[]
nf=len(files)
for i,f in enumerate(files,1):
    if (i % 100) == 0 or (i == 1):
        print >>stderr,f,"%s/%s" % (i,nf)
    d=eu.io.read(f)
    if d['error_string'] == '':
        goodlist.append(d)
    else:
        print >>stderr,d['stat'],'\\n',d['error_string']
        badlist.append(d)
goodfile=deswl.files.collated_path('{serun}', 'goodlist')
badfile=deswl.files.collated_path('{serun}', 'badlist')
eu.ostools.makedirs_fromfile(goodfile)

print >>stderr,"Found %s/%s problems" % (len(badlist),nf)

print >>stderr,"Writing goodlist:",goodfile
eu.io.write(goodfile,goodlist)
print >>stderr,"Writing badlist:",badfile
eu.io.write(badfile,badlist)
    """.format(serun=serun)

    d=deswl.files.wq_dir(serun)
    f=os.path.join(d,'check-reduce.py')
    print >>stderr,"writing check reduce script:",f
    with open(f,'w') as fobj:
        fobj.write(text)




def main():

    options, args = parser.parse_args(sys.argv[1:])
    if len(args) < 1:
        parser.print_help()
        sys.exit(45)

    serun=args[0]

    if options.byccd:
        by_expname=False
        modnum=1000
    else:
        write_reducer(serun)
        return
        by_expname=True
        modnum=20

    rc=deswl.files.Runconfig(serun)
    rc.verify()
    sfobj=ShearFiles(serun, fs=options.fs)
    all_files=sfobj.get_flist(by_expname=by_expname)
    nf=len(all_files)

    print >>stderr,'processing:',nf
    for i,el in enumerate(all_files):
        if ((i+1) % modnum) == 0:
            verbose=True
            print '%d/%d' % (i+1,nf)
        else:
            verbose=False

        if by_expname:
            fdict=all_files[el]
        else:
            fdict=el

        sj=SEWQJob(serun,fdict,groups=options.groups,verbose=verbose)
        sj.write()

main()

