#!/usr/bin/env python
"""
_prodAgent-config_

Command line tool for generating a ProdAgentConfiguration.

Requires that PRODAGENT_CONFIG be set to provide the location
of the configuration file being created

"""

import os
import socket
import sys
import getopt
import time

from ProdAgentCore.Configuration import ProdAgentConfiguration

_Timestamp = time.strftime("%d-%M-%Y")

def usage():
    """print usage info"""
    strg = "Usage: prodAgent-new-config <options>\n"
    strg += " --config=<configFileName> : Config will be written into\n"
    strg += "   file provided, else it will be written to $PRODAGENT_CONFIG\n"
    strg += " --component=comp1,comp2,comp3\n"
    strg += "  If no options are provided the default component list is used\n"
    strg += "  If --components is supplied, the list of components is\n"
    strg += "  taken to be a comma seperated list of components to be used\n"
    
    print strg

valid = ['components=', "config="]
try:
    opts, args = getopt.getopt(sys.argv[1:], "", valid)
except getopt.GetoptError, ex:
    print str(ex)
    usage()
    sys.exit(1)

configFile = None
componentList = []

for opt, arg in opts:
    if opt == "--components":
        compList = arg.split(',')
        for item in compList:
            componentList.append(item.strip())
    if opt == "--config":
        configFile = arg



if configFile == None:
    configFile = os.environ.get("PRODAGENT_CONFIG", None)
    if configFile == None:
        print "Configfile not found:"
        print "Must be provided either via $PRODAGENT_CONFIG env var"
        print "Or via --config option"
        sys.exit(1)

homeDir=os.environ.get("PRODAGENT_ROOT")
if  homeDir == None:
    print "PRODAGENT_ROOT variable not found:"
    print "Must be provided either via $PRODAGENT_ROOT env var"
    sys.exit(1)


coreFields = {
    #  //
    # // Core pieces: ProdAgent, ProdAgentDB, MessageService, JobStates
    #//               Local scope DBS
    "ProdAgent": {
    "ProdAgentWorkDir": os.getcwd(),
    "ProdAgentName" : "ProdAgent@%s" % socket.gethostname(),
    "ProdAgentCert" : "/home/fvlingen/.globus/client.pem",
    "ProdAgentKey"  : "/home/fvlingen/.globus/clientkey.pem",
    "ProdAgentRunOffset": '10',
    },
        

    "ProdAgentDB": {
    'dbType' : 'mysql',
    'dbName':'ProdAgentDB',
    'host':'localhost',
    'user':'ProdAgentUser',
    'passwd':'ProdAgentPass',
    'socketFileLocation':'/var/lib/mysql/mysql.sock',
    'portNr':'',
    'refreshPeriod' : 4*3600 ,
    'maxConnectionAttempts' : 5,
    'dbWaitingTime' : 10 ,
    'schemaLocation': "$PRODAGENT_ROOT/share/ProdAgentDB.sql"
    },

    "BOSS": {
    'BossLiteschemaLocation': "$PRODCOMMON_ROOT/share/setupDatabase.sql"
    },
    
    "MessageService" : {
    "pollInterval" : 5, 
    },

    "JobStates" : {
    "maxRetries":10,
    "mergeMaxRetries":10
    },

    'LocalDBS' :{
    "DBSURL": "None",
    "ReadDBSURL": "None",
    },

    "GlobalDBSDLS" : {
    "DBSURL": "https://cmsdbsprod.cern.ch:8443/cms_dbs_prod_global_writer/servlet/DBSServlet",
    "ReadDBSURL": "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet", 
    },

    "PhEDExConfig" : {
    "PhEDExDropBox" : None,
    "DBPARAM" : None,
    },
    
    }# end core Fields


componentFields = {

    "JobCreator" : {
    "ComponentDir" : None,
    "CreatorName" : "testCreator",
    "GeneratorName" : "Default",
    "CreatorPluginConfig": os.path.join(os.path.dirname(configFile), "CreatorPluginConfig.xml"),
    "RssFeed": "no",

    },
    "JobSubmitter" : {
    "ComponentDir" : None,
    "SubmitterName" : "noSubmit",
    "SubmitterPluginConfig" : os.path.join(os.path.dirname(configFile),"SubmitterPluginConfig.xml"),
    "RssFeed": "no",
    },

    "CleanUpScheduler" : {
    "ComponentDir" : None,
    "QueueJobMode" : "True",
    "PollActive"   : "True",
    "LFNLimit"     : "10",
    "CleanUpInterval" : "3600"
    },


    "DBSInterface" : {
    "ComponentDir" : None,
    # Local DBS settings come from LocalDBS core block 
    "CloseBlockSize" : None,
    "CloseBlockFiles": 100,
    "skipPhEDExInjection" : "True",
    "skipGlobalMigration" : "False",
    "DropBranches"        : "True",
    "RssFeed": "no",
    },

  

    "JobTracking" : {
    "ComponentDir" : None,
    "PollInterval" : 300,
    "QueryInterval" : 3,
    "jobsToPoll" : 2000,
    "PoolThreadsSize" : 5,
    "RssFeed": "no",
    },

    'MergeSensor' : {
    "ComponentDir" : None,
    "StartMode" : "warm",
    "PollInterval" : 60,
    "MaxMergeFileSize" : 2000000000,
    "MinMergeFileSize" : 1500000000,
    "MergeSiteWhitelist" : None,
    "MergeSiteBlacklist" : None,
    "FastMerge" : "no",
    "CleanUp" : "no",
    "MaxInputAccessFailures" : 1,
    "QueueJobMode": "True",
    "MergePolicy": "RunNumberPolicy",
    "RssFeed": "no",
    # Local DBS settings come from LocalDBS core block 
    },

    'MergeAccountant' : {
    "ComponentDir" : None,
    "Enabled" : "yes",
    "RssFeed": "no",
    },

    'ErrorHandler' : {
    "ComponentDir" : None,
    "MaxCacheDirSizeMB" : 80,
    "DelayFactor" : 100,
    "ReportAction" : "noMove",
    "RssFeed": "no",
    "QueueFailures": False,
    },

    'JobCleanup'   : {
    "ComponentDir" : None,
    "FailureArchive":None,
    "SuccessArchive":None,
    "RssFeed": "no",
    },

    'AdminControl' : {
    "ComponentDir" : None,
    "AdminControlHost" : "127.0.0.1",
    "AdminControlPort" : 8081,
    "RssFeed": "no",
    },


    "ProdMgrInterface" : {
    "ComponentDir" : None,
    "JobSpecDir" : '/tmp',
    "WorkflowSpecDir" : '/tmp',
    "JobSize" : '10',
    "JobCutSize" : '500',
    "JobInjection" : 'buffer',
    "ParallelRequests" : "1",
    "ProdMgrs":'https://lxgate42.cern.ch:8443/clarens/',
    "QueueLow" : "100",
    "QueueHigh" : "1000",
    "RetrievalInterval":"01:00:00",
    "QueueInterval":"00:01:00",
    "AllocationSize" : "10",
    "AgentTag":"please insert a proper value",
    "Locations":"please insert a proper value",
    "ProdMgrFeedback":"direct",
    "RssFeed": "no",
    },

    "JobQueue" : {
    "ComponentDir" : None,
    "BulkMode": "False",
    "VerifySites": "False",
    },
                                                                                
    "RssFeeder" : {
    "ComponentDir" : None,
    "ItemListLength" : 100,
    "Port" : 8100,
    },
    
    "ProdMon" : {
    "ComponentDir" : None,
    "RssFeed" : "no",
    "Team" : "Unknown",
    "exportMaxBatchSize" : "500",
    "exportInterval" : "00:05:00",
    "exportEnabled" : "False",
    "DashboardURL" : "http://dashb-cmspa.cern.ch/dashboard/request.py/getPAinfo",
    "expireEnabled" : False,
    "expireInterval" : "01:00:00",
    "expireRecordAfter" : "96:00:00",
    "expireUnexported" : False,
    },

    "JobKiller" : {
    "ComponentDir" : None,
    "KillerName" : None,
    "RssFeed" : "no",
    },

	"ResourceMonitor" : {
	"ComponentDir" : None,
	"MonitorPluginConfig" : os.path.join(os.path.dirname(configFile),"MonitorPluginConfig.xml"),
	"MonitorName" : None,
	"PollInterval" : "01:00:00",
	},

    "GetOutput" : {
    "ComponentDir" : None,
    "PollInterval" : 300,
    "GetOutputPoolThreadsSize" : 5,
    "jobsToPoll" : 1000,
    "OutputLocation" : "local",
    'maxGetOutputAttempts' : 3,
    'retryDelay' : 12,
    'skipWMSAuth' : True,
    "RssFeed" : "no",
    },

	"JobEmulator" : {
	"ComponentDir" : None,
	"JobAllocationPlugin" : "LoadBalanceAllocationPlugin",
	"JobCompletionPlugin" : "RandomCompletionPlugin",
	"JobReportPlugin" : "EmulatorReportPlugin",
	"avgCompletionTime" : "01:00:00",
	"avgCompletionPercentage" : "0.90",
	"avgEventProcessingRate" : "0.95",
	},
	
	"LogCollector" : {
	"ComponentDir" : None,
	"Enabled" : False,
	"QueueJobMode" : False,
	"pollInterval" : "96:00:00",
	"logLifetime" : "24:00:00",
	"maxErrors" : 3,
	"maxLogs" : 200,
	},
	
	"WorkflowInjector" : {
	  "ComponentDir" : None,
	  "Plugin" : "RequestFeeder"
	  },
	
    }# end componentFields

#  //
# // Map of block names to comments for the block.
#//  Comments get inserted into the config file as XML comments
#  //and provide docs for people who want to poke around in there
# //
#//
comments = {
    "ProdAgentDB" : \
"""
You should only supply either the portNr OR socketFileLocation
If you use ports put either leave the value parameter in 
'prodAgent-edit-config' of socketFileLocation empty, or if
you directly edit the xml config file, use a double double quote in the value attribute
of the socketFileLocation parameter.

If the mysqldb is not on the same machine, make sure the 
permissions are set correctly (
grant all privileges on *.* to 'root'@'machine.domain' identified by 
'pass' with grant option;
flush privileges;
show grants for 'root'@'machine.domain';

Beware usernames in mysql can be too long.
""",
   "JobSubmitter" : \
"""
SubmitterName values you can use: condorg, condor, lcg, lsf, lxb1125, and noSubmit
""",
   "CleanUpScheduler" : \
"""
Configuration for cleanupscheduler component to initiate cleanup jobs
-PollActive : If True then this component will be enabled otherwise it won't initiate any cleanup job
-QueueJobMode :  If True then cleanup jobs will run through JobQueue component
-cleanUpInterval : Number of seconds after which cleanup cycle be recalled
-LFNLimit : Number of max lfn's in one cleanup jobspec      

""",      

   "ErrorHandler" : \
"""
-MaxCacheDirSizeMB : The maximum size a cache dir can have 
before it is pruned, to prevent taking up to much space.
If it reaches this size the next submision/run failure will
-trigger an intermediate cleanup event.
-DelayFactor: A factor (in seconds) multiplied with the number of failure
of the job that is used to submit new submission events with a delay.
-ReportAction: When a framework report comes in the error handler can decide
to ignore it (noMove), or to move it (move). Default is set to "noMove".
Caution! when setting it to: "move". Multiple components might read the 
FrameworkJobReport.xml. This action is not yet secured using the trigger
synchronization.
""",
  "JobStates"      : \
"""
maxRetries: The maximum number of retries for a certain job(spec). If there
is a failure in creation,submission,running,etc.. a failure event will be triggered
and the job will restart (retry). Once the maximum is reached a general job failure
will be triggered. NOTE: this number needs to be bigger than 0 (1 and up). 
mergeMaxRetries: The maximum number of retries for a certain merge job(spec). If there
is a failure in creation,submission,running,etc.. a failure event will be triggered
and the job will restart (retry). Once the maximum is reached a general job failure
will be triggered. NOTE: this number needs to be bigger than 0 (1 and up). 
""" ,
  "JobCleanup"     :\
"""
FailureArchive : If jobs fail to many times you want to tar the job cache and move it
to another location (dir) for post mortem inspection and to minimize the risk of the 
job cache directory growing to large (to many files) and slowing down production.
SuccessArchive : After a job successfully finishes remainders of its log files
are tarred and moved to an archive for possible post mortem analysis.
"""  ,
  "ProdMgrInterface":\
"""
-JobSpecDir: The location to which the job specs will be downloaded created
at the prodmgr.
-(deprecated!) JobSize: size (in events). ProdAgent retrieves jobspecs from the prodmgr 
associated to an allocation. If the JobSize is set the jobspecs associated
to an allocation will have this number of events (or less if there are not
enough events in the allocation. If the parameter is set to -1 it will use the 
size of the allocation the job is associated to.
-JobCutSize: size (in events) the job associated to an allocation, retrieved
 from the prodmgr will be cut to. E.g.: If JobCutSize=12, and the jobspec
retrieved from the prodmgr associated to an allocation is 100, this will result
in 9 jobs. 8 with 12 events and one with 4 events.
-ProdMgrs: The prodmgrs this prodagent will receive requests from. This is a comma
separated list of urls.
-JobInjection : If set to 'direct' , CreateJob events are thrown when retrieving
work from the prodmanager (retrieved work goes directly to the submission queue. 
If set to 'buffer', QueueJob events are thrown when retrieving
work form the prodmganager (retrieved work is buffered in the jobqueue.
-RetrievalInterval: The time between retrievals of requests from prodmgrs using the 
HH:MM:SS format. Every so often the prodagent will contact its associated prodmgrs
to retrieve requests if available. 
-QueueInterval. The time between two events that check the queue lenght for 
determining if we need to acquire more work.
-AllcoationSize. The maximum number of jobs per allocation. This is to prevent
large allocations (unless this is set to very large), and prevent slow updates
at the prodmgr level.
-ParallelRequests : The number of requests the prodagent will process in parallel.
e.g. if it is 4, it will acquire requests from the prodmgr using a round robin 
approach on the requests.
-QueueLow : the threshold after which the queue will be automatically filled by the 
ProdMgrInterface
-QueueHigh: the maximum number of jobs allowed in the queue.
-AgentTag: the id used to identify the agent. Although the DN of the ProdAgent certificate
uniquely identifies the prodagent, the AgentTag is a short hand version that is used for
request retrieval. Potentially multiple prodagents can have the same agent tag (which means
they will be handling the same requests)
-Locations: This parameter contains a comma separated lists of locations (storage elements)
this prodagent has access to. The elements in this list have to correspond with the entries
found in the DLS associated to fileblocks (otherwise this prodagent will not get work
for file based requests).
-ProdMgrFeedback: This parameters if set to "direct" will imediately relay information back
to the prodmgr after a job is successfully processed and before the data is safely stored. If
it is set to "delay" it will wait until the data is properly merged.
""",
  "MergeAccountant"     :\
"""
-Enabled: Set this parameter to -yes- to get full accounting support. When it is set to
-no-, only triggering for cleanups of files will be generated, with no updates on
internal database.
""",
  "ProdMon"              :\
  """
   -Team: production team name 
   -DisableExport: Set to true to disable monitoring data export.
   -DashboardURL: external monitoring export URL.
   -exportMaxBatchSize: Max number of jobs to export in each cycle.
   -exportInterval: Time between export cycles in HH:MM:SS.
   -expireEnabled: Remove old jobs from the local ProdMon Database.
   -expireRecordAfter: Age before a job record is considered for deletion.
   -expireUnexported: If False only consider records that have been
     exported to dashboard for deletion.
   -expireInterval: Time between expire cycles in HH:MM:SS.
   """,
  "JobEmulator"              :\
  """
  To run job emulator, JobSumitter component's SubmitterName need to be set to JobEmulatorBulkSubmitter.
  Also, CondorTracker component's TrackerPlugin has to be set to JobEmulatorTracker.
  Followings are parameters for JobEmulator component.
   -JobReportPlugin: Set Job report format for jobEmulator, currently support EmulatorReportPlugin 
   which uses framework report.
   -JobAllocationPlugin: production team name 
   -JobCompletionPlugin: two options are Available RandomCompletionPlugin and BlakHoleNodePlugin.
   RandomCompletion Plugin completes the job with a given average completion time (avgCompletionTime)
   and a given average completion percentage (avgCompletionPercentage).
   BlackHoleNodePlugin is the same as RandomeCompletionPlugin except jobs fail right away all the time
   in one node.
   -avgCompletionTime: average job completion time. Format: "00:02:00" - 2 minutes.
   -avgCompletionPercentage: average job completion percentage. Format: "0.9" - 90 %
   -avgEventProcessingRate: the rate indicates the rate of the number of jobs completing all the events
    among all the successful jobs. The number of incomplete events among the jobs follows the gauss distribution 
    with maximun number (totalEvent -1, minimun 1) the mean value (70% of total event) and 
    standard deviation (15% of the width of total event) is hard coded. If it is necessary they can be parameterized. 
   """,
   "LogCollector"			:\
   """
   Will periodically poll for logs that can be collected at sites for the active workflows
    -Enabled: Whether or not to poll ofr logs to collect
    -QueueJobMode: Set to True to use the JobQueue
    -pollInterval: How often to poll for logs
    -logLifetime: The period of time for which a log will be left at a site
    -maxErrors: The number of jobs which will attmept to collect a log before it is considered lost
    -maxLogs: How many logs to collect in one job
   """,
    }

if componentList == []:
    componentList = componentFields.keys()

print "Writing Config File: %s" % configFile
print "Using Components:"
for item in componentList:
    print "  ",item

if os.path.exists(configFile):
    print "Config File Already Exists: %s" % configFile
    backup = "%s.BAK.%s" % (configFile, _Timestamp)
    os.system("/bin/cp %s %s" % (configFile, backup))
    print "Config File backed up to:"
    print " ", backup



config = ProdAgentConfiguration()

#  //
# // Process core fields
#//
for name, values in coreFields.items():
    cfgDict = config.getConfig(name)
    cfgDict.update(values)
    if comments.has_key(name):
        cfgDict.comment = comments[name]


prodAgentBlock = config.getConfig("ProdAgent")

# NOTE: boss post processing
# NOTE: needs to be done better (how, withouth designating BOSS a component?)
cfgDict = config.getConfig("BOSS")
cfgDict['configDir']=os.path.join(prodAgentBlock['ProdAgentWorkDir'],"BOSS/config")


def processField(fieldName, fieldDict):
    """
    _processField_

    for a given field create a config component for it

    """
    cfg = config.newComponentConfig(fieldName)
    cfg.update(fieldDict)
    if comments.has_key(fieldName):
        cfg.comment = comments[fieldName]

    #  //
    # // Ensure that ComponentDir param is always the unholy
    #//  union of ProdAgent['ProdAgentWorkDir'] and the component
    #  //name. That is it cannot be set by the user.
    # //
    #//
    cfg['ComponentDir'] = os.path.join(prodAgentBlock['ProdAgentWorkDir'],
                                       fieldName)
    
    return

#  //
# // Process components
#//
for component in componentList:
    compDict = componentFields.get(component, None)
    if compDict == None:
        msg = "Unknown Component: %s\n" % component
        msg += "Valid Components are:\n"
        msg += "%s\n" % componentFields.keys()
        print msg
        sys.exit(1)

        
    processField(component, compDict)


#  //
# // Save config file
#//
config.saveToFile(configFile)



#  //
# // Generate plugin configs
#//
os.system("prodAgent-new-pluginconfig")







