# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
# Project:     DBASE/MCStatTools
# Name:        GenXmlLogParser.py
# Purpose:     Parse GeneratorLog.xml and provide a mechanism for merging the 
#              the generator counters. Parse jobDescription.xml files to give
#              detailed informations about the corresponding production setup
#
# Author:      Alex T. GRECU <alexandru.grecu@gmail.com>
#
# Created:     29.07.2013 [dd.mm.yyyy]
# License:     see http://docs.python.org/license.html
# TODO:        review name to id mapping algorithms for GenStats and Gen* classes; 
#              develop JDParser as needed
# Last mod:    26.11.2013 [dd.mm.yyyy] by Alex Grecu
#-------------------------------------------------------------------------------
#!/usr/bin/env python

import os
import sys
import xml.dom.minidom as xdom
import re
import math
import urllib
import shutil

#to correct malformed XML on-the-fly
correctiveRes = {
                 r'crosssection\s+id = (\d+)\s*>':r'crosssection id="\1">', #may be removed when all available logs are properly formed!
                 r'[\x00-\x1f\x7f-\xff]':'' #keep this filter until utf-8 is properly supported if ever
                }
# fill this list with object names for which merging must be done regardless of their presence in the GenStats to be merged
buggyStatObjects = []
bSO_defaults = range(401,420) + [201, 202, 221]
buggyStatObjects += map(lambda x: "pp%d" % (x), bSO_defaults)

class GenXmlNode(object):
    xElementName = None
    xAttribNames = []
    xMandatoryAttribNames = []
    xChildNames = []
    ###
    def __init__(self):
        self._xElem = None
        self._attribs = {}
        self._children = {}
        self._value = '' # property will contain additive quantities only
        self._id = None
    ###
    def getAttribute(self, attrId):
        ret = None
        if self.hasAttribute(attrId):
            ret = self._attribs[attrId]
        return ret
    ###
    def hasAttribute(self, attrId):
        return self._attribs.has_key(attrId) and not self._attribs[attrId] is None
    ###
    def hasChild(self, childId):
        childId = unicode(childId)
        return self._children.has_key(childId) and not self._children[childId] is None
    ###
    def getChild(self, childId):
        ret = None
        childId = unicode(childId)
        if self.hasChild(childId):
            ret = self._children[childId]
        return ret
    ###
    def getId(self):
        return self._id
    ###
    def genericId(self, **params):
        '''
        Returns the generic (hard-coded) counter id based on params passed 
        as a dictionary with key names given by corresponding XML node 
        attributes. If no data is provided this is just an alias for getId()
        '''
        raise NotImplementedError('This method is only implemented by subclasses.')
    ###
    def getValue(self):
        return self._value
    ###
    def isBoundToXml(self):
        return (not self._xElem is None)
    ###
    def _getInt(self, value):
        if value is None:
            return None
        if isinstance(value, basestring):
            value = value.strip()
        try:
            ret = int(value)
        except ValueError, exx:
            ret = None
        return ret
    ###
    def _getFloat(self, value):
        if value is None:
            return None
        if isinstance(value, basestring):
            value = value.strip()
        try:
            ret = float(value)
        except ValueError, exx:
            ret = None
        return ret
    ###
    def __convertProperties(self):
        raise NotImplementedError('This method is only implemented in derived classes.')
    ###
    def fromXmlElement(self, xElem):
        if not isinstance(xElem, xdom.Element):
            raise TypeError('Invalid argument, xml.dom.minidom.Element expected.')
        if xElem.nodeType != xdom.Element.ELEMENT_NODE or \
           xElem.nodeName != self.xElementName or \
           not xElem.hasChildNodes():
            raise ValueError('Invalid or malformed <%s> node.' % self.xElementName)
        for an in self.xAttribNames:
            if not xElem.hasAttribute(an):
                msg = '<%s> element missing \'%s\' attribute.' % (self.xElementName, an)
                if not an in self.xMandatoryAttribNames:
                    print(msg)
                else:
                    raise ValueError('Malformed ' + msg)
        self._xElem = xElem
        cn = xElem.firstChild
        while not cn is None:
            if cn.nodeType == xdom.Element.TEXT_NODE and not cn.nodeValue is None and len(cn.nodeValue.strip()) > 0:
                self._value += cn.nodeValue.strip() # will append!
            if cn.nodeType != xdom.Element.ELEMENT_NODE:
                cn = cn.nextSibling
                continue
            if not cn.nodeName in self.xChildNames:
                print('Unknown child <%s> of <%s> detected. Ignoring...' % (cn.nodeName, self.xElementName))
            else:
                nv = None
                if cn.hasChildNodes():
                    vnn = cn.firstChild
                    while not vnn is None and vnn.nodeType != xdom.Element.TEXT_NODE:
                        vnn = vnn.nextSibling
                    nv = vnn.nodeValue.strip()
                self._children[cn.nodeName] = nv
            cn = cn.nextSibling
        for an in self.xAttribNames:
            self._attribs[an] = None
            if xElem.hasAttribute(an):
                self._attribs[an] = xElem.getAttribute(an)
    ###
    def hasChildElement(self, ceName, getList=False):
        if not self.isBoundToXml() or not self._xElem.hasChildNodes():
            if getList:
                return []
            else:
                return False
        passChNodes = []
        for cn in self._xElem.childNodes:
            if cn.nodeType != xdom.Element.ELEMENT_NODE:
                continue
            if cn.nodeName == ceName:
                passChNodes.append(cn)
        if getList:
            ret = passChNodes
        else:
            ret = (len(passChNodes) > 0)
        return ret
    ###
    def toXmlElement(self, XNode):
        if not self.isBoundToXml():
            self._xElem = XNode.ownerDocument.createElement(self.xElementName)
            XNode.appendChild(self._xElem)
        if not self.isBoundToXml():
            raise ValueError('Invalid value for associated XML node.')
        if len(self.xMandatoryAttribNames) > 0:
            for an in self.xMandatoryAttribNames:
                self._xElem.setAttribute(an, 'not set')
        for cn in self.xChildNames:
            tcns = self.hasChildElement(cn, True)
            if len(tcns) == 0:
                tce = self._xElem.ownerDocument.createElement(cn)
                self._xElem.appendChild(tce)
            elif len(tcns) > 1:
                for ctn in tcns[1:]:
                    self._xElem.removeChild(ctn)
                tce = tcns[0]
            else:
                tce = tcns[0]
            tln = list(tce.childNodes)
            for ctn in tln:
                tce.removeChild(ctn)
    ###
    
class GenGlobalParam(GenXmlNode):
    '''
    Implements the XML-Python interface for global parameters in the .xml log file.
    On 'addition' retains value of first operand and a warning is printed if 
    the value of second operand (as stripped, case insensitive string) differs. May be used to 
    enforce validity of logs being merged. The Id property is computed by default 
    by adding the 'xLog_' prefix (e.g. version -> xLog_version) and used as key 
    in the final GenStats dictionary.
    '''
    def __init__(self, xNode=None):
        for bt in GenCounter.__bases__:
            bt.__init__(self)
        if not xNode is None:
            self.fromXmlElement(xNode)
    ###
    def fromXmlElement(self, xElem):
        self.xElementName = getattr(xElem, 'nodeName', None)
        if self.xElementName is None:
            raise TypeError('Can get tag name for XML element of type %s.' % (str(xElem.__class__)))
        super(GenGlobalParam, self).fromXmlElement(xElem)
        self._id = self.genericId()
    ###
    def genericId(self, **params):
        '''
        See GenXmlNode method for help.
        '''
        return ('xLog_' + self.xElementName)
    ###
    def __add__(self, other):
        if other is None and self._id in buggyStatObjects:
            return self
        ov = other.getValue()
        if not isinstance(ov, basestring):
            print("Value type (%s) of <%s> node differs from previous file. Will ignore change..." % (str(ov.__class__),self.xElementName))
        tv = self.getValue()
        if ov.strip().lower() != tv.strip().lower():
            print("Value of <%s> node differs from previous file. \n (this) \"%s\" vs. (new) \"%s\"\nWill ignore change..." % (self.xElementName, tv, ov))
        return self
    ###
    def recompute(self, coll):
        pass
    ###
    def getCounters(self): #to ensure information is copied in the dict use to generate the web page
        return [(self.getId(), self.getValue()),]
    ###
    def toXmlElement(self, xNode):
        xDoc = xNode.ownerDocument
        if not self.isBoundToXml() or self._xElem.ownerDocument != xDoc:
            self._xElem = xDoc.createElement(self.xElementName)
            xNode.appendChild(self._xElem)
        if not self.isBoundToXml():
            raise ValueError('Invalid value for associated XML node.')
        tvn = xDoc.createTextNode(self.getValue().strip())
        self._xElem.appendChild(tvn)
    ###

class GenCounter(GenXmlNode):
    '''
    XML-Python interface for <counter> nodes. The Id in the final dictionary is 
    given by reverse mapping of the mandatory name attribute from idmap dictionary. 
    '''
    idmap = { 'L1' : "all events (including empty events)",
             'L2' : "events with 0 interaction",
             'N1' : "generated events", 'N2' : "generated interactions",
             'N3' : "accepted events", 'N4' : "interactions in accepted events",
             'N5' : "generated interactions with >= 1b",
             'N6' : "generated interactions with >= 3b",
             'N7' : "generated interactions with 1 prompt B",
             'N8' : "generated interactions with >= 1c",
             'N9' : "generated interactions with >= 3c",
             'N10' : "generated interactions with >= prompt C",
             'N11' : "generated interactions with b and c",
             'N12' : "accepted interactions with >= 1b",
             'N13' : "accepted interactions with >= 3b",
             'N14' : "accepted interactions with 1 prompt B",
             'N15' : "accepted interactions with >= 1c",
             'N16' : "accepted interactions with >= 3c",
             'N17' : "accepted interactions with >= prompt C",
             'N18' : "accepted interactions with b and c",
             'N21' : "z-inverted events",
             'I13' : "generated (bb)", 'I24' : "accepted (bb)",
             'I33' : "generated (cc)", 'I42' : "accepted (cc)"
            }
    xElementName = 'counter'
    xAttribNames = ['name',]
    xMandatoryAttribNames = ['name',]
    xChildNames = ['value',]
    ###
    def __init__ (self, xNode = None, id='NS0', name=None, value=-1):
        for bt in GenCounter.__bases__:
            bt.__init__(self)
        if not xNode is None:
            self.fromXmlElement(xNode)
        else:
            self._id = id
            if not name is None:
                self._attribs['name'] = name
            if value > -1:
                self._children['value'] = value
                self._value = value
    ###
    def genericId(self, **params):
        '''
        See GenXmlNode method for help.
        '''
        if len(params) == 0 or not params.has_key('name'):
            return self.getId()
        id = ''
        name = params['name']
        for (id, nn) in GenCounter.idmap.items():
            if name.lower() == nn.lower():
                break
        if len(id) == 0:
            return self.getId()
        return id
    ###
    def __convertProperties(self):
        xmlName = self.getAttribute('name')
        self._id = self.genericId(name=xmlName)
        sv = self.getChild('value')
        rt = self._getInt(sv)
        if rt is None:
            rt = 0
        self._children['value'] = rt
        self._value = self._children['value']
    ###
    def fromXmlElement(self, xElem):
        super(GenCounter, self).fromXmlElement(xElem)
        self.__convertProperties()
    ###
    def getCounters(self):
        return [(self.getId(), self._children['value']),]
    ###
    def __add__ (self, other): #result will be unbound to any xml node!
        # enforce check of counter being on the 'specials' list before pseudo-merging
        if other is None and self._id in buggyStatObjects:
            return self
        if not isinstance(other, GenCounter):
            raise TypeError('You cannot add GenCounter instances to %s.' % (other.__class__))
        if self.getId() != other.getId():
            raise ValueError('Makes no sense to add counter \'%s\' to counter \'%s\'.' % (self.getId(), other.getId()))
        vname = self.getAttribute('name')
        if vname != other.getAttribute('name'):
            print('\'name\' attributes differ. Will consider value from first operand: \'%s\'.' % (vname))
        sumvalue = self.getValue() + other.getValue()
        return GenCounter(id=self.getId(), name=vname, value=sumvalue)
    ###
    def recompute(self, coll):
        pass    #for compatibility with other counter types
    ###
    def toXmlElement(self, xNode):
        super(GenCounter, self).toXmlElement(xNode)
        self._xElem.setAttribute('name',GenCounter.idmap[self._id])
        vn = self.hasChildElement('value', True)
        tvn = vn[0]
        dn = self._xElem.ownerDocument.createTextNode(str(self.getChild('value')))
        tvn.appendChild(dn)
    ###

class GenXSection(GenXmlNode):
    '''
    XML-Python interface for <crosssection> nodes. The Id in the final dictionary is 
    computed by adding 'pp' prefix to the mandatory 'id' attribute. The final 'value'
    is computed as mean of intermediary values stored in _children['tvalues']! The 
    'description' value is kept from the first parsed log file.
    '''
    xElementName = 'crosssection'
    xAttribNames = ['id',]
    xMandatoryAttribNames = ['id',]
    xChildNames = ['description','generated','value']
    ###
    def __init__(self):
        for bt in GenXSection.__bases__:
            bt.__init__(self)
        self._value=-1.0 # the medium value when summed
        self._procCount = -1
    ###
    def setProcId(self, procId):
        if isinstance(procId, basestring):
            procId = int(procId.strip())
        self._procid = procId
        self._id = self.genericId(id=self._procid)
        self._attribs['id'] = procId
    ###
    def getProcId(self):
        '''
        Value of 'id' XML node attribute with no prefix added.
        '''
        return self._procid
    ###
    def genericId(self, **params):
        '''
        See GenXmlNode method for help.
        '''
        if len(params) == 0 or not params.has_key('id'):
            return self.getId()
        return 'pp%s' % (str(params['id']))
    ###
    def __convertProperties(self):
        self.setProcId(self.getAttribute('id'))
        dd = self.getChild('description')
        if not dd is None:      #cleaning the string of tailling non-ascii characters
            self._children['description'] = dd.strip('" ')
        rt = self._getInt(self.getChild('generated'))
        if rt is None:
            rt = 0
        self._children['generated'] = rt
        rt = self._getFloat(self.getChild('value'))
        if rt is None:
            rt = 0.0
        self._children['value'] = rt
    ###
    def fromXmlElement(self, xElem):
        super(GenXSection, self).fromXmlElement(xElem)
        self.__convertProperties()
    ###
    def getCounters(self):
        return [(self.getId(), self._children['value']),]
    ###
    def __add__ (self, other):
        #very important to maintain correct number of xsection values being added -> correct mean value
        if other is None and self._id in buggyStatObjects:
            return self
        if not isinstance(other, GenXSection):
            raise TypeError('You cannot add GenXSection instances to %s. Counter \'%s\' not on ignored list.' % (other.__class__, self._id))
        if self.getId() != other.getId():
            raise ValueError('Makes no sense to add cross-sections for processes \'%d\' and \'%d\'.' % (self.getProcId(), other.getProcId()))
        ret = GenXSection()
        ret.setProcId(self.getProcId())
        ret._children = dict(self._children)
        sumBefore = ret._children['generated']
        ret._children['generated'] += other._children['generated']
        sumAfter = ret._children['generated']
        if ret._children.has_key('tvalues'):
            ret._children['tvalues'].append(other._children['value'])
        else:
            ret._children['tvalues'] = [self._children['value'], other._children['value']]
        if ( 0 != sumAfter ):
            ret._children['value'] = (float(sumBefore)*ret._children['value']+float(other._children['generated'])*other._children['value'])/float(sumAfter)
        else:
            ret._children['value'] = 0.
        ret._value = -1.0 #use recompute to get final value in this field.
        return ret
    ###
    def toXmlElement(self, xElem):
        super(GenXSection, self).toXmlElement(xElem)
        self._xElem.setAttribute('id', str(self.getProcId()))
        for ten in GenXSection.xChildNames[0:2]:
            tn = self.hasChildElement(ten, True)[0]
            ttn = self._xElem.ownerDocument.createTextNode(str(self.getChild(ten)))
            tn.appendChild(ttn)
        tn = self.hasChildElement('value', True)[0]
        ttn = self._xElem.ownerDocument.createTextNode('%.5g' % (self.getChild('value')))
        tn.appendChild(ttn)
    ###
    def recompute(self, coll):
        if self._children.has_key('tvalues'):
            sum = 0.0
            for vv in self._children['tvalues']:
                sum += vv
            sum = sum/len(self._children['tvalues'])
            self._children['value'] = sum
            self._value = sum
            del self._children['tvalues']
    ###

class GenEfficiency(GenXmlNode):
    '''
    XML-Python interface for <efficiency> nodes. The Id in the final dictionary is 
    given by reverse mapping of the mandatory name attribute from idmap dictionary.
    'value' and 'error' are computed on !final! summed up values. 
    '''
    idmap = { 'E0' : "full event cut",
              'E1' : "generator level cut",
              'E2' : "generator particle level cut",
              'E3' : "generator anti-particle level cut"
            }
    cntNameMap = {'E0':('N19','N20'), 'E1':('I1','I2'), 'E2':('S1','S2'), 'E3':('S3','S4')}
    xElementName = 'efficiency'
    xAttribNames = ['name',]
    xMandatoryAttribNames = ['name',]
    xChildNames = ['before','after','value','error']
    ###
    def __init__(self):
        for bt in GenEfficiency.__bases__:
            bt.__init__(self)
    ###
    def getCounters(self):
        dd = dict(zip(GenEfficiency.xChildNames[0:2],GenEfficiency.cntNameMap[self.getId()]))
        return [(v,self._getInt(self._children[k])) for (k,v) in dd.items()]
            
    ###
    def genericId(self, **params):
        '''
        See GenXmlNode method for help.
        '''
        if len(params) == 0 or not params.has_key('name'):
            return self.getId()
        id = ''
        xname = params['name']
        for (id, nn) in GenEfficiency.idmap.items():
            if xname.lower() == nn.lower():
                break
        if len(id) == 0:
            return self.getId()
        return id
    ###
    def __convertProperties(self):
        nn = self.getAttribute('name')
        self._id = self.genericId(name=nn)
        rt = self._getInt(self.getChild('before'))
        if rt is None:
            rt = 0
        self._children['before'] = rt
        rt = self._getInt(self.getChild('after'))
        if rt is None:
            rt = 0
        self._children['after'] = rt
        rt = self._getFloat(self.getChild('value'))
        self._children['value'] = rt
        if rt is None:
            rt = 0.0
        rt = self._getFloat(self.getChild('error'))
        if rt is None:
            rt = 0.0
        self._children['error'] = rt
    ###
    def fromXmlElement(self, xElem):
        super(GenEfficiency, self).fromXmlElement(xElem)
        self.__convertProperties()
    ###
    def __add__ (self, other):
        if other is None and self._id in buggyStatObjects:
            return self
        if not isinstance(other, GenEfficiency):
            raise TypeError('You cannot add GenEfficiency instances to %s.' % (other.__class__))
        if self.getId() != other.getId():
            raise ValueError('Makes no sense to add different efficiencies \'%s\' and \'%s\'.' % (self.getId(), other.getId()))
        if self.getAttribute('name') != other.getAttribute('name'):
            raise ValueError('Name attributes for GenEfficiency objects may not differ.')
        ret = GenEfficiency()
        ret._id = self.getId()
        ret._attribs = dict(self._attribs)
        ret._children = dict(self._children)
        ret._children['before'] += other._children['before']
        ret._children['after'] += other._children['after']
        ret._children['value'] = -1.0
        ret._children['error'] = -1.0
        ret._value = -1.0 #use recompute to get final value in this field.
        return ret
    ###
    def recompute(self, coll):
        if self._value == -1.0 and self._children['value'] == self._children['error']:
            i1 = self.getChild('before')
            i2 = self.getChild('after')
            if i1 == 0:
                self._value = 0.0
                self._children['error'] = 0.0
            else:
                self._value = float(i2)/float(i1)
                self._children['error'] = math.sqrt(i2*(i1-i2)/i1/i1/i1)
            self._children['value'] = self._value
     ###
    def toXmlElement(self, xElem):
        super(GenEfficiency, self).toXmlElement(xElem)
        self._xElem.setAttribute('name', GenEfficiency.idmap[self._id])
        for ten in GenEfficiency.xChildNames[0:2]:
            tn = self.hasChildElement(ten, True)[0]
            ttn = self._xElem.ownerDocument.createTextNode(str(self.getChild(ten)))
            tn.appendChild(ttn)
        for ten in GenEfficiency.xChildNames[2:]:
            tn = self.hasChildElement(ten, True)[0]
            ttn = self._xElem.ownerDocument.createTextNode('%.5g' % (self.getChild(ten)))
            tn.appendChild(ttn)
    ###
class GenFraction(GenXmlNode):
    '''
    XML-Python interface for <fraction> nodes. The Id in the final dictionary is 
    given by reverse mapping of the mandatory name attribute from dictionary selected
    from 'groups' using the idmap() method return. 'value' and 'error' are recomputed 
    on final summed up 'number' values. See counter documentation for details. 
    '''
    groups = {'b':{'gp_':(('B0','B+','Bs0','b-Baryon'),(14,2),(3,2)),
                   'gp~':(('anti-B0','B-','anti-Bs0','anti-b-Baryon'),(15,2),(4,2)),
                   'g1x':(('Bc+','Bc-'),(22,1),(11,1)), 
                   'gp*':(('B(L=0,J=0)','B* (L=0, J=1)','B** (L=1, J=0,1,2)'),(46,1),(43,1))},
              'c':{'gp_':(('D0','D+','Ds+','c-Baryon'),(34,2),(25,2)),
                   'gp~':(('anti-D0','D-','Ds-','anti-c-Baryon'),(35,2),(26,2)),
                   'gp*':(('D(L=0,J=0)','D* (L=0, J=1)','D** (L=1, J=0,1,2)'),(52,1),(49,1))}
             }
    xElementName = 'fraction'
    xAttribNames = ['name',]
    xMandatoryAttribNames = ['name',]
    xChildNames = ['number','value','error']
    ###
    def __init__(self):
        GenXmlNode.__init__(self)
    ###
    def getCounters(self):
        return [(self.getId(), self._getInt(self._children['number'])),]
    ###
    def isAntiParticle(self):
        return (self._pgroup == 'gp~') or (self._part.find('anti-') > -1) or (self._part.find('~') > -1)
    ###
    def __idmap(self, fType, qgrp, pgrp, idx, pName='missing particle name'):
        cName = 'C_unknown'
        tdidx = (fType == 'accepted' and 1) or (fType == 'generated' and 2)
        if fType == 'signal':
            cName = 'I_%s' % (pName)
        else:
            idx0 = GenFraction.groups[qgrp][pgrp][tdidx][0]
            mul  = GenFraction.groups[qgrp][pgrp][tdidx][1]
            cName = 'I%d' % (idx0 + idx*mul)
        return cName
    def idmap(self, name):
        if isinstance(name, unicode):
            name = str(name)
        tokens = name.split(' ')
        if len(tokens) <= 1:
            raise ValueError('Could not find GenFraction id as name might be invalid.')
        _type = tokens[0].strip()
        _part = tokens[1].strip()
        _pgroup = 'zzz'
        _mq = 'q'
        cl = 'I%d'
        cn = -1
        if len(tokens) > 2:
            suff = ' '.join(tokens[2:])
            if suff != 'in sample':
                _part += (' ' + suff)
        if _type != "signal":
            for (q,di) in GenFraction.groups.items():
                for (igp, plist) in di.items():
                    if _part in plist[0]:
                        cn = plist[0].index(_part)
                        _pgroup = igp
                        break
                if cn > -1:
                    _mq = q
                    break
        cl = self.__idmap(_type, _mq, _pgroup, cn, _part)
        return (cl,_type,_part,_mq,_pgroup,cn)
    ###
    def genericId(self, **params):
        '''
        See GenXmlNode method for help. Duplicates GenFraction::idmap().
        '''
        if len(params) == 0 or not params.has_key('name'):
            return self.getId()
        xname = params['name']
        id = self.idmap(xname)[0]
        if len(id) == 0:
            return self.getId()
        return id
    ###
    def __convertProperties(self):
        nn = self.getAttribute('name')
        (self._id, self._type, self._part, self._mq, self._pgroup, iii) = self.idmap(nn)
        rt = self._getInt(self.getChild('number'))
        if rt is None:
            rt = 0
        self._children['number'] = rt
        rt = self._getFloat(self.getChild('value'))
        if rt is None:
            rt = 0.0
        self._children['value'] = rt
        rt = self._getFloat(self.getChild('error'))
        if rt is None:
            rt = 0.0
        self._children['error'] = rt
    ###
    def fromXmlElement(self, xElem):
        super(GenFraction, self).fromXmlElement(xElem)
        self.__convertProperties()
    ###
    def __add__ (self, other):
        if other is None and self._id in buggyStatObjects:
            return self
        if not isinstance(other, GenFraction):
            raise TypeError('You cannot add GenFraction instances to %s.' % (other.__class__))
        if self.getId() != other.getId():
            raise ValueError('Makes no sense to add different fractions with id \'%s\' and \'%s\'.' % (self.getId(), other.getId()))
        if self.getAttribute('name') != other.getAttribute('name'):
            raise ValueError('Name attributes for GenFraction objects may not differ.')
        ret = GenFraction()
        ret._id = self.getId()
        ret._type = self._type
        ret._part = self._part
        ret._pgroup = self._pgroup
        ret._mq = self._mq
        ret._attribs = dict(self._attribs)
        ret._children = dict(self._children)
        ret._children['number'] += other._children['number']
        ret._children['value'] = -1.0
        ret._children['error'] = -1.0
        ret._value = -1.0 #use recompute to get final value in this field.
        return ret
    ###
    def recompute(self, coll):
        if self._value > -1.0:
            return
        isum = self.getCounters()[0][1]
        (myqc, mygrp, myii) = self.getGroupInfo()
        if myii > -1:
            for ii in xrange(0, len(GenFraction.groups[myqc][mygrp][0])):
                if ii == myii:
                    continue
                oid = self.__idmap(self.getFracType(), myqc, mygrp, ii)
                if not coll.has_key(oid):
                    raise ValueError('Cannot compute fraction \'%s\' since counter \'%s\' is missing.' % (self.getAttribute('name'), oid))
                ps = coll[oid].getCounters()[0][1]
                if ps < 0:
                    raise ValueError('Invalid counter value for counter \'%s\'.' % (oid))
                isum += ps
        else: #for signal counters
            pn = self.getPartName()
            if self.isAntiParticle(): #change rule from particle name to anti-particle name is not clear
                pn = pn.replace('~','')
            isum = 0.0
            for fid in coll.getIdsByType('fraction'):
                pfid = fid.replace('~','') 
                if pfid.find(pn) > -1:
                    isum += coll[fid].getCounters()[0][1]
        cval = float(self.getCounters()[0][1])
        if isum == 0:
            err = 0.0
            isum = 1
        else:
            err = math.sqrt(cval*(isum - cval)/isum/isum/isum)
        self._value = cval/float(isum)
        self._children['error'] = err
        self._children['value'] = self._value
    ###
    def getPartName(self):
        return self._part
    ###
    def getFracType(self):
        return self._type
    ###
    def getGroupInfo(self):
        ii = -1
        if GenFraction.groups.has_key(self._mq) and GenFraction.groups[self._mq].has_key(self._pgroup):
            grp = GenFraction.groups[self._mq][self._pgroup][0]
            if self._part in grp:
                ii = grp.index(self._part)
        return (self._mq, self._pgroup, ii)
    ###
    def getNameFromTokens(self, type=None, part=None):
        ret = None
        if type is None:
            type = self._type
        if part is None:
            part = self._part
        if type == 'signal':
            ret = '%s %s in sample' % (type, part)
        else:
            ret = '%s %s' % (type, part)
        return ret
    ###
    def toXmlElement(self, xElem):
        super(GenFraction, self).toXmlElement(xElem)
        self._xElem.setAttribute('name', self.getNameFromTokens())
        tn = self.hasChildElement('number', True)[0]
        ttn = self._xElem.ownerDocument.createTextNode(str(self.getChild('number')))
        tn.appendChild(ttn)
        for ten in GenFraction.xChildNames[1:]:
            tn = self.hasChildElement(ten, True)[0]
            ttn = self._xElem.ownerDocument.createTextNode('%.5g' % (self.getChild(ten)))
            tn.appendChild(ttn)
    ###
class GenStats(object):
    '''
    XML-Python interface parsing/merging/saving GeneratorLog*.xml files. Parsing is perfomed by 
    GenXmlNode derived classes. Node name to class mapping is given by 'ChildrenMap' static property.
    Implements __add__ operator and partial dict class interface for addressing counters.
    '''
    RootElementName = 'generatorCounters'
    ChildrenMap = {'counter':GenCounter,
                   'crosssection':GenXSection,
                   'efficiency':GenEfficiency,
                   'fraction': GenFraction,
                   'version':GenGlobalParam,
                   'eventType':GenGlobalParam,
                   'method':GenGlobalParam,
                   'generator':GenGlobalParam
                  }
    
    ###
    def __init__(self, filepath=None):
        self._fileCounter = 1
        self._xdoc = None
        self.children = dict([(GenStats.ChildrenMap.keys()[ii],[]) for ii in xrange(0,len(GenStats.ChildrenMap.keys()))])
        self._fullpath = None
        if not filepath is None:
            if not os.path.isfile(filepath) and (filepath.startswith('http') or filepath.startswith('ftp')): #local path or URI?
                try: #if URI retrieve it and move temporary file to local dir with unique id from tmp file appended
                    ret = urllib.urlretrieve(filepath)
                    if os.path.isfile(ret[0]):
                        p = filepath.find('?')
                        turl = filepath
                        if p > -1:
                            turl = filepath[0:p]
                        basefn = os.path.split(turl)[1]
                        (basefn, text) = os.path.splitext(basefn)
                        if text.lower() != '.xml':
                            print('Potentially unknown file type \'%s\' will be passed to XML parser.' % (text))
                        basefn += '_%s' % (os.path.split(ret[0]).trim('tmp'))
                        filepath = os.path.join(os.path.abspath('.'), basefn + text)
                        shutil.copy2(ret[0], filepath)
                except IOError, iexx:
                    print(iexx.message)
                    filepath = None
            if not os.path.isfile(filepath):
                print('File \'%s\' not found. Please, check that paths provided to the parser actually exist.' % (filepath))
                filepath = None
            if not filepath is None:
                self._fullpath = filepath
                self.parseFile(self._fullpath)
    ###
    def parseFile(self, filepath):
        if not os.path.isfile(filepath):
            raise ValueError('Invalid local .xml file path.')
        try:
            xdoc = xdom.parse(filepath)
        except Exception, exx: #try to recover by correcting the XML on the fly!
            fp = open(filepath, 'rb')
            sxContent = fp.read()
            fp.close()
            sxFormedContent = sxContent
            for (reExpr, reRepl) in correctiveRes.items():
                sxFormedContent = re.sub(reExpr, reRepl, sxFormedContent)
            #print('Applied corrections to file %s ...' % (filepath))
            #print(sxFormedContent)
            xdoc = xdom.parseString(sxFormedContent)
        if xdoc.documentElement is None or xdoc.documentElement.nodeName != GenStats.RootElementName:
            raise IOError('.xml file seems invalid!',filepath)
        if not xdoc.documentElement.hasChildNodes():
            raise IOError('.xml file seems truncated. Did Gauss crash?', filepath)
        self._xdoc = xdoc
        child = xdoc.documentElement.firstChild
        obj = None
        while not child is None:
            if child.nodeType != xdom.Element.ELEMENT_NODE:
                child = child.nextSibling
                continue
            if not GenStats.ChildrenMap.has_key(child.nodeName):
                print('No class mapping for element <%s>. Ignoring...' % (child.nodeName))
                child = child.nextSibling
                continue
            obj = GenStats.ChildrenMap[child.nodeName]()
            obj.fromXmlElement(child)
            if self.has_key(obj.getId()):
                print('Counter \'%s\' already registered. Will ignore latest value!' % (obj.getId()))
            else:
                self.children[obj.xElementName].append(obj)
            #del obj
            child = child.nextSibling
    ###
    def getXmlFileName(self):
        if self._fullpath is None or not os.path.isfile(self._fullpath):
            return 'empty/invalid GenStats object'
        return os.path.split(self._fullpath)[1]
    ###
    def __getitem__(self, id):
        iloc = self.has_key(id, True)
        if iloc is None:
            return None
        return self.children[iloc[1]][iloc[0]]
    ###
    ## def __setitem__(self, id, value):
        ## iloc = self.has_key(id, True)
        ## if iloc is None:
            ## return
        ## obj = self.children[iloc[1]][iloc[0]]
        ## if not isinstance(value, obj.__class__):
            ## return
        ## self.children[iloc[1]][iloc[0]] = value
    ###
    def has_key(self, name, getLocation=False):
        ret = None
        for k in self.children.keys():
            for ii in xrange(0,len(self.children[k])):
                obj = self.children[k][ii]
                id = obj.getId()
                #if isinstance(obj, GenXSection):
                #    id = obj.getProcId()
                if name == id:
                    ret = (ii, k)
                    break
            if not ret is None:
                break
        if not getLocation:
            ret = (not ret is None)
        return ret
    ###
    def getIdsByType(self, statName, sorted=False):
        if not statName in GenStats.ChildrenMap.keys():
            raise ValueError('No statistics of type \'%s\' is known to this object.' % (statName))
        ret = [self.children[statName][i].getId() for i in xrange(0,len(self.children[statName]))]
        if sorted:
            ret.sort()
        return ret
    ###
    def isCompatible(self, other):
        global buggyStatObjects
        gFailed = False
        for kk in GenStats.ChildrenMap.keys():
            bFailed = False
            thisCNames = self.getIdsByType(kk)
            otherCNames = other.getIdsByType(kk)
            diffThis2Other = filter(lambda x: not x in otherCNames, thisCNames)
            diffOther2This = filter(lambda x: not x in thisCNames, otherCNames)
            if len(diffThis2Other) > 0 or len(diffOther2This) > 0:
                bFailed = True
                print('Diffs between %s counters in file 1::\'%s\' and 2::\'%s\':' % (kk, self.getXmlFileName(), other.getXmlFileName()))
                # allow merging for specified counters that may be missing
                ignoreThis2Other = filter(lambda x: x in buggyStatObjects, diffThis2Other)
                if len(ignoreThis2Other) > 0:
                    diffThis2Other = filter(lambda x: x not in ignoreThis2Other, diffThis2Other)
                ignoreOther2This = filter(lambda x: x in buggyStatObjects, diffOther2This)
                if len(ignoreOther2This) > 0:
                    diffOther2This = filter(lambda x: x not in ignoreOther2This, diffOther2This)
                if len(buggyStatObjects) > 0 and len(diffThis2Other) == 0 and len(diffOther2This) == 0:
                    print('No remaining diffs. Following missing counters were ignored: %s.' % (buggyStatObjects))
                    bFailed = False
                else:
                    print('Diffs between counters after filtering allowed missing counters:')
            if len(diffThis2Other) > 0:
                print('- %s list in 1 and not in 2: %s.' % (kk, ', '.join(diffThis2Other)))
            if len(diffOther2This) > 0:
                print('- %s list in 2 and not in 1: %s.' % (kk, ', '.join(diffOther2This)))
            gFailed = gFailed or bFailed
        return not gFailed
    ###
    def findCompatibles(self, others):
        if not isinstance(others, list) or len(others)==0 or not isinstance(others[0], GenStats):
            return []
        return filter(lambda x: self.isCompatible(x), others)
    ###
    def __selftest(self):
        for kk in GenStats.ChildrenMap.keys():
            sidlist = self.getIdsByType(kk)
            for cn in sidlist:
                if self[cn] is None:
                    print('Counter %s :: %s is None.' % (kk,cn))
        pass
    ###
    def __add__ (self, other):
        try:
            if not self.isCompatible(other):        # test compatibility between statistics!
                print('This suite of files cannot be merged until buggy counters are specified correctly. Will fail at next merging...')
                # keep failing till all missing counters are correctly vetoed
                return None
            bFailed = False
            osum = GenStats()
            # TODO: change isCompatible to avoid refiltering which is performed below
            for kk in GenStats.ChildrenMap.keys():
                sidlist = self.getIdsByType(kk)
                oidlist = other.getIdsByType(kk)
                if len(oidlist) > 0:
                    newids = filter(lambda x: x not in sidlist, oidlist)
                elif len(sidlist) > 0:
                    newids = sidlist
                else:
                    newids = []
                if len(newids) > 0:     #debug
                    print('%s : %s' % (kk,str(newids)))
                for ii in sidlist:
                    osum.children[kk].append(self[ii]+other[ii])
                for ii in newids:
                    osum.children[kk].append(other[ii]+self[ii])
        except Exception, exx:
            print(exx.message)
            bFailed = True
            print(exx.args)
        if bFailed:
            return None
        osum.__selftest()
        osum._fileCounter = self._fileCounter + 1
        return osum
    ###
    def brRecompute(self):
        for kk in self.children.keys():
            for jj in xrange(0,len(self.children[kk])):
                self.children[kk][jj].recompute(self)
    ###
    def saveFile(self, filepath):
        if os.path.isfile(filepath):
            print('File \'%s\' exists. Will overwrite it...' % (os.path.split(filepath)[1]))
        if self._xdoc is None:
            self._xdoc = xdom.Document()
            de = self._xdoc.createElement(self.RootElementName)
            self._xdoc.appendChild(de)
            cTypes = GenStats.ChildrenMap.keys()
            cTypes.sort()
            for i in xrange(0,len(cTypes)):
                kk = cTypes[i]
                for ocnt in self.children[kk]:
                    ocnt.recompute(self)
                    ocnt.toXmlElement(self._xdoc.documentElement)
        fp = open(filepath, 'wb')
        fp.write(self._xdoc.toprettyxml('  ','\n'))
        fp.flush()
        fp.close()
    ###
    def makeGenerationResults(self):
        ret = {}
        for kk in GenStats.ChildrenMap.keys():
            for ocnt in self.children[kk]:
                for el in ocnt.getCounters():
                    try:
                        ret[el[0]] = float(el[1])
                    except ValueError, vex:
                        ret[el[0]] = el[1]
        ret['C1'] = ret['pp0']
        del ret['pp0']
        if ret.has_key('N2'):
            ret['C1N2'] = ret['C1']*ret['N2']
        return dict(ret)
    ###
    def __eq__(self, other):
        if not self.isCompatible(other):
            return False
        mycnts = self.makeGenerationResults()
        otcnts = other.makeGenerationResults()
        ret = True
        for (k,v) in mycnts.items():
            if v != otcnts[k]:
                ret = False
                break
        return ret
    ###
    def getAdditionCounter(self):
        return self._fileCounter
    ###
class JDParser(object):
    '''
    XML-Python interface for parsing jobDescription.xml files. Reads all first-level
    <Parameter> child nodes and generated 'name' to 'value' dictionary in getMainParams() method.
    '''
    docElemName = u'Workflow'
    def __init__(self, filename):
        self._fn = filename
        self._xdoc = xdom.parse(filename)
        if not isinstance(self._xdoc, xdom.Document):
            raise ValueError('Job description XML file could not be properly parsed.')
        if self._xdoc.documentElement.nodeName != self.docElemName or len(self._xdoc.documentElement.getElementsByTagName('Parameter')) == 0 :
            raise ValueError('Provided XML may not be a valid Dirac JDL file.')
    ###
    ## Returns a dictionary of Parameter@name : Parameter/value/CDATA.data or None if no main parameters found;
    ## when multiple <value> nodes are found a tuple is returned as value of the corresponding key/name
    def getMainParams(self):
        de = self._xdoc.documentElement
        params = de.getElementsByTagName('Parameter') #returns all nodes
        params = filter(lambda n: n.parentNode.nodeName == self.docElemName, params) #get only document element children
        ret = None
        for pp in params:
            if not pp.hasAttribute('name'):
                print(pp, 'has no \'name\' attribute. Skipping...')
                continue
            if len(pp.getElementsByTagName('value')) == 0:
                print(pp, 'has no <value> children nodes. Skipping...')
                continue
            vals = self.__getParamsValues(pp.getElementsByTagName('value'))
            nn = pp.getAttribute('name')
            if ret is None:
                ret = {}
            if len(vals) == 0:
                vals = None
            elif len(vals) == 1:
                vals = vals[0]
            else:
                vals = tuple(vals)
            ret[nn] = vals
        return ret
    ###
    def __getParamsValues(self, vNodes):
        if not isinstance(vNodes, list):
            vNodes = [vNodes,]
        ret = []
        for vn in vNodes:
            if not vn.hasChildNodes():
                continue
            for cn in vn.childNodes:
                if cn.nodeType != xdom.Element.CDATA_SECTION_NODE and cn.nodeType != xdom.Element.TEXT_NODE:
                    continue
                #print(cn.nodeType)
                if cn.nodeType == xdom.Element.CDATA_SECTION_NODE:
                    ret.append(cn.data)
                else:
                    ret.append(cn.nodeValue.strip())
        #print(ret)
        return ret
            
#####
if __name__ == '__main__':
    print("This is a Python module and should not be used as a script.")
    print("Import and use it in your scripts in the following way:")
    print(" from GenXmlLogParser import GenStats" % ())
    print(" statObj1 = GenStats(filepath1) # filepath1 can also be an URL using http(s) or ftp")
    print(" statObj2 = GenStats(filepath2)")
    print(" statMerged = statObj1 + statObj2")
    print(" statMerged.brRecompute()  # recomputing is also done automatically when saving the merged XML")
    print("then use saveFile or makeGenerationResults to save merged \"counters\" to XML")
    print("or create a data dictionary to use in GaussStat.py to generate statistics web pages.")
    print("Additionally you can use the JDParser class in a similar way to parse JobDescription.xml files.")
