#!/usr/bin/python3
# coding: utf-8     # (reference: https://www.python.org/dev/peps/pep-0263/)

# Read a Paratext files (.lds and .ssf), convert the relevant data to LDML, and insert into an LDML file.
# Given the right options, can also download the Paratext files from the DBL.

import sys, traceback
import codecs, copy
import os, re
import logging
import multiprocessing
import json
from io import StringIO
from configparser import RawConfigParser
from xml.etree import ElementTree as etree
import xml.etree.cElementTree as et
from sldr import UnicodeSets, ducet

# For calling DBLAuthV1 class:
# The requests module may need to be installed separately; it does not appear to be part of the standard Python.
from datetime import datetime
from time import mktime
from langtag import lookup, langtag
from sldr.ldml_exemplars import Exemplars, UCD
from sldr.ldml import Ldml, draftratings
from sldr.collation import Collation, CollElement

try:
    import newdbl
except ImportError:
    relpath = os.path.join(os.path.dirname(__file__), '..', 'lib', 'wstools')
    sys.path.append(os.path.abspath(relpath))
    import newdbl

silns = {'sil' : "urn://www.sil.org/ldml/0.1" }
gendraft = draftratings.get('generated', 5)

def gettext(e):
    return e.text if e is not None else ''

class Puamap:

    start = 0xF134
    mapstr = "⌊⌋                                                          ᷂᷊᷽᷄᷅᷆᷇᷈᷉̈͜͞    ᵐᶮᵑᵃᵄᵅᵇᵈᵉᵊᵋᶟᵍᵏᵒᵓᵖᵗᵘᵚᵛˋˈˊꜗꜘꜙꜚꜛꜜꜝꜞᶛ𐞃ᶝ𐞎𐞏ᶠᶢᶤᶦᶡᶩ𐞢ᶱꟹ𐞣ᶴᶶᶷᶭᶺ𐞲𐞑ᶻᶽᶾᶜᶞᶣ𐞕ᶨᶪᶫᶬᶯᶰᶲᶳᶵᶹᶼˀᶿᶥᶧᶸ 𐞠 ꜒꜓꜔꜕꜖꜈꜉꜊꜋꜌꜍꜎꜏꜐꜑꜀꜂꜄꜆꜁꜃꜅꜇ꞈˬ꞉꞊                     ᴀȡᴇȴȵȶʮʯⱭⱰɃȼᶑ ⱡⱠᵽɋɊɍɌⱤᶘƷɄɅⱳⱲᶚꞌɂȽɇɆⱨⱧᶀꞔᶁᶂᶃᶄᶅᶆᶇᶈᶉᶊᶋᶌᶍᶎꟆ𝼘ᶏᶐᶒᶓᶔᶕᶖᶗᶙᵾȸȹⱢɏɎꜫꜭⓐⒶᵬᵭᵮᵯᵰᵱᵲᵳᵴᵵᵶᵻᵿȿɀ  ꜧⱮⱣᵼⱴⱱɉɈⱪⱩⱬⱫꞎ𝼆𝼄𝼈ꞋꞍꞲ                                                                                                                                                   ׆ׅ                              ӶӷԒԓӼӽ  ӾӿԐԑԦ"
    def __init__(self):
        ranges = []
        s = None
        for i, c in enumerate(self.mapstr):
            if c == " ":
                if s is not None:
                    ranges.append([s, i])
                s = None
            elif s is None:
                s = i
        self.re = re.compile("["+"".join("{}-{}".format(chr(self.start+r[0]), chr(self.start+r[1])) for r in ranges) + "]")
        self.ucd = UCD()

    def __call__(self, s):
        return self.ucd.normalize('NFC', self.re.sub(lambda m:self.mapstr[ord(m.group(0)) - self.start], s))

puamap = Puamap()

ssfmap = {
    # LanguageIsoCode is passed in
    'DefaultFont': ('special/sil:external-resources/sil:font[@types="default"]/@name', ''),
    'DefaultFontSize': ('special/sil:external-resources/sil:font[@types="default"]/@size', '0'),
    # ValidCharacters is unused
    'Pairs': ('delimiters/special/sil:matched-pairs', lambda x: " ".join("{}/{}".format(a.get('open',''), a.get('close','')) for a in x)),
    'Quotes': ('delimiters', lambda x: " ".join(gettext(x.find('quotation'+a)) for a in ('Start', 'End'))),
    'InnerQuotes': ('delimiters', lambda x: " ".join(gettext(x.find('alternateQuotation'+a)) for a in ('Start', 'End'))),
    'InnerInnerQuotes': ('delimiters/special/sil:quotation-marks/sil:quotation[@level="3"]', lambda x: " ".join(x.get(a) or '' for a in ('open', 'close'))),
    'ContinueQuotes': ('delimiters/special/sil:quotation-marks/sil:quotation[@level="1"]/@continue', ''),
    'ContinueInnerQuotes': ('delimiters/special/sil:quotation-marks/sil:quotation[@level="2"]/@continue', '')
    # nothing else is used
    }

class Ssf:
    def __init__(self):
        pass

    def parseSsf(self, ssfFilename):
        if ssfFilename is None:
            return
        if isinstance(ssfFilename, str):  # Python 2 uses basestring
            ssfFile = open(ssfFilename, 'rb')
        else:
            ssfFile = ssfFilename

        self.ssfLangData = {  # initializing everything to None simplifies the processing
            'LanguageIsoCode': None,
            'DefaultFont': None,
            'DefaultFontSize': None,
            'ValidCharacters': None,
            'Pairs': None,
            'Quotes': None,
            'InnerQuotes': None,
            'InnerInnerQuotes': None,
            'ContinueQuotes': None,
            'ContinueInnerQuotes': None,
            'Continuer': None,
            'InnerContinuer': None,
            'InnerInnerContinuer': None,
            'VerboseQuotes': None,
            'ValidPunctuation': None}
        
        for (event, e) in etree.iterparse(ssfFile, events=('start', 'end')) :
            ##print(event + ": " + e.tag)
            if event == 'start' :
                if e.tag in ssfLangData :
                    self.ssfLangData[e.tag] = e.text
        s = self.ssfLangData.get('LanguageIsoCode', '').rstrip(':')
        self.ssfLangData['LanguageIsoCode'] = s.replace(':', '-')

    def fromLdml(self, ldml, langcode):
        self.ssfLangData = {'LanguageIsoCode': langcode}
        for k, v in ssfmap.items():
            e = ldml.find(v[0])
            if e is None and isinstance(v[1], str):
                res = v[1]
            elif e is not None:
                res = v[1](e)
            else:
                res = ''
            self.ssfLangData[k] = res

    def get_lang(self):
        return self.ssfLangData['LanguageIsoCode']

    def process(self, ldml):
        if self.ssfLangData['DefaultFont'] or self.ssfLangData['DefaultFontSize'] :
            _addLdsSsfDataFont(ldml, self.ssfLangData['DefaultFont'], float(self.ssfLangData['DefaultFontSize']) * 14)
        if self.ssfLangData['Pairs'] :
            self._addSsfDataPairs(ldml, self.ssfLangData['Pairs'])
        if self.ssfLangData['Quotes'] :
            self._addSsfDataQuotes(ldml, self.ssfLangData['Quotes'])
        if self.ssfLangData['InnerQuotes'] :
            self._addSsfDataInnerQuotes(ldml, self.ssfLangData['InnerQuotes'])
        if self.ssfLangData['InnerInnerQuotes'] :
            self._addSsfDataInnerInnerQuotes(ldml, self.ssfLangData['InnerInnerQuotes'])

    def _addSsfDataPairs(self, ldml, pairsValue) :
        # Pairs ->
        # delimiters/special/sil:matched-pairs/sil:matched-pair/@open, @close
        pairs = set(tuple(x.split('/')) for x in pairsValue.split(' '))
        if not len(pairs):
            return
        matchedElem = ldml.ensure_path('delimiters/special/sil:matched-pairs')[0]
        if ldml.get_draft(matchedElem) < gendraft:
            return
        for e in list(matchedElem):
            t = (e.get('open', ''), e.get('close', ''))
            if t not in pairs:
                matchedElem.remove(e)
            else:
                pairs.remove(t)
        for p in sorted(pairs):
            ldml.addnode(matchedElem, 'sil:matched-pair', open=p[0], close=p[1])

    def _addSsfDataQuotes(self, ldml, quotesValue) :
        # Quotes ->
        # delimiters/quotationStart, delimiters/quotationEnd
        (qStart, qEnd) = quotesValue.split(' ')
        if not qStart and not qEnd:
            return
        qStartElem = ldml.ensure_path('delimiters/quotationStart', text=qStart)
        qEndElem = ldml.ensure_path('delimiters/quotationEnd', text=qEnd)

    def _addSsfDataInnerQuotes(self, ldml, quotesValue) :
        # InnerQuotes ->
        # delimiters/alternateQuotationStart, delimiters/alternateQuotationEnd
        (qStart, qEnd) = quotesValue.split(' ')
        if not qStart and not qEnd:
            return
        qStartElem = ldml.ensure_path('delimiters/alternateQuotationStart', text=qStart)
        qEndElem = ldml.ensure_path('delimiters/alternateQuotationEnd', text=qEnd)

    def _addSsfDataInnerInnerQuotes(self, ldml, quotesValue) :
        # InnerInnerQuotes ->
        # delimiters/special/sil:quotation-marks[@level="3"]/@open, @close
        (qStart, qEnd) = quotesValue.split(' ')
        if not qStart and not qEnd:
            return
        qMark3Elem = ldml.ensure_path('delimiters/special/sil:quotation-marks/'
                    'sil:quotation[@level="3"][@open="{}"][@close="{}"]'.format(qStart, qEnd))[0]

    def _addSsfDataContinueQuotes(self, ldml, contValue, contInnerValue) :
        # ContinueQuotes ->
        # delimiters/special/sil:quotation-marks[@paraContinueType]/@open, @close
        if _valueNotNo(contValue):
            ldml.ensure_path('delimiters/special/sil:quotation-marks/sil:quotation[@level="1"][@continue="{}"]'.format(contValue))
        if _valueNotNo(contInnerValue):
            ldml.ensure_path('delimiters/special/sil:quotation-marks/sil:quotation[@level="2"][@continue="{}"]'.format(contInnerValue))


def _valueNotNo(value) :
    if value is None :
        return False
    if value.lower() == "no" :
        return False
    if value.lower() == "false" :
        return False
    return True

def processLds(ldml, ldsFile, ducetDict, replaceColl):
    if ldsFile.peek(3)[:3] == b"\xef\xbb\xbf":
        ldsFile.read(1)
    ldsConfig = RawConfigParser()
    ldsConfig.read_file(ldsFile)

    if ldsConfig.has_section('General'):
        fontValue = None
        sizeValue = None
        if ldsConfig.has_option('General', 'font'):
            fontValue = ldsConfig.get('General', 'font')
        if ldsConfig.has_option('General', 'size'):
            sizeValue = ldsConfig.get('General', 'size')

        _addLdsSsfDataFont(ldml, fontValue, float(sizeValue or 0))

    if replaceColl:
        simpleColls = readLdsCollation(ldsConfig)
        (sortColl, alphabet) = calcSortLdsData(ldml, simpleColls, ducetDict)
        return (sortColl, alphabet)
    return (None, None)


def _addLdsSsfDataFont(ldml, defaultFontValue, defaultSizeValue) :
    # DefaultFont, DefaultFontSize ->
    # special/sil:external-resources/sil:fontrole[@types="default"]/sil:font[@name, @size]
    # TODO: this isn't quite right since it should handle an existing sil:fontrole[@types="default heading"]
    #fontElemp = ldml.ensure_path('special/sil:external-resources/sil:fontrole[@types="default"]')[0]
    # Or (more likely) this might come from the .lds file.

    xpath = 'special/sil:external-resources/sil:font[@types="default"][@name="{}"]'.format(defaultFontValue)
    allfonts = set([e.get('name') for e in ldml.findall('special/sil:external-resources/sil:font')])
    if defaultFontValue in allfonts:
        return

    defaultSizeFactor = 1.0
    if defaultSizeValue:
        # We treat Times New Roman size 14 as the standard.
        defaultSizeFactor = int(defaultSizeValue) / 14
        # Round to the nearest .05:
        defaultSizeFactor = (int(defaultSizeFactor * 20)) / 20
        # Factors close to 1.0 are not really significant.
        if defaultSizeFactor > .92 and defaultSizeFactor < 1.08:
            defaultSizeFactor = 1.0
    exResNode = ldml.ensure_path('special/sil:external-resources')[0]
    fontElemNode = None
    for e in ldml.findall('special/sil:external-resources/sil:font'):
        if 'default' in e.get("types", ""):
            fontElemNode = e
            if (defaultFontValue == "" or e.get('name') == defaultFontValue) \
                    and (defaultSizeValue < 0.001 or float(e.get('size', '1.0')) == defaultSizeFactor):
                return

    if defaultSizeFactor != 1.0:
        xpath += '[@size="{}"]'.format(defaultSizeFactor)

    if fontElemNode is None:
        if defaultFontValue == "":
            return
        fontElemGen = ldml.ensure_path(xpath, draft="generated")[0]
    else:
        fontElemGen = None
        fontNamePrev = fontElemNode.get('name', "")
        fontSizePrev = fontElemNode.get('size', None)

        if fontNamePrev == "":
            # Throw away old empty value.
            if defaultFontValue is None or defaultFontValue == '':
                # Remove altogether.
                if fontElemNode is not None:
                    exResNode.remove(fontElemNode)
            else:
                fontElemNode = ldml.ensure_path(xpath, draft="generated")[0]
        elif fontNamePrev != defaultFontValue:
            # mismatch; old value is marked suspect and alt=proposed-dbl, new value is draft
            fontElemSuspect = fontElemNode    #ldml.ensure_path(xpath)[0]
            for alt in getattr(fontElemNode, 'alternates', {}).values():
                if alt.get('name', '') == defaultFontValue and alt.get('size', '1.0') == str(defaultSizeFactor):
                    break
            else:
                fontElemSuspect.parent.remove(fontElemSuspect)
            fontElemNode = ldml.ensure_path(xpath, draft="generated", matchdraft="draft")[0]
        else:
            # Font values match.
            if fontSizePrev is not None and defaultSizeFactor == 1.0:
                del fontElemNode.attrib['size']
            elif defaultSizeFactor != 1.0:
                fontElemNode.set('size', str(defaultSizeFactor))

def processMetadata(ldml, fname):
    etree = et.parse(fname)
    langNode = etree.getroot().find('language')
    if langNode is None:
        return
    ltagnameNode = langNode.find('ldml')
    if ltagnameNode is None:
        ltagnameNode = ldml.find('identity/language')
        ltagname = ltagnameNode.get("type", "")
    else:
        ltagname = ltagnameNode.text or ""
    stagnameNode = langNode.find('scriptCode')
    localnameNode = langNode.find('nameLocal')
    if localnameNode is not None and ltagnameNode is not None:
        ltag = langtag(ltagname)
        ldml.ensure_path('localeDisplayNames/languages/language[@type="{}"]'.format(ltag.lang),
                text=localnameNode.text)
    engnameNode = langNode.find('name')
    if engnameNode is not None or ldml.get_draft(engnameNode) >= gendraft:
        ldml.ensure_path('localeDisplayNames/special/sil:names/sil:name[@xml:lang="en"]',
                text=engnameNode.text)
    res = ltagname + ("-"+stagnameNode.text if stagnameNode else "")
    return res

def readLdsCollation(ldsConfig):
    valueList = []
    # Read sorted characters lists from .lds file.
    if ldsConfig.has_section('Characters') :
        cntr = 1
        keepGoing = True
        while keepGoing :  ### and cntr < 100:
            strCntr = str(cntr)
            if len(strCntr) < 2 : strCntr = '0' + strCntr
            key = 'Chr' + strCntr
            if not ldsConfig.has_option('Characters', key) :
                keepGoing = False
            else :
                value = ldsConfig.get('Characters', key)
                if value == "'/'":
                    value = "'"  # kludge; why does the get() function return the wrong value?
                valueList.append(value)
            cntr = cntr + 1
    return valueList

def calcSortLdsData(ldml, valueList, ducetDict) :
    # Generate a data structure similar to a sort tailoring for the list of characters.
    sortResult = []
    collObj = Collation(ducetDict)
    alphabet = []
    simple = True
    simplelist = list("abcdefghijklmnopqrstuvwxyz'")
    if len(valueList) > 0 :
        currBase = None
        for value in valueList :
            uValue = value  # Python 2: .decode('utf-8')
            spaceItems = uValue.split(' ')
            if len(spaceItems) == 2 and spaceItems[0].lower() == spaceItems[1].lower():
                # Kludge: deal with a limitation of Paratext. Since these items are case equivalent, the user probably
                # intended x/X rather than x X and was not permitted by Paratext.
                value = value.replace(' ', '/')
                uValue = value  # Python 2: .decode('utf-8')
                spaceItems = uValue.split(' ')
            sortSpecItems = []
            spaceSep = "&"
            prevSlashItems = None
            currLevel = 1
            for spaceItem in spaceItems :
                slashItems = spaceItem.split('/')
                # Kludge to handle something like xX which should really be x/X
                if len(slashItems) == 1 and len(slashItems[0]) == 2 :
                    c1 = (slashItems[0])[0:1]
                    c2 = (slashItems[0])[1:]
                    if ducet.ducetCompare(ducetDict, c1, c2) == 3 : # case equivalent with x <<< X
                        # Assume a typo where they left out the slash.
                        slashItems = [c1, c2]
                if len(slashItems) == 1 and len(slashItems[0]) > 1 and len(slashItems[0]) < 10:
                    s = slashItems[0]
                    slashSet = set()
                    if sum(1 if c.lower() == c else 0 for c in s) == len(s):
                        for i in range(2 ** len(s)):
                            slashSet.add("".join(c.upper() if (i & (1 << j)) != 0 else c for j,c in enumerate(s)))
                        slashSet.remove(s)
                        slashItems.extend(slashSet)
                slashItems.sort(key=ducet.keyfn(ducetDict, 3))
                if simple:
                    if not len(simplelist) or slashItems[0] != simplelist.pop(0):
                        simple = False
                for s in slashItems:
                    if currBase is not None:
                        try:
                            collObj[s] = CollElement(currBase, currLevel)
                        except KeyError:
                            continue
                    currLevel = 3
                    currBase = s
                    alphabet.append(s)
                currLevel = 2
    # print(collObj.asICU())
    return (collObj, alphabet) if not simple else (None, None)

def cleanUpOldSortSpec(oldValue):
    newValue = oldValue.replace('\/', '/')
    return newValue

def _debugStr(item):
    if isinstance(item, int):
        item = [item]
    result = "".join(map(unichr, item))
    result += " =" + " ".join(map(hex, item))
    return repr(result)

def processOneProject(filename, outputPath, ducetDict, langCode, sldrPath=None):
    dblObj = newdbl.DBL()
    dblObj.open_project(filename)

    filenames = {}
    for n in dblObj.namelist():
        for ext in ('ldml', 'lds', 'ssf'):
            if n.endswith("."+ext):
                filenames[ext] = n
    hasMetaDataFile = 'metadata.xml' in dblObj.namelist()

    mainChTextOrig = None
    auxChTextOrig = None
    indexChTextOrig = None
    punctChTextOrig = None

    exemplarinfo = {}
    ldml = None
    dblLdml = None
    ltagp = None
    ssf = None
    if 'ldml' in filenames:
        with dblObj.project.open(filenames['ldml']) as inf:
            dblLdml = Ldml(inf)
    if 'ssf' in filenames:
        with dblObj.project.open(filenames['ssf']) as inf:
            ssf = Ssf()
            ssf.parseSsf(inf)
            ltagp = ssf.get_lang()
    elif dblLdml is not None:
        ssf = Ssf()
        ssf.fromLdml(dblLdml, langCode)
    try:
        ltag = langtag(str(lookup(ltagp or langCode).tag))
    except KeyError:
        ltag = langtag(ltagp or langCode)
    outfname = str(ltag).replace("-","_")+".xml"

    hasldml = False
    if sldrPath is not None:
        testpath = os.path.join(sldrPath, outfname[0], outfname)
        if os.path.exists(testpath):
            ldml = Ldml(testpath)
            identity = ldml.find(".//{{{}}}identity".format(silns['sil']))
            if identity is not None and identity.get("source", "") == "cldr":
                logging.debug("Skipping {} since in CLDR".format(filename))
                return
            hasldml = True
    if ldml is None:
        ldml = dblLdml if dblLdml is not None else Ldml(None)
        ldml.use_draft = 'generated'
        ldml.default_draft = 'generated'
    chElems = ldml.root.findall('characters/exemplarCharacters')
    for chEl in chElems:
        exemType = chEl.get('type')
        exemplarinfo[chEl.get('type', '')] = (chEl.text, ldml.get_draft(chEl))

    ldml.uid = "dbl"  # generates alt="proposed-dbl"

    # Any <cr> element in <collation> is suspect. In fact, just go ahead and delete it.
    nosort = True
    sortColl = None
    collationNode = ldml.find('collations/collation[@type="standard"]')
    collationElem = None
    if dblLdml is not None:
        collationElem = dblLdml.find('collations/collation[@type="standard"]/special/sil:simple')
        nosort = False
    if ldml.get_draft(collationNode) >= gendraft:
        replaceColl = True
        if collationNode is not None and ldml.get_draft(collationNode) >= gendraft:
            collCrNode = collationNode.find('cr')
            if collCrNode is not None and collationNode.find('special/{{{}}}simple'.format(ldml.silns)) is not None:
                collationNode.remove(collCrNode)
    else:
        replaceColl = False

    sortSpecString = ''
    if 'lds' in filenames:
        with codecs.getreader("utf-8")(dblObj.project.open(filenames['lds'])) as inf:
           (sortColl, alphabet) = processLds(ldml, inf, ducetDict, replaceColl)
    elif replaceColl and collationElem is not None:
        vs = re.sub(r"(\s*\n)+", "\n", collationElem.text)
        if vs is not None:
            vstrings = [puamap(s) for s in vs.split("\n")]
            (sortColl, alphabet) = calcSortLdsData(ldml, vstrings, ducetDict)
        if alphabet is not None:
            se = ldml.ensure_path('collations/collation[@type="standard"]/special/sil:simple')[0]
            se.text = puamap(vs)
    if sortColl is not None:
        thisColl = copy.deepcopy(sortColl)
        thisColl.minimise(alphabet)
        sortSpecString = thisColl.asICU()
        if len(sortSpecString):
            collationElem = ldml.ensure_path('collations/collation[@type="standard"]')[0]
            crElem = ldml.addnode(collationElem, 'cr', returnnew=True)
            crElem.text = sortSpecString
    elif not nosort:   # delete the collation
        # only delete collation if the original was a simple sort. If hand crafted, keep it.
        if collationNode is not None and collationNode.find('special/{{{}}}simple'.format(ldml.silns)) is not None:
            collNode = collationNode.parent
            collNode.remove(collationNode)
            if not len(collNode):
                collNode.parent.remove(collNode)

    exemplars = Exemplars()
    exemplars.frequent = 0.0
    for t in dblObj.analyze_text():
        exemplars.process(t)
    exemplars.analyze()
    exemplars.normalize("NFC")
    if sortColl is not None:
        sortColl._setSortKeys()
        exemplars.collator = sortColl

    for t in ('auxiliary', 'main', 'index', 'punctuation'):     # order is important
        if t == 'main':
            key = ''
            exemplars._main -= exemplars._auxiliary
        key = '' if t == 'main' else t
        xpath = 'characters/exemplarCharacters[@type="' + key + '"]'
        oldValue, oldDraft = exemplarinfo.get(key, ('', 6))
        newValue = puamap(getattr(exemplars, t))        # sort these perhaps in analyze?
        oldSets = UnicodeSets.parse(oldValue)
        oldSet = oldSets[0].asSet() if len(oldSets) else set()
        newSet = getattr(exemplars, "_"+t)
        chElem = None
        if oldValue == '':
            if newValue != "[]":
                # no previous value; mark new value generated
                chElem = ldml.ensure_path(xpath)[0]
        elif newSet != oldSet:
            if oldValue == "[]" or oldDraft >= gendraft:
                # Throw away old empty value
                chElem = ldml.ensure_path(xpath)[0]
                ldml.change_draft(chElem, "generated")
            elif t == 'auxiliary' and oldDraft < gendraft:
                setattr(exemplars, t, oldValue)
        if chElem is not None:
            chElem.text = newValue

    # Delete empty data from original LDML
    likelyEmptyAttrs = ['crossrefs', 'diacritics', 'footnotes', 'verseSegments', 'wordBreaks', 'wordFormingPunctuation']
    specialNode = ldml.find('characters/special')
    for attr in likelyEmptyAttrs:
        node = ldml.find('characters/special/sil:exemplarCharacters[@type="' + attr + '"]')
        if node is not None and (node.text == '' or node.text == '[]'):
            specialNode.remove(node)
    if specialNode is None:
        pass
    elif len(specialNode) == 0:  # do we still need this case?
        charNode = ldml.find('characters')
        charNode.remove(specialNode)

    if ssf is not None:
        ssf.process(ldml)


    if hasMetaDataFile:
        with dblObj.project.open("metadata.xml") as inf:
            ltagp = processMetadata(ldml, inf)

    dblObj.close_project()

    # Version
    if not hasldml:
        # only sort out identity if we are creating rather than editing sldr file
        dateNow = datetime.utcnow()
        genTimeValue = datetime.strftime(dateNow, "%Y%m%d.%H%M")
        ldml.remove_path("identity")
        versionNode = ldml.ensure_path('identity/version', draft="unconfirmed")[0]
        versionNode.set('number', genTimeValue)
        ldml.ensure_path('identity/language[@type="{}"]'.format(ltag.lang), draft="unconfirmed")
        if ltag.script is not None:
            ldml.ensure_path('identity/script[@type="{}"]'.format(ltag.script), draft="unconfirmed")
        if ltag.region is not None:
            ldml.ensure_path('identity/territory[@type="{}"]'.format(ltag.region), draft="unconfirmed")
        if ltag.vars or ltag.ns:
            bits = []
            if ltag.vars is not None:
                bits.extend(ltag.vars)
            if ltag.ns:
                for k, v in sorted(ltag.ns.items()):
                    bits.extend([k] + v)
            ldml.ensure_path('identity/variant[@type="{}"]'.format("-".join(bits)), draft="unconfirmed")
        identity = ldml.ensure_path('identity/special/sil:identity', draft="generated")[0]
        try:
            ltagset = lookup(str(ltag))
            if ltagset.region is not None:
                identity.set('defaultRegion', ltagset.region)
            if ltagset.script is not None and ltag.script is None:
                identity.set('script', ltagset.script)
        except KeyError:
            pass

    outdir = os.path.join(outputPath, str(ltag)[0])
    if not os.path.exists(outdir):
        os.makedirs(outdir, exist_ok=True)
    ldmlOutputFilename = os.path.join(outdir, str(ltag).replace("-", "_") + ".xml")
    ldml.normalise()
    ldml.save_as(ldmlOutputFilename)

# end of processOneProject

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('outpath',help="Directory to store generated ldml files in if not -l")
    parser.add_argument('-d','--dblpath',help="Path to local zips of DBL")
    parser.add_argument('-s','--sldrpath',help="Path to SLDR root for testing for CLDR files not to process")
    parser.add_argument('-u','--update',action='store_true',help='Update .zip files in dblpath')
    parser.add_argument('-m','--map',help="paratext project to langtag map .json")
    parser.add_argument('-L','--lang',help='Only process given language')
    parser.add_argument('-j','--jobs',type=int,default=1,help="Number of parallel processes to run, 0 = default = number of processors")
    parser.add_argument('-z','--zipfile',help='Process a specific .zip file')
    parser.add_argument('--ldml',help='input LDML base file to directly process')
    parser.add_argument('--ssf',help='input SSF file to directly process')
    parser.add_argument('--lds',help='input LDS file to directly process')
    parser.add_argument('-S','--start',help='skip up to an including this langtag')
    parser.add_argument('-l','--loglevel',help='Set logging level')
    parser.add_argument('-D','--debug',action="store_true",help="Enable debug")
    parser.add_argument('-Z','--zdebug',default=0,type=int,help="bitfield: 1=don't download zips, 2=skip existing")

    args = parser.parse_args()

    import sys
    if args.loglevel:
        logging.basicConfig(stream=sys.stdout, level=args.loglevel.upper(),
                format="%(levelname)s:%(module)s %(message)s")

    def processfile(f, l, ducetDict):
        logging.info("Processing file: {}".format(f))
        try:
            processOneProject(f, args.outpath, ducetDict, l, sldrPath=args.sldrpath)
        except Exception as e:
            bt = traceback.format_exc(limit=5)
            logging.error("Error in {}, {}\nType: {} Args: {}".format(f, e, type(e), e.args))
            logging.error(bt)
            if args.debug:
                raise e
        return True

    if args.jobs == 1:
        pool = None
    else:
        pool = multiprocessing.Pool(processes=args.jobs)
    if args.update:
        rdr = newdbl.DBLReader()
        rdr.download(args.dblpath, lang=args.lang, nozips=args.zdebug & 1, mapfile=args.map, pool=pool)

    if args.sldrpath is not None:
        ducetDict = ducet.readDucet()

        if args.zipfile is None:
            filelist = [os.path.join(args.dblpath, f) for f in os.listdir(args.dblpath) if f.endswith(".zip")]
        else:
            filelist = [args.zipfile]
        jobs = [j+(ducetDict, ) for j in sorted(newdbl.process_projects(filelist, args.lang))]
        if args.start:
            for i, j in enumerate(jobs):
                if j[1] == args.start:
                    jobs = jobs[i+1:]
                    break
        if (args.zdebug & 2) != 0:
            jobs = [j for j in jobs if not os.path.exists(os.path.join(args.outpath, j[1][0], j[1].replace("-","_")+".xml"))]
        if pool is None:
            [processfile(*j) for j in jobs]
        else:
            asyncres = pool.starmap_async(processfile, jobs).get()
    if False:
        # Just process one set of files that is already present.
        ssfFile = args.ssf
        ldsFile = args.lds
        outFile = args.outpath

        if ldmlFile:
            ldml = Ldml(args.ldml)
        else:
            ldml = Ldml(None)
        ldml.use_draft = 'generated'
        if args.ssf:
            processSsf(ldml, args.ssf)
        if args.lds:
            import collation
            processLds(ldml, args.lds, ducetDict)
        if args.outfile:
            outf = codecs.open(args.outfile, 'w', encoding="utf-8")
        else:
            outf = sys.stdout
        ldml.serialize_xml(outf.write)
