#!/usr/bin/python3

# Read a Paratext files (.lds and .ssf), convert the relevant data to LDML, and insert into an LDML file.
# Given the right options, can also download the Paratext files from the DBL.

import sys
import collections
import codecs, logging
import copy
import os, io
import unicodedata
import xml.etree.cElementTree as et

# For calling DBLAuthV1 class:
# The requests module may need to be installed separately; it does not appear to be part of the standard Python.
import requests
from datetime import datetime
from wsgiref.handlers import format_date_time
from time import mktime

# The authorization key is stored in a different file:
try:
    import dblauthkey
except ImportError:
    sys.path.append(os.path.abspath(os.path.dirname(__file__)))
    import dblauthkey

try:
    import iso639
except ImportError:
    sys.path.append(os.path.abspath(os.path.dirname(__file__)))
    import iso639

##libPath = "../../../sldr/python/lib/"
libPath = os.path.join("..", "..", "..", "palaso-python", "lib")
sys.path.append(os.path.abspath(os.path.dirname(__file__) + libPath))

try:
    from olddbl import DBL
except ImportError:
    relpath = os.path.join(os.path.dirname(__file__), '..', 'lib', 'wstools')
    sys.path.append(sys.path.append(os.path.abspath(relpath)))
    from olddbl import DBL

import json

from configparser import RawConfigParser

try:
    from palaso.sldr.ldml import Ldml
except ImportError:
    sys.path.append(os.path.abspath(os.path.dirname(__file__) + libPath))
    from palaso.sldr.ldml import Ldml

from sldr.collation import Collation, CollElement
###from sldr.langtags import LangTag
from langtag import LangTag
try:
    import palaso.sldr.ducet as ducet
except:
    print("ducet not imported")


##import xml.etree.cElementTree as etree
# lxml preserves comments, also handles namespaces better:
from xml.etree import ElementTree as etree
##from cStringIO import StringIO # Python 2.7
from io import StringIO

silns = {'sil' : "urn://www.sil.org/ldml/0.1" }


class DBLAuthV1(requests.auth.AuthBase):
    authorization_header = 'X-DBL-Authorization'

    def __init__(self, api_token, private_key):
        super(DBLAuthV1, self).__init__()
        self.api_token = api_token.lower()
        self.private_key = private_key.lower()

    def __call__(self, r):
        r.headers[self.authorization_header] = self.make_authorization_header(r)
        return r

    def make_authorization_header(self, request):
        import hmac
        import hashlib

        mac = hmac.new(bytes(self.api_token, "ascii"), None, hashlib.sha1)
        mac.update(bytes(self.signing_string_from_request(request), 'utf-8)'))
        mac.update(bytes(self.private_key.lower(), "ascii"))
        tokenStr = self.api_token
        res = 'version=v1,token=%s,signature=%s' % (tokenStr, mac.hexdigest().lower())
        print(res)
        return res

    def signing_string_from_request(self, request):
        dbl_header_prefix = 'x-dbl-'
        signing_headers = ['content-type', 'date']

        method = request.method
        # use request uri, but not any of the arguments.
        path = request.path_url.split('?')[0]
        collected_headers = {}

        for key, value in request.headers.items():
            if key == self.authorization_header:
                continue
            k = key.lower()
            if k in signing_headers or k.startswith(dbl_header_prefix):
                collected_headers[k] = value.strip()

        # these keys get empty strings if they don't exist
        if 'content-type' not in collected_headers:
            collected_headers['content-type'] = ''
        if 'date' not in collected_headers:
            collected_headers['date'] = ''

        sorted_header_keys = sorted(collected_headers.keys())

        buf = "%s %s\n" % (method, path)
        for key in sorted_header_keys:
            val = collected_headers[key]
            if key.startswith(dbl_header_prefix):
                buf += "%s:%s\n" % (key, val)
            else:
                buf += "%s\n" % val
        return buf


# end of class DBLAuthV1


class DBLReader(object):

    def __init__(self):
        import dblauthkey
        (key1, key2) = dblauthkey.authkey()
        self.secretKey = key2
        self.auth = DBLAuthV1(key1, key2)
        print(f"{key1=} {key2=}")

    def testAccess(self):
        response = requests.get('https://thedigitalbiblelibrary.org',
                                auth=self.auth, headers=self._jsonHeaders())
        return response.status_code

    def getLicenses(self):
        response = requests.get('https://thedigitalbiblelibrary.org/api/licenses',
                                auth=self.auth, headers=self._jsonHeaders())
        if response.status_code == 200:
            licensesJson = response.content
            result = json.loads(licensesJson)
        return response.status_code


    def getEntries(self):

        fullResult = {}
        httpResult = 1000

        #accessTypeKeys = ['publishable', 'public', 'open_access', 'owned']
        accessTypeKeys = ['owned']
        logging.getLogger("urllib3").setLevel(logging.WARNING)
        print(self.auth, self._jsonHeaders())
        for accessType in accessTypeKeys:
            response = requests.get('https://thedigitalbiblelibrary.org/api/' + accessType + '_entries_list',
                                auth=self.auth, headers=self._jsonHeaders())
            if response.status_code == 200:
                entriesJson = response.content
                entriesDict = json.loads(entriesJson)
                for entry in entriesDict['list']:
                    id = entry['id']
                    langCode = entry['language_code']
                    if id in fullResult:
                        (bogus, oldAccess) = fullResult[id]
                        if oldAccess != accessType:
                            print("changing " + langCode + "_" + id + " from " + oldAccess + " to " + accessType)
                    fullResult[id] = (langCode, accessType)
            else:
                print(type(response), response.headers, getattr(response, 'text', ''))
                l = int(response.headers.get('Content-Length', 0))
                if l > 0:
                    for i in response.iter_lines():
                        print(i)
            httpResult = min(httpResult, response.status_code)

        if httpResult != 200:
            return httpResult
        else:
            return fullResult

    # end of getEntries


    def downloadOneEntry(self, entryId, langCode, accessType, downloadPath):

        # Get the metadata that includes the license key for reading.
        response = requests.get('https://thedigitalbiblelibrary.org/api/entries/' + entryId,
                                auth=self.auth, headers=self._jsonHeaders())
        if response.status_code == 200:
            entryJson = response.content
            entryMetaData = json.loads(entryJson)
            licenses = entryMetaData['licenses']
            if accessType == 'owned':
                licenseId = "owner"
            elif len(licenses) > 0:
                licenseId = str((licenses[0])['id'])
                #oldStyleAuth = "&email=sharon_correll@sil.org&key=" + '08325c5e2be189f3361e58c038a04015'  ### self.secretKey
            else:
                licenseId = 'none'

            if licenseId != 'none':
                entryUrl = 'https://thedigitalbiblelibrary.org/api/entries/' + entryId + "/revisions/latest/licenses/" + licenseId
                response = requests.get(entryUrl, auth=self.auth, headers=self._jsonHeaders())
                if response.status_code == 200:
                    entryJson = response.content
                    entryData = json.loads(entryJson)

                    result = {}

                    urlList = entryData['urls']
                    downloadZip = False
                    for url in urlList:
                        path = url['path']
                        downloadUrl = None
                        if path[-4:] == ".lds":
                            downloadUrl = url['url']
                            ext = ".lds"
                            key = "lds"
                        elif path[-4:] == ".ssf":
                            downloadUrl = url['url']
                            ext = ".ssf"
                            key = "ssf"
                        elif path[-5:] == ".ldml":
                            downloadUrl = url['url']
                            ext = ".ldml"
                            key = "ldml"
                        elif path[-4:] == ".usx":
                            downloadZip = True
                            break

                        if path[-4:] == ".sfm" or path[-5:] == ".usfm" or path[-4:] == ".ptx":
                            print("Found SFM file!!")

                        # if downloadUrl is not None:
                        #     downloadUrl = downloadUrl.replace("http:", "https:")
                        #     # KLUDGE to account for bug:
                        #     downloadUrl = downloadUrl.replace("licenses/owner/", "licenses/owner/release/")
                        #     response = requests.get(downloadUrl, auth=self.auth, headers=self._jsonHeaders())
                        #     if response.status_code == 200:
                        #         downloadFileName = langCode + "_" + entryId + ext
                        #         #self._saveDownloadedFile(downloadPath, downloadFileName, response.content)
                        #         result[key] = downloadPath + "/" + downloadFileName
                        #         print(langCode + " - DOWNLOADED " + downloadUrl)
                            ###else:
                            ### print(langCode + " - can't download file " + downloadUrl + ": " + str(response.status_code))

                    # end for...urlList

                    if downloadZip:
                        response = requests.get(
                            'https://thedigitalbiblelibrary.org/api/entries/' + entryId + "/revisions/latest/licenses/"
                                        + licenseId + ".zip",
                            auth=self.auth, headers=self._jsonHeaders())
                        if response.status_code == 200:
                            downloadFileName = langCode + "_" + entryId + ".zip"
                            self._saveDownloadedFile(downloadPath, downloadFileName, response.content)
                            result['usx'] = downloadPath + "/" + downloadFileName
                            print(langCode + " - DOWNLOADED " + downloadFileName)
                        else:
                            print(langCode + " - can't download zip file" + ": " + str(response.status_code))
                    else:
                        print(langCode + " - no usx files")

                    return result
                else:
                    print(langCode + " - can't access files")
                    return response.status_code
                # end if response.status_code == 200
            else:
                print(langCode + "_" + entryId + " - no licenses")
                return 403  # forbidden
        else:
            return response.status_code

    # end of downloadOneEntry

    #def getEntryFiles(self, entryId):


    def _jsonHeaders(self):
        return {'Date': format_date_time(mktime(datetime.now().timetuple())),
                'Content-Type': 'application/json'}


    def _saveDownloadedFile(self, dirPath, filename, contents):
        self._ensureDir(dirPath)
        fullname = dirPath + "/" + filename
        outf = open(fullname, 'wb')
        outf.write(contents)
        outf.close()


    def _ensureDir(self, dirPath):
        if not os.path.exists(dirPath):
            os.makedirs(dirPath)

#end of class DBLReader


def processSsf(ldml, ssfFilename):
    if ssfFilename is None:
        return
    if isinstance(ssfFilename, str):  # Python 2 uses basestring
        ssfFile = open(ssfFilename, 'rb')
    else:
        ssfFile = ssfFilename

    ssfLangData = {  # initializing everything to None simplifies the processing
        'DefaultFont': None,
        'DefaultFontSize': None,
        'ValidCharacters': None,
        'Pairs': None,
        'Quotes': None,
        'InnerQuotes': None,
        'InnerInnerQuotes': None,
        'ContinueQuotes': None,
        'ContinueInnerQuotes': None,
        'Continuer': None,
        'InnerContinuer': None,
        'InnerInnerContinuer': None,
        'VerboseQuotes': None,
        'ValidPunctuation': None}
    
    for (event, e) in etree.iterparse(ssfFile, events=('start', 'end')) :
        ##print(event + ": " + e.tag)
        if event == 'start' :
            if e.tag in ssfLangData :
                ssfLangData[e.tag] = e.text

    if ssfLangData['DefaultFont'] or ssfLangData['DefaultFontSize'] :
        _addSsfDataFont(ldml, ssfLangData['DefaultFont'], ssfLangData['DefaultFontSize'])
    if ssfLangData['Pairs'] :
        _addSsfDataPairs(ldml, ssfLangData['Pairs'])
    if ssfLangData['Quotes'] :
        _addSsfDataQuotes(ldml, ssfLangData['Quotes'])
    if ssfLangData['InnerQuotes'] :
        _addSsfDataInnerQuotes(ldml, ssfLangData['InnerQuotes'])
    if ssfLangData['InnerInnerQuotes'] :
        _addSsfDataInnerInnerQuotes(ldml, ssfLangData['InnerInnerQuotes'])


def processLds(ldml, ldsFilename, ducetDict):
    ldsConfig = RawConfigParser()

    if isinstance(ldsFilename, str):  # Python 2 uses basestring
        ldsFile = codecs.open(ldsFilename, 'r', encoding="utf-8")
        # The RawConfigParser chokes on the BOM.
        #try:
        ldsFile = _removeBom(ldsFile, ldsFilename, "utf-8")
        ldsConfig.read(ldsFilename, encoding="utf-8")
        #except:
        #    # Try again with latin1 encoding.
        #    ldsFile = codecs.open(ldsFilename, mode='r', encoding="latin_1")
        #    ldsFile = _removeBom(ldsFile, ldsFilename, "latin_1")
        #    ldsConfig.read(ldsFilename, "latin_1")
    else:
        ldsFile = ldsFilename

    # Sort specification

    sortSpecString = calcSortLdsData(ldml, ldsConfig, ducetDict)

    if len(sortSpecString):
        collationElem = ldml.ensure_path('collations/collation[@type="standard"]')[0]
        crElem = ldml.addnode(collationElem, 'cr')
        crElem.text = sortSpecString
        # CDATA handling done by ldml object, we hope

    # Font name and size

    if ldsConfig.has_section('General'):
        fontValue = None
        sizeValue = None
        if ldsConfig.has_option('General', 'font'):
            fontValue = ldsConfig.get('General', 'font')
        if ldsConfig.has_option('General', 'size'):
            sizeValue = ldsConfig.get('General', 'size')

        _addLdsSsfDataFont(ldml, fontValue, sizeValue)


def _addLdsSsfDataFont(ldml, defaultFontValue, defaultSizeValue) :
    # DefaultFont, DefaultFontSize ->
    # special/sil:external-resources/sil:fontrole[@types="default"]/sil:font[@name, @size]
    # TODO: this isn't quite right since it should handle an existing sil:fontrole[@types="default heading"]
    #fontElemp = ldml.ensure_path('special/sil:external-resources/sil:fontrole[@types="default"]')[0]
    # Or (more likely) this might come from the .lds file.

    xpath = 'special/sil:external-resources/sil:font[@types=\"default\"]'

    exResNode = ldml.ensure_path('special/sil:external-resources')[0]
    fontElemNode = ldml.ensure_path('special/sil:external-resources/sil:font')[0]
    #fontElemNode = ldml.ensure_path(xpath)[0]

    defaultSizeFactor = 1.0
    if defaultSizeValue:
        # We treat Times New Roman size 14 as the standard.
        defaultSizeFactor = int(defaultSizeValue) / 14
        # Round to the nearest .05:
        defaultSizeFactor = (int(defaultSizeFactor * 20)) / 20
        # Factors close to 1.0 are not really significant.
        if defaultSizeFactor > .92 and defaultSizeFactor < 1.08:
            defaultSizeFactor = 1.0

    if fontElemNode is None:
        fontElemGen = ldml.ensure_path(xpath, draft="generated")[0]

    else:
        fontElemAttrs = fontElemNode.attrib
        fontNamePrev = fontElemAttrs['name'] if 'name' in fontElemAttrs else None
        fontSizePrev = fontElemAttrs['size'] if 'size' in fontElemAttrs else None

        if fontNamePrev is None or fontNamePrev == "":
            # Throw away old empty value.
            if defaultFontValue is None or defaultFontValue == '':
                # Remove altogether.
                exResNode.remove(fontElemNode)
                fontElemGen = None
            else:
                fontElemGen = ldml.ensure_path(xpath, draft="generated")[0]
                ldml.change_draft(fontElem, "generated")
        elif fontNamePrev != defaultFontValue:
            # mismatch; old value is marked suspect and alt=proposed-dbl, new value is draft
            fontElemSuspect = fontElemNode    #ldml.ensure_path(xpath)[0]
            if not 'types' in fontElemSuspect.attrib:
                fontElemSuspect.attrib['types'] = "default"
            #ldml.change_draft(fontElemSuspect, "suspect")
            fontElemGen = ldml.ensure_path(xpath, draft="generated", matchdraft="draft")[0]
        else:
            # Font values match.
            if defaultSizeValue is None or defaultSizeFactor == fontSizePrev or defaultSizeFactor == 1.0:
                fontElemGen = None  # no change
                # Maybe we should clear the size attribute if it is there, but none of these
                # files actually have size in them, so don't bother.
            else:
                # There is a size to set, but don't bother creating a new node, use the existing one.
                # Set types="default" on existing node.
                if not 'types' in fontElemNode.attrib:
                    fontElemNode.attrib['types'] = "default"
                fontElemGen = ldml.ensure_path(xpath)[0]  # fontElemGen should = fontElemNode

    if fontElemGen is not None:
        # Set up the new font node.
        attrs = {}
        if defaultFontValue:
            attrs['name'] = defaultFontValue
        if defaultSizeFactor != 1.0:
            attrs['size'] = str(defaultSizeFactor)

        attrs['types'] = 'default'
        attrs['draft'] = 'generated'
        fontElemGen.attrib = attrs

#end of _addLdsSsfDataFont


def _addSsfDataPairs(ldml, pairsValue) :
    # Pairs ->
    # delimiters/special/sil:matched-pairs/sil:matched-pair/@open, @close
    matchedElem = ldml.ensure_path('delimiters/special/sil:matched-pairs')[0]
    for pair in pairsValue.split(' '):
        (openVal, closeVal) = pair.split('/')
        openVal = openVal.strip()
        closeVal = closeVal.strip()
        if not _findPair(matchedElem, openVal, closeVal) :
            matchElem = ldml.addnode(matchedElem, 'sil:matched-pair', open=openVal, close=closeVal)

def _findPair(pairElements, openValue, closeValue) :
    for elem in pairElements :
        if elem.get('open') == openValue and elem.get('close') == closeValue :
            return True
    return False

    
def _addSsfDataQuotes(ldml, quotesValue) :
    # Quotes ->
    # delimiters/quotationStart, delimiters/quotationEnd
    qStartElem = ldml.ensure_path('delimiters/quotationStart')[0]
    qEndElem = ldml.ensure_path('delimiters/quotationEnd')[0]
    (qStart, qEnd) = quotesValue.split(' ')
    qStartElem.text = qStart
    qEndElem.text = qEnd


def _addSsfDataInnerQuotes(ldml, quotesValue) :
    # InnerQuotes ->
    # delimiters/alternateQuotationStart, delimiters/alternateQuotationEnd
    qStartElem = ldml.ensure_path('delimiters/alternateQuotationStart')[0]
    qEndElem = ldml.ensure_path('delimiters/alternateQuotationEnd')[0]
    (qStart, qEnd) = quotesValue.split(' ')
    qStartElem.text = qStart
    qEndElem.text = qEnd


def _addSsfDataInnerInnerQuotes(ldml, quotesValue) :
    # InnerInnerQuotes ->
    # delimiters/special/sil:quotation-marks[@level="3"]/@open, @close
    qMark3Elem = ldml.ensure_path('delimiters/special/sil:quotation-marks/sil:quotation[@level="3"]')[0]
    (qStart, qEnd) = quotesValue.split(' ')
    qMark3Elem.set('open', qStart)
    qMark3Elem.set('close', qEnd)


def _addSsfDataContinueQuotes(ldml, contValue, contInnerValue) :
    # ContinueQuotes ->
    # delimiters/special/sil:quotation-marks[@paraContinueType]/@open, @close
    if _valueNotNo(contValue):
        ldml.ensure_path('delimiters/special/sil:quotation-marks/sil:quotation[@level="1"][@continue="{}"]'.format(contValue))
    if _valueNotNo(contInnerValue):
        ldml.ensure_path('delimiters/special/sil:quotation-marks/sil:quotation[@level="2"][@continue="{}"]'.format(contInnerValue))

def _valueNotNo(value) :
    if value is None :
        return False
    if value.lower() == "no" :
        return False
    if value.lower() == "false" :
        return False
    return True

def _removeBom(openFile, filename, enc="default"):
    openFile.close()
    binaryFile = codecs.open(filename, 'rb')  # open in binary mode, to avoid encoding issues
    bomMaybe = binaryFile.read(3)
    if bomMaybe[0] == 239 and bomMaybe[1] == 187 and bomMaybe[2] == 191:
        # Create a new file with no BOM.
        filenameNoBom = filename + "_nobom"
        fileNoBom = open(filenameNoBom, 'wb')
        cc = 3;
        while True:
            c = binaryFile.read(1)
            cc = cc + 1;
            if not c: break
            fileNoBom.write(c)
        fileNoBom.close()
        binaryFile.close()
        os.remove(filename)
        os.rename(filenameNoBom, filename)
        # Open it like it was when the method started.
        if enc == "default":
            newOpenFile = codecs.open(filename, 'r')
        else:
            newOpenFile = codecs.open(filename, 'r', enc)
        return newOpenFile
    else:
        return openFile

def processMetadata(ldml, fname):
    etree = et.parse(fname)
    langNode = etree.getroot().find('language')
    if langNode is None:
        return
    ltagnameNode = langNode.find('ldml')
    if ltagnameNode is None:
        ltagnameNode = ldml.find('identity/language/@type')
    localnameNode = langNode.find('nameLocal')
    if localnameNode is not None and ltagnameNode is not None:
        ltag = LangTag(ltagnameNode.text)
        elocalenameNode = ldml.ensure_path('localeDisplayName/languages/language[@type="{}"]'.format(ltag.lang))[0]
        elocalenameNode.text = localnameNode.text
    engnameNode = langNode.find('name')
    if engnameNode is not None:
        eengnameNode = ldml.ensure_path('localeDisplayName/sil:special/sil:names/sil:name[@xml:lang="en"]')[0]
        eengnameNode.text = engnameNode.text

def calcSortLdsData(ldml, ldsConfig, ducetDict) :
    valueList = []

    # Read sorted characters lists from .lds file.
    if ldsConfig.has_section('Characters') :
        cntr = 1
        keepGoing = True
        while keepGoing :  ### and cntr < 100:
            strCntr = str(cntr)
            if len(strCntr) < 2 : strCntr = '0' + strCntr
            key = 'Chr' + strCntr
            
            if not ldsConfig.has_option('Characters', key) :
                keepGoing = False
            else :
                value = ldsConfig.get('Characters', key)
                if value == "'/'":
                    value = "'"  # kludge; why does the get() function return the wrong value?
                valueList.append(value)
                
            cntr = cntr + 1

    # Generate a data structure similar to a sort tailoring for the list of characters.
    sortResult = []
    collObj = Collation(ducetDict)
    alphabet = []
    if len(valueList) > 0 :
        currBase = None
        for value in valueList :
            uValue = value  # Python 2: .decode('utf-8')
            spaceItems = uValue.split(' ')

            if len(spaceItems) == 2 and spaceItems[0].lower() == spaceItems[1].lower():
                # Kludge: deal with a limitation of Paratext. Since these items are case equivalent, the user probably
                # intended x/X rather than x X and was not permitted by Paratext.
                value = value.replace(' ', '/')
                uValue = value  # Python 2: .decode('utf-8')
                spaceItems = uValue.split(' ')

            sortSpecItems = []
            spaceSep = "&"
            prevSlashItems = None
            currLevel = 1
            for spaceItem in spaceItems :
                slashItems = spaceItem.split('/')

                # Kludge to handle something like xX which should really be x/X
                if len(slashItems) == 1 and len(slashItems[0]) == 2 :
                    c1 = (slashItems[0])[0:1]
                    c2 = (slashItems[0])[1:]
                    if ducet.ducetCompare(ducetDict, c1, c2) == 3 : # case equivalent with x <<< X
                        # Assume a typo where they left out the slash.
                        slashItems = [c1, c2]


                for s in slashItems:
                    if currBase is not None:
                        try:
                            collObj[s] = CollElement(currBase, currLevel)
                        except KeyError:
                            continue
                    currLevel = 3
                    currBase = s
                    alphabet.append(s)
                currLevel = 2
            # end for spaceItems
        # end for valueList
    # end if

                        
    collObj.minimise(alphabet)
    print(collObj.asICU())
    return collObj.asICU()

# end of calcSortLdsData


def cleanUpOldSortSpec(oldValue):
    newValue = oldValue.replace('\/', '/')
    return newValue


# OBSOLETE
def _minimizeSortSpec(sortSpec, ducetDict):
    # First, look at each line individually and see which have information we need to retain.
    # Note that we don't try to minimize the lines themselves, just the set of lines.
    needLines = {}
    for iline in range(len(sortSpec)):
        thisLine = sortSpec[iline]
        needThisLine = False
        prevItems = None
        # Try to find something in this line that doesn't fit the DUCET; if so, we definitely need this line.
        for iitem in range(len(thisLine)):
            cExpected = _compareToken(thisLine[iitem])
            if cExpected >= 0:
                cVal = ducet.ducetCompare(ducetDict, _intToUnichr(thisLine[iitem-1]), _intToUnichr(thisLine[iitem+1]))
                if cVal != cExpected:
                    needThisLine = True
                    break # out of loop over items
        needLines[iline] = needThisLine

    ampIline = 0
    ampItem = (sortSpec[0])[1]  # most recent item before an &

    for iline in range(len(sortSpec)-1):
        thisLine = sortSpec[iline]
        nextLine = sortSpec[iline+1]
        debugThis = _debugStr(thisLine[1])
        debugNext = _debugStr(nextLine[1])
        # Compare end of this line and start of next:
        cVal = ducet.ducetCompare(ducetDict, _intToUnichr(thisLine[-1]), _intToUnichr(nextLine[1]))
        if cVal != 1 or needLines[iline]: # something funny about this line
            firstItem = thisLine[1] if thisLine[0] == "&"  else ampItem
            # Compare start of this line (current &) and start of next:
            cValAmp = ducet.ducetCompare(ducetDict, _intToUnichr(firstItem), _intToUnichr(nextLine[1]))
        else:
            cValAmp = cVal

        if cVal == 1 and cValAmp == 1:
            # Order and level match.
            ilineFirstOfGroup = iline
        elif cVal > 1 and cValAmp > 0:
            # Order matches, but not level; merge.
            nextLine[0] = "<"
            needLines[iline] = True
            needLines[iline+1] = True
            ilineFirstOfGroup = iline
        elif cVal < 0 and cValAmp > 0:
            # Initial items are in order, but this line not so much; merge them.
            needLines[iline] = True
            needLines[iline+1] = True
            nextLine[0] = "<"
            ilineFirstOfGroup = iline
        else:
            howFarThis = _howFarOff(sortSpec, _intToUnichr(thisLine[1]), iline, 1, ducetDict)
            howFarNext = _howFarOff(sortSpec, _intToUnichr(nextLine[1]), iline+1, -1, ducetDict)
            if howFarThis == 1 :
                # Next line is out of order; we need: this < next < nextnext.
                nextLine[0] = "<"
                needLines[iline] = True
                needLines[iline+1] = True
                if iline < len(sortSpec) - 2:
                    nextNextLine = sortSpec[iline+2]
                    nextNextLine[0] = "<"
                    needLines[iline+2] = True
                ilineFirstOfGroup = iline
            elif howFarNext == -1:
                # This line is out of order.
                if iline == 0 :
                    # Do something special for the first line.
                    thisLine[0] = "&[before 1]a <"
                    # DON'T merge with the following line; it is not necessary and has the effect of creating a long chain.
                    needLines[iline] = True
                    ilineFirstOfGroup = iline
                else:
                    # We need: prev < this < next.
                    thisLine[0] = "<"
                    nextLine[0] = "<"
                    needLines[iline - 1] = True
                    needLines[iline] = True
                    needLines[iline + 1] = True
                    ilineFirstOfGroup = iline - 1
            else:
                # We don't know what we can assume; so we need: prev < this < next < nextnext.
                nextLine[0] = "<"
                needLines[iline] = True
                needLines[iline+1] = True
                if iline > 0 :
                    thisLine[0] = "<"
                    needLines[iline-1] = True
                    ilineFirstOfGroup = iline - 1
                else:
                    ilineFirstOfGroup = iline
                if iline < len(sortSpec) - 1:
                    nextNextLine = sortSpec[iline+2]
                    nextNextLine[0] = "<"
                    needLines[iline+2] = True

        # Remember the current last &.
        fogLine = sortSpec[ilineFirstOfGroup]
        if fogLine[0] == "&":
            ampIline = ilineFirstOfGroup
            ampItem = fogLine[1]

    # Retain only the lines we need.
    minSpec = []
    for iline in range(len(sortSpec)):
        if needLines[iline]:
            minSpec.append(sortSpec[iline])

    return minSpec

# end of minimizeSortSpec

# OBSOLETE
def _howFarOff(sortSpec, uChar, iline, dir, ducetDict):
    if dir == 1:
        rng = range(iline + 1, len(sortSpec))
    else:
        rng = range(iline - 1, -1, -1)

    for iline2 in rng:
        if (sortSpec[iline2])[0] == "&" :
            cVal = ducet.ducetCompare(ducetDict, uChar, _intToUnichr((sortSpec[iline2])[1]))
            if cVal >= (1 * dir):
                return iline2 - iline

    return iline2 - iline

# OBSOLETE
def _compareToken(token):
    if token == "<": return 1
    elif token == "<<": return 2
    elif token == "<<<": return 3
    elif token == "=": return 0
    return -1  # not a token

# OBSOLETE
def _charToInt(charsStr):
    # There's a codec that will do this
    result = []
    i = 0
    while i < len(charsStr):
        if charsStr[i:i+2] == "\\u":
            result.append(int(charsStr[i+2:i+6], 16))
            i = i + 5
        elif charsStr[i:i+2] == "\\U":
            result.append(int(charsStr[i+2:i+10], 16))
            i = i + 9
        else : ### isinstance(charItem, list) :
            result.append(ord(charsStr[i]))
        i = i + 1
    # end while

    return result

# OBSOLETE
def _unicodeEntity(ordValue):
    if isinstance(ordValue, int):
        ordValue = [ordValue]

    result = ""
    for x in ordValue:
        # what about \, &, < and other escaped chars (e.g. < 33)
        if (x) < 128:
            result += chr(x)
        # and \U support?
        else:
            uitem = unichr(x)
            hexStr = hex(ord(uitem))
            hexStr = hexStr.replace('0x', '')
            while len(hexStr) < 4:
                hexStr = '0' + hexStr
            hexStr = '\\u' + hexStr
            result += hexStr

    return result

# OBSOLETE
def _intToUnichr(item):
    if isinstance(item, int):
        return unichr(item)

    result = []
    for i in range(len(item)):
        x = item[i]
        if item[i] == ord('\\') and item[i+1] == ord('u'):
            sval = "".join(item[i+2,i+6])
            result.append(unichr(int(sval, 16)))
            i = i + 5
        elif item[i] == ord('\\') and item[i+1] == ord('U'):
            sval = "".join(item[i+2,i+10])
            result.append(unichr(int(sval, 16)))
            i = i + 9
        else:
            result.append(unichr(x))
    return result


def _debugStr(item):
    if isinstance(item, int):
        item = [item]
    result = "".join(map(unichr, item))
    result += " =" + " ".join(map(hex, item))
    return repr(result)



def downloadFromDbl(downloadPath, langCode = ''):
    dblreader = DBLReader()

    ###code = dblreader.testAccess()
    ###t = dblreader.getLicenses()
    ###if code != 200:
    ###    print("ERROR in accessing DBL; HTTP response code = ",code)

    entriesDict = dblreader.getEntries()
    if isinstance(entriesDict, int):
        print("ERROR in obtaining DBL entries; HTTP response code = ",entriesDict)
        return False
    else:
        for (entryId, entryInfo) in entriesDict.items():
            (entryLangCode, entryAccessType) = entryInfo
            if entryLangCode != 'eng' and entryLangCode != 'en':
                print("Downloading: " + entryId + " - " + entryLangCode)
                entryInfo = dblreader.downloadOneEntry(entryId, entryLangCode, entryAccessType, downloadPath)
        return True


def processOneProject(filename, outputPath, ducetDict, langCode):
    fullFilename = filename # should already include path
    dblObj = DBL()
    dblObj.open_project(fullFilename)

    """
    ldmlFilename = ldsFilename = ssfFilename = None
    ldmlContent = dblObj.file_contents_with_ext('ldml')
    if ldmlContent is not None:
        ldmlFilename = fullFilename.replace('.zip', '.ldml')
        outf = open(ldmlFilename, 'wb')
        outf.write(ldmlContent)
        outf.close()
    ldsContent = dblObj.file_contents_with_ext('lds')
    if ldsContent is not None:
        ldsFilename = fullFilename.replace('.zip','.lds')
        outf = open(ldsFilename, 'wb')
        if ord(ldsContent[0]) == 239 and ord(ldsContent[1]) == 187 and ord(ldsContent[2]) == 191:  # byte order mark
            ldsContent = ldsContent[3:]
        outf.write(ldsContent)
        outf.close()
    ssfContent = dblObj.file_contents_with_ext('ssf')
    if ssfContent is not None:
        ssfFilename = fullFilename.replace('.zip', '.ssf')
        outf = open(ssfFilename, 'wb')
        outf.write(ssfContent)
        outf.close()
    """

    ldmlFilename = fullFilename.replace('.zip', '.ldml')
    dblObj.extract_file_with_ext('ldml', ldmlFilename)
    ldsFilename = fullFilename.replace('.zip', '.lds')
    dblObj.extract_file_with_ext('lds', ldsFilename)
    ssfFilename = fullFilename.replace('.zip', '.ssf')
    dblObj.extract_file_with_ext('ssf', ssfFilename)
    hasMetaDataFile = dblObj.extract_file('metadata.xml')

    mainChTextOrig = None
    auxChTextOrig = None
    indexChTextOrig = None
    punctChTextOrig = None

    if ldmlFilename is not None and os.path.exists(ldmlFilename):
        ldml = Ldml(ldmlFilename, ducetDict)
        ldml.use_draft = 'generated'  # use @draft='generated' for newly created data

        mainChTextOrig = None
        auxChTextOrig = ldml.find('characters/exemplarCharacters[@type="auxiliary"]')
        indexChTextOrig = ldml.find('characters/exemplarCharacters[@type="index"]')
        punctChTextOrig = ldml.find('characters/exemplarCharacters[@type="punctuation"]')

        # Initialize the DBO analyzer with the existing exemplars.
        # We've decided not to do this due to the poor quality of the data.
        # But do remember the values.
        #dblObj.exemplars.main = ''
        #dblObj.exemplars.auxiliary = ''
        #dblObj.exemplars.index = ''
        #dblObj.exemplars.punctuation = ''

        chElems = ldml.root.findall('characters/exemplarCharacters')
        for chEl in chElems:
            exemType = chEl.get('type')
            if exemType is None:
                #dblObj.exemplars.main = chEl.text
                mainChTextOrig = chEl.text
            elif exemType == 'auxiliary':
                #dblObj.exemplars.auxiliary = chEl.text
                auxChTextOrig = chEl.text
            elif exemType == "index":
                #dblObj.exemplars.index = chEl.text
                indexChTextOrig = chEl.text
            elif exemType == "punctuation":
                #dblObj.exemplars.punctuation = chEl.text
                punctChTextOrig = chEl.text

        # chElemOrig = ldml.ensure_path('characters/exemplarCharacters')
        # mainChElemOrig = None
        # auxChElemOrig = ldml.find('characters/exemplarCharacters[@type="auxiliary"]')
        # indexChElemOrig = ldml.find('characters/exemplarCharacters[@type="index"]')
        # punctChElemOrig = ldml.find('characters/exemplarCharacters[@type="punctuation"]')

    else:
        ldml = Ldml(None, ducetDict)
#        dblObj.exemplars.main = ''
#        dblObj.exemplars.auxiliary = ''
#        dblObj.exemplars.index = ''
#        dblObj.exemplars.punctuation = ''

    ldml.uid = "dbl"  # generates alt="proposed-dbl"

    try:
        dblObj.process_project()
        dblObj.close_project()
        dblObj.analyze_projects()
    except Exception as err:
        print("ERROR processing exemplars: " + str(err))
        print(type(err))
        print(err.args)
        dblObj.close_project()

    print(dblObj.exemplars.main)
    print(dblObj.exemplars.auxiliary)
    print(dblObj.exemplars.index)
    print(dblObj.exemplars.punctuation)
    print(dblObj.exemplars.script)

    for (ldmlType, oldValue, newValue) in\
                    (('', mainChTextOrig, dblObj.exemplars.main),
                     ('auxiliary', auxChTextOrig, dblObj.exemplars.auxiliary),
                     ('index', indexChTextOrig, dblObj.exemplars.index),
                     ('punctuation', punctChTextOrig, dblObj.exemplars.punctuation)):

        xpath = 'characters/exemplarCharacters[@type="' + ldmlType + '"]'
        if oldValue is None or oldValue == "":
            if newValue == "[]":
                chElem = None  # ignore new empty value
            else:
                # no previous value; mark new value generated
                chElem = ldml.ensure_path(xpath, draft="generated")[0]
        elif newValue != oldValue:
            if oldValue == "[]":
                # Throw away old empty value
                chElem = ldml.ensure_path(xpath, draft="generated")[0]
                ldml.change_draft(chElem, "generated")
            else:
                # mismatch; old value is marked suspect and alt=proposed-dbl, new value is draft
                chElemSuspect = ldml.ensure_path(xpath)[0]
                chElemSuspect.text = oldValue
                #ldml.change_draft(chElemSuspect, "suspect")
                chElem = ldml.ensure_path(xpath, draft="generated", matchdraft="draft")[0]
        else:
            # values match; no change
            chElem = ldml.ensure_path(xpath)[0]
        if chElem is not None:
            chElem.text = newValue

    # Delete empty data from original LDML
    likelyEmptyAttrs = ['crossrefs', 'diacritics', 'footnotes', 'verseSegments', 'wordBreaks', 'wordFormingPunctuation']
    specialNode = ldml.find('characters/special')
    for attr in likelyEmptyAttrs:
        node = ldml.find('characters/special/sil:exemplarCharacters[@type="' + attr + '"]')
        if node is not None and (node.text == '' or node.text == '[]'):
            specialNode.remove(node)
    if specialNode is None:
        pass
    elif len(specialNode) == 0:  # do we still need this case?
        charNode = ldml.find('characters')
        charNode.remove(specialNode)

    scriptCode = dblObj.exemplars.script

    scriptElem = ldml.ensure_path('identity/script')[0]
    scriptElem.attrib['type'] = scriptCode
    langElem = ldml.ensure_path('identity/language')[0]
    isoCode = iso639.iso639_3_2(langCode)
    isoCode = isoCode if isoCode else langCode
    langElem.attrib['type'] = isoCode

    if ssfFilename is not None and os.path.isfile(ssfFilename):
        try:
            processSsf(ldml, ssfFilename)
        except Exception as err:
            print("ERROR processing .ssf file: " + str(err))
            print(type(err))
            print(err.args)

    # Any <cr> element in <collation> is suspect. In fact, just go ahead and delete it.
    collationNode = ldml.find('collations/collation')
    if collationNode is not None:
        collCrNode = collationNode.find('cr')
        if collCrNode is not None:
            collationNode.remove(collCrNode)
            #ldml.change_draft(collCrNode, "suspect")
            #collCrNode.text = cleanUpOldSortSpec(collCrNode.text)

    if ldsFilename is not None and  os.path.isfile(ldsFilename):
        try:
            processLds(ldml, ldsFilename, ducetDict)
        except Exception as err:
            print("ERROR processing .lds file: " + str(err))
            print(type(err))
            print(err.args)

    if hasMetaDataFile:
        try:
            processMetadata(ldml, 'metadata.xml')
        except Exception as err:
            print("ERROR processing metadata.xml file: " + str(err))
            print(type(err))
            print(err.args)

    # Version
    dateNow = datetime.utcnow()
    genTimeValue = datetime.strftime(dateNow, "%Y%m%d.%H%M")
    versionNode = ldml.ensure_path('identity/version')[0]
    versionNode.set('number', genTimeValue)
    versionNode.text = 'Generated by dbl2ldml'

    #versionNode = ldml.find('identity/version')
    #if versionNode is not None:
    #    idNode = ldml.find('identity')
    #    idNode.remove(versionNode)

    # Generation time - deprecated and also duplicated by version
    genTimeNode = ldml.find('identity/generation')
    if genTimeNode is not None:
        idNode = ldml.find('identity')
        idNode.remove(genTimeNode)
    #genTimeValue = datetime.strftime(dateNow, "%Y-%m-%d %H:%M:%S")
    #genTimeNode = ldml.ensure_path('identity/generation')[0]
    #genTimeNode.set('date', genTimeValue)

    if scriptCode is None or scriptCode == '':
        ldmlOutputFilename = outputPath + "/" + langCode + ".xml"
    else:
        ldmlOutputFilename = outputPath + "/" + langCode + "_" + scriptCode + ".xml"
    ldmlOutFile = codecs.open(ldmlOutputFilename, 'w', encoding="utf-8")
    ldml.normalise()
    ldml.serialize_xml(ldmlOutFile.write)

# end of processOneProject


def processProjects(inputPath, outputPath, filterLangCode = '', filterZipFile = ''):
    import sys
    if sys.maxunicode == 0x10FFFF:
        print('Python built with UCS4 (wide unicode) support')
    else:
        print('Python built with UCS2 (narrow unicode) support')

    ducetDict = ducet.readDucet()

    max = 10
    count = 0
    if filterZipFile == None or filterZipFile == '':
        filelist = os.listdir(inputPath)
        if inputPath[:-1] != '/':
            inputPath = inputPath + '/'
    else:
        basename = os.path.basename(filterZipFile)
        filterPath = filterZipFile.replace(basename, '')
        filelist = [basename]
        inputPath = inputPath + filterPath

    for filename in filelist:
        if filename.endswith('.zip'):   # and os.path.isfile(filename)
            basename = os.path.basename(filename)
            langCode = basename[0:3]
            if langCode[2:3] == "_":
                langCode = basename[0:2]

            #if langCode != "anv": continue   # TEMPORARY ------------------
            if filterLangCode == '' or langCode == filterLangCode:
                try:
                    print("")
                    print("-----------")
                    print("Processing: " + filename)
                    filePlusPath = inputPath + filename
                    processOneProject(filePlusPath, outputPath, ducetDict, langCode)
                except Exception as err:
                    print("ERROR: " + str(err))
                    print(type(err))
                    print(err.args)
                count = count + 1

            #if filterZipFile is not None and filterZipFile != '':
            #    break  # we've handled the one file specified

            #if count > max:
            #    break
        # end if filename ends with zip
    # end for files

# end of processProjects


def downloadAndProcess(updateDblSource, zipFile, dblPath, inputPath, outputPath):

    if updateDblSource:
        try:
            ok = downloadFromDbl(inputPath)
        except Exception as err:
            ok = False
            print("ERROR: " + str(err))
            print(type(err))
            print(err.args)
    else:
        ok = True

    if ok:
        langCode = ''
        #langCode = 'aai'  # debugging
        processProjects(inputPath, outputPath, langCode, zipFile)

# end of downloadAndProcess


if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-D','--dbl', default='true')  # values are 'false', 'true', 'update'
    parser.add_argument('-z','--zipfile')
    parser.add_argument('-l','--ldml',help='input LDML base file')
    parser.add_argument('-s','--ssf',help='input SSF file')
    parser.add_argument('-d','--lds',default="C:/ChordTeam/DBL2LDML/testdata/zzz/zzz.lds",help='input LDS file')
    parser.add_argument('-o','--outfile',default="C:/ChordTeam/DBL2LDML/testdata/zzz/zzz_Latn_out.xml",help='output LDML file')
    parser.add_argument('-O','--outpath',default="./temp_output",help="Directory to store generated ldml files in if not -l")
    parser.add_argument('-P','--dblpath',default="C:/ChordTeam/DBL2LDML",help="Path to local zips of DBL")
    parser.add_argument('-T','--tempdownload',default="./temp_download",help="Where to download a DBL zip file to, or where to find them")

    args = parser.parse_args()

    if args.dbl == 'true' or args.dbl == 'update':
        downloadAndProcess((args.dbl == 'update'), args.zipfile, args.dblpath, args.tempdownload, args.outpath)

    else:
        # Just process one set of files that is already present.
        ldmlFile = args.ldml
        ssfFile = args.ssf
        ldsFile = args.lds
        outFile = args.outfile

        if ldmlFile:
            ldml = Ldml(ldmlFile)
        else:
            ldml = Ldml(None)
        ldml.use_draft = 'generated'
        if ssfFile:
            processSsf(ldml, ssfFile)
        if ldsFile:
            #import ducet
            import sldr.collation
            ducetDict = ducet.readDucet()
            processLds(ldml, ldsFile, ducetDict)
        if outFile:
            outf = codecs.open(outFile, 'w', encoding="utf-8")
        else:
            outf = sys.stdout
        ldml.serialize_xml(outf.write)
