#!/usr/bin/env python3

import os, sys, csv, json, re, datetime
from argparse import ArgumentParser
import xml.etree.ElementTree as et
import csv, codecs
import urllib.request as urllib2
from collections import namedtuple
from multiprocessing import Pool
import sldr.langtags_full as lt
from palaso.unicode.ucd import UCD

# A useless comment

silns = "urn://www.sil.org/ldml/0.1"

hacks = {
    "aeb-Arab-TN": (lambda n:n['tags'].extend(("aeb-Arab-IL", "ar-aeb-Arab-IL", "ar-aeb-Arab-TN")), lambda n:n['regions'].remove('IL')),
#    "aeb-Hebr-IL": (lambda n:n['tags'].extend(("ar-aeb-Hebr", "ar-aeb-Hebr-IL")),),
#    "aeb-Latn-TN": (lambda n:n.setdefault('tags', []).extend(("ar-aeb-Latn", "ar-aeb-Latn-TN")),)
}

try: unicode
except NameError:
    unicode = str

Autonym = namedtuple('Autonym', ['langtag', 'name', 'roman'])
ucds = UCD()

def read_autonyms(filename):
    def stripinvisibles(s):
        return re.sub("[\u200E]", "", s)
    res = {}
    with open(filename) as inf:
        rdr = csv.reader(inf)
        next(rdr)
        for row in rdr:
            lang = row[0].strip()
            name = row[2].strip()
            if len(row) > 4:
                roman = row[4].strip()
            # handle missing comma after T
            elif len(row) > 3 and row[3].startswith("T"):
                roman = row[3][1:].strip()
            else:
                roman = ""
            script = script_analyze(name)
            lt = "{}-{}".format(lang, script)
            a = Autonym(lt, stripinvisibles(name), stripinvisibles(roman))
            res.setdefault(lt, []).append(a)
    return res

def script_analyze(s):
    res = {}
    for c in s:
        for sc in ucds.get(ord(c), 'scx').split():
            if not sc.startswith('Z'):
                res[sc] = res.setdefault(sc, 0) + 1
    return sorted(res.keys(), key=lambda x:-res[x])[0]

def find_file(tagstr, root='.', alt='.') :
    fname = tagstr.replace('-', '_') + '.xml'
    for a in (root, alt, '.'):
        for f in (fname, os.path.join(fname[0], fname)):
            testf = os.path.join(a, f)
            if os.path.exists(testf):
                return testf
    return None

def getlocalename(testf, getnames):
    try:
        doc = et.parse(testf)
    except:
        return None
    if len(doc.getroot()) > 1 :
        for getname in getnames:
            e = doc.find('./localeDisplayNames/languages/language[@type="{}"]'.format(getname.replace("-","_")))
            if e is not None:
                return unicode(e.text)
    return None

def issimple(testf):
    try:
        doc = et.parse(testf)
    except:
        return False
    root = doc.getroot()
    if len(root) < 2:
        return True
    elif len(root) > 2:
        return False
    elif root[-1].tag == "special" and all(x.tag.startswith(f"{{{silns}}}") for x in root[-1]):
        return True
    return False

def process_cldrentry(s, lts, allentries, root='.'):
    s = s.replace("_", "-")
    if s in lts:
        return
    l = lt.LangTag(s)
    if l.variants is not None and 'posix' in l.variants:
        return
    l.parentsame = False
    if l.region is not None:
        t = lt.LangTag(tag=None, lang=l.lang, script=l.script, variants=l.variants, extensions=l.extensions)
        n = None
        if str(t) in lts:
            n = lts[str(t)]
        elif str(t) in allentries:
            if str(l.region) in allentries[str(t)][1]:
                n = allentries[str(t)][0]
        if n is not None:
            if l.script is None:
                l.script = n.script
            if n.hidescript and l.lang+"-"+l.region not in lts:
                l.hidescript = True
        testf = find_file(str(t), root=root)
        if testf is not None:
            l.parent = t
            l.parentsame = issimple(find_file(s, root))
    if not l.parentsame:
        lts.add(l)
    return l

def addnames(n, b, names, regions, getname=True):
    alllrefs = set()
    regionlist = [b.region] + sorted(list(set(r for r in getattr(b, 'regions', []) if r != b.region)))
    for reg in regionlist:
        if reg is not None:
            regions.add(reg)
        lname = "{}-{}".format(getattr(b, 'iso639', b.lang), reg)
        alllrefs.add(lname)
        if lname in names:
            if 'name' in n or not getname:
                n['names'].append(names[lname][0][0])
            else:
                n['name'] = names[lname][0][0]
            n['names'].extend(names[lname][1])
    return alllrefs

def get_region_scriptnames():
    regs = {}
    scripts = {}
    fname = os.path.join(os.path.dirname(lt.__file__), "language-subtag-registry.txt")
    with open(fname) as f :
        currlang = None
        currscr = None
        mode = None
        deprecated = False
        for l in f.readlines() :
            l = l.strip()
            if l.startswith("Type: ") :
                mode = l[6:]
                if currlang is not None and currname is not None:
                    regs[currlang] = currname
                elif currscr is not None and currname is not None:
                    scripts[currscr] = currname
                currlang = None
                currscr = None
                currname = None
            elif l.startswith("Subtag: ") :
                if mode == "region":
                    currlang = l[8:]
                elif mode == "script":
                    currscr = l[8:]
            elif l.startswith("Description: "):
                if mode in ("region", "script"):
                    currname = l[13:]
        if currlang is not None:
            regs[currlang] = currname
        elif currscr is not None:
            scripts[currscr] = currname
    regs['TW'] = 'China-Taiwan'  # A tricky political choice
    return regs, scripts

def fromiso639(t, isomap):
    b = t.split("-")
    b[0] = isomap.get(b[0], b[0])
    return "-".join(b)

parser = ArgumentParser()
parser.add_argument('infile', help='Input CSV file from Google drive')
parser.add_argument('outfile', help='alltags.txt output file')
parser.add_argument('-f','--flatdir', default='.', help='Directory containing full sldr tree, for language names')
parser.add_argument('-i','--indir', default='.', help='Directory containing sldr tree with inheritance')
parser.add_argument('-p','--operators',action="store_true",default=False,help="Output complex operators")
parser.add_argument('-L','--langindex',help="Use local LanguageInfo.tab")
parser.add_argument('-M','--macros',help="Use ISO639-3 Macrolanguage mappings file")
parser.add_argument('-a','--autonyms',help="CSV of Autonyms from Ethnologue")
parser.add_argument('-q','--quiet',action='store_false',default=True,help='Be noisy')
parser.add_argument('-j','--jobs',type=int,default=0,help='Number of parallel jobs to run, 0 = number of processors')
parser.add_argument('-Z','--debug',help="Break inside for the given lang tag prefix")
args = parser.parse_args()

if args.debug is not None:
    args.jobs = 1

if args.flatdir and not os.path.exists(args.flatdir):
    print(f"Can't find {args.flatdir}")

lts = lt.LangTags(noalltags=True)
deprecated = {}
iananames = {}
suppress = {}
preferences = {}
cldrs = {}
for k,v in lts.items():
    if getattr(v, 'deprecated', False):
        deprecated[v.lang] = "1"
    if hasattr(v, 'desc'):
        iananames[k] = v.desc
    if getattr(v, 'suppress', False):
        suppress[k] = v.script
    if getattr(v, 'isCldr', False):
        cldrs[k] = True
    pref = getattr(v, 'preferred', None)
    if pref is not None:
        preferences.setdefault(pref, []).append(k)
regionnames, scriptnames = get_region_scriptnames()
lts.clear()
allentries = {}
allmacros = {}
iso639map = {}
if not args.quiet:
    print("Reading CSV")
with open(args.infile) as csvfile :
    reader = csv.DictReader(csvfile)
    for row in reader :
#        if 'CLDR' in row['confirmed'] : continue
        lid = row['Lang_Id']
        ls = row['likely_subtag']
        i6 = row['ISO 639-3']
        d = row['deprecated']
        rod = row['ROD']
        variants = row['variants'].split()
        macro = row['Macro']
        t = lt.LangTag(ls)
        b = lt.LangTag(lid)
        if b.lang == "000":
            continue
        if args.debug is not None and str(b).startswith(args.debug):
            import pdb; pdb.set_trace()
        if d != "":
            deprecated[t.lang] = d
            if (4 > len(d) > 1 or "-" in d) and macro == "":
                macro = d
        if b.script in [None, 'Zyyy', 'Qaax']:
            t.hidescript = True
        if b.region is None :
            t.hideregion = True
        t.hideboth |= (t.hideregion and t.hidescript)
        if b.variants is None and t.variants is not None:
            t.hidevariants = t.variants[:]
        if b.extensions is None and t.extensions is not None:
            t.hideextensions = {k: v[:] for k, v in t.extensions.items()}
        if t.lang in deprecated:
            t.deprecated = deprecated[t.lang]
        t.regions = row['regions'].split()
        t.name = [row['LangName']]
        t = lts.add(t)
        if repr(t) in lts:
            lts[repr(t)].iso639 = i6
        if i6 != t.lang:
            iso639map[i6] = t.lang
        allentries[lid] = (t, row['regions'].split(' '))
        if macro != "":
            s = str(t)
            m = lt.LangTag(macro)
            if macro in t.allforms():
                t.primary = macro
            else:
                m.merge_equivalent(t)
            allmacros[s] = repr(m)
        t.nofonipa = not t.hidescript and t.script != 'Latn'
        t.extra_variants = variants
        t.rod = rod
        t.obsolete = row['obsolete'] != ""
        t.unwritten = row['unwritten'] != ""

if not args.quiet:
    print("Reading sldr")
for l in os.listdir(args.indir) :
    if l.endswith('.xml') :
        if 1 < len(l.split('_', 1)[0]) < 4 :
            process_cldrentry(l[:-4], lts, allentries, root=args.indir)
    elif len(l) == 1 and os.path.isdir(os.path.join(args.indir, l)) :
        for s in os.listdir(os.path.join(args.indir, l)) :
            if s.endswith('.xml') :
                if 1 < len(s.split('_', 1)[0]) < 4 :
                    process_cldrentry(s[:-4], lts, allentries, root=args.indir)

for k, v in allmacros.items():
    try:
        base = lts[k]
        macro = lts[v]
    except KeyError:
        macro = lt.LangTag(v)
        lts.add(macro)
        macro.iso639 = base.iso639
        if base.script is None:
            macro.hidescript = True
        if base.region is None:
            macro.hideregion = True
    if base != macro:
        base.skip = True
    if not hasattr(macro, 'regions'):
        macro.regions = []
    if hasattr(base, 'regions'):
        macro.regions.extend(base.regions)
    ev = getattr(base, 'extra_variants', [])
    if len(ev):
        macro.extra_variants = sorted(set(getattr(macro, 'extra_variants', []) + ev))
    macro.base.append(base)

    
names = {}
pejoratives = {}
if args.langindex:
    namef = open(args.langindex, "r")
else:
    req = urllib2.Request(url="https://www.ethnologue.com/codes/LanguageIndex.tab",
                          headers = {"User-Agent": "Mozilla"})
    namef = urllib2.urlopen(req)
reader = csv.DictReader(namef, dialect=csv.excel_tab)
for r in reader:
    if r['CountryID'] == 'ET' and r['NameType'] == 'LA':     # hack fix for buggy Ethnologue data. To Be Reviewed 21/Feb/2019, 2020
        pejoratives.setdefault(r['LangID']+"-"+r['CountryID'], []).append(r['Name'].lower())
    elif r['NameType'].startswith("L") and r['CountryID'] and not r['NameType'].endswith("P"):
        names.setdefault(r['LangID']+"-"+r['CountryID'], [[],[]])[0 if r['NameType'] == 'L' else 1].append(r['Name'])
    elif r['NameType'] == 'LP' and r['CountryID']:
        pejoratives.setdefault(r['LangID']+"-"+r['CountryID'], []).append(r['Name'].lower())
namef.close()

macros = {}
if args.macros:
    namef = open(args.macros, "rb")
else:
    req = urllib2.Request(url="https://iso639-3.sil.org/sites/iso639-3/files/downloads/iso-639-3-macrolanguages.tab",
                            headers = {"User-Agent": "Mozilla"})
    namef = urllib2.urlopen(req)
fields = namef.readline().decode("utf-8").split("\t")
for l in namef.readlines():
    b = l.decode("utf-8").strip().split("\t")
    r = {fields[i]: b[i] for i in range(len(fields))}
    macros[r['I_Id']] = r['M_Id']
namef.close()


collisions = {}
res = []
alllangreg = set()
allscripts = {}
allregions = {}
langregions = {}
usedregions = set()
usedscripts = set()

for t in set(lts.values()):
    x = lt.LangTag(lang=t.lang, region=t.region, variants=t.variants, extensions=t.extensions)
    allregions.setdefault(str(x), set()).add(t)
    usedregions.add(t.region)
    usedscripts.add(t.script)
    y = t.copy()
    #y.hidescript = True
    #y.hideboth = True
    y.hideregion = True
    langregions.setdefault(str(y), set()).add(t.region)
    x = lt.LangTag(lang=t.lang, script=t.script, variants=t.variants, extensions=t.extensions)
    allscripts.setdefault(str(x), set()).add(t)

if args.autonyms:
    autonyms = read_autonyms(args.autonyms)
else:
    autonyms = {}

def make_record(t):
    if t.skip:
        return None
    r = t.allforms()
    if not len(r[0]):
        return None
    if r[0][0] == "q" and r[0][1] in "abcdefghijklmnopqrst":
        return None
    if t.region is None:
        print("Missing Region for {}".format(str(t)))
        return None
    if hasattr(t, 'primary'):
        r = [t.primary] + [x for x in r if x != t.primary]
    # Add an entry for the region (if region hidden and this is the only tagset for this region in this lang)
    if t.hideregion and not t.hidescript:
        x = lt.LangTag(lang=t.lang, region=t.region, variants=t.variants, extensions=t.extensions)
        if len(allregions.get(str(x), [])) < 2:
            r.insert(1, str(x))
    # Add an entry for the script (if script hidden and this is the only tagset for this script in this lang)
    elif t.hidescript and not t.hideregion:
        x = lt.LangTag(lang=t.lang, script=t.script, variants=t.variants, extensions=t.extensions)
        if len(allscripts.get(str(x), [])) < 2:
            r.insert(1, str(x))
    # Fixup for Windows getting confused by reserved device names in DOS
    if r[0] in ('con', 'prn', 'aux', 'nul'):
        ts = f"{r[0]}-Arab" if r[0] == "prn" else f"{r[0]}-Latn"
        if ts in r:
            r.remove(ts)
        r.insert(0, ts)
    n = {'tag': r[0], 'full': r[-1], 'script': t.script, 'region': t.region}
    wintag = t.copy()
    if suppress.get(str(t), '') == t.script:
        n['suppress'] = True
        wintag.hidescript = True
    else:
        wintag.hidescript = False
    if len(langregions.get(t.lang, [])) > 1:
#        if t.lang.startswith("a"):
#            print(f"{t.lang}: {langregions[t.lang]}")
        wintag.hideregion = False
    n['windows'] = str(wintag)
    if t.region in regionnames:
        n['regionname'] = regionnames[t.region]
    tags = set(r[1:-1])
    hassldr = False
    # n['primary'] = True
    if False and "-x-" in n['tag']:
        stripx = n['tag'].find("-x-")
        s = n['tag'][:stripx] if stripx >=0 else n['tag']
        finald = s.rfind("-")
        if finald >= 0:
            test = s[:finald]
            testl = lts.get(test, None)
            if testl is not None and testl != t:
                n['primary'] = False

    # A hack to resolve ICU's used of zh and industry use of zh-CN
    # Code positioned here to use zh when hunting localname
    if r[0] == 'zh':
        n['tag'] = 'zh-CN'
        tags.discard('zh-CN')
        tags.add("zh")
    for a in [n['tag'], n['full']] + list(tags):
        atag = lt.LangTag(a)
        al = atag.lang
        if al not in preferences:
            continue
        for ap in preferences[al]:
            ntag = lt.LangTag(a)
            apt = lt.LangTag(ap)
            if apt.script is not None:
                if t.script != apt.script:
                    continue
                ntag.script = apt.script
            elif ntag.script is not None and apt.script is not None and ntag.script != apt.script:
                continue
            if apt.region is not None:
                if apt.region != t.region:
                    continue
                ntag.region = apt.region
            ntag.lang = apt.lang
            tags.add(str(ntag))
    if len(tags):
        n['tags'] = sorted(tags)
    if hasattr(t, 'iso639'):
        n['iso639_3'] = t.iso639
    elif t.base is not None and len(t.base):
        for b in t.base:
            if hasattr(n, 'iso639'):
                n['iso639_3'] = b.iso639
                break
    elif t.lang in lts:
        x = lts[t.lang]
        if hasattr(x, 'iso639'):
            n['iso639_3'] = x.iso639
    if 'iso639_3' in n:
        macro = macros.get(n['iso639_3'], None)
        if macro is not None:
            n['macrolang'] = iso639map.get(macro, macro)
    auto = set()
    for l in r:
        x = lt.LangTag(l)
        # Pending a need:
        # if str(x) in cldrs:
        #     n['fromCldr'] = True
        if not x.script:
            continue
        s = "{}-{}".format(x.lang, x.script)
        if s in autonyms:
            auto.update(autonyms[s])
        if 'iso639_3' in n:
            s = "{}-{}".format(n['iso639_3'], x.script)
            if s in autonyms:
                auto.update(autonyms[s])
    if len(auto):
        lauto = sorted(auto)
        n['localnames'] = [a.name for a in lauto]
        n['latnnames'] = [a.roman for a in lauto]
        if not any(n['latnnames']):
            del n['latnnames']
    for rf in r:
        tf = find_file(rf, root=args.flatdir, alt=args.indir)
        if tf is not None:
            hassldr = True
            getnames = [n['full']]
            if len(r) > 2:
                getnames.extend(n['tags'])
            getnames.extend([n['tag'], t.lang])
            name = getlocalename(tf, getnames)
            if name is not None and name != "" and 'localname' not in n:
                n['localname'] = name
    n['sldr'] = hassldr
    n['names'] = []
    regions = set()
    alllrefs = addnames(n, t, names, regions)
    for b in t.base:
        alllrefs.update(addnames(n, b, names, regions, getname=False))
    if t.lang in iananames:
        extlang = "-".join([t.lang] + t._extensions())
        ns = iananames.get(extlang, getattr(t, 'name', iananames[t.lang]))[:]
        if len(ns):
            if 'name' not in n:
                n['name'] = ns[0]
            n['iana'] = ns
            if len(ns) > 0 :
                n['names'].extend(ns)
    if 'name' in n:
        n['names'] = sorted(x for x in set(n['names']) - set([n['name']]) if len(x))
        n['names'] = [x for x in n['names'] if x.lower() not in pejoratives.get(n.get('iso639_3', t.lang)+"-"+t.region, [])]
    elif hasattr(t, 'name'):
        n['name'] = t.name[0]
    else:
        print("Missing name entry for {} = {}".format(n['tag'], n['full']))
    if not len(n['names']):
        del n['names']
    if not len(n.get('iso639_3', "fred")):
        del n['iso639_3']
    if t.region is not None:
        regions.discard(t.region)
    regions.discard('XX')
    testtag = re.sub('-[A-Z]{2}(?:-|$)', '-{}', n['full'])
    regions = [r for r in regions if testtag.format(r) not in lts]
    if len(regions):
        n['regions'] = sorted(regions)
#            n['regions'] = " ".join(sorted(regions))
    if getattr(t, 'nofonipa', False):
        n['nophonvars'] = True
    if getattr(t, 'extra_variants', ""):
        n['variants'] = t.extra_variants
    if getattr(t, 'rod', ""):
        n['rod'] = t.rod
    if getattr(t, 'obsolete', False):
        n['obsolete'] = True
    if getattr(t, 'unwritten', False):
        n['unwritten'] = True
    if n['full'] in hacks:
        for c in hacks[n['full']]:
            c(n)

    if n['tag'] in collisions:
        print("Multiple entries for: {} = {}, {}".format(n['tag'], n['full'], collisions[n['tag']]))
    else:
        collisions[n['tag']] = n['full']
    for k, v in n.items():
        if v is None:
            print("Null entry for {} in {}".format(k, n['tag']))
    return n, alllrefs

if not args.quiet:
    print("Making records")
# special record for passing standard variants
conformance = {'tag': "_conformance"}
extrascripts = set((x for x in scriptnames.keys() if len(x) == 4))
extrascripts.update(('Qaa'+chr(97+a) for a in range(26)))
extrascripts.update(('Qab'+chr(97+a) for a in range(24)))
conformance['scripts'] = sorted(extrascripts - usedscripts)
res.append({'tag': '_globalvar', 'variants': ['simple']})
res.append({'tag': "_phonvar", 'variants': ['alalc97', 'fonipa', 'fonkirsh', 'fonnapa', 'fonupa', 'fonxsamp']})
res.append(conformance)
res.append({'tag': "_version", 'api': '1.3.1', 'date': str(datetime.date.today())})
if args.jobs == 1:
    for t in sorted(set(lts.values())):
        r = make_record(t)
        if r is not None:
            res.append(r[0])
            alllangreg.update(r[1])
else:
    pool = Pool(processes=args.jobs) if args.jobs else Pool()
    tempres = pool.imap_unordered(make_record, sorted(set(lts.values())))
    for r in sorted(tempres, key=lambda t:t[0]['full'] if t is not None else ""):
        if r is not None:
            res.append(r[0])
            alllangreg.update(r[1])

for t in res:
    usedregions.update(getattr(t, 'regions', []))
possibleregions = set((x for x in regionnames.keys() if len(x) == 2))
possibleregions.update(('Q' + chr(65+a) for a in range(12, 26)))
possibleregions.update(('X' + chr(65+a) for a in range(26)))
conformance['regions'] = sorted(possibleregions - usedregions)

def mergelines(m):
    return "[" + " ".join(x.strip() for x in m.group(1).split("\n")) + "]"

if not args.quiet:
    print("Output results")
output = json.dumps(sorted(res, key=lambda x:x['tag']), sort_keys=True, indent=4, ensure_ascii=False)
output = "[" + re.sub(r'\[([^\]]+)\]', mergelines, output[1:-1]) + "]\n"
with codecs.open(args.outfile, "w", encoding="utf-8") as fh:
    # use encoding="utf8" (not "utf-8") as a hack around a bug in json: https://stackoverflow.com/a/40777484/10488020
    fh.write(output)
missing = set(names.keys()) - alllangreg
missing = set(fromiso639(x, iso639map) for x in missing)
if len(missing):
    print("Missing language regions ({}): ".format(len(missing)) + " ".join(sorted(missing)))

