#! /usr/bin/env python
# -*- coding: utf-8 -*-
# License: CC BY-SA
# Python: 3.3.2
# Author: Aleksi J. Sahala (University of Helsinki)
# https://helsinki.academia.edu/AleksiSahala

__version__ = "2016-04-27"

"""
This script
  1) converts ETCSL (http://etcsl.orinst.ox.ac.uk/) transliteration
  into traditional notation used in most Sumerian grammars and
  textbooks. Following the classical convention sign names are not
  accented.

  2) generates a frequency list containing:
    - Lemma frequency (with translation and POS tagging).
    - Wordform frequency with cross references to all unique
      occurrences.
    - Cross references from wordforms to actual text.

  3) generates an interactive cuneiform version of the text.

Please note:
  Use for educational purposes only. See the disclaimer on using the
  cuneiform signs at http://etcsl.orinst.ox.ac.uk/edition2/signlist.php

  The HTML generated by this script will not work properly in offline
  mode as the javascript, stylesheets and sign bitmaps are loaded from
  ETCSL.

To do:
  - Include morphological analyzer.
  - Tidy up messy html-generator code
  - Add generator for numerals
  - Handle lines with multiple different attestations properly

Known bugs:
  - Certain ETCSL pages may not load properly or at all.
  - Some transliterated numbers will appear in cuneiform as arabic numerals
    instead of cuneiform

HOW TO USE:
   Replace self.url with the link to the desired ETCSL transliteration.
   Use Unicode version of the text (make sure that the url ends with
   &charenc=gcirc).
"""

import etcsl_head
import urllib.request
import re
import sys

class XLITconv:

    def __init__(self):
        self.url = 'http://etcsl.orinst.ox.ac.uk/cgi-bin/etcsl.cgi?text=c.4.08.08&display=Crit&charenc=gcirc'
        self.hr = '<hr width=100%>'
        self.base_style = etcsl_head.head
        self.scripts = etcsl_head.scripts
        self.message = (self.hr + '<p>Generated from'
            ' <a href="{u}">ETCSL</a> with xlitconv</p>'.format(u=self.url))
        self.settings = {'make_freqlist': True,
                         'make_cuneiform': True,
                         'convert_indices': True,
                         'cuneiform_wordspacing': True}
        self.name = re.sub('.*text=(.+?)&.*', r'\1', self.url).replace('.', '')
        self.signlist = {}
        self.inv_signlist = {}
        self.output_html = []
        self.cuneiform = []
        self.trans_table = []
        self.load_signlist()
        html = self.html_request()
        self.parse_html(html)

    def load_signlist(self):
        print('reading sign list...')
        with open('signlist2.txt', 'r') as f:
            signlist = f.read().splitlines()

        for line in signlist:
            values, link = line.split('\t')
            for value in values.split(','):
                #self.signlist[re.sub('<\/?sub>', '', value.lower())] = link
                self.signlist[value.lower()] = link

                if link not in self.inv_signlist.keys():
                    self.inv_signlist[link] = [value]
                else:
                    self.inv_signlist[link].append(value)

    def html_request(self):
        """ Read HTML from ETCSL """
        try:
            with urllib.request.urlopen(self.url) as data:
                return data.read().decode('utf-8').splitlines()
        except:
            print('Cannot read %s' %self.url)
            sys.exit(1)

    def write_file(self, data, suffix):
        """ Write transliteration, freq list and cuneiform """
        x = open(self.name + suffix + '.html', 'w')
        x.writelines('\n'.join(data) + self.message + '</body></html>')
        x.close()
        print('created: ' + self.name + suffix + '.html')

    def translate_accents(self, line):
        for item in self.trans_table:
            line = re.sub(re.escape(item[1]), item[0], line)
        return line

    def generate_freqlist(self, wordlist):
        """ Generate frequency list """

        def def_link(lineno):
            link = '<a href="%s#%s">%s</a>'\
                   % (self.name + '_xlit.html', lineno, lineno)
            #(self.url[0:-1]+'&lineid='+lineno+'#'+lineno, lineno)
            return(link)

        stats = []
        stats.append('<html>' + self.base_style\
                     + '<body><h1>Frequency list</h1>' + self.hr)

        """ Define custom alphabetic sort order. X, x = for broken signs """
        sort_order = [str(i) for i in range(0, 10)] +\
                     [('a', 'A', '&a'), ('b', 'B'), ('d', 'D'),
                      ('e', '&e', 'E'), ('g', 'G'), (u'&#x011'), (u'&#x1E2'),
                      ('i', '&i', 'I'), ('k', 'K'), ('l', 'L'), ('m', 'M'),
                      ('n', 'N'), ('p', 'P'), ('q', 'Q'), ('r', 'R'),
                      ('s', 'S'), (u'&#x016'), ('t', 'T'), ('u', '&u', 'U'),
                      ('w', 'W'), ('z', 'Z'), ('x', 'X')]

        for letter in sort_order:
            for key in sorted(wordlist.keys()):
                if key.startswith(letter):
                    lemma, pos, trans = re.sub('(\([A-Z].*?\)|\(\))',
                                               r'__\1__', key).split('__')
                    freq = wordlist[key]['freq']

                    """ Count wordforms, their freqs and unique occurrences """
                    wf_freqs = {}
                    for wf in wordlist[key]['wfs']:
                        if wf[0] not in wf_freqs.keys():
                            wf_freqs[wf[0]] = {'freq': 1,
                                               'attested': [def_link(wf[1])]}
                        else:
                            wf_freqs[wf[0]]['freq'] += 1
                            wf_freqs[wf[0]]['attested'].append(def_link(wf[1]))

                    wfrows = ''
                    for wf in wf_freqs.keys():
                        frequency = '(%s)' % str(wf_freqs[wf]['freq'])
                        attested = ', '.join(wf_freqs[wf]['attested'])
                        wfrows += '<tr><td width="200">'+frequency+' '+wf\
                                  +'</td><td>'+attested+'</td></tr>'

                    row = ''.join(['<td>'+k+'</td>'\
                                   for k in ['(%s) '%str(freq)\
                                             +'<b>'+lemma+'</b>',
                                             '&quot;'+trans.strip()+'&quot;',
                                             pos]])

                    stats.append('<table><tr>' + row + '</tr></table>')
                    stats.append('<blockquote><table>'
                                 + wfrows + '</table></blockquote>')
                else:
                    pass

        self.write_file(stats, '_freqlist')

    def find_tokens(self, line, printlineno):
        """ Find all tokens from line and get line number """
        data = re.findall('event.+?<\/span>', line)
        if '<a name=' in line:
            lineno = re.sub('.*<a name=\'(.+?)\'.*', r'\1', line)
            if printlineno:
                self.cuneiform.append('<a name="%s" href="%s#%s">%s</a>'\
                    % (lineno, self.name + '_xlit.html', lineno, lineno))
                #(self.url[0:-1]+'&lineid='+lineno+'#'+lineno, lineno))
        for item in data:
            item = re.sub('&#x1E2B;ide', 'hide', item) #Fixes an ETCSL bug
            tmp = item[8:].split('" onMouseout="hideTip()">')
            yield {'lineno': lineno,
                   'lemma': tmp[0][0:-2],
                   'wf': tmp[1]}

    def generate_cuneiform(self, data):
        """ Generate cuneiform representation """

        self.cuneiform.append('<html>')
        self.cuneiform.append(self.base_style)
        self.cuneiform.append('<h1>Cuneiform</h1>' + self.hr)

        def split_signs(token):
            """ Remove html-tags and meta signs, but preserve <sub> </sub>"""
            #todo: handle alternative lines and fractions correctly
            removable = [r'[\/\\\?\!\[\]]', r'(&[gl]t;|&[lr]aquo;)',
                         r'\(.+?\)', '<\/?span>']
            token = re.sub('<\/sub>', '¤', token)
            for item in removable:
                token = re.sub(item, '', token)
            token = re.sub(r'<\/?sup>', '-', token)
            token = re.sub('¤', '</sub>', token)
            #token = re.sub('(&gt;|&lt;)', '', token)
            return token.strip('-').split('-')

        def make_tooltip(sign, token):
            """ Generate tooltip information for cuneiform signs """
            info = '<h3>{s}</h3><h5>{lemma}</h5>{vals}'.format(
                s=self.translate_accents(sign),
                lemma=token['lemma'],
                vals='<br/><b>Values:</b> '\
                    + ', '.join(self.inv_signlist[self.signlist[sign.lower()]]))
            code = """<span onMouseover="doTooltip(event, """\
                    """'{info}')" onMouseout="hideTip()">""".format(info=info)
            return (code)

        for line in data:
            if 'class="gap"' in line:
                lacuna_type = re.sub('.*class="gap">(.+?)<.*', r'\1', line)
                self.cuneiform.append('<br/><p>' + lacuna_type + '</p><br/>')
            tokens = self.find_tokens(line, True)
            for token in tokens:
                signs = split_signs(token['wf'])
                if self.settings['cuneiform_wordspacing']:
                    self.cuneiform.append('<img src="empty.png"/>')
                for sign in signs:
                    if sign:
                        sign = sign.strip()
                        if sign.lower() in self.signlist.keys():
                            self.cuneiform.append(make_tooltip(sign, token))
                            self.cuneiform.append(self.signlist[sign.lower()])
                            self.cuneiform.append('</span>')
                        elif sign == 'X':
                            self.cuneiform.append('<img src="broken.png"/>')
                        elif sign == '&hellip;':
                            self.cuneiform.append('<img src="brokenlong.png"/>')
                        else:
                            self.cuneiform.append(sign)
            if 'onMouseover' in line:
                self.cuneiform.append('<br/>')

        self.cuneiform.append(self.scripts + '</body>')
        self.write_file(self.cuneiform, '_cuneiform')

    def parse_html(self, html):
        print('parsing html...')
        wordlist = {}
        raw_cuneiform = []

        def count_freq(line):
            """ Count lemmas and their wordforms """
            tokens = self.find_tokens(line, False)
            for token in tokens:
                lemma = token['lemma']
                wordform = token['wf']
                lineno = token['lineno']
                if lemma not in wordlist.keys():
                    wordlist[lemma] = {'freq' : 1,
                                       'wfs' : [[wordform, lineno]]}
                else:
                    wordlist[lemma]['freq'] += 1
                    wordlist[lemma]['wfs'].append([wordform, lineno])

        def redirect_links(line):
            """ Fix relative links """
            base_url = "http://etcsl.orinst.ox.ac.uk"
            line = re.sub("a href=\'",
                          "a href='%s/cgi-bin/" %base_url, line)
            line = re.sub("='\.\.",
                          "='%s" %base_url, line)
            return line

        def traverse_back(pos, line):
            """ Traverse back to find a sign delimiter or HTML code
            and calculate sign's position in line (the list of delimiters
            is not perfect but should cover most cases)."""
            init_pos = pos
            while pos > 0:
                if line[pos] in ['>', "'", "-", ".", " ", "+"]:
                    sign = line[pos+1:init_pos]
                    break
                pos -= 1
            return sign

        def make_translation_table(signs):
            v_replace = {'a': [u'&aacute;', u'&agrave;'],
                        'e': [u'&eacute;', u'&egrave;'],
                        'i': [u'&iacute;', u'&igrave;'],
                        'u': [u'&uacute;', u'&ugrave;']}
            """ Build translation tables for 2/3-indexed sign values """
            trans_table = []
            for pair in signs:
                substitute = True
                newsign = ''
                for char in pair[0]:
                    if char in v_replace.keys() and substitute:
                        newsign += v_replace[char][int(pair[1])-2]
                        substitute = False
                    else:
                        newsign += char
                trans_table.append([newsign, pair[0]+'<sub>'+pair[1]+'</sub>'])
            return trans_table

        """ Redirect relative links and clean up unwanted html code """
        for line in html:
            line = redirect_links(line)
            line = re.sub('<\/(html|body)>' ,'', line)
            # Remove javascript preventing cross-page anchors
            line = re.sub('<body onLoad=.+?>', '<body>', line)
            # Create croess-references to the cuneiform
            line = re.sub("<a name='(.+?)'></a>",
                    r"<a name='\1'></a>[<a href='%s_cuneiform.html#\1'>C</a>]"\
                    %self.name, line)

            """ Store indices for cuneiform generation if wanted """
            if self.settings['make_cuneiform']:
                raw_cuneiform.append(line)

            """ Detect 2/3 indexed signs """
            if self.settings['convert_indices']:
                signs = []
                i = 0
                while i < len(line) - 12:
                    chunk = line[i:i+12]
                    if re.match('<sub>[23]</sub>', chunk):
                        signs.append([traverse_back(i, line), chunk[5]])
                    i += 1

                """ Translate indices into accents if needed """
                if signs:
                    trans_table = make_translation_table(signs)
                    self.trans_table += trans_table
                    line = self.translate_accents(line)

            self.output_html.append(line)

            """ Count frequencies if such option is chosen """
            if self.settings['make_freqlist']:
                count_freq(line)

        """ Build frequency list (if such option is chosen) """
        if self.settings['make_freqlist']:
            self.generate_freqlist(wordlist)

        """ Make cuneiform if wanted """
        if self.settings['make_cuneiform']:
            self.generate_cuneiform(raw_cuneiform)

        """ Make traditionalized transliteration """
        self.write_file(self.output_html, '_xlit')

def main():
    XLITconv()

if __name__ == "__main__":
    main()
