
import json
import sys
import os

from urllib.request import urlopen
from urllib.parse import quote
from bs4 import BeautifulSoup

def save_wordmap(wordmap):
    with open('wordmap.json', 'w', encoding='utf-8') as f:
        json.dump(wordmap, f, ensure_ascii=False)

def youdao_query(word):
    url = "http://www.youdao.com/w/eng/" + word
    c1 = urlopen(url).read()
    c2 = c1.decode('utf-8')
    return c2

def youdao_normalize(html):
    soup = BeautifulSoup(html, features='html.parser')
    c2 = {"pronounces": [], "explains": [], "synonyms": [], "relwords": []}
    keyword = soup.select('.wordbook-js .keyword')
    if len(keyword) == 0:
        return None
    else:
        c2["keyword"] = keyword[0].string
    explains = soup.select('#phrsListTab .trans-container li')
    if len(explains) == 0:
        return None
    else:
        for v in explains:
            c2["explains"].append(v.string)
    pronounces = soup.select('#phrsListTab .pronounce .phonetic')
    for v in pronounces:
        c2["pronounces"].append(v.string)
    synonyms = soup.select('#synonyms a')
    for v in synonyms:
        c2["synonyms"].append(v.string)
    relwords = soup.select('#relWordTab a')
    for v in relwords:
        c2["relwords"].append(v.string)
    return c2

def get_wordlist():
    wordlist = []
    with open('./wordlist.txt', 'r') as f:
        for line in f:
            wordlist.append(line.split()[0])

    a = []
    try:
        with open('./append.txt', 'r') as f:
            for line in f:
                a.append(line.split()[0])
    except:
        pass

    for word in a:
        if word not in wordlist:
            wordlist.append(word)

    b = []
    try:
        with open('./ignore.txt', 'r') as f:
            for line in f:
                b.append(line.split()[0])
    except:
        pass

    for word in b:
        if word in wordlist:
            wordlist.remove(word)
    return wordlist

def create_wordmap(wordlist):
    wordmap = {}
    try:
        with open('wordmap.json', 'r') as f:
            wordmap = json.load(f)
    except:
        pass
    for i, w in enumerate(wordlist):
        if w not in wordmap:
            wordmap[w] = {}
        wordmap[w]["word"] = w
        wordmap[w]["list_index"] = i + 1
    
    wordmap2 = {}
    for k in wordmap:
        if k in wordlist:
            wordmap2[k] = wordmap[k]
    save_wordmap(wordmap2)
    return wordmap2

def youdao(wordmap):
    for k in wordmap:
        if "explains" not in wordmap[k] or len(wordmap[k]["explains"]) == 0:
            print("youdao.com: " + k)
            r1 = youdao_query(k)
            r2 = youdao_normalize(r1)
            if not r2:
                print("youdao.com not found " + k)
                continue
            if r2["keyword"] != k:
                print("youdao.com not match " + k + " " + r2["keyword"])
                continue
            wordmap[k]["pronounces"] = r2["pronounces"]
            wordmap[k]["explains"] = r2["explains"]
            wordmap[k]["synonyms"] = r2["synonyms"]
            wordmap[k]["relwords"] = r2["relwords"]
            save_wordmap(wordmap)

def __getfamily(html):
    soup = BeautifulSoup(html, features='html.parser')
    c2 = soup.find('vcom:wordfamily')
    if not c2:
        return None
    return c2.get('data')

def query_vocabulary(word, entry):
    url = "http://www.vocabulary.com/dictionary/" + word
    r1 = os.popen("curl -L -s " + url)
    r2 = __getfamily(r1.read())
    if r2:
        r3 = json.loads(r2)
        for a in r3:
            if a["word"] == word:
                entry["freq"] = a["freq"]
                entry["ffreq"] = a["ffreq"]
                if "parent" in a:
                    entry["parent"] = a["parent"]
                print(word + ' found vocabulary.com ' + str(a["freq"]))
                return True
    print(word + ' not found vocabulary.com !!!')
    return False

def vocabulary(wordmap):
    for k in wordmap:
        if "freq" not in wordmap[k]:
            print("vocabulary.com: " + k)
            if query_vocabulary(k, wordmap[k]):
                save_wordmap(wordmap)

def kv(d, k):
    if k in d:
        return d[k]
    return 0

def convert_freq_to_seq(wordmap, k):
    wordsort1 = []
    for word in wordmap:
        wordmap[word]["word"] = word
        wordsort1.append(wordmap[word])
    wordsort1.sort(key=lambda entry: kv(entry, k), reverse=True)
    for i, word in enumerate(wordsort1):
        wordmap[word["word"]][k + "_index"] = i

def bnc_seq(wordmap):
    bnc = []
    try:
        with open('./bnc15knum.txt', 'r') as f:
            for line in f:
                bnc.append(line.split()[0])
    except:
        return
    for i, word in enumerate(bnc):
        if word in wordmap:
            wordmap[word]["bnc_index"] = i

def manual_adjust_seq(wordmap):
    adjust = {}
    try:
        with open('./append.txt', 'r') as f:
            for line in f:
                xx = line.split()
                if len(xx) > 1:
                    adjust[xx[0]] = int(xx[1])
    except:
        return
    for word in adjust:
        if word in wordmap:
            wordmap[word]["adjust_index"] = adjust[word]

def mix_sort(elem):
    res = 80000
    if "list_index" in elem:
        res = min(elem["list_index"], res)
    if "ffreq_index" in elem:
        res = min(elem["ffreq_index"], res)
    if "freq_index" in elem:
        res = min(elem["freq_index"], res)
    if "bnc_index" in elem:
        res = min(elem["bnc_index"], res)
    return res

def fin_sort(elem):
    res = elem["seq"]
    if "adjust_index" in elem:
        res = elem["adjust_index"]
    return res

def sequence(wordmap):
    convert_freq_to_seq(wordmap, 'freq')
    convert_freq_to_seq(wordmap, 'ffreq')
    bnc_seq(wordmap)
    manual_adjust_seq(wordmap)
    wordsort1 = []
    for word in wordmap:
        wordsort1.append(wordmap[word])
    wordsort1.sort(key=mix_sort, reverse=False)
    for i, word in enumerate(wordsort1):
        word["seq"] = i + 1
    wordsort1.sort(key=fin_sort, reverse=False)
    for i, word in enumerate(wordsort1):
        word["seq"] = i + 1
    return wordsort1
    
def build_family(wordmap):
    for word in wordmap:
        wordmap[word]["family"] = []
        if "parent" in wordmap[word]:
            wordmap[word]["family"].append(wordmap[word]["parent"])
        if "relwords" in wordmap[word]:
            for x in wordmap[word]["relwords"]:
                if x not in wordmap[word]["family"]:
                    wordmap[word]["family"].append(x)
    for word in wordmap:
        if "parent" in wordmap[word]:
            p = wordmap[word]["parent"]
            if p in wordmap:
                if word not in wordmap[p]["family"]:
                    wordmap[p]["family"].append(word)

def generate(wordsort):
    wordlist = []
    for i, word in enumerate(wordsort):
        t = {}
        t["w"] = word["word"]
        t["p"] = word["pronounces"]
        t["i"] = word["seq"]
        t["e"] = []
        for y in word["explains"]:
            if y != None:
                t["e"].append(y)
        if "synonyms" in word and 0 < len(word["synonyms"]):
            t["s"] = word["synonyms"]
        if "family" in word and 0 < len(word["family"]):
            t["f"] = word["family"]
        wordlist.append(t)
    with open('../wordlist.json', 'w') as f:
        json.dump(wordlist, f, ensure_ascii=False)

#main()
wordlist = get_wordlist()
wordmap = create_wordmap(wordlist)
youdao(wordmap)
vocabulary(wordmap)
build_family(wordmap)
wordsort = sequence(wordmap)
generate(wordsort)
