# -*- coding: utf-8 -*-
"""
Created on Fri May  5 21:11:25 2017

@author: Kazuki
"""

import requests
import re
import time
from random import randint
import pandas as pd
import utils
from collections import Counter

#tf = pd.read_csv('../nlp_source/tf.csv.gz')
train, test = utils.load(0)
all_q = pd.concat([train.q1, train.q2, test.q1, test.q2], ignore_index=1)
all_q = pd.DataFrame(all_q.unique(), columns=['q'])

word_tbl = Counter()

all_q['is_search'] = 0

raise

for i,q in enumerate(all_q.q):
    words = q.split()
    if sum([1 for w in words if w not in word_tbl])>0:
        all_q.ix[i, 'is_search'] = 1
    word_tbl += Counter(words)




#==============================================================================
# google
#==============================================================================
START_SPELL_CHECK="<span class=\"spell\">Showing results for</span>"
END_SPELL_CHECK="<br><span class=\"spell_orig\">Search instead for"

HTML_Codes = (
		("'", '&#39;'),
		('"', '&quot;'),
		('>', '&gt;'),
		('<', '&lt;'),
		('&', '&amp;'),
)

def spell_check(s):
	q = '+'.join(s.split())
	time.sleep(  randint(0,2) ) #relax and don't let google be angry
	r = requests.get("https://www.google.co.uk/search?q="+q)
	content = r.text
	start=content.find(START_SPELL_CHECK) 
	if ( start > -1 ):
		start = start + len(START_SPELL_CHECK)
		end=content.find(END_SPELL_CHECK)
		search= content[start:end]
		search = re.sub(r'<[^>]+>', '', search)
		for code in HTML_Codes:
			search = search.replace(code[1], code[0])
		search = search[1:]
	else:
		search = s
	return search ;


for q in all_q:
    q_ = spell_check(q)
    if q != q_:
        print(q + " -> " + q_)


#==============================================================================
# http://norvig.com/spell-correct.html
#==============================================================================
from collections import Counter

def words(text): return re.findall(r'\w+', text.lower())

WORDS = Counter(words(open('../nlp_source/big.txt').read()))

def P(word, N=sum(WORDS.values())): 
    "Probability of `word`."
    return WORDS[word] / N

def correction(word): 
    "Most probable spelling correction for word."
    return max(candidates(word), key=P)

def candidates(word): 
    "Generate possible spelling corrections for word."
    return (known([word]) or known(edits1(word)) or known(edits2(word)) or [word])

def known(words): 
    "The subset of `words` that appear in the dictionary of WORDS."
    return set(w for w in words if w in WORDS)

def edits1(word):
    "All edits that are one edit away from `word`."
    letters    = 'abcdefghijklmnopqrstuvwxyz'
    splits     = [(word[:i], word[i:])    for i in range(len(word) + 1)]
    deletes    = [L + R[1:]               for L, R in splits if R]
    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
    replaces   = [L + c + R[1:]           for L, R in splits if R for c in letters]
    inserts    = [L + c + R               for L, R in splits for c in letters]
    return set(deletes + transposes + replaces + inserts)

def edits2(word): 
    "All edits that are two edits away from `word`."
    return (e2 for e1 in edits1(word) for e2 in edits1(e1))


for w in tf.word:
    w_ = correction(w)
    if w != w_:
        print(w, '->', w_)



print("""#==============================================================================
# SUCCESS !!! {}
#==============================================================================
""".format(__file__))

