#!/usr/bin/env python
 # -*- coding: utf-8 -*-
import argparse # optparse is deprecated
from itertools import islice # slicing for iterators
from scipy import stats
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction import DictVectorizer
import numpy as np
from nltk.tokenize import word_tokenize
from nltk.util import ngrams
from gensim.models import Word2Vec
from nltk.corpus import brown

''' 
class LogRegModel:
    def __init__(self):
      self.model = LogisticRegression()
      self.vec = DictVectorizer()
      self.allFeatures = []
      self.allCorrect = []

    def extract_features(self, h1, h2, ref):
      featureSet = {}
      precision1 = 0
      recall1 = 0
      total_p1 = 0
      total_r1 = 0
      precision2 = 0
      recall2 = 0
      total_p2 = 0
      total_r2 = 0
      ngram_prec1 = 0
      ngram_prec2 = 0
      for n in range(0,3):
        for word in h1[n]:
          total_p1 += 1
          if word in ref[n]:
            precision1 += 1
        for word in h2[n]:
          total_p2 += 1
          if word in ref[n]:
            precision2 += 1
        for word in ref[n]:
          total_r1 += 1
          total_r2 += 1
          if word in h1[n]:
            recall1 += 1
          if word in h2[n]:
            recall2 += 1
        
        try:
          ngram_prec1 += float(precision1)/float(total_p1)
        except:
          ngram_prec1 += 0.0
        try:
          ngram_prec2 += float(precision2)/float(total_p2)
        except:
          ngram_prec2 += 0.0 
        
        try:
          featureSet[str(n)+"-precision-h1"] = float(precision1)/float(total_p1)
        except:
           featureSet[str(n)+"-precision-h1"] = "x"
        try:
          featureSet[str(n)+"-precision-h2"] = float(precision2)/float(total_p2)
        except:
          featureSet[str(n)+"-precision-h2"] = "x"
        #this feature only = 0.436534
        
        try:
          featureSet[str(n)+"-recall-h1"] = float(recall1)/float(total_r1)
        except:
          featureSet[str(n)+"-recall-h1"] = "x" 
        try:
          featureSet[str(n)+"-recall-h2"] = float(recall2)/float(total_r2)
        except:
          featureSet[str(n)+"-recall-h2"] = "x"
        #this feature only = 0.435084
        
        try:
          featureSet[str(n)+"-harmonic mean-h1"] = stats.hmean([float(precision1)/float(total_p1), float(recall1)/float(total_r1)])
        except:
          featureSet[str(n)+"-harmonic mean-h1"] = "x"
        try:
          featureSet[str(n)+"-harmonic mean-h2"] = stats.hmean([float(precision2)/float(total_p2), float(recall2)/float(total_r2)])
        except:
          featureSet[str(n)+"-harmonic mean-h2"] = "x"
        #this feature only = 0.437822  
       
        precision1 = 0
        recall1 = 0
        total_p1 = 0
        total_r1 = 0
        precision2 = 0
        recall2 = 0
        total_p2 = 0
        total_r2 = 0
      
      #featureSet["average-ngram-precision1"] = float(ngram_prec1)/len(h1) #only this feature = 0.435084
      #featureSet["average-ngram-precision2"] = float(ngram_prec2)/len(h2)
      
      featureSet["wordcount1"] = len(h1[0]) #only this feature = 0.437339
      featureSet["wordcount2"] = len(h2[0])
      
      fx_words1 = [word for word in h1[0] if len(word) <= 4]
      fx_words2 = [word for word in h2[0] if len(word) <= 4]
      featureSet["fxwordcount1"] = len(fx_words1) #only this feature = 0.431701
      featureSet["fxwordcount2"] = len(fx_words2)
      
      non_words1 = [word for word in h1[0] if word.isalpha() != True]
      non_words2 = [word for word in h2[0] if word.isalpha() != True]
      featureSet["nonwordcount1"] = len(non_words1) #only this feature = 0.431701
      featureSet["nonwordcount2"] = len(non_words2)
      
      content_words1 = [word for word in h1[0] if len(word) > 4]
      content_words2 = [word for word in h2[0] if len(word) > 4]
      featureSet["contentwordcount1"] = len(content_words1) 
      featureSet["contentwordcount2"] = len(content_words2)
      #only this feature = 0.430896
      #this + nonwordcount = 0.435406
      #this + nonwordcount + fxwordcount = 0.438144
      #this + nonwordcount + fxwordcount + wordcount = 0.438789
      #this + nonwordcount + fxwordcount + wordcount + avg ngram prec = 0.433956
      #this + nonwordcount + fxwordcount + wordcount + hmean = 0.440077
      #this + nonwordcount + fxwordcount + wordcount + hmean + recall = 0.436534
      #this + nonwordcount + fxwordcount + wordcount + hmean + precision = 0.432829
      #all = 0.436856
      return featureSet

    def learn(self, h1, h2, ref, correct_class):
      features = self.extract_features(h1, h2, ref)
      self.allFeatures.append(features)
      self.allCorrect.append(correct_class)

    def fitModel(self):
      X = self.vec.fit_transform(self.allFeatures).toarray()
      y = np.array(self.allCorrect)
      self.model.fit(X, y)

    def predict(self, h1, h2, ref):
      features = self.extract_features(h1, h2, ref)
      f = self.vec.transform(features).toarray()
      prediction = self.model.predict(f)
      prob = self.model.predict_proba(f)
      print self.model.predict(f)[0]


def dependency_matches(h, ref, dependency_parser):
    precision = 0
    recall = 0
    total_p = 0
    total_r = 0
    h = unicode(" ".join(h).lower(), 'utf-8')
    ref = unicode(" ".join(ref).lower(), 'utf-8')
    h_result = dependency_parser.raw_parse(h)
    h_dep = h_result.next()
    try:
      h_deps = list(h_dep.triples())
    except:
      return 0.0
    try:
      ref_result = dependency_parser.raw_parse(ref)
    except:
      return 0.0
    ref_dep = ref_result.next()
    try:
      ref_deps = list(ref_dep.triples())
    except:
      return 0.0  
    for dep in h_deps:
      total_p += 1
      h_triple = dep[0][0]+"-"+dep[1]
      for r in ref_deps:
        r_triple = r[0][0]+"-"+r[1]
        if h_triple == r_triple:
          precision += 1
    for dep in ref_deps:
      total_r += 1
      r_triple = dep[0][0]+"-"+dep[1]
      for h in h_deps:
        h_triple = h[0][0]+"-"+h[1]
        if h_triple == r_triple:
          recall += 1
    try:
      p = float(precision)/float(total_p)
    except:
      p = 0
    try:
      r = float(recall)/float(total_r)
    except:
      r = 0
    try:
      out = stats.hmean([p, r])
    except:
      out = 0
    return out


def word_matches(h, ref):
    return sum(1 for w in h if w in ref)
    # or sum(w in ref for w in f) # cast bool -> int
    # or sum(map(ref.__contains__, h)) # ugly!
''' 

def meteorScore(hyp, ref, b):
    hyp = " ".join(hyp)
    hyp = hyp.decode('utf-8')
    hypToken = [w.lower().strip() for w in word_tokenize(hyp) if w.isalnum()] # any (c.isalnum() for c in w)
    ref = " ".join(ref)
    ref = ref.decode('utf-8')
    refToken = [w.lower().strip() for w in word_tokenize(ref) if w.isalnum()]
    sent_score = 0
    for word1 in hypToken:
      sims = []
      for word2 in refToken:
        try:
          sims.append(b.similarity(word1, word2))
        except:
          sims.append(0.0) 
      score = max(sims)
      sent_score += score
    try:
      return float(sent_score)/float(len(hypToken))
    except:
      return 0.0

    '''
    try:
        precision = m / len(hypToken)
        recall = m / len(refToken)
        fmean = (10.0*precision*recall) / (7.0*recall + 3.0 * precision)
        #fmean = stats.hmean([precision, recall])
        return fmean
    except:
        return 0
    '''
    
def bleuScore(hyp, ref):
    hyp = " ".join(hyp)
    hyp = hyp.decode('utf-8')
    hypToken = [w.lower().strip() for w in word_tokenize(hyp) if w.isalnum() ] # any (c.isalnum() for c in w)
    bigrams = [b for b in zip(hypToken[:-1], hypToken[1:])]
    ref = " ".join(ref)
    ref = ref.decode('utf-8')
    refToken = [w.lower().strip() for w in word_tokenize(ref) if w.isalnum() ]
    m = sum([1.0 for w in hypToken if w in refToken])

    try:
        precision = m / len(hypToken)
        
    except:
        return 0
def main():
    parser = argparse.ArgumentParser(description='Evaluate translation hypotheses.')
    # PEP8: use ' and not " for strings
    parser.add_argument('-i', '--input', default='data/train-test.hyp1-hyp2-ref',
            help='input file (default data/train-test.hyp1-hyp2-ref)')
    parser.add_argument('-n', '--num_sentences', default=None, type=int,
            help='Number of hypothesis pairs to evaluate')
    # note that if x == [1, 2, 3], then x[:None] == x[:] == x (copy); no need for sys.maxint
    opts = parser.parse_args()

    #path_to_jar = 'stanford-parser-full-2014-08-27/stanford-parser.jar'
    #path_to_models_jar = 'stanford-parser-full-2014-08-27/stanford-parser-3.4.1-models.jar'
    #dependency_parser = StanfordDependencyParser(path_to_jar=path_to_jar, path_to_models_jar=path_to_models_jar)
 
    # we create a generator and avoid loading all sentences into a list
    def sentences(afile):
        with open(afile) as f:
            for pair in f:
                yield [sentence.strip().split() for sentence in pair.split(' ||| ')]

    '''
    def get_ngrams(text, n ):
      n_grams = ngrams(word_tokenize(text), n)
      return [ ' '.join(grams) for grams in n_grams]
    '''
    #model = LogRegModel()
    b = Word2Vec(brown.sents())
    #print "getting data..."
    # note: the -n option does not work in the original code
    for h1, h2, ref in islice(sentences(opts.input), opts.num_sentences):
        #rset = set(ref)
        '''
        corr = "".join(corr)
        h1sent = unicode(" ".join(h1),'utf-8')
        h2sent = unicode(" ".join(h2),'utf-8')
        refsent = unicode(" ".join(ref),'utf-8')
        allgrams1 = [get_ngrams(h1sent, 1)] + [get_ngrams(h1sent, 2)] + [get_ngrams(h1sent, 3)] + [get_ngrams(h1sent, 4)]
        allgrams2 = [get_ngrams(h2sent, 1)] + [get_ngrams(h2sent, 2)] + [get_ngrams(h2sent, 3)] + [get_ngrams(h2sent, 4)]
        allgrams_ref = [get_ngrams(refsent, 1)] + [get_ngrams(refsent, 2)] + [get_ngrams(refsent, 3)] + [get_ngrams(refsent, 4)]
        model.learn(allgrams1, allgrams2, allgrams_ref, corr)
        #h1_match = word_matches(h1, rset)
        #h2_match = word_matches(h2, rset)
        #h1_match = dependency_matches(h1, ref, dependency_parser)
        #h2_match = dependency_matches(h2, ref, dependency_parser)
        '''
        h1_match = meteorScore(h1, ref, b)
        h2_match = meteorScore(h2, ref, b)
        #bleu1 = bleuScore(h1, ref)
        #print h1_match, h2_match
        print(-1 if h1_match > h2_match else # \begin{cases}
                (0 if h1_match == h2_match
                    else 1)) # \end{cases}

    #print "fitting model..."
    #model.fitModel()
    '''
    for h1, h2, ref in islice(sentences("data/dev.sents.txt"), opts.num_sentences):
      h1sent = unicode(" ".join(h1),'utf-8')
      h2sent = unicode(" ".join(h2),'utf-8')
      refsent = unicode(" ".join(ref),'utf-8')  
      allgrams1 = [get_ngrams(h1sent, 1)] + [get_ngrams(h1sent, 2)] + [get_ngrams(h1sent, 3)] + [get_ngrams(h1sent, 4)]
      allgrams2 = [get_ngrams(h2sent, 1)] + [get_ngrams(h2sent, 2)] + [get_ngrams(h2sent, 3)] + [get_ngrams(h2sent, 4)]
      allgrams_ref = [get_ngrams(refsent, 1)] + [get_ngrams(refsent, 2)] + [get_ngrams(refsent, 3)] + [get_ngrams(refsent, 4)]
      model.predict(allgrams1, allgrams2, allgrams_ref)
    '''
# convention to allow import of this file as a module
if __name__ == '__main__':
    main()
