#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
The MIT License (MIT)
Copyright (c) 2012-2013 Karsten Jeschkies <jeskar@web.de>

Permission is hereby granted, free of charge, to any person obtaining a copy of 
this software and associated documentation files (the "Software"), to deal in 
the Software without restriction, including without limitation the rights to use, 
copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the 
Software, and to permit persons to whom the Software is furnished to do so, 
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all 
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
'''

'''
Created on 23.11.2012

@author: karsten jeschkies <jeskar@web.de>

Implementation of a modified ESA model for gensim which uses consine similarity 
instead of sums. 
See my master's thesis: http://www.blackmagiclabs.com/portfolio/work/master-thesis.html

For details on the ESA model see: 
"Computing semantic relatedness using wikipedia-based explicit semantic analysis"
by Gabrilovich, E. and Markovitch, S. 
in "Proceedings of the 20th international joint conference on artificial intelligence"
'''

from collections import defaultdict
from gensim.similarities import Similarity, MatrixSimilarity
from itertools import izip
import logging
import itertools
from kmedoids import KMedoids 
import math
import numpy as np
import scipy
from selectkbest import iSelectKBest, if_classif

from gensim import interfaces, matutils, utils, similarities


logger = logging.getLogger('gensim.models.esamodel')

class DocumentTitles(object):
    '''
    Loads a list of document titles form a text file.
    Each line is considered to be a title.
    '''
    
    def __init__(self):
        self.document_titles = []
    
    @classmethod
    def load(cls, file_path):
        logger.info("Loading concept titles from %s" % file_path)
        
        result = DocumentTitles()
        with open(file_path, "r") as file:
            for line in file:
                doc_title = line.strip("\n").decode("UTF-8")
                result.document_titles.append(doc_title)
        
        logger.info("Loaded %d concept titles." % len(result.document_titles))
               
        return result
    
    def append(self, value):
        self.document_titles.append(value)
    
    def __iter__(self):
        for title in self.document_titles: yield title
        
    def __getitem__(self, key):
        return self.document_titles[key]
    
    def __len__(self):
        return len(self.document_titles)

class CosineEsaModel(interfaces.TransformationABC):
    """
    The cosine ESA, cESA, model is a modified version of the Explicit Semantic Analysis
    model.
    
    ESA uses the product of a document and a concept to calculate their similarity.
    The cESA model uses the cosine similarity.
    
    Originally ESA uses all Wikipedia concepts which have a certain amount of
    incoming inter-Wikipedia links. Denpending on the settings one is still left
    with well over 1 million concepts. To reduce this number cESA uses feature
    selection.
    """
    def __init__(self, corpus, document_titles, 
                 test_corpus, test_corpus_targets, num_test_corpus,
                 num_best_features = 1000,
                 num_features = None,
                 tmp_path = 'complete_similarity'):
        """
        The similarity between a document and each document of the corpus is
        the feature created.
        
        The corpus is filtered for significant features.
        
        Parameters
        ----------
        corpus : The corpus which contains all concepts. E.g. English Wikipedia
                 in TF-IDF space.
        test_corpus : The test corpus is used to select features. 
                      All documents in this corous should be classified.
        test_corpus_targets : The target classes of each document in the 
                              test corpus.
        num_test_corpus : Number of documents in the test corpus.
        document_titles : The names of each concept (doc) in corpus.
        num_features : The number of features of corpus
        num_best_features : Number of features which should be selected for cESA model.
                            If one wants to use all concepts as features the she has
                            to set num_best_features to the size of corpus.
        """
        
        if num_features is None:
            logger.info("scanning corpus to determine the number of features")
            num_features = 1 + utils.get_max_id(corpus)
            
        self.num_features = num_features
        
        #create similarity index of complete corpus
        complete_similarity_index = Similarity(output_prefix = tmp_path,
                                                    corpus = corpus,
                                                    num_features = self.num_features)
        
        #reduce concept count by feature selection
        self.selector = iSelectKBest(if_classif, k = num_best_features)
        
        #transform each document of test_corpus
        logger.info("Test corpus of %d documents..." % num_test_corpus)
            
        transformed_test_corpus = (complete_similarity_index[doc]
                                   for doc 
                                   in test_corpus)

        logger.info("Select best features...")
        X_y = izip(transformed_test_corpus, test_corpus_targets)
        self.selector.fit(X_y, len(document_titles))
        
        logger.info("Done selecting.")
        
        #reduce similarity index
        selected_documents = [doc 
                              for doc, mask 
                              in itertools.izip(corpus, self.selector.get_support())
                              if mask]
        self.similarity_index = MatrixSimilarity(corpus = selected_documents,
                                                 num_features = self.num_features)
        
        #reduce document titles
        self.document_titles = DocumentTitles()
        for doc_title, mask in itertools.izip(document_titles, self.selector.get_support()):
            if mask:
                self.document_titles.append(doc_title)
  
        #print doc titles
        for title in self.document_titles:
            logger.debug("%s" % title)
            

    def __str__(self):
        return " \n".join(self.document_titles)
    
    def get_concept_titles(self, doc_vec):
        '''
        Converts ids from document vector to concept titles.
        '''
        return [(self.document_titles[concept_id], weight) 
                for concept_id, weight in doc_vec]

    def __getitem__(self, bow, eps=1e-12):
        """
        Return esa representation of the input vector and/or corpus.
        
        bow should already be weights, e.g. with TF-IDF
        """
        # if the input vector is in fact a corpus, return a transformed corpus 
        # as a result
        is_corpus, bow = utils.is_corpus(bow)
        if is_corpus:
            return self._apply(bow)

        #use similarity index to calculate similarity with each vector of corpus
        vector = self.similarity_index[bow]
        
        #consine similarity is in [-1, 1] shift and scale to make it [0, 1]
        vector += 1
        vector /= 2

        #normalize
        vector = matutils.unitvec(vector)

        # make sure there are no explicit zeroes in the vector (must be sparse)
        vector = [(concept_id, weight) 
                  for concept_id, weight 
                  in enumerate(vector) 
                  if abs(weight) > eps]
        return vector
    
    def save(self, fname):
        '''
        See MatrixSimilarity.save()
        '''
        logger.info("storing %s object to %s and %s" % (self.__class__.__name__, 
                                                        fname, 
                                                        fname + '.index'))
        # first, remove the similarity index from self., so it doesn't get pickled
        sim = self.similarity_index
        del self.similarity_index
        try:
            sim.save(fname + ".index")
            utils.pickle(self, fname) # store index-less object
        finally:
            self.similarity_index = sim
        

    @classmethod
    def load(cls, fname):
        """
        Load a previously saved object from file (also see `save`).
        """
        logger.info("loading %s object from %s and %s" % (cls.__name__, 
                                                          fname, 
                                                          fname + ".index"))
        result = utils.unpickle(fname)
        result.similarity_index = MatrixSimilarity.load(fname + ".index")
        return result
#endclass CosineEsaModel
