from __future__ import division
import re
class tokenise:
    def __init__(self,corpus):
        self.unigram=dict()
        self.bigram=dict()
        self.trigram=dict()
        self.remove_dets = [",", ".", "*", "'", '"', "/", "!", "=", "[","]", "(",")", "{","}", "@", "#", "$", "%", "^", "-", "_", ":", ";", "?", "\\", "|", "<", ">", "~"] # define desired replacements here
        self.corpus_file=corpus
        #self.unigram_create()
    def unigram_create(self):
        fp=open(self.corpus_file,"r")
        self.chEp=0
        self.n1Ep=0
        for l in fp:
            l=re.sub("[^a-zA-Z0-9]", " ", l)
            l=re.split(" |\n|\t",l)
            for w in l:
                #for d in self.remove_dets:
                    #w=w.replace(d,"")
                if w in self.unigram:
                    self.unigram[w]+=1
                    self.chEp+=1
                else:
                    self.unigram[w]=1
                    self.chEp+=1
                    self.n1Ep+=1

        #for w in self.unigram:
        #    print(w,self.unigram[w])

    def bigram_create(self):
        fp=open(self.corpus_file,"r")
        self.n1w1=dict()
        prev_word="$"
        for sent in fp:
            sent=re.sub("[^a-zA-Z0-9]", " ",sent)
            sent=re.split("\. ",sent)
            for l in sent:
                if "#" not in self.unigram:
                    self.unigram["#"]=1
                else:
                    self.unigram["#"]+=1
                prev_word="#"
                first_word = True
                l=re.split(" |\n|\t",l)
                for w in l:
                    #for d in self.remove_dets:
                        #w=w.replace(d,"")
                    if(w):
                        tup=(prev_word,w);
                        if tup in self.bigram:
                            self.bigram[tup]+=1
                        else:
                            self.bigram[tup]=1
                            if prev_word in self.n1w1:
                                self.n1w1[prev_word]+=1
                            else:
                                self.n1w1[prev_word]=1
                    if(w):
                        prev_word=w
                if  "%" not in self.unigram:
                    self.unigram["%"]=1
                else:
                    self.unigram["%"]+=1

                tup1=(prev_word,"%");
                if tup1 not in self.bigram:
                    self.bigram[tup1]=1
                else:
                    self.bigram[tup1]+=1
                    if prev_word in self.n1w1:
                        self.n1w1[prev_word]+=1
                    else:
                        self.n1w1[prev_word]=1
        #for w in self.bigram:
        #    print(w,self.bigram[w])

    def trigram_create(self):
        fp=open(self.corpus_file,"r")
        self.n1w1w2=dict()
        for sent in fp:
            sent=re.sub("[^a-zA-Z0-9]", " ",sent)
            sent=re.split("\. ",sent)
            for l in sent:
                prev_word1=""
                prev_word2="#"
                first_word = True
                second_word = True
                l=re.split(" |\n|\t",l)
                for w in l:
                    #for d in self.remove_dets:
                        #w=w.replace(d,"")
                    if(w):
                        if first_word:
                            first_word=False
                        else:
                            tup=(prev_word1,prev_word2,w);
                            tup1=(prev_word1,prev_word2);
                            if tup in self.trigram:
                                self.trigram[tup]+=1
                            else:
                                self.trigram[tup]=1
                                if tup1 in self.n1w1w2:
                                    self.n1w1w2[tup1]+=1
                                else:
                                    self.n1w1w2[tup1]=1
                    if(w):
                        prev_word1=prev_word2
                        prev_word2=w
                if prev_word1 and prev_word2:
                    tup=(prev_word1,prev_word2,"%");
                    tup1=(prev_word1,prev_word2);
                    if tup not in self.trigram:
                        self.trigram[tup]=1
                    else:
                        self.trigram[tup]+=1    
                        if tup1 in self.n1w1w2:
                            self.n1w1w2[tup1]+=1
                        else:
                            self.n1w1w2[tup1]=1

        #for w in self.trigram:
        #    print(w,self.trigram[w])
