'''
Markov NN for smart word generation
Author: Christian M. Adams

Create a word generator for a language whose example words are {spare, spear, pares, peers, reaps, peaks, speaker, keeper, pester, paste, tapas, pasta, past, straps, tears, terse, steer, street, stare, rates, streak, taste, tapa, peat, eat, ate, tea, seat}.
       Use your program to generate at least 100 random words.
        Are any words repeated?

        What is the probability of generation for each word?

        What are the shortest and longest words generated?

        What is the longest word that can be generated and what is the probability of its occurrence? If you cannot answer this directly, can you provide a bound for the probability of occurrence?

       Use the words generated by your program to create a second Markov model and compare the transition probabilities of the two models you created.
        Use you program to create two Markov models based on 500-word (or larger) writing samples from two, different languages
        (say, English and French, American and German, or possibly American and English). The writing samples are to be taken
          from web sites using different modern languages that are written using Latin letters.
          Compare the transition probabilities of the two models you create.

'''

import string
import numpy as np
import matplotlib.pyplot as plt
from random import randint, choice

#Variables
WORD_LIST = []

#English Alphabet
ENGLISH = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i',
                   'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
                   's', 't', 'u', 'v', 'w', 'x', 'y', 'z']
ENGLISH_END = ['$','a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i',
                   'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
                   's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'END']

print "ENGLISH TABLE: ", ENGLISH_END

#Make lookup table for use with 2d matrices
ENG_LOOKUP = {}
count = 0
ENG_LOOKUP['$'] = count
count += 1
for let in ENGLISH:
	ENG_LOOKUP[let] = count
	count += 1
ENG_LOOKUP['END'] = count

print "English Lookup table: \n", ENG_LOOKUP

# Sample Training Words
WORDS = ['spare', 'spear', 'pares', 'peers', 'reaps', 'peaks',
			'speaker', 'keeper', 'pester', 'paste', 'tapas', 'pasta',
			'past', 'straps', 'tears', 'terse', 'steer', 'street',
			'stare', 'rates', 'streak', 'taste', 'tapa', 'peat', 'eat',
			'ate', 'tea', 'seat']


#Reads in article using each word as input
def readArticle(path):
    with open(path, 'r') as f:
        for line in f:
            for word in line.split():
                cleanWord = word.translate(None, string.punctuation)
                WORDS.append(cleanWord)
    return WORDS

WORDS = readArticle('dictionary.txt')


#Create 2D Matrix for transition probabilities (including last letter-->END transtition)
n = len(ENGLISH_END)
print "N is: ", n
transition_prob = [[0] * n for i in range(n)]


#Populate 2D transition probability matrix
for word in WORDS:
	for i in range(len(word) - 1):
		transition_prob[ENG_LOOKUP[word[i]]][ENG_LOOKUP[word[i + 1]]] += 1
   	transition_prob[ENG_LOOKUP['$']][ENG_LOOKUP[word[0]]] += 1
        transition_prob[ENG_LOOKUP[word[-1]]][ENG_LOOKUP['END']] += 1

# print "Transition Probability Matrix\n\n", transition_prob, "\n\n"


#Write to CSV for later analysis in Excel
with open('csvfile.csv','wb') as file:
    for i in range(27):
        for value in transition_prob[i]:
            value = str(value) + ','
            file.write(value)
        file.write('\n')


def gen_prob():
    return randint(0,1000)/1000.


def pick_next_letter(prev_letter):
    let_bank = []
    for i in range(28):
        temp = transition_prob[ENG_LOOKUP[prev_letter]][i]
        letter = ENGLISH_END[i]
        while temp > 0:
            let_bank.append(letter)
            temp -= 1
    return choice(let_bank)


#Roulette wheel selection approach to generating a word
def generate_word():

    # new_word = pick_first_letter()
    new_word = '$'
    n_letter = ''
    new_word = new_word + n_letter
    while n_letter != 'END':
        new_word = new_word + n_letter
        n_letter = pick_next_letter(new_word[-1])
    new_word = new_word[1:]
    WORD_LIST.append(new_word)
    return new_word


def make_report():
    largest_word = 0
    for item in WORD_LIST:
        if len(item) > largest_word:
            largest_word = len(item)

    #Calculate average word length
    word_length = []
    total_letters = 0
    length_table = [0] * (largest_word + 1)

    # print WORD_LIST
    for item in WORD_LIST:
        length = len(item)
        length_table[length] += 1
        word_length.append(length)
        total_letters += length


    print "\n\nNumber of words: ", len(word_length)
    print "Average Word-length: ", float(total_letters) / len(word_length)
    print "Below are the following length percentages: \n"
    for i in range(len(length_table)):
        print "Length:", i, "     Percentage:", (float(length_table[i])/len(word_length)) * 100


count = 0
while count < 20:
    print generate_word()
    count += 1

make_report()
