import numpy as np
import tensorflow as tf
import re
import time

# Importing the dataset
lines = open('/Users/neel/My_Conda/anaconda3/Deep Learning and Chatbot/movie_lines.txt', encoding = 'utf-8', errors = 'ignore').read().split('\n')
conversations = open('/Users/neel/My_Conda/anaconda3/Deep Learning and Chatbot/movie_conversations.txt', encoding = 'utf-8', errors = 'ignore').read().split('\n')
 
# Creating a dictionary that maps each line and its id
id2line = {}
for line in lines:
    _line = line.split(' +++$+++ ')
    if len(_line) == 5:
        id2line[_line[0]] = _line[4]
# Creating a list of all of the conversations
        
conversations_ids = []
for conversation in conversations[:-1]:
    _conversation = conversation.split(' +++$+++ ')[-1][1:-1].replace("'", "").replace(" ", "")
    conversations_ids.append(_conversation.split(','))
    
    
 # Getting separately the questions and the answers
questions = []
answers = []
for conversation in conversations_ids:
    for i in range(len(conversation) - 1):
        questions.append(id2line[conversation[i]])
        answers.append(id2line[conversation[i+1]])       
 
    
# Cleaning of text
def clean_text(text):
    text = text.lower()
    text = re.sub(r"i'm","i am",text)
    text = re.sub(r"it's","it is",text)
    text = re.sub(r"he's","he is",text)
    text = re.sub(r"she's","she is",text)
    text = re.sub(r"that's","that is",text)
    text = re.sub(r"what's","what is",text)
    text = re.sub(r"where's","where is",text)
    text = re.sub(r"there's","there is",text)
    text = re.sub(r"don't","do not",text)
    text = re.sub(r"\'ve","have",text)
    text = re.sub(r"\'re","are",text)
    text = re.sub(r"\'d","would",text)
    text = re.sub(r"\'ll","will",text)
    text = re.sub(r"won't","will not",text)
    text = re.sub(r"can't","cannot",text)
    text = re.sub(r"[-()\"#/@;:<>{}+=~|.?,]","",text)
    return text   

# Cleaning the questions
clean_questions = []
for question in questions:
    clean_questions.append(clean_text(question))
 
# Cleaning the answers  
clean_answers = []
for answer in answers:
    clean_answers.append(clean_text(answer))  
    
# Time to create to dictionary to map each word with its no of occurences
    
word2count ={}
for question in clean_questions:
    for word in question.split():
        if word not in word2count:
            word2count[word]=1
        else:
            word2count[word] +=1
            
# For Answers

for answer in clean_answers:
    for word in answer.split():
        if word not in word2count:
            word2count[word]=1
        else:
            word2count[word] +=1     
            
# Creating two dictionaries that map with the questions words and answers words to a unique integer
threshold =20
questionswords2int={}
word_number =0
for word, count in word2count.items():
    if count >= threshold:
        questionswords2int[word]=word_number
        word_number += 1
answerswords2int={}
word_number =0
for word, count in word2count.items():
    if count >= threshold:
        answerswords2int[word]=word_number
        word_number += 1   
# Adding last tokens to these dictionaries which is essential for seq2seq model
tokens= ['<PAD>','<EOS>','<OUT>','<SOS>']
for token in tokens:
    questionswords2int[token]= len(questionswords2int)+1
for token in tokens:
    answerswords2int[token]= len(answerswords2int)+1    
# Creating inverse dictionary of answers2word dictionary    
    
answersints2word= {w_i:w for w,w_i in answerswords2int.items()}  
# Adding end of string token to every clean answers,which is essential for sec2sec model in each layer
for i in range(len(clean_answers)):
    clean_answers[i] += ' <EOS>'
# Translating all questions and answers to integers    
# replacing all the words filtered out by <out>
questions_into_int= []
for question in clean_questions:
    ints=[]
    for word in question.split():
        if word not in questionswords2int:
            ints.append(questionswords2int['<OUT>'])
        else:
            ints.append(questionswords2int[word])
            questions_into_int.append(ints)
            
answers_into_int= []
for answer in clean_answers:
    ints=[]
    for word in question.split():
        if word not in answerswords2int:
            ints.append(answerswords2int['<OUT>'])
        else:
            ints.append(answerswords2int[word])
            answers_into_int.append(ints)  
            
#Sorting questions and Answers by length of questions
sorted_clean_questions=[] 
sorted_clean_answers=[] 
for length in range(1,25+1):
    for i in enumerate(questions_into_int):
        if len(i[1])== length:
            sorted_clean_questions.append(questions_into_int[i[0]])
            sorted_clean_answers.append(answers_into_int[i[0]])
            
           
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
    
    

































  
    

        
            
    
    





































         