DRIVE = False
PREPROCESS = False
KNN_VAL = False
KNN_TEST = False
NB_VAL = False
NB_TEST = False
T_STAT = True

# -*- coding: utf-8 -*-
"""Final version of ML_offline_2.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1WKZyTReeojb4nvxiHXb-aUp_1s9Qa97F
"""

# from google.colab import drive
# drive.mount('/drive')

if DRIVE:
    from google.colab import drive
    drive.mount('/content/drive')

    import os
    os.chdir('drive/My Drive/ML Offline 2')

"""# Preprocessing

## Text preprocessing
"""

import numpy as np
import string
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.stem import PorterStemmer
import re
import datetime
import pickle
import os

# Import BeautifulSoup
from bs4 import BeautifulSoup as bs

def process_document(document):
    """
    Standard text preprocessing steps.
    
    Convert document to list of words.
    
    Parameters:
    document (str): String containing multiple sentences
    
    Returns:
    list of strings: list of words in document
    """
    
    #Lowercase the text
    document = document.lower()
    #Number Removal
    document = re.sub(r'[-+]?\d+', '', document)
    #Remove punctuations
    document=document.translate((str.maketrans('','',string.punctuation)))
    #Tokenize
    document = word_tokenize(document)
    #Remove stopwords
    stop_words = set(stopwords.words('english'))
    document = [word for word in document if not word in stop_words]
    #Lemmatize tokens
    lemmatizer=WordNetLemmatizer()
    document = [lemmatizer.lemmatize(word) for word in document]
    #Stemming tokens
    stemmer= PorterStemmer()
    document = [stemmer.stem(word) for word in document]
    return document

def preprocess_text(xml_filepath, start_index, end_index):
    """ 
    Preprocesses xml text into array of array of words. 
      
    Reads the xml file and converts into array of rows. Then takes the subarray according to index. Then cleans and tokenizes the rows.
  
    Parameters: 
    xml_filepath (str): path to xml file
    start_index (int): from which row to start
    end_index (int): cover until which row
    
  
    Returns: 
    list of list of strings: list of words in each document. 
  
    """
    
    with open(xml_filepath, "r") as file:
        #lines = file.readlines()
        
        #instead of reading the whole file we go for (end_index+100) lines
        number_of_lines = end_index+200 #Extra lines for initial non-row lines and blank bodies
        lines = []
        for i in range(number_of_lines):
            line = file.readline()
            if(len(line)==0): #EOF
                break
            lines.append(line)
            
        lines = "".join(lines)
        lines_bs = bs(lines, "lxml")
        
        row_bodies = [x.get("body") for x in lines_bs.find_all("row")]
        row_bodies_tag_free = []

        for row_body in row_bodies:
            #text = bs(row_body,'html.parser').get_text().strip()
            text = bs(row_body,'lxml').get_text().strip()
            if(len(text)>0): #Discarding empty bodies
                row_bodies_tag_free.append(text)
        
        row_bodies_tag_free = row_bodies_tag_free[start_index:end_index] #array of documents<each in form of string>
        
        
        tokenized_documents = []
        for row_body_tag_free in row_bodies_tag_free:
            tokenized_documents.append(process_document(row_body_tag_free))
        
        return tokenized_documents

"""## Prepare training, test and validation set"""
if PREPROCESS:
    nltk.download('punkt')
    nltk.download('stopwords')
    nltk.download('wordnet')

    # MAY IGNORE RUNNING THIS CELL IF WE WANT TO USE WORD PICKLES
    SHOW_TIME = False

    trainX_words = []
    valX_words = []
    testX_words = []

    trainY_words = []
    valY_words = []
    testY_words = []

    xml_files = []
    exclude_files = ['3d_Printer']
    breakpoints = [0,500,700,1200] #indices to split for train val test set

    with open('Data/topics.txt', "r") as topicfile:
        filenames = topicfile.readlines()
        topic_xml_files = [filename.strip() for filename in filenames]

    for topic_xml_file in topic_xml_files:
        if topic_xml_file not in exclude_files:
            xml_files.append(topic_xml_file)

    start_time = datetime.datetime.now()

    if SHOW_TIME:
        print('Start building training data: ', datetime.datetime.now().time())
        
    #Training data
    for xml_file in xml_files:
        print('Processing training ' + xml_file + '.xml...')
        xml_filepath = 'Data/Training/' + xml_file + '.xml'
        tmp = preprocess_text(xml_filepath, breakpoints[0], breakpoints[1])
        trainX_words.extend(tmp)
        trainY_words.extend([xml_file]*(breakpoints[1]-breakpoints[0]))
        if SHOW_TIME:
            print('Done at: ', datetime.datetime.now().time())


    if SHOW_TIME:
        print('Start building validation data: ', datetime.datetime.now().time())
        
    #Validation data
    for xml_file in xml_files:
        print('Processing validation ' + xml_file + '.xml...')
        xml_filepath = 'Data/Training/' + xml_file + '.xml'
        tmp = preprocess_text(xml_filepath, breakpoints[1], breakpoints[2])
        valX_words.extend(tmp)
        valY_words.extend([xml_file]*(breakpoints[2]-breakpoints[1]))
        if SHOW_TIME:
            print('Done at: ', datetime.datetime.now().time())



    if SHOW_TIME:
        print('Start building testing data: ', datetime.datetime.now().time())
        
    #Training data
    for xml_file in xml_files:
        print('Processing testing ' + xml_file + '.xml...')
        xml_filepath = 'Data/Training/' + xml_file + '.xml'
        tmp = preprocess_text(xml_filepath, breakpoints[2], breakpoints[3])
        testX_words.extend(tmp)
        testY_words.extend([xml_file]*(breakpoints[3]-breakpoints[2]))
        if SHOW_TIME:
            print('Done at: ', datetime.datetime.now().time())
        
    #Writing pickles
    try:
        os.mkdir('pickles')
    except:
        pass

    pickle.dump(trainX_words, open('pickles/trainX_words.p', 'wb'))
    pickle.dump(trainY_words, open('pickles/trainY_words.p', 'wb'))
    pickle.dump(valX_words, open('pickles/valX_words.p', 'wb'))
    pickle.dump(valY_words, open('pickles/valY_words.p', 'wb'))
    pickle.dump(testX_words, open('pickles/testX_words.p', 'wb'))
    pickle.dump(testY_words, open('pickles/testY_words.p', 'wb'))

    print('All done at: ', datetime.datetime.now() - start_time)

    LOAD_PICKLE = False
    if LOAD_PICKLE:
        trainX_words = pickle.load(open('pickles/trainX_words.p', 'rb'))
        trainY_words = pickle.load(open('pickles/trainY_words.p', 'rb'))
        valX_words = pickle.load(open('pickles/valX_words.p', 'rb'))
        valY_words = pickle.load(open('pickles/valY_words.p', 'rb'))
        testX_words = pickle.load(open('pickles/testX_words.p', 'rb'))
        testY_words = pickle.load(open('pickles/testY_words.p', 'rb'))

        print(len(trainX_words))
        print(len(trainY_words))
        print(len(valX_words))
        print(len(valY_words))
        print(len(testX_words))
        print(len(testY_words))

"""## Making the vocabulary"""

def build_vocab(words):
    """
    Builds the vocabulary from words array.
    
    Get all the unique words from words array and sort them lexicographically. Then build a dictionary which gives the position of words in the sorted list.
    
    Parameters:
    words (list of list of str): The collection of documents in tokenized format.
    
    Returns:
    dictionary: Given a word, get its position.
    """
    
    list_of_words = []
    for line in words:
        for word in line:
            if word not in list_of_words:
                list_of_words.append(word)
    list_of_words.sort()
    
    position_of_word_in_list = {}
    pos=0
    for word in list_of_words:
        position_of_word_in_list[word]=pos
        pos+=1
    
    return position_of_word_in_list

def get_position_in_dictionary(dictionary, word):
    """
    Checks if word exists in dictionary.
    
    If exists, returns its index. Else returns -1
    
    Parameters:
    dictionary (dict): word as key - position as value
    word (str): word to find
    
    Returns: 
    int: postion of word.
    """
    if word in dictionary:
        return dictionary[word]
    else:
        return -1
    

def get_word_count_vector(dictionary, document):
    """
    Builds the feature vector as count of each word.
    
    Get a vector of size len(dictionary) with each word count.
    
    Parameters:
    dictionary (dict): word as key - position as value
    document (list of str): list of words in the document
    
    Returns:
    numpy array: feature vector
    """
    n_feat = len(dictionary)
    feat_vec = np.zeros((n_feat))
    
    for word in document:
        pos = get_position_in_dictionary(dictionary, word)
        if pos !=-1:
            feat_vec[pos] += 1
    
    return feat_vec

if PREPROCESS:
    position_dictionary = build_vocab(trainX_words)

    """#### Preparing trainX_vectors, valX_vectors, testX_vectors(count of each word in documents)"""

    trainX_vectors = []
    for document in trainX_words:
        feat_vec = get_word_count_vector(position_dictionary, document)
        trainX_vectors.append(feat_vec)
        
    trainX_vectors = np.array(trainX_vectors)
    print(trainX_vectors.shape)

    valX_vectors = []
    for document in valX_words:
        feat_vec = get_word_count_vector(position_dictionary, document)
        valX_vectors.append(feat_vec)
        
    valX_vectors = np.array(valX_vectors)
    print(valX_vectors.shape)

    testX_vectors = []
    for document in testX_words:
        feat_vec = get_word_count_vector(position_dictionary, document)
        testX_vectors.append(feat_vec)
        
    testX_vectors = np.array(testX_vectors)
    print(testX_vectors.shape)

    #dumping all the vectors

    pickle.dump(trainX_vectors, open('pickles/trainX_vectors.p', 'wb'))
    pickle.dump(valX_vectors, open('pickles/valX_vectors.p', 'wb'))
    pickle.dump(testX_vectors, open('pickles/testX_vectors.p', 'wb'))

"""#k-NN"""

import pickle as pickle
import numpy as np
from datetime import datetime
from tqdm import tqdm

trainX_vectors = pickle.load(open('pickles/trainX_vectors.p', 'rb'))
trainY_words = pickle.load(open('pickles/trainY_words.p', 'rb'))
valX_vectors = pickle.load(open('pickles/valX_vectors.p', 'rb'))
valY_words = pickle.load(open('pickles/valY_words.p', 'rb'))
testX_vectors = pickle.load(open('pickles/testX_vectors.p', 'rb'))
testY_words = pickle.load(open('pickles/testY_words.p', 'rb'))

"""### Distance functions"""

#a and b both numpy arrays(feature vector)
def hamming_distance(a,b):
    # a = a>0
    # b = b>0
    c = (a^b)*1 #xor to find mismatches
    dist = np.sum(c)
    return max(dist, 0.000000001) #In case of 0 we return eps to avoid divide by zero

def euclidean_distance(a,b):
    dist = np.sqrt(np.sum(np.square(a-b)))
    return max(dist, 0.000000001) #In case of 0 we return eps to avoid divide by zero

def get_IDF_array(trainX_vectors):
    trainX_vectors_bool = (trainX_vectors>0)*1
    document_count_for_each_word = np.sum(trainX_vectors_bool,axis=0)
    IDF = (trainX_vectors_bool.shape[0]/document_count_for_each_word)
    IDF = np.log(IDF)
    IDF = np.where(IDF==0, 0.000000001, IDF) #Take 1e-9 if idf 0, else whatever idf has
    return IDF
    
def get_TF_IDF(a, IDF):
    TF_a = a/np.sum(a)
    TF_IDF_a = np.multiply(TF_a,IDF)
    return TF_IDF_a

def cosine_similarity(p,q,IDF):
    # p = get_TF_IDF(a,IDF)
    # q = get_TF_IDF(b,IDF)
    
    pdotq = np.dot(p,q)
    p_norm = np.sqrt(np.sum(np.square(p)))
    q_norm = np.sqrt(np.sum(np.square(q)))
    
    if (p_norm==0 or q_norm==0):
        #simply means no common word with the train instance, zero similarity, so pdotq should be 0 and so cos=0
        cos = 0
    else:
        cos = pdotq/(p_norm*q_norm)
    
    return cos

if (KNN_VAL or KNN_TEST):
    IDF = get_IDF_array(trainX_vectors)

"""### Prediction and validation"""

#Make prediction of the test points using training points
def prediction(X_train, Y_train, X_test, n_neighbors, dist_method):
    allPredictedOutputs =[]
    
    #Determine Number of unique class lebels
    uniqueOutputLabels = []
    for label in Y_train:
        if label not in uniqueOutputLabels:
            uniqueOutputLabels.append(label)
    uniqueOutputCount = len(uniqueOutputLabels)
    
    #Making a dictionary to get position in uniqueOutputLabels from the actual Label which is a string
    label_dict = {}
    pos=0
    for uniqueOutputLabel in uniqueOutputLabels:
        label_dict[uniqueOutputLabel] = pos
        pos+=1
    
    #calculate for earch test data points
    for testInput in tqdm(X_test):
        allDistances = []
        for trainInput, trainActualOutput in zip(X_train, Y_train):
            if dist_method==1:
                distance = hamming_distance(testInput, trainInput)
            elif dist_method==2:
                distance = euclidean_distance(testInput, trainInput)
            elif dist_method==3:
                distance = -cosine_similarity(testInput, trainInput, IDF) #negative for distance
            allDistances.append((trainActualOutput, distance))
        #Sort (in ascending order) the training data points based on distances from the test point     
        allDistances.sort(key=lambda x: x[1])
        
        
        #Assuming output labels are from 0 to uniqueOutputCount-1
        voteCount = np.zeros(uniqueOutputCount)
        
        for n in range(n_neighbors):
            class_label = int(label_dict[allDistances[n][0]])
            
            WEIGHTED_VOTING = True
            
            if WEIGHTED_VOTING:
                if dist_method==3:
                    voteCount[class_label] += (-allDistances[n][1])
                else:
                    voteCount[class_label] += (1/allDistances[n][1])
            else:
                voteCount[class_label] += 1
        
        #Determine the Majority Voting 
        predictedOutputIndex = np.argmax(voteCount)
        predictedOutput = uniqueOutputLabels[predictedOutputIndex]
        
        allPredictedOutputs.append(predictedOutput)
        
    return allPredictedOutputs

if KNN_VAL:
    start_time = datetime.now()

    k = 3
    dist_method = 3
    if (dist_method==1):
        trainX_vectors_bool = (trainX_vectors>0)*1
        valX_vectors_bool = (valX_vectors>0)*1
        predictedOutputs = prediction(trainX_vectors_bool, trainY_words, valX_vectors_bool, n_neighbors=k, dist_method=dist_method)

    elif (dist_method==2):
        predictedOutputs = prediction(trainX_vectors, trainY_words, valX_vectors, n_neighbors=k, dist_method=dist_method)

    elif (dist_method==3):
        trainX_vectors_tfidf = []
        valX_vectors_tfidf = []
        for train_x in trainX_vectors:
            trainX_vectors_tfidf.append(get_TF_IDF(train_x, IDF))
        for val_x in valX_vectors:
            valX_vectors_tfidf.append(get_TF_IDF(val_x, IDF))
        trainX_vectors_tfidf = np.array(trainX_vectors_tfidf)
        valX_vectors_tfidf = np.array(valX_vectors_tfidf)
        
        predictedOutputs = prediction(trainX_vectors_tfidf, trainY_words, valX_vectors_tfidf, n_neighbors=k, dist_method=dist_method)

    print("Time elapsed: ", datetime.now()-start_time)

    corr = 0
    for i in range(len(predictedOutputs)):
        if predictedOutputs[i]==valY_words[i]:
            corr += 1

    acc = corr/len(predictedOutputs)*100
    print("Accuracy: ", acc)

    if False:
        #accuracy of validation write to file
        with open("knn_validation_accuracy.txt", "a") as f:
            f.write("k: " + str(k) + " distance measure: " + str(dist_method) + " -> Accuracy: " + str("{:.6f}".format(acc)) + "\n")

"""## Test set accuracy (incomplete)"""
if KNN_TEST:
    if False: #Whole test set avg
        start_time = datetime.now()

        k = 5
        dist_method = 3

        if (dist_method==1):
            trainX_vectors_bool = (trainX_vectors>0)*1
            testX_vectors_bool = (testX_vectors>0)*1
            predictedOutputs = prediction(trainX_vectors_bool, trainY_words, testX_vectors_bool, n_neighbors=k, dist_method=dist_method)

        elif (dist_method==2):
            predictedOutputs = prediction(trainX_vectors, trainY_words, testX_vectors, n_neighbors=k, dist_method=dist_method)

        elif (dist_method==3):
            trainX_vectors_tfidf = []
            testX_vectors_tfidf = []
            for train_x in trainX_vectors:
                trainX_vectors_tfidf.append(get_TF_IDF(train_x, IDF))
            for test_x in testX_vectors:
                testX_vectors_tfidf.append(get_TF_IDF(test_x, IDF))
            trainX_vectors_tfidf = np.array(trainX_vectors_tfidf)
            testX_vectors_tfidf = np.array(testX_vectors_tfidf)
            
            predictedOutputs = prediction(trainX_vectors_tfidf, trainY_words, testX_vectors_tfidf, n_neighbors=k, dist_method=dist_method)

        print("Time elapsed: ", datetime.now()-start_time)

        corr = 0
        for i in range(len(predictedOutputs)):
            if predictedOutputs[i]==testY_words[i]:
                corr += 1

        acc = corr/len(predictedOutputs)*100
        print("Accuracy: ", acc)

    start_time = datetime.now()

    #Best validation hyperparameters
    # k = 5
    # dist_method = 3

    with open("Data/topics.txt","r") as f:
        topic_count = (len(f.readlines())-1)

    trainX_vectors_tfidf = []
    testX_vectors_tfidf = []
    for train_x in trainX_vectors:
        trainX_vectors_tfidf.append(get_TF_IDF(train_x, IDF))
    for test_x in testX_vectors:
        testX_vectors_tfidf.append(get_TF_IDF(test_x, IDF))
    trainX_vectors_tfidf = np.array(trainX_vectors_tfidf)
    testX_vectors_tfidf = np.array(testX_vectors_tfidf)

    batch_acc_list = []

    for i in tqdm(range(50)): # run 50 iterations
        testX_mini = []
        testY_mini = []
        for j in range(topic_count):
            start_idx = 10*i+500*j
            end_idx = start_idx + 10
            testX_mini.extend(testX_vectors_tfidf[start_idx:end_idx])
            testY_mini.extend(testY_words[start_idx:end_idx])
            
        predictedOutputs = prediction(trainX_vectors_tfidf, trainY_words, testX_mini, 5, 3)
        
        corr = 0
        for pred_i in range(len(predictedOutputs)):
            if predictedOutputs[pred_i]==testY_mini[pred_i]:
                corr += 1

        acc = corr/len(predictedOutputs)*100
        print("Iteration ", i, "->Accuracy: ", acc)
        batch_acc_list.append(acc)



    print("Time elapsed: ", datetime.now()-start_time)

    print(np.mean(batch_acc_list))
    with open("kNN_test_output.txt", "w") as knn_file:
        for i in range(len(batch_acc_list)):
            knn_file.write(str(batch_acc_list[i])+'\n')

"""# NB"""

import pickle as pickle
import numpy as np
from tqdm import tqdm
from datetime import datetime

if (NB_VAL or NB_TEST):
    trainX_words = pickle.load(open('pickles/trainX_words.p', 'rb'))
    trainY_words = pickle.load(open('pickles/trainY_words.p', 'rb'))
    valX_words = pickle.load(open('pickles/valX_words.p', 'rb'))
    valY_words = pickle.load(open('pickles/valY_words.p', 'rb'))
    testX_words = pickle.load(open('pickles/testX_words.p', 'rb'))
    testY_words = pickle.load(open('pickles/testY_words.p', 'rb'))

    print(len(trainX_words))
    print(len(trainY_words))
    print(len(valX_words))
    print(len(valY_words))
    print(len(testX_words))
    print(len(testY_words))

    """### Unique labels list and dictionary to get position in the list from label"""

    #Determine Number of unique class lebels
    uniqueOutputLabels = []
    for label in trainY_words:
        if label not in uniqueOutputLabels:
            uniqueOutputLabels.append(label)
    uniqueOutputCount = len(uniqueOutputLabels)

    #Making a dictionary to get position in uniqueOutputLabels from the actual Label which is a string
    label_dict = {}
    pos=0
    for uniqueOutputLabel in uniqueOutputLabels:
        label_dict[uniqueOutputLabel] = pos
        pos+=1

    """### Preparing lists and dictionaries for naive bayes algorithm"""

    vocabulary = []
    doc_count_topicwise = np.zeros(uniqueOutputCount)
    word_count_topicwise = np.zeros(uniqueOutputCount)
    specific_word_count_topicwise = {} #dictionary with key as (word, label_dict[label]) Example: specific_word_count_topicwise['microroast', label_dict['Coffee']]
    for train_x, train_y in tqdm(zip(trainX_words, trainY_words), total = len(trainY_words)):
        target_index = label_dict[train_y]
        doc_count_topicwise[target_index] += 1
        word_count_topicwise[target_index] += len(train_x)
        for word in train_x:
            if word not in vocabulary:
                vocabulary.append(word)
                
            key = (word,target_index)
            if key in specific_word_count_topicwise:
                specific_word_count_topicwise[key] += 1
            else:
                specific_word_count_topicwise[key] = 1
    print(doc_count_topicwise)
    print(word_count_topicwise)
    print(len(vocabulary))
    print(len(specific_word_count_topicwise))

"""### Prediction function"""

def prediction(testX, alpha, vocab_size, doc_count_topicwise, word_count_topicwise, specific_word_count_topicwise, uniqueOutputLabels):
    #returns the indices of target label for testX
    allPredictions = []
    #for test_x in tqdm(testX):
    for test_x in (testX):
        topic_count = len(doc_count_topicwise)
        prediction_topicwise = np.zeros(topic_count)
        
        for target_index in range(topic_count):
            p = np.log(doc_count_topicwise[target_index]/np.sum(doc_count_topicwise)) #ok
            
            for word in test_x:
                key = (word,target_index)
                if key in specific_word_count_topicwise:
                    numerator = specific_word_count_topicwise[key]
                else:
                    numerator = 0
                denominator = word_count_topicwise[target_index]
                prob = np.log((numerator+alpha)/(denominator+alpha*vocab_size))
                p += prob
            prediction_topicwise[target_index] = p

            
        decision_target_index = np.argmax(prediction_topicwise)
        prediction = uniqueOutputLabels[decision_target_index]
        allPredictions.append(prediction)
    return allPredictions

"""### Run on validation set"""

if NB_VAL:
    # alpha_min = 0.001
    # alpha_max = 1.0
    # diff = (alpha_max-alpha_min)/10
    # alpha = alpha_min
    how_many_alpha = 20

    alpha = 1
    # while(alpha <= alpha_max):
    while(how_many_alpha):
        start_time = datetime.now()
        #alpha = 1
        predictedOutputs = prediction(valX_words, alpha, len(vocabulary), doc_count_topicwise, word_count_topicwise, specific_word_count_topicwise, uniqueOutputLabels)
        #print("Time elapsed: ", datetime.now()-start_time)


        corr = 0
        for i in range(len(predictedOutputs)):
            if predictedOutputs[i]==valY_words[i]:
                corr += 1

        acc = corr/len(predictedOutputs)*100
        print("alpha: {} ->Accuracy: {}".format( alpha, acc))
        # print("{} & {}".format( alpha, acc))
        # print("\hline")
        how_many_alpha-=1
        if how_many_alpha%2:
            alpha *= (4/5)
        else:
            alpha *= (5/8)

"""## Test set accuracy"""
if NB_TEST:
    if False: #Whole test set average
        start_time = datetime.now()

        alpha = 0.03125

        predictedOutputs = prediction(testX_words, alpha, len(vocabulary), doc_count_topicwise, word_count_topicwise, specific_word_count_topicwise, uniqueOutputLabels)


        corr = 0
        for i in range(len(predictedOutputs)):
            if predictedOutputs[i]==testY_words[i]:
                corr += 1

        acc = corr/len(predictedOutputs)*100
        print("alpha: ", alpha, "->Accuracy: ", acc)

        print("Time elapsed: ", datetime.now()-start_time)

    start_time = datetime.now()

    #Best validation hyperparameters
    alpha = 0.03125

    with open("Data/topics.txt","r") as f:
        topic_count = (len(f.readlines())-1)

    batch_acc_list = []

    for i in tqdm(range(50)): # run 50 iterations
        testX_mini = []
        testY_mini = []
        for j in range(topic_count):
            start_idx = 10*i+500*j
            end_idx = start_idx + 10
            testX_mini.extend(testX_words[start_idx:end_idx])
            testY_mini.extend(testY_words[start_idx:end_idx])
            
        predictedOutputs = prediction(testX_mini, alpha, len(vocabulary), doc_count_topicwise, word_count_topicwise, specific_word_count_topicwise, uniqueOutputLabels)
        
        corr = 0
        for pred_i in range(len(predictedOutputs)):
            if predictedOutputs[pred_i]==testY_mini[pred_i]:
                corr += 1

        acc = corr/len(predictedOutputs)*100
        print("Iteration ", i, "->Accuracy: ", acc)
        batch_acc_list.append(acc)



    print("Time elapsed: ", datetime.now()-start_time)

    print(np.mean(batch_acc_list))
    with open("NB_test_output.txt", "w") as knn_file:
        for i in range(len(batch_acc_list)):
            knn_file.write(str(batch_acc_list[i])+'\n')

"""# T-statistics"""
if T_STAT:
    from scipy import stats
    import numpy as np

    with open("kNN_test_output.txt", "r") as f:
        knn_acc = f.readlines()
    knn_acc = [float(x.strip()) for x in knn_acc]
    print("knn mean: ", np.mean(knn_acc))
    print("knn sd: ", np.std(knn_acc))

    with open("NB_test_output.txt", "r") as f:
        nb_acc = f.readlines()
    nb_acc = [float(x.strip()) for x in nb_acc]
    print("nb mean: ", np.mean(nb_acc))
    print("nb sd: ", np.std(nb_acc))

    tstat, p_val = stats.ttest_rel(knn_acc, nb_acc)
    print(tstat, p_val)

    if False:
        #for latex table
        for i in range(50):
            print("$" + str(i+1) + "$ & $" + str(knn_acc[i]) + "$ & $" + str(nb_acc[i]) + "$\\\\")
            print("\\hline")