import random
from functools import partial
from itertools import izip
from sklearn.metrics import  jaccard_similarity_score
import numpy as np
from itertools import permutations,combinations
from collections import defaultdict



def gen_hash(a,b,c,row):
    hashed = (a*row +b)%c
    return hashed

def load_text_as_matrix(data_path):
    dt = np.loadtxt(fname=data_path, dtype="i", delimiter=",")
    return dt

def __create_hash_func_val__(matrix_rows,num_of_hash_func,next_big_prime):

    """

    :param matrix_rows: the number of rows in the matrix generated by load_matrix_data
    :param next_big_prime : the next prime number greater than matrix_rows
    :param num_of_hash_func: total number of hash functions needed
    :return: matrix of num_of_hash_func*matrix_rows
    """
    # create choices should be 2* num_of_hash_functions
    totals = 2*num_of_hash_func
    choices = np.random.choice(next_big_prime,totals)
    [choicesA ,choicesB] = np.split(choices,2)
    # generator of hashed functions where I provide a,b,c value to hash function and store the partial func
    # this will help in creating the matrix in next step
    funct_list = [partial(gen_hash, a=i, b=j, c=6007) for i, j in izip(choicesA, choicesB)]
    # create the numpymatrix and initialize it to zeroes will populate
    sizes = (num_of_hash_func,matrix_rows)
    hashed_matrix  = np.zeros(sizes)
    for i in range(0,num_of_hash_func):
        for j in range(0,matrix_rows):
            hashed_matrix[i,j]= funct_list[i](row=j)
    return  hashed_matrix


def gen_sig_matrix(main_matrix,num_hash_func,next_prime=6007):

    """

    :param main_matrix: number of rows in input matrix
    :param ncols:  number of cols /documents in input matrix
    :param num_hash_func: hash func required
    :return: signature matrix
    """
    nrows = main_matrix.shape[0]

    ncols = main_matrix.shape[1]

    sig_matrix_size = (num_hash_func, ncols)

    sig_matrix = np.full(sig_matrix_size, np.inf)

    hashed_matrix = __create_hash_func_val__(nrows,num_hash_func,next_prime)

    for m in range(0,nrows):
        for n in range(0,ncols):
            if main_matrix[m,n]==1:
                for i in range(0,num_hash_func):
                    sig_matrix[i, n] = min(sig_matrix[i, n],hashed_matrix[i, m])


            else:
                continue

    return  sig_matrix








def hashed(ar, hash_val):
    # print "Here is the array",ar
    r = np.dot(ar, range(1,ar.size+1)) % hash_val
    return r


def hash_split_array(array, hash_val=10000):
    """
    hash the columns of the bands to different hash buckets based on the hash function then
    hash bucket is a list which has columns it has hashed to
    then for every key in hash bucket pairs

    """
    # print "array in question is ",array
    hash_dict = defaultdict(list)
    cand_set_temp = []
    for i in range(array.shape[1]):
        c = array[:, i]
        hash_dict[hashed(c, hash_val)].append(i)
    for k in hash_dict:
        mj = permutations(hash_dict[k], 2)
        for i in mj:
            if tuple(sorted(i)) not in cand_set_temp:
                cand_set_temp.append(tuple(sorted(i)))
    # print cand_set_temp
    return cand_set_temp


def lsh_band(s_mat, b):
    """ divide each sig matrix
    by b band and hash them to bucket of 10000
    for each band create

    """
    splitted_array = np.vsplit(s_mat, b)
    # print splitted_array
    #### create hashed bucket ####

    candidate_set = defaultdict(int)
    for i in splitted_array:
        # print i.shape
        mkn = hash_split_array(i, 10000)
        for i in mkn:
            if i in candidate_set:
                candidate_set[i] += 1
            else:
                candidate_set[i] = 1
    return candidate_set




def main():

    """
    load matrix

    :return:
    """
    num_of_permutation = 100
    next_big_prime = 6007
    np.random.seed(9001)
    random.seed(9001)
    data_path = "D:\\741A2\\data_Assignment2.txt"
    input_matrix =  load_text_as_matrix(data_path)
    matrix_rows = input_matrix.shape[0]
    ncols = input_matrix.shape[1]
    # hashed_matrix = __create_hash_func_val__(matrix_rows,num_of_permutation,next_big_prime)
    sig_matrix= gen_sig_matrix(input_matrix,num_hash_func=num_of_permutation,next_prime=next_big_prime)
    # create different bands


    score_list = []
    col_list = [i for i in range(0, 1000)]
    pairs = combinations(col_list, 2)
    # true_candidate_input_matrix = []
    for i in pairs:
        # print "pair is", i
        skc = jaccard_similarity_score(y_true=sig_matrix[:, i[0]], y_pred=sig_matrix[:, i[1]])
        # print "score is ", skc
        if skc >= 0.3:
            score_list.append(i)
        else:
            continue

    hashed_column_count_band_25 = lsh_band(sig_matrix, 25)
    hashed_column_count_band_20 = lsh_band(sig_matrix, 20)
    hashed_column_count_band_10 = lsh_band(sig_matrix, 10)
    hashed_column_count_band_5 = lsh_band(sig_matrix, 5)
    hashed_column_count_band_4 = lsh_band(sig_matrix, 4)
    fps_5 = [i for i in hashed_column_count_band_5.keys() if i not in score_list]
    fps_4 = [i for i in hashed_column_count_band_4.keys() if i not in score_list]
    fps_10 = [i for i in hashed_column_count_band_10.keys() if i not in score_list]
    fps_20 = [i for i in hashed_column_count_band_20.keys() if i not in score_list]
    fps_25 = [i for i in hashed_column_count_band_25.keys() if i not in score_list]
    fns_4 = [i for i in score_list if i not in hashed_column_count_band_4]
    fns_5 = [i for i in score_list if i not in hashed_column_count_band_5]
    fns_10 = [i for i in score_list if i not in hashed_column_count_band_10]
    fns_20 = [i for i in score_list if i not in hashed_column_count_band_20]
    fns_25 = [i for i in score_list if i not in hashed_column_count_band_25]
    print "false positive rate for 5 bands", len(fps_5)
    print "false positive rate for 4 bands", len(fps_4)
    print "false positive rate for 10 bands", len(fps_10)
    print "false positive rate for 20 bands", len(fps_20)
    print "false positive rate for 25 bands", len(fps_25)

    print "false neg rate for 4 bands", len(fns_4)
    print "false neg rate for 5 bands", len(fns_5)
    print "false neg rate for 10 bands", len(fns_10)
    print "false neg rate for 20 bands", len(fns_20)
    print "false neg rate for 25 bands", len(fns_25)

    print "signature matrix candidate pair",len(score_list)

    print "500 hash pairs starting now :"

    sig_matrix_500 = gen_sig_matrix(input_matrix,num_hash_func=500,next_prime=next_big_prime)
    hashed_500_column_count_band_100 = lsh_band(sig_matrix_500, 100)
    hashed_500_column_count_band_50 =  lsh_band(sig_matrix_500, 50)
    hashed_500_column_count_band_25 = lsh_band(sig_matrix_500, 25)
    hashed_500_column_count_band_20 = lsh_band(sig_matrix_500, 20)
    hashed_500_column_count_band_125 = lsh_band(sig_matrix_500, 125)
    #hashed_500_column_count_band_5 = lsh_band(sig_matrix, 5)
    #hashed_500_column_count_band_4 = lsh_band(sig_matrix, 4)
    fps_500_50 = [i for i in hashed_500_column_count_band_50.keys() if i not in score_list]
    fps_500_100 = [i for i in hashed_500_column_count_band_100.keys() if i not in score_list]
    fps_500_125 = [i for i in hashed_500_column_count_band_125.keys() if i not in score_list]
    fps_500_20 = [i for i in hashed_500_column_count_band_20.keys() if i not in score_list]
    fps_500_25 = [i for i in hashed_500_column_count_band_25.keys() if i not in score_list]
    fns_500_100 = [i for i in score_list if i not in hashed_500_column_count_band_100]
    fns_500_50 = [i for i in score_list if i not in hashed_500_column_count_band_50]
    fns_500_125 = [i for i in score_list if i not in hashed_500_column_count_band_125]
    fns_500_20 = [i for i in score_list if i not in hashed_500_column_count_band_20]
    fns_500_25 = [i for i in score_list if i not in hashed_500_column_count_band_25]
    print "false positive rate for 50 bands", len(fps_500_50)
    print "false positive rate for 100 bands", len(fps_500_100)
    print "false positive rate for 125 bands", len(fps_500_125)
    print "false positive rate for 20 bands", len(fps_20)
    print "false positive rate for 25 bands", len(fps_25)

    print "false neg rate for 50 bands", len(fns_500_50)
    print "false neg rate for 100 bands", len(fns_500_100)
    print "false neg rate for 125 bands", len(fns_500_125)
    print "false neg rate for 20 bands", len(fns_500_20)
    print "false neg rate for 25 bands", len(fns_500_25)

    print "signature matrix candidate pair", len(score_list)


if __name__ == "__main__":
    main()
