#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jul 25 18:54:15 2019

@author: marcdimeo
"""
import scipy.io as spio 
import histone_utility
import numpy as np
import pandas as pd
#import vis

"""
CODE TO GET CCLE NORMALIZED
"""

df1 = histone_utility.read_csv_df("./../../../CCLE_RNAseq_rsem_genes_tpm_20180929.txt")
df1 = histone_utility.remove_columns(df1, 'transcript_ids')

gene_id_list = []

for gene_id in df1['gene_id']:
    gene_id = gene_id.split('.')
    gene_id_list.append(gene_id[0])
    
df1['gene_id'] = gene_id_list

ginfo = histone_utility.gene_conversion(df1['gene_id'], 'ensembl.gene', 'symbol', 'human')

# Remove any repeated gene_ids from ginfo.
# In this case there is a repeat at 30544.
ginfo.pop(30544)

gene = histone_utility.gene_symbols(ginfo)

# Format the my df to my liking based on the gene names.
df1['gene_id'] = gene
df1.rename(columns={'gene_id' : 'Gene'}, inplace = True)

# Retrieve the tissue dictionary in case you need it.
tissue_dictionary, cellline_list, cell_df = histone_utility.tissue_dict(df1)

# Normalize the expression results
df1_normalized = histone_utility.normalize(df1, 'Gene', cellline_list)

"""
CODE TO GET H3 NORMALIZED
"""

histone_utility.change_filepath('/Users/marcdimeo/Desktop/University of Michigan Research/methylation-gem/matlab/new_var')

df2_celllines = np.array(spio.loadmat('correlation_value', squeeze_me=True)["h3_ccle_names_python"])
df2_celllines = df2_celllines.tolist()
df2_markers =   list(dict.fromkeys(spio.loadmat('correlation_value', squeeze_me=True)["h3_marks_python"]))
df2_expression = spio.loadmat('h3_relval', squeeze_me=True)["h3_relval"]

# Create the dataframe.
df2 = pd.DataFrame(df2_expression)
df2 = df2.T
df2.insert(0, 'Histone', df2_markers, True)
df2.columns = ['Histone'] + df2_celllines

df2_normalized = histone_utility.normalize(df2, 'Histone', df2_celllines)

"""
CODE TO GET LEROY AS DATAFRAME
"""
histone_utility.change_filepath('/Users/marcdimeo/Desktop/University of Michigan Research/methylation-gem/matlab/vars')

df3_celllines =  spio.loadmat('supplementary_software_code', squeeze_me=True)["acetlevellist"]
df3_celllines = df3_celllines.tolist()
df3_markers = spio.loadmat('methylation_proteomics_validation_data', squeeze_me=True)["acet_meth_list_rowlab"]
df3_markers = df3_markers.tolist()
df3_expression=  spio.loadmat('hist_proteomics', squeeze_me=True)["acet_meth_listval"]

# Slight formating to keep the H3 and LeRoy data consistant 
# by making sure the histones are represented the same.
i = 0
for markers in df3_markers:
    df3_markers[i] = "H3" + markers
    i=i+1
    
i = 0
for markers in df3_markers:
    if 'un' in markers:
        df3_markers[i] = markers.replace('un', 'ac0')
    if 'ac' in markers:
        df3_markers[i] = markers.replace('ac', 'ac1')
    i = i+1
 
# Create the dataframe.
df3 = pd.DataFrame(df3_expression)
df3.insert(0, 'Histones', df3_markers , True)
df3 = df3.set_index('Histones')
df3.columns = df3_celllines
# This is already normalized.

"""
H3 AND CCLE COMPARISON
"""
# List of genes found on KEGG database
kegg_genes = ['HDAC6', 'HDAC5', 'HDAC1', 'HDAC2', 'HDAC7', 'HDAC8', 'HDAC11', 
              'HDAC10', 'HDAC3', 'HDAC9', 'HDAC4', 'KAT5', 'OGA', 'KAT7', 
              'CDYL2', 'TAF1L', 'CREBBP', 'EP300', 'CDY2B', 'KAT6B', 'CDY1B', 
              'KAT2A', 'ELP3', 'TAF1', 'KAT6A', 'NCOA3', 'KAT8', 'HAT1', 
              'NCOA1', 'KAT2B', 'CDY1', 'GTF3C4', 'CDYL', 'CDY2A', 'CLOCK', 
              'EHMT2', 'PRDM7', 'EZH1', 'EZH2', 'SETD1B', 'SETD2', 'KMT5A', 
              'KMT2A', 'KMT5B', 'NSD3', 'ASH1L', 'KMT2E', 'PRDM9', 'KMT2C', 
              'SETMAR', 'NSD1', 'SUV39H1', 'NSD2', 'PRDM2', 'SUV39H2', 'EHMT1',
              'KMT2D', 'SETD7', 'SETDB2', 'SETD3', 'DOT1L', 'KMT5C', 'PRDM6', 
              'SETD1A', 'KMT2B', 'SETDB1', 'KDM2A', 'PHF8', 'RIOX1', 'KDM8', 
              'KDM2B']

h3_ccle_cellline = histone_utility.common(cellline_list, df2_celllines) 
h3_ccle_cellline.sort()
     
df12 = df1_normalized[['Gene']+ h3_ccle_cellline]
df12 = df12.set_index('Gene')

df21= df2_normalized[['Histone']+ h3_ccle_cellline]
df21= df21.set_index('Histone')  

df21 = df21.fillna(method='ffill')
df12 = df12.fillna(method='ffill')

# H3 and CCLE R-value dataframe which can be graphed
h3_ccle = histone_utility.pearson_dfs(kegg_genes, df2_markers,
                                      df12, df21, 'df')[0]

"""
LEROY AND CCLE COMPARISON
"""
leroy_ccle_cellline = histone_utility.common(cellline_list, df3_celllines) 
leroy_ccle_cellline.sort()

df13 = df1_normalized[['Gene']+ leroy_ccle_cellline]
df13 = df13.set_index('Gene')
df31 = df3[leroy_ccle_cellline]

# LeRoy and CCLE R-value dataframe which can be graphed   
leroy_ccle = histone_utility.pearson_dfs(kegg_genes, df3_markers,
                                                df13, df31, 'df')[0]  
"""
RECON1 GENE LIST
"""
# Create a list of all the Recon1 genes
recon1_list = []
recon1_genes = pd.read_excel(r'/Users/marcdimeo/Desktop/University of Michigan Research/methylation-gem/data/RECON1_genes.xlsx')
for genes in recon1_genes['Genes']:
    genes = genes.split('\'')
    recon1_list.append(genes[1])
    
i = 0
for genes in recon1_list:
    genes = genes.split('_')
    recon1_list[i] = genes[0]
    i = i+1


recon1info = histone_utility.gene_conversion(recon1_list, 'entrezgene', 'symbol', 'human')

recon1_list = []
for entry in recon1info:
    if entry.get('symbol') != None:
        recon1_list.append(entry['symbol'])
    else:
        pass
    
recon1_list = pd.DataFrame(recon1_list)

    
recon1_list = histone_utility.remove_duplicate_entries(recon1_list)
recon1_list = recon1_list.values.tolist()
recon1_genes =[]
for entry in recon1_list:
    for item in entry:
        recon1_genes.append(item)

recon1_genes = histone_utility.common(gene, recon1_genes)
    
"""
LEROY AND CCLE WITH RECON1 GENES
"""

leroy_ccle_r1 = histone_utility.pearson_dfs(recon1_genes, df3_markers, 
                                            df13, df31, 'df')[0]

"""
TISSUE ANALYSIS: Only run if you have to it takes a very long time approx 2 hours
"""

cellline_tissue = cell_df.values
tissue_dict ={}
for data in cellline_tissue:
    tissue_dict[data[0]] = data[1]

h3_ccle_lung = histone_utility.tissue_analysis(cell_df, h3_ccle_cellline, df2_markers, recon1_genes, df12, df21,'LUNG')
#h3_ccle_ovary = histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df,'OVARY')
#h3_ccle_li = histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df,'LARGE INTESTINE')
#h3_ccle_cns = histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df,'CENTRAL NERVOUS SYSTEM')
#h3_ccle_hlt = histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df,'HAEMATOPOIETIC AND LYMPHOID TISSUE')
#h3_ccle_pancreas = histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df,'PANCREAS')
#h3_ccle_uat =  histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df, 'UPPER AERODIGESTIVE TRACT')
#h3_ccle_breast =  histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df, 'BREAST')
#h3_ccle_prostate =  histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df,'PROSTATE')
#h3_ccle_stomach =  histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df, 'STOMACH')
#h3_ccle_endometrium =   histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df,'ENDOMETRIUM')
#h3_ccle_bone =  histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df, 'BONE')
#h3_ccle_skin = histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df, 'SKIN')
#h3_ccle_liver =  histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df, 'LIVER')
#h3_ccle_fibroblast =  histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df, 'FIBROBLAST')
#h3_ccle_st = histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df,'SOFT TISSUE')
#h3_ccle_bt=  histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df,  'BILIARY TRACT')
#h3_ccle_ag = histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df, 'AUTONOMIC GANGLIA')
#h3_ccle_pleura = histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df, 'PLEURA')
#h3_ccle_ut = histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df, 'URINARY TRACT')
#h3_ccle_kidney = histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df,'KIDNEY')
#h3_ccle_oesophagus =  histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df,'OESOPHAGUS')
#h3_ccle_thyroid = histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df, 'THYROID')
#h3_ccle_sg = histone_utility.TissueAnalysis(tissue_dict, h3_ccle_cellline, ccle_h3_df, h3_ccle_df, 'SALIVARY GLAND')

"""
VISUALIZING
"""

plotlyheat(h3_ccle, None, 'H3 and CCLE Data', df2_markers, kegg_genes)
plotlyheat(leroy_ccle, None, 'LeRoy and CCLE Data', df3_markers, kegg_genes)
#vis.plotlyheat(leroy_ccle_r1, (1000,10000), 'LeRoy and CCLE Data with all Recon1 Genes', df3_markers, recon1_genes)

