#!/usr/bin/env python
# coding: utf-8

# In[1]:


#from graphviz import Digraph
from math import sqrt
from matplotlib import pyplot
from scipy.stats import norm
from scipy.stats import skew
from sklearn import tree
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor,  GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC
from sklearn.linear_model import Ridge, RidgeCV, ElasticNet, LassoCV, LassoLarsCV
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import make_scorer, accuracy_score, roc_auc_score
from sklearn.metrics import mean_absolute_error, mean_squared_error
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_squared_log_error
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import StandardScaler
from sklearn.tree import export_graphviz
from sqlalchemy import create_engine
import plotly.express as px
import pandas.api.types as ptypes
from pandas.api.types import is_numeric_dtype
import csv
#import graphviz
import json
#import lightgbm as lgb
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
#import pydot
import re
import scipy
import scipy.stats as stats
import seaborn as sns
import sklearn
import sys
import warnings
import pickle
import plotly.io as pio
import pandas.plotting
import plotly.offline as pyo
from pandas.plotting import scatter_matrix
import plotly.graph_objs as go
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
import pyarrow.parquet as pq


# In[2]:


print('matplotlib: {}'.format(matplotlib.__version__))
print('sklearn: {}'.format(sklearn.__version__))
print('scipy: {}'.format(scipy.__version__))
print('seaborn: {}'.format(sns.__version__))
print('pandas: {}'.format(pd.__version__))
print('numpy: {}'.format(np.__version__))
print('Python: {}'.format(sys.version))


# In[3]:


pd.set_option('display.float_format', lambda x: '%.3f' % x)
pd.set_option('display.max_rows', 10000)
pd.set_option('display.max_columns', 1500)
pd.set_option('display.width', 10000)
sns.set(style='white', context='notebook', palette='deep')
# warnings.filterwarnings('ignore')
get_ipython().run_line_magic('matplotlib', 'inline')
plt.rcParams.update({'figure.max_open_warning': 0})


# In[4]:





# In[5]:


#df = pd.read_csv('C:/Users/CBU6BU/Desktop/ProjeDosyaları/data/sample5K.csv',nrows=50, sep=';',encoding='utf-8')


# In[8]:



df = pq.read_table(r'C:/Users/CBU6BU/Desktop/ProjeDosyaları/processed_last100K.parquet').to_pandas()


# In[9]:


df.info()


# In[10]:


df.shape


# In[11]:


df.head()


# In[12]:


df.describe()


# In[13]:


df.dtypes.value_counts()


# In[14]:


df.dtypes


# ## Scatter Plot

# In[15]:


#row number by features plots
def scatterplots_features_and_row(df_in):
    process_names = df_in.columns
    for index in process_names:
        plt.figure(figsize=(16, 12), dpi=80)
        plt.ylabel(index)
        plt.xlabel('Row Number')
        fig = plt.scatter((list(range(len(df[index])))),df[index])
        #plt.gcf().autofmt_xdate()
        #plt.savefig("C:/Users/CBU6BU/Desktop/Proje Dosyaları/images-by-row/{}.jpeg".format(index))
        plt.show()


# In[16]:


scatterplots_features_and_row(df)


# In[17]:


#dictionary for outputs
labels_target = {'target1' : 'PN_155301001_Result_MeasResults_Cvo_CvoA_Qber_Qber',
                 'target2' : 'PN_155301001_Result_MeasResults_S2SDispersal_RangeS2STabTiCurve2',
                 'target3' : 'PN_155301001_Result_MeasResults_Cvo_CvoC_Qber_Qber'}


# In[18]:


#plot functions target by features
def scatter_plots(df_in,target_name):
    """
    df_in  :  input DataFrame
    target_name : target column name
    
    """
    Col_Names_df_in = df_in.columns
    for index in Col_Names_df_in:
        plt.figure(figsize=(16, 12), dpi=80)
        plt.ylabel(index)
        plt.xlabel(labels_target[target_name])
        fig = plt.scatter(df_in[labels_target[target_name]],df_in[index])
        #plt.gcf().autofmt_xdate()
        #plt.savefig("C:/Users/CBU6BU/Desktop/Proje Dosyaları/images-by-row/{}.jpeg".format(index))
        plt.show()


# In[19]:


scatter_plots(df,'target1')


# ## Box Plots

# ### First remove object types -- like Result Date

# In[20]:


#to eliminate object types
def find_features_without_object_list(df_in):
    """
    df_in  :  input DataFrame
    obj_list :  object type labels like Result Date
    col_list :  DataFrame column names list
    list_without_obj : process list without object types
    """
    obj_list = list(df_in.select_dtypes(include=['object']).columns)
    col_list = list(df_in.columns)
    list_without_obj=list(set(col_list)-set(obj_list))
    list_without_obj = sorted(list_without_obj)
    return list_without_obj


# In[21]:


#headers : list of headers without object types
headers = find_features_without_object_list(df)


# In[22]:


def box_plots(df_in):
    for index in headers:
        fig = df_in[[index]].boxplot() 
        #plt.savefig("C:/Users/CBU6BU/Desktop/Proje Dosyaları/images-by-row/{}.jpeg".format(index))
        plt.show()



# In[23]:


box_plots(df)


# In[24]:


def box_plot_seaborn(df_in):
    for index in headers:        
        plt.figure(figsize=(10, 10), dpi=60)
        plt.ylabel(index,fontsize=15)
        sns.boxplot(y = df_in[index], data=df_in)
        #plt.savefig("C:/Users/CBU6BU/Desktop/Proje Dosyaları/images-by-row/{}.jpeg".format(index))


# In[25]:


box_plot_seaborn(df)


# ## Scatter Matrix
# #to define small df which contains target label in these small df
# # Target Column should be in df range

# In[26]:


def column_index(df_in, query_cols):
    cols = df.columns.values
    sidx = np.argsort(cols)
    return sidx[np.searchsorted(cols,query_cols,sorter=sidx)]


# In[27]:


column_index(df,['PN_155301001_Result_MeasResults_Cvo_CvoA_Qber_Qber',
                 'PN_155301001_Result_MeasResults_S2SDispersal_RangeS2STabTiCurve2',
                 'PN_155301001_Result_MeasResults_Cvo_CvoC_Qber_Qber'])


# In[29]:


#df_CvoA = df[df.columns[655:775]]


# In[33]:


#df_S2S = df[df.columns[:810]]


# In[34]:


#df_CvoC = df[df.columns[655:662]]


# In[28]:


df_targets = df[df.columns[657:662]]


# # actual scatter matrix function
# 

# In[38]:



#df_in is input df, target_name is target label
#def scatterplot_matrix(df_in,target_name):
    #sns.set(style="ticks")
    #sns.pairplot(df_in , hue = labels_target[target_name],palette="husl",height=6)


# In[39]:


#scatterplot_matrix(df_targets,'target2')


# # ScatterMatrix Plotly 

# In[40]:


#Plotly ScatterMatrix
def ScatterplotMatrix_Plotly(df_in,target_name):
    """
    df_in : input DataFrame
    target_name : target column name
    """
    fig = px.scatter_matrix(df_in,dimensions=list(df_in.columns), color = labels_target[target_name])
    fig.update_layout(width=3000,height=3000)
    fig.show()


# In[46]:


ScatterplotMatrix_Plotly(df_targets,'target1')


# # Scatterplot Matrix Pandas

# In[62]:


def Scatterplot_Matrix_Pandas(df_in,first_col=0,last_col=30):
    """
    df_in  :  input DataFrame
    target_name : target column name
    col_numb : input number of columns, default = 30 
    df_i : first number of col_numb
    """
    df_i=df_in[df_in.columns[first_col:last_col]]
    
    pd.plotting.scatter_matrix(df_i,figsize=(30,30))
    #pd.plotting.scatter_matrix(df_i,figsize=(30,30), diagonal='kde')
    
    plt.figure()


# In[63]:


Scatterplot_Matrix_Pandas(df,0,10)


# # Corr Matrix Seaborn

# In[35]:


def corr_matrix(df_in,first_col_numb=0,last_col_numb=30):
    #if data has no NaN or missing values, no need to use below part
    """
    #drop non unique columns:
    nunique = df_in.apply(pd.Series.nunique)
    cols_to_drop = nunique[nunique == 1].index
    df_in=df_in.drop(cols_to_drop,axis=1)
    
    #drop full nan columns
    df_i=df_in.dropna(axis='columns',how='all')
    df_d=df_i[df_i.columns[:col_numb]]
    """
    nunique = df_in.apply(pd.Series.nunique)
    cols_to_drop = nunique[nunique == 1].index
    df_in=df_in.drop(cols_to_drop,axis=1)
    
    df_i=df_in.dropna(axis='columns',how='all')
    df_d=df_i[df_i.columns[first_col_numb:last_col_numb]]

    #----sns.heatmap(df_in.corr(),  annot=True, fmt=".2f") 
    plt.figure(figsize=(30,30)) # CORRELATION HEATMAP 
    sns.heatmap(df_d.corr(),  annot=True, fmt=".2f") 
    plt.show()


# In[36]:


corr_matrix(df,400,420)


# # Corr Matrix Alternative 

# In[37]:


def corr_matrix2(df_in,first_col_numb=0,last_col_numb=30):
    sns.set(style="white")

# Compute the correlation matrix
    df_d = df_in[df_in.columns[first_col_numb:last_col_numb]]
    corr = df_d.corr()

# Generate a mask for the upper triangle
    mask = np.triu(np.ones_like(corr, dtype=np.bool))

# Set up the matplotlib figure
    f, ax = plt.subplots(figsize=(11, 11))

# Generate a custom diverging colormap
    cmap = sns.diverging_palette(220, 10, as_cmap=True)

# Draw the heatmap with the mask and correct aspect ratio
    sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})
    


# In[38]:


corr_matrix2(df,400,420)


# # Correlation with other columns plot

# In[39]:



from pandas.api.types import is_numeric_dtype

target_labels = ["PN_155301001_Result_MeasResults_Cvo_CvoA_Qber_Qber",
                 "PN_155301001_Result_MeasResults_Cvo_CvoC_Qber_Qber",
                 "PN_155301001_Result_MeasResults_S2SDispersal_RangeS2STabTiCurve2"]

def plot_correlation_with_other_cols(df_in: df, target_cols: list, correlation_value: float = 0.3):
    x_cols = [col for col in df_in.columns if col not in target_cols if is_numeric_dtype(df_in[col])]

    for targ_col in target_cols:
        labels = []
        values = []
        for col in x_cols:
            labels.append(col)
            values.append(np.corrcoef(df_in[col].values, df_in[targ_col].values)[0, 1])
        corr_df = pd.DataFrame({'col_labels':labels, 'corr_values':values})
        corr_df = corr_df.sort_values(by='corr_values')
        corr_df = corr_df.loc[abs(corr_df['corr_values']) > correlation_value]
        ind = np.arange(len(corr_df))
        width = 0.9
        fig, ax = plt.subplots(figsize=(12,60))
        rects = ax.barh(ind, np.array(corr_df.corr_values.values), color='y')
        ax.set_yticks(ind)
        ax.set_yticklabels(corr_df.col_labels.values, rotation='horizontal')
        ax.set_xlabel("Correlation coefficient")
        ax.set_title("Correlation coefficient of "+ targ_col)
    #autolabel(rects)
    plt.show()



# In[40]:


plot_correlation_with_other_cols(df, target_labels, 0.3)


# ## Histogram Seaborn

# In[34]:





# In[32]:





# In[35]:


#First run function --> #def find_features_without_object_list(df_in):
#headers = find_features_without_object_list(df)

def hist_without_objects(df_in):
    headers = find_object_list(df)
    for index in headers:
        plt.figure(figsize=(10, 10), dpi=60)
        plt.xlabel(index)
        plt.ylabel("Frequency",fontsize=12)
        sns.distplot(df_in[index],kde=False,bins=10)
        #plt.savefig("C:/Users/CBU6BU/Desktop/Proje Dosyaları/images-by-row/{}.jpeg".format(index))


# In[36]:


hist_without_objects(df)


# # Histogram Matplot

# In[46]:


def histogram_plot(df_in,col_numb=30):
    #to drop non-unique columns in df
    
    nunique = df_in.apply(pd.Series.nunique)
    cols_to_drop = nunique[nunique == 1].index
    df_in=df_in.drop(cols_to_drop,axis=1)
    
    #to drop full nan columns
    
    df_i=df_in.dropna(axis='columns',how='all')
    df_d=df_i[df_i.columns[:col_numb]]
    df_d.hist(figsize=(30,30)) 
    plt.figure() 


# In[47]:


histogram_plot(df,20)


# In[ ]:




