import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
import os
agg=pd.read_csv('aggregate.csv')
print(agg.shape)
# feature creation
agg['storage_memory_executor']=agg['storage_memory_executor']-agg['storage_memory_driver']
agg['executor_instances']=agg['executor_instances']-1
agg['appName_target_median']=agg.groupby('app_name')['job_time'].transform('median')
agg['appName_target_std']=agg.groupby('app_name')['job_time'].transform('std')
agg['median_time_job_ratio']=agg['appName_target_median']/agg['total_jobs']

# seperating the input features and target label
input_params=agg.drop(['application_id', 'job_time'],axis=1)
target_feature=agg.job_time

# train test split for model selection and hyper parameters selection
x_train, x_test, y_train, y_test = train_test_split(input_params, target_feature, test_size=.2, random_state=21)

dt=DecisionTreeRegressor(min_samples_leaf=4)

dt.fit(x_train,y_train)
print('RMSE using decision Tree on training data is', np.sqrt(mean_squared_error(y_train ,dt.predict(x_train))))

print('RMSE using Decision Tree on test data is', np.sqrt(mean_squared_error(y_test ,dt.predict(x_test))))

# plotting the Desired and Predicted values of the target label
temp=pd.DataFrame([x_train.index, y_train, dt.predict(x_train)]).T
temp.sort_values(0,inplace=True)
plt.figure(figsize=(10,10))
plt.plot(temp[0], temp[1])
plt.plot(temp[0],temp[2])
plt.legend(['Actual values on training data','predicted values on training data'], loc='upper left')
plt.xlabel('Indexes of training data')
plt.ylabel('Time taken(in min)')
plt.show()

temp=pd.DataFrame([x_test.index, y_test, dt.predict(x_test)]).T
temp.sort_values(0,inplace=True)
plt.figure(figsize=(10,10))
plt.plot(temp[0], temp[1])
plt.plot(temp[0],temp[2])
plt.legend(['actual values on test data','predicted values on test data'], loc='upper left')
plt.xlabel('Indexes of test data')
plt.ylabel('Time taken(in min)')
plt.show()

# model training on whole data after hyper parameter tuning
dt.fit(input_params, target_feature)
# dumping the trained model in a file named dt.joblib
#dump(dt,'dt.joblib')

# feature importance calculated by the Decision Tree
pd.DataFrame(dt.feature_importances_,index=x_train.columns).plot(kind='bar')

# from sklearn.linear_model import LinearRegression
# from sklearn import linear_model
# from sklearn.metrics import mean_squared_error
# import numpy as np
# lr=linear_model.Ridge(alpha=0.1)
# lr.fit(x_train,y_train)
# print('RMSE using Linear regression on training data is',np.sqrt(mean_squared_error(y_train ,lr.predict(x_train))))

# print('RMSE using Linear regression on test data is',np.sqrt(mean_squared_error(y_test ,lr.predict(x_test))))
# print('coefficients of the linear regression are:', lr.coef_)




