#! python3
# coding: utf-8

import datetime
import pandas as pd
import numpy as np
import os

# DRAGONS
import xgboost as xgb
import lightgbm as lgb
import catboost as cat


import matplotlib.pyplot as plt

# pandas / plt options
pd.options.display.max_columns = 999
plt.rcParams['figure.figsize'] = (14, 7)
font = {'family' : 'verdana',
        'weight' : 'bold',
        'size'   : 14}
plt.rc('font', **font)

# remove warnings
import warnings
warnings.simplefilter("ignore")

# garbage collector
import gc
gc.enable()


#input_file_path = '/Users/rick/Dropbox/python_projects/data_science/Kaggle/GA_customer_revenue_prediction/input'
input_file_path = "C:/Users/p003230/PycharmProjects/python_projects/kaggle/GoogleAnalyticsCustomerRevenuePrediction/input"

#output_path = '/Users/rick/Dropbox/python_projects/data_science/Kaggle/GA_customer_revenue_prediction/output/'
output_path = 'C:/Users/p003230/PycharmProjects/python_projects/kaggle/GoogleAnalyticsCustomerRevenuePrediction/output'

#  Load Files
train = pd.read_csv(input_file_path + '/extracted_fields_train.gz', dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, 'visitId': np.int64})
test = pd.read_csv(input_file_path + '/extracted_fields_test.gz', dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, 'visitId': np.int64})
print(train.shape, test.shape)

# Getting data from leak
train_store_1 = pd.read_csv(input_file_path + '/Train_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
train_store_2 = pd.read_csv(input_file_path + '/Train_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
test_store_1 = pd.read_csv(input_file_path + '/Test_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
test_store_2 = pd.read_csv(input_file_path + '/Test_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})

# Getting VisitId from Google Analytics...
print("Getting VisitId from Google Analytics...")
for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
    df["visitId"] = df["Client Id"].apply(lambda x: x.split('.', 1)[1]).astype(np.int64)


# Merge with train/test data
print("Merging with train/test data...")
train = train.merge(pd.concat([train_store_1, train_store_2], sort=False), how="left", on="visitId")
test = test.merge(pd.concat([test_store_1, test_store_2], sort=False), how="left", on="visitId")

# Drop Client Id
print("Dropping Client ID...")
for df in [train, test]:
    df.drop("Client Id", 1, inplace=True)

# Cleaning Revenue
print("Cleaning Revenue")
for df in [train, test]:
    df["Revenue"].fillna('$', inplace=True)
    df["Revenue"] = df["Revenue"].apply(lambda x: x.replace('$', '').replace(',', ''))
    df["Revenue"] = pd.to_numeric(df["Revenue"], errors="coerce")
    df["Revenue"].fillna(0.0, inplace=True)

#
for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
    del df
gc.collect()

# Preprocessing
# Setting time as index and saving time as feature
train.visitStartTime = pd.to_datetime(train.visitStartTime, unit='s')
test.visitStartTime = pd.to_datetime(test.visitStartTime, unit='s')
train["date"] = train.visitStartTime
test["date"] = test.visitStartTime

#
train.set_index("visitStartTime", inplace=True)
test.set_index("visitStartTime", inplace=True)
train.sort_index(inplace=True)
test.sort_index(inplace=True)


# Clearing rare categoriesa and setting 0 to NaNs
def clearRare(columnname, limit=1000):

    # you may search for rare categories in train, train&test, or just test
    # vc = pd.concat([train[columnname], test[columnname]], sort=False).value_counts()
    vc = test[columnname].value_counts()

    common = vc > limit
    common = set(common.index[common].values)  # limitより多い重複の値を持ってるカラムをインデックスに設定
    print("Set", sum(vc <= limit), columnname, "categories to 'other';", end=" ")

    train.loc[train[columnname].map(lambda x: x not in common), columnname] = 'other'
    test.loc[test[columnname].map(lambda x: x not in common), columnname] = 'other'
    print("now there are", train[columnname].nunique(), "categories in train")


train.fillna(0, inplace=True)
test.fillna(0, inplace=True)

clearRare("device.browser")
clearRare("device.operatingSystem")
clearRare("geoNetwork.country")
clearRare("geoNetwork.city")
clearRare("geoNetwork.metro")
clearRare("geoNetwork.networkDomain")
clearRare("geoNetwork.region")
clearRare("geoNetwork.subContinent")
clearRare("trafficSource.adContent")
clearRare("trafficSource.campaign")
clearRare("trafficSource.keyword")
clearRare("trafficSource.medium")
clearRare("trafficSource.referralPath")
clearRare("trafficSource.source")

# Clearing leaked data:
for df in [train, test]:
    df["Avg. Session Duration"][df["Avg. Session Duration"] == 0] = "00:00:00"
    df["Avg. Session Duration"] = df["Avg. Session Duration"].str.split(':').apply(lambda x: int(x[0]) * 60 + int(x[1]))
    df["Bounce Rate"] = df["Bounce Rate"].astype(str).apply(lambda x: x.replace('%', '')).astype(float)
    df["Goal Conversion Rate"] = df["Goal Conversion Rate"].astype(str).apply(lambda x: x.replace('%', '')).astype(float)

# Features
# Based on strange things in Dataset
for df in [train, test]:
    # remember these features were equal, but not always? May be it means something...
    df["id_incoherence"] = pd.to_datetime(df.visitId, unit='s') != df.date
    # remember visitId dublicates?
    df["visitId_dublicates"] = df.visitId.map(df.visitId.value_counts())
    # remember session dublicates?
    df["session_dublicates"] = df.sessionId.map(df.sessionId.value_counts())

# Basic time features
for df in [train, test]:
    df['weekday'] = df['date'].dt.dayofweek.astype(object)
    df['time'] = df['date'].dt.second + df['date'].dt.minute*60 + df['date'].dt.hour*3600
    #df['month'] = df['date'].dt.month   # it must not be included in features during learning!
    df['day'] = df['date'].dt.date       # it must not be included in features during learning!

# Looking to future features
df = pd.concat([train, test])
df.sort_values(['fullVisitorId', 'date'], ascending=True, inplace=True)
df['prev_session'] = (df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(1)).astype(np.int64) // 1e9 // 60 // 60
df['next_session'] = (df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(-1)).astype(np.int64) // 1e9 // 60 // 60
df.sort_index(inplace=True)

train = df[:len(train)]
test = df[len(train):]

# Paired categories from "teach lightgbm to sum predictions" kernel
for df in [train, test]:
    df['source.country'] = df['trafficSource.source'] + '_' + df['geoNetwork.country']
    df['campaign.medium'] = df['trafficSource.campaign'] + '_' + df['trafficSource.medium']
    df['browser.category'] = df['device.browser'] + '_' + df['device.deviceCategory']
    df['browser.os'] = df['device.browser'] + '_' + df['device.operatingSystem']

#
for df in [train, test]:
    df['device_deviceCategory_channelGrouping'] = df['device.deviceCategory'] + "_" + df['channelGrouping']
    df['channelGrouping_browser'] = df['device.browser'] + "_" + df['channelGrouping']
    df['channelGrouping_OS'] = df['device.operatingSystem'] + "_" + df['channelGrouping']

    for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country', 'geoNetwork.metro',
              'geoNetwork.networkDomain', 'geoNetwork.region', 'geoNetwork.subContinent']:
        for j in ['device.browser', 'device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
            df[i + "_" + j] = df[i] + "_" + df[j]

    df['content.source'] = df['trafficSource.adContent'].astype(str) + "_" + df['source.country']
    df['medium.source'] = df['trafficSource.medium'] + "_" + df['source.country']

# User aggregating features
for feature in ["totals.hits", "totals.pageviews"]:
    info = pd.concat([train, test], sort=False).groupby("fullVisitorId")[feature].mean()
    train["usermean_" + feature] = train.fullVisitorId.map(info)
    test["usermean_" + feature] = test.fullVisitorId.map(info)

for feature in ["visitNumber"]:
    info = pd.concat([train, test], sort=False).groupby("fullVisitorId")[feature].max()
    train["usermax_" + feature] = train.fullVisitorId.map(info)
    test["usermax_" + feature] = test.fullVisitorId.map(info)

# Encoding Features
excluded = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime',
            'month', 'day', 'help']

cat_cols = [f for f in train.columns if (train[f].dtype == 'object' and f not in excluded)]
real_cols = [f for f in train.columns if (not f in cat_cols and f not in excluded)]

# ユニークな要素の個数を整数int型で返す
train[cat_cols].nunique()

#  文字データを数値に変換する
from sklearn.preprocessing import LabelEncoder
for col in cat_cols:
    lbl = LabelEncoder()
    lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))
    train[col] = lbl.transform(list(train[col].values.astype('str')))
    test[col] = lbl.transform(list(test[col].values.astype('str')))

# change data type to float
# 数字はフロートに変換
for col in real_cols:
    train[col] = train[col].astype(float)
    test[col] = test[col].astype(float)

#　不要なカラムの削除
for to_del in ["date", "sessionId", "visitId", "day"]:
    del train[to_del]
    del test[to_del]

# Preparing Validations
excluded = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime', "month", "help"]

# int64でexcludedにないカラムはカテゴリカラムに設定
cat_cols = [f for f in train.columns if (train[f].dtype == 'int64' and f not in excluded)]
# カテゴリカラムとexcludedにないカラムにないカラムを設定
real_cols = [f for f in train.columns if (not f in cat_cols and f not in excluded)]

# Function to tell us the score using the metric we actually care about
from sklearn.metrics import mean_squared_error


def score(data, y):
    validation_res = pd.DataFrame(
    {"fullVisitorId": data["fullVisitorId"].values,
     "transactionRevenue": data["totals.transactionRevenue"].values,
     "predictedRevenue": np.expm1(y)})

    validation_res = validation_res.groupby("fullVisitorId")["transactionRevenue", "predictedRevenue"].sum().reset_index()
    return np.sqrt(mean_squared_error(np.log1p(validation_res["transactionRevenue"].values),
                                     np.log1p(validation_res["predictedRevenue"].values)))


# Cute function to validate and prepare stacking
from sklearn.model_selection import GroupKFold


class KFoldValidation():
    def __init__(self, data, n_splits=5):
        unique_vis = np.array(sorted(data['fullVisitorId'].astype(str).unique()))
        folds = GroupKFold(n_splits)
        ids = np.arange(data.shape[0])

        self.fold_ids = []
        for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):
            self.fold_ids.append([
                ids[data['fullVisitorId'].astype(str).isin(unique_vis[trn_vis])],
                ids[data['fullVisitorId'].astype(str).isin(unique_vis[val_vis])]
            ])

    def validate(self, train, test, features, model, name="", prepare_stacking=False,
                 fit_params={"early_stopping_rounds": 50, "verbose": 100, "eval_metric": "rmse"}):  # changed from 50, verbose changed from 100
        model.FI = pd.DataFrame(index=features)
        full_score = 0

        if prepare_stacking:
            test[name] = 0
            train[name] = np.NaN

        for fold_id, (trn, val) in enumerate(self.fold_ids):
            devel = train[features].iloc[trn]
            y_devel = np.log1p(train["totals.transactionRevenue"].iloc[trn])
            valid = train[features].iloc[val]
            y_valid = np.log1p(train["totals.transactionRevenue"].iloc[val])

            print("Fold ", fold_id, ":")
            model.fit(devel, y_devel, eval_set=[(valid, y_valid)], **fit_params)

            if len(model.feature_importances_) == len(features):  # some bugs in catboost?
                model.FI['fold' + str(fold_id)] = model.feature_importances_ / model.feature_importances_.sum()

            predictions = model.predict(valid)
            predictions[predictions < 0] = 0
            print("Fold ", fold_id, " error: ", mean_squared_error(y_valid, predictions) ** 0.5)

            fold_score = score(train.iloc[val], predictions)
            full_score += fold_score / len(self.fold_ids)
            print("Fold ", fold_id, " score: ", fold_score)

            if prepare_stacking:
                train[name].iloc[val] = predictions

                test_predictions = model.predict(test[features])
                test_predictions[test_predictions < 0] = 0
                test[name] += test_predictions / len(self.fold_ids)

        print("Final score: ", full_score)
        return full_score


Kfolder = KFoldValidation(train)

# LGBM Prams
lgbmodel = lgb.LGBMRegressor(n_estimators=1000, objective="regression", metric="rmse", num_leaves=31, min_child_samples=100,
                      learning_rate=0.03, bagging_fraction=0.7, feature_fraction=0.5, min_data_in_leaf=5, bagging_frequency=5,  # changed from min_child_weight=15
                      bagging_seed=2019, subsample=.9, colsample_bytree=.9, use_best_model=True)


# Execute
print("-"* 20 + "LightGBM Training" + "-"* 20)
Kfolder.validate(train, test, real_cols + cat_cols, lgbmodel, "lgbpred", prepare_stacking=True)

# User-Level
# Make one user one object:
# * all features are averaged
# * we hope, that categorical features do not change for one user (that's not true :/ )
# * categoricals labels are averaged (!!!) and are treated as numerical features (o_O)
# * predictions are averaged in multiple ways...


def create_user_df(df):
    agg_data = df[real_cols + cat_cols + ['fullVisitorId']].groupby('fullVisitorId').mean()

    pred_list = df[['fullVisitorId', 'lgbpred']].groupby('fullVisitorId').apply(
        lambda visitor_df: list(visitor_df.lgbpred)) \
        .apply(lambda x: {'pred_' + str(i): pred for i, pred in enumerate(x)})
    all_predictions = pd.DataFrame(list(pred_list.values), index=agg_data.index)
    feats = all_predictions.columns

    all_predictions['t_mean'] = all_predictions.mean(axis=1)
    all_predictions['t_median'] = all_predictions.median(axis=1)  # including t_mean as one of the elements? well, ok
    all_predictions['t_sum_log'] = all_predictions.sum(axis=1)
    all_predictions['t_sum_act'] = all_predictions.fillna(0).sum(axis=1)
    all_predictions['t_nb_sess'] = all_predictions.isnull().sum(axis=1)

    full_data = pd.concat([agg_data, all_predictions], axis=1).astype(float)
    full_data['fullVisitorId'] = full_data.index
    del agg_data, all_predictions
    gc.collect()
    return full_data

user_train = create_user_df(train)
user_test = create_user_df(test)

#
features = list(user_train.columns)[:-1]  # don't include "fullVisitorId"
user_train["totals.transactionRevenue"] = train[['fullVisitorId', 'totals.transactionRevenue']].groupby('fullVisitorId').sum()

# featureがuser_test.columnになかったらNaNにする
for f in features:
    if f not in user_test.columns:
        user_test[f] = np.nan


# Meta-models
Kfolder = KFoldValidation(user_train)

# LightGBM
lgbmodel = lgb.LGBMRegressor(n_estimators=1000, objective="regression", metric="rmse", num_leaves=31, min_child_samples=100,
                      learning_rate=0.03, bagging_fraction=0.7, feature_fraction=0.5, min_data_in_leaf=5, bagging_frequency=5,  # changed from min_data_in_leaf 15
                      bagging_seed=2019, subsample=.9, colsample_bytree=.9, use_best_model=True)

print("-"* 20 + "User-Level LightGBM Training" + "-" * 20)
Kfolder.validate(user_train, user_test, features, lgbmodel, name="lgbfinal", prepare_stacking=True)
print("-"* 20 + "Done Training" + "-"* 20)

# XGBoost
xgbmodel = xgb.XGBRegressor(max_depth=22, learning_rate=0.02, n_estimators=1000,
                                         objective='reg:linear', gamma=1.45, seed=2019, silent=True,  # min_child_weight 15
                                        subsample=0.67, colsample_bytree=0.054, colsample_bylevel=0.50)

print("-" * 20 + "User-Level XGBoost Training" + "-" * 20)
Kfolder.validate(user_train, user_test, features, xgbmodel, name="xgbfinal", prepare_stacking=True)
print("-"* 20 + "Done Training" + "-"* 20)

# CatBoost
catmodel = cat.CatBoostRegressor(iterations=500, learning_rate=0.2, depth=5, random_seed=2019)

print("-" * 20 + "User-Level CatBoost Training" + "-" * 20)
Kfolder.validate(user_train, user_test, features, catmodel, name="catfinal", prepare_stacking=True,
                fit_params={"use_best_model": True, "verbose": 100})
print("-" * 20 + "Done Training" + "-" * 20)
# Ensembling dragons
user_train['PredictedLogRevenue'] = 0.4 * user_train["lgbfinal"] + \
                                    0.2 * user_train["xgbfinal"] + \
                                    0.4 * user_train["catfinal"]

score(user_train, user_train.PredictedLogRevenue)

print('Saving result to csv...')
today = datetime.datetime.today().strftime("%Y_%m_%d")  # フォーマットの指定

user_test['PredictedLogRevenue'] = 0.4 * user_test["lgbfinal"] + 0.4 * user_test["catfinal"] + 0.2 * user_test["xgbfinal"]
os.chdir(output_path)
user_test[['PredictedLogRevenue']].to_csv('three_boosters_submission_' + today + '.csv', index=True)

print("Combined: ", score(user_train, user_train.PredictedLogRevenue))
