#!/usr/bin/python

import re
import pandas as pd
import numpy as np
import sklearn.preprocessing as preprocessing
from sklearn.linear_model import LogisticRegression
from sklearn import cross_validation
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.svm import LinearSVC


def fill_missing_ages(data):
    """Fill missing age using RandomForestRegressor
    The Regressor is trained with the data which has ages,
    The features for training are Age, SibSP, Parch, Fare, and Pclass
    Return the data filled with ages, and also the trained Regressor
    """

    # Selete features for training
    age_data = data[['Age', 'SibSp', 'Parch', 'Fare', 'Pclass']]

    # Divide passengers into with and without age
    with_age = age_data.loc[age_data['Age'].notnull()]
    without_age = age_data.loc[age_data['Age'].isnull()]
    # In the case that predict does not work  when withou_age is null
    if without_age.shape[0] == 0:
        without_age = pd.DataFrame({'Age': None, 'SibSp': {'1': 0},
                                    'Parch': {'1': 0}, 'Fare': {'1': 0},
                                    'Pclass': {'1': 0}})
    # X is the features
    X = with_age.iloc[:, 1:]
    Y = with_age.iloc[:, 0]

    # Train RandomForestRegressor
    alg = RandomForestRegressor(n_estimators=1000, n_jobs=-1, random_state=0)
    alg.fit(X, Y)

    # Predict age fill age
    data.loc[data['Age'].isnull(), 'Age'] = alg.predict(without_age.iloc[:, 1:])

    return data, alg


def set_cabin_type(data):
    """Set cabin into 2 types: with or without value
    Return the data whose Cabin column is replace by Yes
    """

    data.loc[data['Cabin'].notnull(), 'Cabin'] = 'Yes'
    data.loc[data['Cabin'].isnull(), 'Cabin'] = 'No'

    return data


def get_title(name):
    """Get title from a passenger's name"""

    title = re.search(' ([A-Za-z]+)\.', name)
    if title:
        return title.group(1)
    return ''


def get_title_feature(data):
    """Get title feature, this idea is from DATAQUEST"""

    titles = data['Name'].apply(get_title)
    title_map = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Dr": 5,
                 "Rev": 6, "Major": 7, "Col": 7, "Mlle": 8, "Mme": 8,
                 "Don": 9, "Lady": 10, "Countess": 10, "Jonkheer": 10,
                 "Sir": 9, "Capt": 7, "Ms": 2, "Dona": 8}

    for title, value in title_map.items():
        titles[titles == title] = value

    data['Title'] = titles

    return titles


def scale_data(data, param):
    """Scale the colomns of  param
    Return scaled colomn of data
    """
    scaler = preprocessing.StandardScaler()
    scale_param = scaler.fit(data[param])
    scaled = pd.DataFrame({param: scaler.transform(data[param], scale_param)})

    return scaled


def extract_features(data):
    """Features extraction and selection method,
    Return features extracted, the features are DataFrame objective
    """

    # Set cabin type
    data = set_cabin_type(data)
    # Fill Embarked missing data with S
    data["Embarked"] = data["Embarked"].fillna("S")
    # Fill Fare missing data with the median
    data['Fare'] = data['Fare'].fillna(data['Fare'].median())

    # Fill missing value of ages
    # Fill null data with age generated by Random Forest
    if data.loc[data['Age'].isnull()].shape[0] != 0:
        data, alg = fill_missing_ages(data)
    # Or we can fill null data with median of age
    # data['Age'] = data['Age'].fillna(data['Age'].median())

    # If a passenger is a Child
    data.loc[data['Age'] < 13, 'Child'] = 1
    data.loc[data['Age'] >= 13, 'Child'] = 0

    # Extract dummies of Pclass, Sex, Cabin, Embarked
    pclass = pd.get_dummies(data['Pclass'], prefix='Pclass')
    sex = pd.get_dummies(data['Sex'], prefix='Sex')
    cabin = pd.get_dummies(data['Cabin'], prefix='Cabin')
    embarked = pd.get_dummies(data['Embarked'], prefix='Embarked')
    # Age category
    age_bins = [0, 12, 18, 25, 35, 60, 100]
    age_groups = ['Child', 'Teenager', 'Youth', 'YoungAdult', 'MiddleAge', 'Senior']
    data['Age_category'] = pd.cut(data['Age'], age_bins, labels=age_groups)
    age_category = pd.get_dummies(data['Age_category'], prefix='Age_category')
    # Other features
    sibsp = data['SibSp']
    parch = data['Parch']
    data['FamilySize'] = data['SibSp'] + data['Parch']
    family_size = data['FamilySize']
    title = get_title_feature(data)

    # Scale features
    age = scale_data(data, 'Age')
    fare = scale_data(data, 'Fare')
    sibsp = scale_data(data, 'SibSp')
    parch = scale_data(data, 'Parch')
    family_size = scale_data(data, 'FamilySize')
    title = scale_data(data, 'Title')

    # Concate features
    features = pd.concat([pclass, sex, cabin, embarked, age, fare,
                          family_size, title, age_category], axis=1)

    return features


def train_model(data, algs):
    """Train model with all data """

    # Extract train features
    features = extract_features(data)

    # Train model
    for alg in algs:
        alg.fit(features, data['Survived'])


def cross_validation_evaluation(data, algs):
    """Train system with cross validation of the data, with the input algorithms
    Then evaluate the trained system,
    Return the accuracy
    """

    # Extract train features
    features = extract_features(data)

    # Set random_state to ensure to get the same splits every time we run this
    kf = cross_validation.KFold(features.shape[0], n_folds=3,
                                shuffle=False, random_state=1)

    predictions = []
    for train, test in kf:
        ensemble_predictions = []
        for alg in algs:
            alg.fit(features.iloc[train, :], data['Survived'].iloc[train])
            ensemble_predictions.append(alg.predict(features.iloc[test, :]))
        test_prediction = ensemble_predictions[0]
        for i in range(1, len(algs)):
            test_prediction += ensemble_predictions[i]
        test_prediction = test_prediction / len(algs)
        test_prediction[test_prediction > 0.5] = 1
        test_prediction[test_prediction <= 0.5] = 0
        predictions.append(test_prediction)

    predictions = np.concatenate(predictions, axis=0)
    # Calculate CV accuracy
    error_nb = sum(abs(predictions - pd.Series.as_matrix(data.Survived)))
    accuracy = 1 - error_nb / data.shape[0]

    return accuracy


def submission(data, classifiers):
    """Evaluate test data"""

    # Extract test features
    features = extract_features(data)
    # Predict test data
    predictions = []
    for classifier in classifiers:
        prediction = classifier.predict(features)
        predictions.append(prediction)

    test_prediction = predictions[0]
    for i in range(1, len(classifiers)):
        test_prediction += predictions[i]
    test_prediction = test_prediction / len(classifiers)
    test_prediction[test_prediction > 0.5] = 1
    test_prediction[test_prediction <= 0.5] = 0
    submission = pd.DataFrame({'PassengerId': data['PassengerId'],
                               'Survived': test_prediction})
    submission.to_csv('Titanic.csv', index=False)
    print('\n\nCongras, you\'v finished the prediction, you can submit it now')


# main
if __name__ == '__main__':

    # Load train data
    train_data = pd.read_csv('data/train.csv')
    # Define ensembling classifiers
    algs = [
        # LogisticRegression(),
        # LinearSVC(),
        GradientBoostingClassifier()
        # RandomForestClassifier()
    ]
    # Use CV to evaluate the accuracy
    accuracy = cross_validation_evaluation(train_data, algs)
    print("accuracy for the training data is:", accuracy)
    """
    # Train with all the data, then make a prediction and create submit file
    train_model(train_data, algs)
    test_data = pd.read_csv('data/test.csv')
    submission(test_data, algs)
    """
