#!/usr/bin/env python
# coding: utf-8

# # https://lp-tech.net/articles/0QUUd の写経

# In[ ]:


# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import os
print(os.listdir("../input"))

# Any results you write to the current directory are saved as output.


# In[ ]:


from sklearn.ensemble import RandomForestClassifier


# In[ ]:


train= pd.read_csv("../input/train.csv")
test= pd.read_csv("../input/test.csv")
train.head(3)


# In[ ]:


train.info()


# In[ ]:


# replaceの第一引数にstringを与えた場合、全文マッチしたセルのみリプレースされる
train= train.replace("male",0).replace("female",1).replace("S",0).replace("C",1).replace("Q",2)
test= test.replace("male",0).replace("female",1).replace("S",0).replace("C",1).replace("Q",2)


# In[ ]:


train.head()


# # 欠損値補完
# AgeとEmbarkedの欠損値を埋めます。そのコードは下のようになります。  
# 今回はそれぞれの平均値meanで埋めました。他にも中央値で埋める方法などもありましたが何度か試した所、結局平均値がいいという結果に私は至りました。

# In[ ]:


#inplace=True→元のオブジェクト自体が変更される。デフォルトでは新しいオブジェクトを返して元のオブジェクトは変更されない
train["Age"].fillna(train.Age.mean(), inplace=True) 
train["Embarked"].fillna(train.Embarked.mean(), inplace=True)


# In[ ]:


combine1 = [train]
for train in combine1: 
        #英字アルファベット1文字以上
        #ドットは特殊文字ではなくそのままドットにマッチする
        train['Salutation'] = train.Name.str.extract(' ([A-Za-z]+).', expand=False) 


# In[ ]:


train['Salutation'] 


# In[ ]:


for train in combine1: 
        train['Salutation'] = train['Salutation'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
        train['Salutation'] = train['Salutation'].replace('Mlle', 'Miss')
        train['Salutation'] = train['Salutation'].replace('Ms', 'Miss')
        train['Salutation'] = train['Salutation'].replace('Mme', 'Mrs')
        del train['Name']
Salutation_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5} 
for train in combine1: 
        train['Salutation'] = train['Salutation'].map(Salutation_mapping) 
        train['Salutation'] = train['Salutation'].fillna(0)


# In[ ]:


for train in combine1: 
        train['Ticket_Lett'] = train['Ticket'].apply(lambda x: str(x)[0])
        train['Ticket_Lett'] = train['Ticket_Lett'].apply(lambda x: str(x)) 
        #第一引数に抽出したい要素の条件を指定し、第二引数以降で条件を満たすとき値と満たさないときの値を指定することができます。 x, yを上手く使うことで、配列の一部の要素だけ変換したいときに便利になります。
        #1,2,3,S,P,C,Aのときはそのまま、それ以外は0に置換
        train['Ticket_Lett'] = np.where((train['Ticket_Lett']).isin(['1', '2', '3', 'S', 'P', 'C', 'A']), train['Ticket_Lett'], np.where((train['Ticket_Lett']).isin(['W', '4', '7', '6', 'L', '5', '8']), '0','0')) 
        train['Ticket_Len'] = train['Ticket'].apply(lambda x: len(x)) 
        del train['Ticket'] 
#なぜこのように置換するのか？
train['Ticket_Lett']=train['Ticket_Lett'].replace("1",1).replace("2",2).replace("3",3).replace("0",0).replace("S",3).replace("P",0).replace("C",3).replace("A",3)


# In[ ]:


train.head(10)


# # 変数追加
# + Sibsp+Parch+1がFamilySize  
# +  また、FamilySizeが1だとIsAlone一人で乗っているかどうかが1となります。

# In[ ]:


combine1


# In[ ]:


train["FamilySize"] = train["SibSp"] + train["Parch"] + 1
for train in combine1:
    train['IsAlone'] = 0
    #familysize=1の行の場合、isaloneに1代入
    train.loc[train['FamilySize'] == 1, 'IsAlone'] = 1


# In[ ]:


#numpy arrayに変換
train_data = train.values
xs = train_data[:, 2:] # Pclass以降の変数
y  = train_data[:, 1]  # 正解データ


# In[ ]:


test.info()


# In[ ]:


test["Age"].fillna(train.Age.mean(), inplace=True)
test["Fare"].fillna(train.Fare.mean(), inplace=True)

combine = [test]
for test in combine:
    test['Salutation'] = test.Name.str.extract(' ([A-Za-z]+)\.', expand=False)
for test in combine:
    test['Salutation'] = test['Salutation'].replace(['Lady', 'Countess','Capt', 'Col',         'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')

    test['Salutation'] = test['Salutation'].replace('Mlle', 'Miss')
    test['Salutation'] = test['Salutation'].replace('Ms', 'Miss')
    test['Salutation'] = test['Salutation'].replace('Mme', 'Mrs')
    del test['Name']
Salutation_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}

for test in combine:
    test['Salutation'] = test['Salutation'].map(Salutation_mapping)
    test['Salutation'] = test['Salutation'].fillna(0)

for test in combine:
        test['Ticket_Lett'] = test['Ticket'].apply(lambda x: str(x)[0])
        test['Ticket_Lett'] = test['Ticket_Lett'].apply(lambda x: str(x))
        test['Ticket_Lett'] = np.where((test['Ticket_Lett']).isin(['1', '2', '3', 'S', 'P', 'C', 'A']), test['Ticket_Lett'],
                                   np.where((test['Ticket_Lett']).isin(['W', '4', '7', '6', 'L', '5', '8']),
                                            '0', '0'))
        test['Ticket_Len'] = test['Ticket'].apply(lambda x: len(x))
        del test['Ticket']
test['Ticket_Lett']=test['Ticket_Lett'].replace("1",1).replace("2",2).replace("3",3).replace("0",0).replace("S",3).replace("P",0).replace("C",3).replace("A",3) 

for test in combine:
        test['Cabin_Lett'] = test['Cabin'].apply(lambda x: str(x)[0])
        test['Cabin_Lett'] = test['Cabin_Lett'].apply(lambda x: str(x))
        #なぜこういう変換？
        test['Cabin_Lett'] = np.where((test['Cabin_Lett']).isin(['T', 'H', 'G', 'F', 'E', 'D', 'C', 'B', 'A']),test['Cabin_Lett'],
                                   np.where((test['Cabin_Lett']).isin(['W', '4', '7', '6', 'L', '5', '8']),
                                            '0','0'))        
        del test['Cabin']
test['Cabin_Lett']=test['Cabin_Lett'].replace("A",1).replace("B",2).replace("C",1).replace("0",0).replace("D",2).replace("E",2).replace("F",1).replace("G",1) 

test["FamilySize"] = train["SibSp"] + train["Parch"] + 1

for test in combine:
    test['IsAlone'] = 0
    test.loc[test['FamilySize'] == 1, 'IsAlone'] = 1
    
test_data = test.values
xs_test = test_data[:, 1:]


# In[ ]:


from sklearn.ensemble import RandomForestClassifier

random_forest=RandomForestClassifier()
random_forest.fit(xs, y)
Y_pred = random_forest.predict(xs_test)


# In[ ]:


train


# # 訓練データでcabinの前処理をしていないためValueError

# In[ ]:


for train in combine1:
        train['Cabin_Lett'] = train['Cabin'].apply(lambda x: str(x)[0])
        train['Cabin_Lett'] = train['Cabin_Lett'].apply(lambda x: str(x))
        train['Cabin_Lett'] = np.where((train['Cabin_Lett']).isin(['T', 'H', 'G', 'F', 'E', 'D', 'C', 'B', 'A']),train['Cabin_Lett'],
                                   np.where((train['Cabin_Lett']).isin(['W', '4', '7', '6', 'L', '5', '8']),
                                            '0','0'))        
        del train['Cabin']
train['Cabin_Lett']=train['Cabin_Lett'].replace("A",1).replace("B",2).replace("C",1).replace("0",0).replace("D",2).replace("E",2).replace("F",1).replace("G",1) 


# ## 再実行

# In[ ]:


#numpy arrayに変換
train_data = train.values
xs = train_data[:, 2:] # Pclass以降の変数
y  = train_data[:, 1]  # 正解データ


# In[ ]:


from sklearn.ensemble import RandomForestClassifier

random_forest=RandomForestClassifier()
random_forest.fit(xs, y)
Y_pred = random_forest.predict(xs_test)


# ## HとTを前処理してないから？

# In[ ]:


train['Cabin_Lett']=train['Cabin_Lett'].replace("A",1).replace("B",2).replace("C",1).replace("0",0).replace("D",2).replace("E",2).replace("F",1).replace("G",1).replace("H",0).replace("T",0)


# In[ ]:


#numpy arrayに変換
train_data = train.values
xs = train_data[:, 2:] # Pclass以降の変数
y  = train_data[:, 1]  # 正解データ


# In[ ]:


from sklearn.ensemble import RandomForestClassifier

random_forest=RandomForestClassifier()
random_forest.fit(xs, y)
Y_pred = random_forest.predict(xs_test)


# # 訓練成功

# In[ ]:


Y_pred


# In[ ]:


import csv
with open("predict_result_data.csv", "w") as f:
    writer = csv.writer(f, lineterminator='\n')
    writer.writerow(["PassengerId", "Survived"])
    for pid, survived in zip(test_data[:,0].astype(int), Y_pred.astype(int)):
        writer.writerow([pid, survived])


# # グリッドサーチ

# In[ ]:


from sklearn.ensemble import RandomForestClassifier
from sklearn import grid_search
from sklearn.grid_search import GridSearchCV

parameters = {
        #木の数
        'n_estimators'      : [10,25,50,75,100],
        'random_state'      : [0],
        #-1 でCPUコア数となる
        'n_jobs'            : [4],
        #整数または小数を指定．デフォルトは None．ノードを分割するために必要な最小サンプルサイズ．整数を指定した場合，その数，小数を指定した場合，全サンプルサイズに対する割合個．
        'min_samples_split' : [5,10, 15, 20,25, 30],
        #整数または None を指定．決定木の深さの最大値を指定．過学習を避けるためにはこれを調節するのが最も重要．
        'max_depth'         : [5, 10, 15,20,25,30]
}

clf = grid_search.GridSearchCV(RandomForestClassifier(), parameters)
clf.fit(xs, y)
 
print(clf.best_estimator_)


# In[ ]:


Y_pred = clf.predict(xs_test)
Y_pred


# In[ ]:


import csv
with open("predict_result_data_rf_gs.csv", "w") as f:
    writer = csv.writer(f, lineterminator='\n')
    writer.writerow(["PassengerId", "Survived"])
    for pid, survived in zip(test_data[:,0].astype(int), Y_pred.astype(int)):
        writer.writerow([pid, survived])


# # *4.データ分析*

# In[ ]:


get_ipython().magic(u'matplotlib inline')
import matplotlib.pyplot as plt
import seaborn as sns
g = sns.factorplot(x="Sex", y="Survived",  data=train,
                   size=6, kind="bar", palette="muted")
#g.despine(left=True)
g = g.set_ylabels("survival probability")


# In[ ]:


sns.countplot(x='Sex', data = train)


# ## 男性が多い

# In[ ]:


g = sns.factorplot(x="Pclass",y="Survived",data=train,kind="bar", size = 6 , palette = "muted")
#g.despine(left=True)
g = g.set_ylabels("survival probability")


# In[ ]:


g = sns.factorplot(x="Pclass", y="Survived", hue="Sex", data=train,
                   size=6, kind="bar", palette="muted")
#g.despine(left=True)
g = g.set_ylabels("survival probability")


# In[ ]:


#g = sns.factorplot(x="Salutation", y="Survived",  data=train, size=6, kind="bar", palette="muted")
g = sns.factorplot(x="Salutation", y="Survived",  data=train, size=6, kind="bar")

#g.despine(left=True)
g = g.set_ylabels("survival probability")


# In[ ]:


colormap = plt.cm.viridis
plt.figure(figsize=(12,12))
plt.title('Pearson Correlation of Features', y=1.05, size=15)
del train['PassengerId']
sns.heatmap(train.astype(float).corr(),linewidths=0.1,vmax=1.0, square=True, cmap=colormap, linecolor='white', annot=True)


# In[ ]:


sns.countplot(x='FamilySize', data = train, hue = 'Survived')


# In[ ]:


sns.countplot(x='FamilySize', data = train,hue = 'Pclass')


# ## 1人の乗客は3等多く、2人の乗客は1等多い

# In[ ]:


t=pd.read_csv("../input/train.csv").replace("S",0).replace("C",1).replace("Q",2)
train['Embarked']= t['Embarked']
g = sns.factorplot(x="Embarked", y="Survived",  data=train,
                   size=6, kind="bar", palette="muted")
#g.despine(left=True)
g = g.set_ylabels("survival probability")


# In[ ]:


sns.countplot(x='Embarked', data = train,hue = 'Pclass')


# In[ ]:


sns.countplot(x='Embarked', data = train,hue = 'Sex')


# In[ ]:


plt.figure()
sns.FacetGrid(data=t, hue="Survived", aspect=4).map(sns.kdeplot, "Age", shade=True)
plt.ylabel('Passenger Density')
plt.title('KDE of Age against Survival')
plt.legend()


# In[ ]:





# In[ ]:





# In[ ]:





# In[ ]:





# In[ ]:
































# In[ ]:





# In[ ]:




