#coding=utf-8
#前面的几次大计算效果不是特别的理想，我觉得问题主要在超参选择过多上面,准备大削超参咯
#然后我现在感觉dropout也是非常需要设置滴，但是我现在还不知道怎么个设置发咯，随缘设置么
import os
import sys
import random
import pickle
import datetime
import numpy as np
import pandas as pd
sys.path.append("D:\\Workspace\\Titanic")

from sklearn import preprocessing
from sklearn.cross_validation import cross_val_score, StratifiedKFold

import torch.nn.init
import torch.nn as nn
import torch.nn.functional as F

import skorch
from skorch import NeuralNetClassifier

import hyperopt
from hyperopt import fmin, tpe, hp, space_eval, rand, Trials, partial, STATUS_OK

def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
    return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)


data_train = pd.read_csv("C:/Users/win7/Desktop/train.csv")
data_test = pd.read_csv("C:/Users/win7/Desktop/test.csv")
combine = [data_train, data_test]

for dataset in combine:
    dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
    dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
    dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
    dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
    dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
    title_map = {'Mr': 1, 'Miss': 2, 'Mrs': 3, 'Master': 4, 'Rare': 5}
    dataset['Title'] = dataset['Title'].map(title_map)
    dataset['Title'] = dataset['Title'].fillna(0)

for dataset in combine:
    dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
    dataset['FamilySizePlus'] = 0
    dataset.loc[dataset['FamilySize'] == 1, 'FamilySizePlus'] = 1
    dataset.loc[dataset['FamilySize'] == 2, 'FamilySizePlus'] = 2
    dataset.loc[dataset['FamilySize'] == 3, 'FamilySizePlus'] = 2
    dataset.loc[dataset['FamilySize'] == 4, 'FamilySizePlus'] = 2
    dataset.loc[dataset['FamilySize'] == 5, 'FamilySizePlus'] = 1
    dataset.loc[dataset['FamilySize'] == 6, 'FamilySizePlus'] = 1
    dataset.loc[dataset['FamilySize'] == 7, 'FamilySizePlus'] = 1

for dataset in combine:
    dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0}).astype(int)

guess_ages = np.zeros((2, 3))
for dataset in combine:
    for i in range(0, 2):
        for j in range(0, 3):
            guess_df = dataset[(dataset['Sex'] == i) & (dataset['Pclass'] == j+1)]['Age'].dropna()
            age_guess = guess_df.median()
            guess_ages[i,j] = int(age_guess / 0.5 + 0.5) * 0.5
    for i in range(0, 2):
        for j in range(0, 3):
            dataset.loc[(dataset.Age.isnull()) & (dataset.Sex == i) & (dataset.Pclass == j + 1), 'Age'] = guess_ages[i, j]
    dataset['Age'] = dataset['Age'].astype(int)
    
for dataset in combine: 
    dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0 
    dataset.loc[(dataset['Age'] > 16) & (dataset['Age'] <= 32), 'Age'] = 1 
    dataset.loc[(dataset['Age'] > 32) & (dataset['Age'] <= 48), 'Age'] = 2 
    dataset.loc[(dataset['Age'] > 48) & (dataset['Age'] <= 64), 'Age'] = 3 
    dataset.loc[ dataset['Age'] > 64, 'Age'] = 4
    
#这里的mode是求解pandas.core.series.Series众数的第一个值（可能有多个众数）
freq_port = data_train.Embarked.dropna().mode()[0]
for dataset in combine:
    dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
for dataset in combine:
    dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2})

#将data_test中的fare元素所缺失的部分由已经包含的数据的中位数决定哈
data_test['Fare'].fillna(data_test['Fare'].dropna().median(), inplace=True)

for dataset in combine:
    dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
    dataset.loc[(dataset['Fare'] > 7.91) & (dataset['Fare'] <= 14.454), 'Fare'] = 1
    dataset.loc[(dataset['Fare'] > 14.454) & (dataset['Fare'] <= 31), 'Fare']   = 2
    dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
    dataset['Fare'] = dataset['Fare'].astype(int)

for dataset in combine:
    dataset.loc[(dataset.Cabin.isnull()), 'Cabin'] = 0
    dataset.loc[(dataset.Cabin.notnull()), 'Cabin'] = 1

#尼玛给你说的这个是贡献船票，原来的英文里面根本就没有这种说法嘛
df = data_train['Ticket'].value_counts()
df = pd.DataFrame(df)
df = df[df['Ticket'] > 1]
#print(df)
df_ticket = df.index.values          #共享船票的票号
tickets = data_train.Ticket.values   #所有的船票
#print(tickets)
result = []
for ticket in tickets:
    if ticket in df_ticket:
        ticket = 1
    else:
        ticket = 0                   #遍历所有船票，在共享船票里面的为1，否则为0
    result.append(ticket)
    
df = data_train['Ticket'].value_counts()
df = pd.DataFrame(df)
df = df[df['Ticket'] > 1]
df_ticket = df.index.values          #共享船票的票号
tickets = data_train.Ticket.values   #所有的船票

result = []
for ticket in tickets:
    if ticket in df_ticket:
        ticket = 1
    else:
        ticket = 0                   #遍历所有船票，在共享船票里面的为1，否则为0
    result.append(ticket)

results = pd.DataFrame(result)
results.columns = ['Ticket_Count']
data_train = pd.concat([data_train, results], axis=1)

df = data_test['Ticket'].value_counts()
df = pd.DataFrame(df)
df = df[df['Ticket'] > 1]
df_ticket = df.index.values          
tickets = data_test.Ticket.values   
result = []
for ticket in tickets:
    if ticket in df_ticket:
        ticket = 1
    else:
        ticket = 0                   
    result.append(ticket)
results = pd.DataFrame(result)
results.columns = ['Ticket_Count']
data_test = pd.concat([data_test, results], axis=1) 

data_train_1 = data_train.copy()
data_test_1  = data_test.copy()
data_test_1 = data_test_1.drop(['PassengerId', 'Name', 'SibSp', 'Parch', 'Ticket', 'FamilySize'], axis=1)

X_train = data_train_1[['Pclass', 'Sex', 'Age', 'Fare', 'Embarked', 'Cabin', 'Title', 'FamilySizePlus', 'Ticket_Count']]
Y_train = data_train_1['Survived']

X_test = data_test_1[['Pclass', 'Sex', 'Age', 'Fare', 'Embarked', 'Cabin', 'Title', 'FamilySizePlus', 'Ticket_Count']]

X_all = pd.concat([X_train, X_test], axis=0)
#我觉得训练集和测试集需要在一起进行特征缩放，所以注释掉了原来的X_train的特征缩放咯
X_all_scaled = pd.DataFrame(preprocessing.scale(X_all), columns = X_train.columns)
#X_train_scaled = pd.DataFrame(preprocessing.scale(X_train), columns = X_train.columns)
X_train_scaled = X_all_scaled[:len(X_train)]
X_test_scaled = X_all_scaled[len(X_train):]

class MyModule1(nn.Module):
    def __init__(self):
        super(MyModule1, self).__init__()

        self.fc1 = nn.Linear(9, 60)
        self.fc2 = nn.Linear(60, 60)
        self.fc3 = nn.Linear(60, 60)
        self.fc4 = nn.Linear(60, 60)
        self.fc5 = nn.Linear(60, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = self.dropout1(X)
        X = F.relu(self.fc4(X))
        X = F.softmax(self.fc5(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            
        return self
    
class MyModule2(nn.Module):
    def __init__(self):
        super(MyModule2, self).__init__()

        self.fc1 = nn.Linear(9, 45)
        self.fc2 = nn.Linear(45, 45)
        self.fc3 = nn.Linear(45, 45)
        self.fc4 = nn.Linear(45, 45)
        self.fc5 = nn.Linear(45, 45)
        self.fc6 = nn.Linear(45, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = F.relu(self.fc4(X))
        X = self.dropout1(X)
        X = F.relu(self.fc5(X))
        X = F.softmax(self.fc6(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            
        return self
    
class MyModule3(nn.Module):
    def __init__(self):
        super(MyModule3, self).__init__()

        self.fc1 = nn.Linear(9, 36)
        self.fc2 = nn.Linear(36, 36)
        self.fc3 = nn.Linear(36, 36)
        self.fc4 = nn.Linear(36, 36)
        self.fc5 = nn.Linear(36, 36)
        self.fc6 = nn.Linear(36, 36)
        self.fc7 = nn.Linear(36, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = F.relu(self.fc4(X))
        X = self.dropout1(X)
        X = F.relu(self.fc5(X))
        X = F.relu(self.fc6(X))
        X = F.softmax(self.fc7(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            
        return self
    
class MyModule4(nn.Module):
    def __init__(self):
        super(MyModule4, self).__init__()

        self.fc1 = nn.Linear(9, 30)
        self.fc2 = nn.Linear(30, 30)
        self.fc3 = nn.Linear(30, 30)
        self.fc4 = nn.Linear(30, 30)
        self.fc5 = nn.Linear(30, 30)
        self.fc6 = nn.Linear(30, 30)
        self.fc7 = nn.Linear(30, 30)
        self.fc8 = nn.Linear(30, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = F.relu(self.fc4(X))
        X = self.dropout1(X)
        X = F.relu(self.fc5(X))
        X = F.relu(self.fc6(X))
        X = self.dropout1(X)
        X = F.relu(self.fc7(X))
        X = F.softmax(self.fc8(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.normal_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
            
        return self

class MyModule5(nn.Module):
    def __init__(self):
        super(MyModule5, self).__init__()

        self.fc1 = nn.Linear(9, 20)
        self.fc2 = nn.Linear(20, 20)
        self.fc3 = nn.Linear(20, 20)
        self.fc4 = nn.Linear(20, 20)
        self.fc5 = nn.Linear(20, 20)
        self.fc6 = nn.Linear(20, 20)
        self.fc7 = nn.Linear(20, 20)
        self.fc8 = nn.Linear(20, 20)
        self.fc9 = nn.Linear(20, 20)
        self.fc10 = nn.Linear(20, 20)
        self.fc11 = nn.Linear(20, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = F.relu(self.fc4(X))
        X = self.dropout1(X)
        X = F.relu(self.fc5(X))
        X = F.relu(self.fc6(X))
        X = self.dropout1(X)
        X = F.relu(self.fc7(X))
        X = F.relu(self.fc8(X))
        X = self.dropout1(X)
        X = F.relu(self.fc9(X))
        X = F.relu(self.fc10(X))
        X = F.softmax(self.fc11(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.normal_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
            torch.nn.init.normal_(self.fc9.weight.data)
            torch.nn.init.constant_(self.fc9.bias.data, bias)
            torch.nn.init.normal_(self.fc10.weight.data)
            torch.nn.init.constant_(self.fc10.bias.data, bias)
            torch.nn.init.normal_(self.fc11.weight.data)
            torch.nn.init.constant_(self.fc11.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc9.weight.data)
            torch.nn.init.constant_(self.fc9.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc10.weight.data)
            torch.nn.init.constant_(self.fc10.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc11.weight.data)
            torch.nn.init.constant_(self.fc11.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc9.weight.data)
            torch.nn.init.constant_(self.fc9.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc10.weight.data)
            torch.nn.init.constant_(self.fc10.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc11.weight.data)
            torch.nn.init.constant_(self.fc11.bias.data, bias)
            
        return self

class MyModule6(nn.Module):
    def __init__(self):
        super(MyModule6, self).__init__()

        self.fc1 = nn.Linear(9, 18)
        self.fc2 = nn.Linear(18, 18)
        self.fc3 = nn.Linear(18, 18)
        self.fc4 = nn.Linear(18, 18)
        self.fc5 = nn.Linear(18, 18)
        self.fc6 = nn.Linear(18, 18)
        self.fc7 = nn.Linear(18, 18)
        self.fc8 = nn.Linear(18, 18)
        self.fc9 = nn.Linear(18, 18)
        self.fc10 = nn.Linear(18, 18)
        self.fc11 = nn.Linear(18, 18)
        self.fc12 = nn.Linear(18, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = F.relu(self.fc4(X))
        X = self.dropout1(X)
        X = F.relu(self.fc5(X))
        X = F.relu(self.fc6(X))
        X = self.dropout1(X)
        X = F.relu(self.fc7(X))
        X = F.relu(self.fc8(X))
        X = self.dropout1(X)
        X = F.relu(self.fc9(X))
        X = F.relu(self.fc10(X))
        X = self.dropout1(X)
        X = F.relu(self.fc11(X))
        X = F.softmax(self.fc12(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.normal_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
            torch.nn.init.normal_(self.fc9.weight.data)
            torch.nn.init.constant_(self.fc9.bias.data, bias)
            torch.nn.init.normal_(self.fc10.weight.data)
            torch.nn.init.constant_(self.fc10.bias.data, bias)
            torch.nn.init.normal_(self.fc11.weight.data)
            torch.nn.init.constant_(self.fc11.bias.data, bias)
            torch.nn.init.normal_(self.fc12.weight.data)
            torch.nn.init.constant_(self.fc12.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc9.weight.data)
            torch.nn.init.constant_(self.fc9.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc10.weight.data)
            torch.nn.init.constant_(self.fc10.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc11.weight.data)
            torch.nn.init.constant_(self.fc11.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc12.weight.data)
            torch.nn.init.constant_(self.fc12.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc9.weight.data)
            torch.nn.init.constant_(self.fc9.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc10.weight.data)
            torch.nn.init.constant_(self.fc10.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc11.weight.data)
            torch.nn.init.constant_(self.fc11.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc12.weight.data)
            torch.nn.init.constant_(self.fc12.bias.data, bias)
            
        return self

class MyModule7(nn.Module):
    def __init__(self):
        super(MyModule7, self).__init__()

        self.fc1 = nn.Linear(9, 24)
        self.fc2 = nn.Linear(24, 24)
        self.fc3 = nn.Linear(24, 24)
        self.fc4 = nn.Linear(24, 24)
        self.fc5 = nn.Linear(24, 24)
        self.fc6 = nn.Linear(24, 24)
        self.fc7 = nn.Linear(24, 24)
        self.fc8 = nn.Linear(24, 24)
        self.fc9 = nn.Linear(24, 24)
        self.fc10 = nn.Linear(24, 24)
        self.fc11 = nn.Linear(24, 24)
        self.fc12 = nn.Linear(24, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = F.relu(self.fc4(X))
        X = self.dropout1(X)
        X = F.relu(self.fc5(X))
        X = F.relu(self.fc6(X))
        X = self.dropout1(X)
        X = F.relu(self.fc7(X))
        X = F.relu(self.fc8(X))
        X = self.dropout1(X)
        X = F.relu(self.fc9(X))
        X = F.relu(self.fc10(X))
        X = self.dropout1(X)
        X = F.relu(self.fc11(X))
        X = F.softmax(self.fc12(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.normal_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
            torch.nn.init.normal_(self.fc9.weight.data)
            torch.nn.init.constant_(self.fc9.bias.data, bias)
            torch.nn.init.normal_(self.fc10.weight.data)
            torch.nn.init.constant_(self.fc10.bias.data, bias)
            torch.nn.init.normal_(self.fc11.weight.data)
            torch.nn.init.constant_(self.fc11.bias.data, bias)
            torch.nn.init.normal_(self.fc12.weight.data)
            torch.nn.init.constant_(self.fc12.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc9.weight.data)
            torch.nn.init.constant_(self.fc9.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc10.weight.data)
            torch.nn.init.constant_(self.fc10.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc11.weight.data)
            torch.nn.init.constant_(self.fc11.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc12.weight.data)
            torch.nn.init.constant_(self.fc12.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc9.weight.data)
            torch.nn.init.constant_(self.fc9.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc10.weight.data)
            torch.nn.init.constant_(self.fc10.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc11.weight.data)
            torch.nn.init.constant_(self.fc11.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc12.weight.data)
            torch.nn.init.constant_(self.fc12.bias.data, bias)
            
        return self

class MyModule8(nn.Module):
    def __init__(self):
        super(MyModule8, self).__init__()

        self.fc1 = nn.Linear(9, 30)
        self.fc2 = nn.Linear(30, 30)
        self.fc3 = nn.Linear(30, 30)
        self.fc4 = nn.Linear(30, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))

        X = F.softmax(self.fc4(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            
        return self
    
class MyModule9(nn.Module):
    def __init__(self):
        super(MyModule9, self).__init__()

        self.fc1 = nn.Linear(9, 20)
        self.fc2 = nn.Linear(20, 20)
        self.fc3 = nn.Linear(20, 20)
        self.fc4 = nn.Linear(20, 20)
        self.fc5 = nn.Linear(20, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = self.dropout1(X)
        X = F.relu(self.fc4(X))
        X = F.softmax(self.fc5(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            
        return self
    
class MyModule10(nn.Module):
    def __init__(self):
        super(MyModule10, self).__init__()

        self.fc1 = nn.Linear(9, 15)
        self.fc2 = nn.Linear(15, 15)
        self.fc3 = nn.Linear(15, 15)
        self.fc4 = nn.Linear(15, 15)
        self.fc5 = nn.Linear(15, 15)
        self.fc6 = nn.Linear(15, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = F.relu(self.fc4(X))
        X = self.dropout1(X)
        X = F.relu(self.fc5(X))
        X = F.softmax(self.fc6(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            
        return self
        
class MyModule11(nn.Module):
    def __init__(self):
        super(MyModule11, self).__init__()

        self.fc1 = nn.Linear(9, 12)
        self.fc2 = nn.Linear(12, 12)
        self.fc3 = nn.Linear(12, 12)
        self.fc4 = nn.Linear(12, 12)
        self.fc5 = nn.Linear(12, 12)
        self.fc6 = nn.Linear(12, 12)
        self.fc7 = nn.Linear(12, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = F.relu(self.fc4(X))
        X = self.dropout1(X)
        X = F.relu(self.fc5(X))
        X = self.dropout1(X)
        X = F.relu(self.fc6(X))
        X = F.softmax(self.fc7(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            
        return self

class MyModule12(nn.Module):
    def __init__(self):
        super(MyModule12, self).__init__()

        self.fc1 = nn.Linear(9, 10)
        self.fc2 = nn.Linear(10, 10)
        self.fc3 = nn.Linear(10, 10)
        self.fc4 = nn.Linear(10, 10)
        self.fc5 = nn.Linear(10, 10)
        self.fc6 = nn.Linear(10, 10)
        self.fc7 = nn.Linear(10, 10)
        self.fc8 = nn.Linear(10, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = F.relu(self.fc4(X))
        X = self.dropout1(X)
        X = F.relu(self.fc5(X))
        X = F.relu(self.fc6(X))
        X = self.dropout1(X)
        X = F.relu(self.fc7(X))
        X = F.softmax(self.fc8(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.normal_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc8.weight.data)
            torch.nn.init.constant_(self.fc8.bias.data, bias)
            
        return self

class MyModule13(nn.Module):
    def __init__(self):
        super(MyModule13, self).__init__()

        self.fc1 = nn.Linear(9, 25)
        self.fc2 = nn.Linear(25, 25)
        self.fc3 = nn.Linear(25, 25)
        self.fc4 = nn.Linear(25, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = F.softmax(self.fc4(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            
        return self
    
class MyModule14(nn.Module):
    def __init__(self):
        super(MyModule14, self).__init__()

        self.fc1 = nn.Linear(9, 17)
        self.fc2 = nn.Linear(17, 17)
        self.fc3 = nn.Linear(17, 17)
        self.fc4 = nn.Linear(17, 17)
        self.fc5 = nn.Linear(17, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = self.dropout1(X)
        X = F.relu(self.fc4(X))
        X = F.softmax(self.fc5(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            
        return self
    
class MyModule15(nn.Module):
    def __init__(self):
        super(MyModule15, self).__init__()

        self.fc1 = nn.Linear(9, 10)
        self.fc2 = nn.Linear(10, 10)
        self.fc3 = nn.Linear(10, 10)
        self.fc4 = nn.Linear(10, 10)
        self.fc5 = nn.Linear(10, 10)
        self.fc6 = nn.Linear(10, 10)
        self.fc7 = nn.Linear(10, 2)  
        self.dropout1 = nn.Dropout(0.1)
        self.dropout2 = nn.Dropout(0.2)
        
    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.dropout1(X)
        X = F.relu(self.fc3(X))
        X = F.relu(self.fc4(X))
        X = self.dropout1(X)
        X = F.relu(self.fc5(X))
        X = F.relu(self.fc6(X))
        X = F.softmax(self.fc7(X), dim=-1)
        return X

    def init_module(self, weight_mode, bias):
        
        if (weight_mode==1):
            pass#就是什么都不做的意思，使用默认值的意思
        
        elif (weight_mode==2):
            torch.nn.init.normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
        
        elif (weight_mode==3):
            torch.nn.init.xavier_normal_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_normal_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
        
        else:
            torch.nn.init.xavier_uniform_(self.fc1.weight.data)
            torch.nn.init.constant_(self.fc1.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc2.weight.data)
            torch.nn.init.constant_(self.fc2.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc3.weight.data)
            torch.nn.init.constant_(self.fc3.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc4.weight.data)
            torch.nn.init.constant_(self.fc4.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc5.weight.data)
            torch.nn.init.constant_(self.fc5.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc6.weight.data)
            torch.nn.init.constant_(self.fc6.bias.data, bias)
            torch.nn.init.xavier_uniform_(self.fc7.weight.data)
            torch.nn.init.constant_(self.fc7.bias.data, bias)
            
        return self

module1 = MyModule1()
module2 = MyModule2()
module3 = MyModule3()
module4 = MyModule4()
module5 = MyModule5()
module6 = MyModule6()
module7 = MyModule7()
module8 = MyModule8()
module9 = MyModule9()
module10 = MyModule10()
module11 = MyModule11()
module12 = MyModule12()
module13 = MyModule13()
module14 = MyModule14()
module15 = MyModule15()

def cal_nnclf_acc(clf, X_train, Y_train):
    
    Y_train_pred = clf.predict(X_train.values.astype(np.float32))
    count = (Y_train_pred == Y_train).sum()
    acc = count/len(Y_train)
    
    return acc

def print_nnclf_acc(acc):
    
    print("the accuracy rate of the model on the whole train dataset is:", acc)
  
def print_best_params_acc(trials):
    
    trials_list =[]
    #从trials中读取最大的准确率信息咯
    #item和result其实指向了一个dict对象
    for item in trials.trials:
        trials_list.append(item)
    
    #按照关键词进行排序，关键词即为item['result']['loss']
    trials_list.sort(key=lambda item: item["result"]["loss"])
    
    print("best parameter is:", trials_list[0])
    print()
    
def exist_files(title):
    
    return os.path.exists(title+"_best_model.pickle")
    
def save_inter_params(trials, space_nodes, best_nodes, title):
 
    files = open(str(title+"_intermediate_parameters.pickle"), "wb")
    pickle.dump([trials, space_nodes, best_nodes], files)
    files.close()

def load_inter_params(title):
  
    files = open(str(title+"_intermediate_parameters.pickle"), "rb")
    trials, space_nodes, best_nodes = pickle.load(files)
    files.close()
    
    return trials, space_nodes ,best_nodes
    
def save_best_model(best_model, title):
    
    files = open(str(title+"_best_model.pickle"), "wb")
    pickle.dump(best_model, files)
    files.close()
    
def load_best_model(title):
    
    files = open(str(title+"_best_model.pickle"), "rb")
    best_model = pickle.load(files)
    files.close()
    
    return best_model
    
def record_best_model_acc(clf, acc, best_model, best_acc):
    
    flag = False
    
    if not isclose(best_acc, acc):
        if best_acc < acc:
            flag = True
            best_acc = acc
            best_model = clf
            
    return best_model, best_acc, flag

def noise_augment_data(mean, std, X_train, Y_train, columns):
    
    X_noise_train = X_train.copy()
    X_noise_train.is_copy = False
    
    row = X_train.shape[0]
    for i in range(0, row):
        for j in columns:
            X_noise_train.iloc[i,[j]] +=  random.gauss(mean, std)

    return X_noise_train, Y_train
    
#我觉得这个中文文档介绍hyperopt还是比较好https://www.jianshu.com/p/35eed1567463
def nn_f(params):
    
    print("mean", params["mean"])
    print("std", params["std"])
    print("lr", params["lr"])
    print("optimizer__weight_decay", params["optimizer__weight_decay"])
    print("criterion", params["criterion"])
    print("batch_size", params["batch_size"])
    print("optimizer__betas", params["optimizer__betas"])
    print("bias", params["bias"])
    print("weight_mode", params["weight_mode"])
    print("patience", params["patience"])
    print("module", params["module"])    
    
    X_noise_train, Y_noise_train = noise_augment_data(params["mean"], params["std"], X_train_scaled, Y_train, columns=[3, 4, 5, 6, 7, 8])
    
    clf = NeuralNetClassifier(lr = params["lr"],
                              optimizer__weight_decay = params["optimizer__weight_decay"],
                              criterion = params["criterion"],
                              batch_size = params["batch_size"],
                              optimizer__betas = params["optimizer__betas"],
                              module=params["module"],
                              max_epochs = params["max_epochs"],
                              callbacks=[skorch.callbacks.EarlyStopping(patience=params["patience"])],
                              device = params["device"],
                              optimizer = params["optimizer"]
                              )
    
    skf = StratifiedKFold(Y_noise_train, n_folds=5, shuffle=True, random_state=None)
    
    clf.module.init_module(params["weight_mode"], params["bias"])
    
    metric = cross_val_score(clf, X_noise_train.values.astype(np.float32), Y_noise_train.values.astype(np.longlong), cv=skf, scoring="accuracy").mean()
    
    print(metric)
    print()    
    return -metric

def display_search_progress(search_times, nn_f):
    
    print("search times:", search_times)
    return nn_f
    
def parse_space(trials, space_nodes):
    
    trials_list =[]
    for item in trials.trials:
        trials_list.append(item)
    trials_list.sort(key=lambda item: item['result']['loss'])
    
    best_nodes = {}
    best_nodes["title"] = space_nodes["title"][trials_list[0]["misc"]["vals"]["title"][0]]
    best_nodes["path"] = space_nodes["path"][trials_list[0]["misc"]["vals"]["path"][0]]
    best_nodes["mean"] = space_nodes["mean"][trials_list[0]["misc"]["vals"]["mean"][0]]
    best_nodes["std"] = space_nodes["std"][trials_list[0]["misc"]["vals"]["std"][0]]
    best_nodes["batch_size"] = space_nodes["batch_size"][trials_list[0]["misc"]["vals"]["batch_size"][0]]
    best_nodes["criterion"] = space_nodes["criterion"][trials_list[0]["misc"]["vals"]["criterion"][0]]
    best_nodes["max_epochs"] = space_nodes["max_epochs"][trials_list[0]["misc"]["vals"]["max_epochs"][0]]
    #修改了lr的类型也需要修改三个地方哟
    #best_nodes["lr"] = trials_list[0]["misc"]["vals"]["lr"][0]
    best_nodes["lr"] = space_nodes["lr"][trials_list[0]["misc"]["vals"]["lr"][0]] 
    best_nodes["module"] = space_nodes["module"][trials_list[0]["misc"]["vals"]["module"][0]] 
    best_nodes["optimizer__betas"] = space_nodes["optimizer__betas"][trials_list[0]["misc"]["vals"]["optimizer__betas"][0]]
    best_nodes["optimizer__weight_decay"] = space_nodes["optimizer__weight_decay"][trials_list[0]["misc"]["vals"]["optimizer__weight_decay"][0]]
    best_nodes["weight_mode"] = space_nodes["weight_mode"][trials_list[0]["misc"]["vals"]["weight_mode"][0]]
    best_nodes["bias"] = space_nodes["bias"][trials_list[0]["misc"]["vals"]["bias"][0]]
    best_nodes["patience"] = space_nodes["patience"][trials_list[0]["misc"]["vals"]["patience"][0]]
    best_nodes["device"] = space_nodes["device"][trials_list[0]["misc"]["vals"]["device"][0]]
    best_nodes["optimizer"] = space_nodes["optimizer"][trials_list[0]["misc"]["vals"]["optimizer"][0]]
    
    return best_nodes
    
def predict(best_nodes, max_evals=10):
    
    best_acc = 0.0
    best_model = 0.0
    #只要module class的定义没有修改就可以继续用，虽然增加了超参bias以及修改了初始化方式
    if (exist_files(best_nodes["title"])):
        best_model = load_best_model(best_nodes["title"])
        best_acc = cal_nnclf_acc(best_model, X_train_scaled, Y_train)
         
    for i in range(0, max_evals):
        
        print(str(i+1)+"/"+str(max_evals)+" prediction progress have been made.")
        
        clf = NeuralNetClassifier(lr = best_nodes["lr"],
                                  optimizer__weight_decay = best_nodes["optimizer__weight_decay"],
                                  criterion = best_nodes["criterion"],
                                  batch_size = best_nodes["batch_size"],
                                  optimizer__betas = best_nodes["optimizer__betas"],
                                  module=best_nodes["module"],
                                  max_epochs = best_nodes["max_epochs"],
                                  callbacks = [skorch.callbacks.EarlyStopping(patience=best_nodes["patience"])],
                                  device = best_nodes["device"],
                                  optimizer = best_nodes["optimizer"]
                                  )
        
        #在这重新初始化一次基本就会得到差异很大的结果吧
        #现在可以看到确实是经过了权重初始化模型重新训练的
        #但是这个best_model的数据怎么还是上一回的数据？
        clf.module.init_module(best_nodes["weight_mode"], best_nodes["bias"])
        
        clf.fit(X_train_scaled.values.astype(np.float32), Y_train.values.astype(np.longlong)) 
        
        metric = cal_nnclf_acc(clf, X_train_scaled, Y_train)
        print_nnclf_acc(metric)
        
        #那么现在的best_model应该就不会被修改了吧
        best_model, best_acc, flag = record_best_model_acc(clf, metric, best_model, best_acc)
        #通过下面的两行代码可以发现clf确实每次都是新创建的，但是module每次都是重复使用的
        #print(id(clf))
        #print(id(clf.module))
    
        if (flag):
            save_best_model(best_model, best_nodes["title"])
            Y_pred = best_model.predict(X_test_scaled.values.astype(np.float32))
            
            data = {"PassengerId":data_test["PassengerId"], "Survived":Y_pred}
            output = pd.DataFrame(data = data)
            #这个path是在我的windows7环境下的，如果用别的机器运行是需要修改的
            output.to_csv(best_nodes["path"], index=False)
            print("prediction file has been written.")
        print()
     
    #因为下面的clf中的module已经被重新训练了，所以已经是新的模型了，还是直接输出best_acc   
    #metric = cal_nnclf_acc(best_model, X_train_scaled, Y_train)
    print("the best accuracy rate of the model on the whole train dataset is:", best_acc)
    
#我真的曹乐，做不做数据集增强好像差别很大哦，不添加噪声准确率高得多呢。。好像也不是
#上回那个貌似是巧合而已，我多运行了几次发现加了噪声好像是要高一点点呢。。
#倒是patience设置为10的时候绝壁没有设置为5的时候效果好呢。。不看运行输出真还不知道
#现在我的代码应用到下一个版本的时候只需要修改space、space_nodes以及best_nodes和parse_space
#predict函数内data = {"PassengerId":data_test["PassengerId"], "Survived":Y_pred}
space = {"title":hp.choice("title", ["titanic"]),
         "path":hp.choice("path", ["C:/Users/win7/Desktop/Titanic_Prediction.csv"]),
         "mean":hp.choice("mean", [0]),
         #"std":hp.choice("std", [0]),
         "std":hp.choice("std", [0, 0.02, 0.04, 0.06, 0.08, 0.10, 0.12, 0.14, 0.16, 0.18, 0.20]),
         "max_epochs":hp.choice("max_epochs",[400]),
         "patience":hp.choice("patience", [4,5,6,7,8,9,10]),
         #将这个参数由范围改成具体的取值应该能够提升模型的准确率的吧
         "lr":hp.choice("lr", [0.00001, 0.00002, 0.00003, 0.00004, 0.00005, 0.00006, 0.00007, 0.00008, 0.00009, 0.00010,
                               0.00011, 0.00012, 0.00013, 0.00014, 0.00015, 0.00016, 0.00017, 0.00018, 0.00019, 0.00020,
                               0.00021, 0.00022, 0.00023, 0.00024, 0.00025, 0.00026, 0.00027, 0.00028, 0.00029, 0.00030,
                               0.00031, 0.00032, 0.00033, 0.00034, 0.00035, 0.00036, 0.00037, 0.00038, 0.00039, 0.00040,
                               0.00041, 0.00042, 0.00043, 0.00044, 0.00045, 0.00046, 0.00047, 0.00048, 0.00049, 0.00050,
                               0.00051, 0.00052, 0.00053, 0.00054, 0.00055, 0.00056, 0.00057, 0.00058, 0.00059, 0.00060,
                               0.00061, 0.00062, 0.00063, 0.00064, 0.00065, 0.00066, 0.00067, 0.00068, 0.00069, 0.00070,
                               0.00071, 0.00072, 0.00073, 0.00074, 0.00075, 0.00076, 0.00077, 0.00078, 0.00079, 0.00080,
                               0.00081, 0.00082, 0.00083, 0.00084, 0.00085, 0.00086, 0.00087, 0.00088, 0.00089, 0.00090,
                               0.00091, 0.00092, 0.00093, 0.00094, 0.00095, 0.00096, 0.00097, 0.00098, 0.00099, 0.00100,
                               0.00101, 0.00102, 0.00103, 0.00104, 0.00105, 0.00106, 0.00107, 0.00108, 0.00109, 0.00110,
                               0.00111, 0.00112, 0.00113, 0.00114, 0.00115, 0.00116, 0.00117, 0.00118, 0.00119, 0.00120,
                               0.00121, 0.00122, 0.00123, 0.00124, 0.00125, 0.00126, 0.00127, 0.00128, 0.00129, 0.00130,
                               0.00131, 0.00132, 0.00133, 0.00134, 0.00135, 0.00136, 0.00137, 0.00138, 0.00139, 0.00140,
                               0.00141, 0.00142, 0.00143, 0.00144, 0.00145, 0.00146, 0.00147, 0.00148, 0.00149, 0.00150,
                               0.00151, 0.00152, 0.00153, 0.00154, 0.00155, 0.00156, 0.00157, 0.00158, 0.00159, 0.00160]),  
         "optimizer__weight_decay":hp.choice("optimizer__weight_decay",
            [0.000, 0.002, 0.004, 0.006, 0.010, 0.012, 0.014, 0.016, 0.018, 0.020]),  
         "criterion":hp.choice("criterion", [torch.nn.NLLLoss, torch.nn.CrossEntropyLoss]),

         "batch_size":hp.choice("batch_size", [32, 64, 128, 256, 512, 1024]),
         "optimizer__betas":hp.choice("optimizer__betas",
                                      [[0.88, 0.9991], [0.88, 0.9993], [0.88, 0.9995], [0.88, 0.9997], [0.88, 0.9999],
                                       [0.90, 0.9991], [0.90, 0.9993], [0.90, 0.9995], [0.90, 0.9997], [0.90, 0.9999],
                                       [0.92, 0.9991], [0.92, 0.9993], [0.92, 0.9995], [0.92, 0.9997], [0.92, 0.9999]]),
         "module":hp.choice("module", [module1,  module2,  module3,  module4,  module5,  module6,  module7,  module8,
                                       module9,  module10, module11, module12, module13, module14, module15]),
         "weight_mode":hp.choice("weight_mode", [1, 2, 3, 4]),
         "bias":hp.choice("bias", [0.20, 0.18, 0.16, 0.14, 0.12, 0.10, 0.08, 0.06, 0.04, 0.02, 0.00, 
                                   -0.02, -0.04, -0.06, -0.08, -0.10, -0.12, -0.14, -0.16, -0.18, -0.20]),
         "device":hp.choice("device", ["cuda"]),
         "optimizer":hp.choice("optimizer", [torch.optim.Adam])
         }

space_nodes = {"title":["titanic"],
               "path":["C:/Users/win7/Desktop/Titanic_Prediction.csv"],
               "mean":[0],
               #"std":[0],
               "std":[0, 0.02, 0.04, 0.06, 0.08, 0.10, 0.12, 0.14, 0.16, 0.18, 0.20],
               "max_epochs":[400],
               "patience":[4,5,6,7,8,9,10],
               #"lr":[0.0001],我修改了lr的类型这边应该也需要修改的吧
               "lr":[0.00001, 0.00002, 0.00003, 0.00004, 0.00005, 0.00006, 0.00007, 0.00008, 0.00009, 0.00010,
                     0.00011, 0.00012, 0.00013, 0.00014, 0.00015, 0.00016, 0.00017, 0.00018, 0.00019, 0.00020,
                     0.00021, 0.00022, 0.00023, 0.00024, 0.00025, 0.00026, 0.00027, 0.00028, 0.00029, 0.00030,
                     0.00031, 0.00032, 0.00033, 0.00034, 0.00035, 0.00036, 0.00037, 0.00038, 0.00039, 0.00040,
                     0.00041, 0.00042, 0.00043, 0.00044, 0.00045, 0.00046, 0.00047, 0.00048, 0.00049, 0.00050,
                     0.00051, 0.00052, 0.00053, 0.00054, 0.00055, 0.00056, 0.00057, 0.00058, 0.00059, 0.00060,
                     0.00061, 0.00062, 0.00063, 0.00064, 0.00065, 0.00066, 0.00067, 0.00068, 0.00069, 0.00070,
                     0.00071, 0.00072, 0.00073, 0.00074, 0.00075, 0.00076, 0.00077, 0.00078, 0.00079, 0.00080,
                     0.00081, 0.00082, 0.00083, 0.00084, 0.00085, 0.00086, 0.00087, 0.00088, 0.00089, 0.00090,
                     0.00091, 0.00092, 0.00093, 0.00094, 0.00095, 0.00096, 0.00097, 0.00098, 0.00099, 0.00100,
                     0.00101, 0.00102, 0.00103, 0.00104, 0.00105, 0.00106, 0.00107, 0.00108, 0.00109, 0.00110,
                     0.00111, 0.00112, 0.00113, 0.00114, 0.00115, 0.00116, 0.00117, 0.00118, 0.00119, 0.00120,
                     0.00121, 0.00122, 0.00123, 0.00124, 0.00125, 0.00126, 0.00127, 0.00128, 0.00129, 0.00130,
                     0.00131, 0.00132, 0.00133, 0.00134, 0.00135, 0.00136, 0.00137, 0.00138, 0.00139, 0.00140,
                     0.00141, 0.00142, 0.00143, 0.00144, 0.00145, 0.00146, 0.00147, 0.00148, 0.00149, 0.00150,
                     0.00151, 0.00152, 0.00153, 0.00154, 0.00155, 0.00156, 0.00157, 0.00158, 0.00159, 0.00160],
               "optimizer__weight_decay":[0.000, 0.002, 0.004, 0.006, 0.010, 0.012, 0.014, 0.016, 0.018, 0.020],
               "criterion":[torch.nn.NLLLoss, torch.nn.CrossEntropyLoss],
               "batch_size":[32, 64, 128, 256, 512, 1024],
               "optimizer__betas":[[0.88, 0.9991], [0.88, 0.9993], [0.88, 0.9995], [0.88, 0.9997], [0.88, 0.9999],
                                   [0.90, 0.9991], [0.90, 0.9993], [0.90, 0.9995], [0.90, 0.9997], [0.90, 0.9999],
                                   [0.92, 0.9991], [0.92, 0.9993], [0.92, 0.9995], [0.92, 0.9997], [0.92, 0.9999]],
               "module":[module1,  module2,  module3,  module4,  module5,  module6,  module7,  module8,
                         module9,  module10, module11, module12, module13, module14, module15],
               "weight_mode":[1, 2, 3, 4],
               "bias":[0.20, 0.18, 0.16, 0.14, 0.12, 0.10, 0.08, 0.06, 0.04, 0.02, 0.00, 
                       -0.02, -0.04, -0.06, -0.08, -0.10, -0.12, -0.14, -0.16, -0.18, -0.20],
               "device":["cuda"],
               "optimizer":[torch.optim.Adam]
               }

"""
best_nodes = {"title":"titanic",
              "path":"path",
              "mean":0,
              "std":0.1,
              "max_epochs":400,
              "patience":5,
              "lr":0.00010,
              "optimizer__weight_decay":0.005,
              "criterion":torch.nn.NLLLoss,
              "batch_size":1,
              "optimizer__betas":[0.86, 0.999],
              "module":module3,
              "weight_mode":1,
              "bias":0.0,
              "device":"cuda",
              "optimizer":torch.optim.Adam
              }
"""

#我觉得这边需要添加一个计算计时的功能
start_time = datetime.datetime.now()

trials = Trials()
algo = partial(tpe.suggest, n_startup_jobs=10)

best_params = fmin(nn_f, space, algo=algo, max_evals=10, trials=trials)
print_best_params_acc(trials)

best_nodes = parse_space(trials, space_nodes)
#save_inter_params保存的是本次搜索到的参数
save_inter_params(trials, space_nodes, best_nodes, "titanic")
trials, space_nodes, best_nodes = load_inter_params("titanic")

#predict中的best_model保存的是本机运行过程中最佳模型
#现在还有一个奇怪的问题，predict中似乎还是不对，因为初始正确率太高了吧
#经过我测试，我发现predict中初始正确率确实有在变化，所以应该木问题吧
#现在就是将所有容易改变的东西都放到space、space_nodes、best_nodes以及parse_space
#这样的做法有利于避免我忘记修改代码细节从而导致影响模型的训练过程
#我感觉除了和模型相关的超参我已经搞定的差不多了，今后主要决策和模型相关的超参
#比如说是模型的层数、每层的节点数、初始化的方式、初始化的范围、偏置的设置值
#明天的工作就先从模型生成器开始咯。。
predict(best_nodes, max_evals=10)

end_time = datetime.datetime.now()
print("time cost", (end_time - start_time))

"""
#模型一旦被修改了之后pickle中的文件再读出来就没用啦
#还好我及时保存了老版本的模型，不然感觉就心酸了。。。
#不过其实没保存问题也不大的吧，直接重新计算一次。。。
#同一日期下的中间结果和最佳模型是配套的，虽然这中间结果未必产生了最佳模型
files = open("titanic_intermediate_parameters.pickle", "rb")
trials, space_nodes, best_nodes = pickle.load(files)
files.close()
print(best_nodes)
#print(space_nodes)

files = open("titanic_best_model.pickle", "rb")
best_model = pickle.load(files)
files.close()
best_acc = cal_nnclf_acc(best_model, X_train_scaled, Y_train)
print(best_acc)
"""