#coding=utf8
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import re
import time
from tqdm import tqdm

train_path='./datasets/round1_ijcai_18_train_20180301.txt'
test_path='./datasets/round1_ijcai_18_test_a_20180301.txt'
train=pd.read_table(train_path,delimiter=' ')
test=pd.read_table(test_path,delimiter=' ')

print('We have {} training rows and {} test rows'.format(train.shape[0],test.shape[0]))
print('We have {} training columns and {} test columns'.format(train.shape[1],test.shape[1]))
print('There are {} duplicate Instance_id in train'.format(train.shape[0]-len(train.instance_id.unique())))
print('There are {} duplicate instance_id in train and test'.format(len(np.intersect1d(train.instance_id.values,test.instance_id.values))))
print('There are not missing values in train') if True not in train.isnull().any().values else print('Exist miss value in train')
print('There are not missing values in test') if True not in test.isnull().any().values else print('Exist miss value in test')
# print(train.head())
# print(test.head())
plt.figure()
plt.hist(train['is_trade'].values, bins=100)
plt.xlabel('trade information')
plt.ylabel('number of trade')
#Ad information
plt.figure()
plt.hist(train['item_id'].values, bins=100)
plt.xlabel('item_id')
plt.ylabel('number if item')
# plt.show()
print('There are {} items in train and {} in test'.format(len(train.item_id.unique()),len(test.item_id.unique())))
print('There are {} intersect in train and test'.format(len(np.intersect1d(train.item_id.values,test.item_id.values))))
train_item_category_list_1=pd.DataFrame([int(i.split(';')[0]) for i in train.item_category_list])
train_item_category_list_2=pd.DataFrame([int(i.split(';')[1]) for i in train.item_category_list])
train_item_category_list_3=pd.DataFrame([int(i.split(';')[2]) for i in train.item_category_list if len(i.split(';'))==3])
test_item_category_list_1=pd.DataFrame([int(i.split(';')[0]) for i in test.item_category_list])
test_item_category_list_2=pd.DataFrame([int(i.split(';')[1]) for i in test.item_category_list])
test_item_category_list_3=pd.DataFrame([int(i.split(';')[2]) for i in test.item_category_list if len(i.split(';'))==3])

print('There are {} item_category_list_1 categories'.format(len(train_item_category_list_1[0].unique())))
print('There are {} item_category_list_2 categories'.format(len(train_item_category_list_2[0].unique())))
print('There are {} item_category_list_3 categories'.format(len(train_item_category_list_3[0].unique())))
print('There are {} item_category_list_3 in train and {} in test'.format(len(train_item_category_list_3[0]),len(test_item_category_list_3[0])))

train_item_property_list=pd.DataFrame([int(len(i.split(';'))) for i in train.item_property_list])
test_item_property_list=pd.DataFrame([int(len(i.split(';'))) for i in test.item_property_list])
plt.figure()
plt.plot(train_item_property_list[0], '.-', label='train')
plt.plot(test_item_property_list[0], '.-', label='test')
plt.title('Number of item property in train and test .')
plt.legend(loc=0)
plt.ylabel('number of property')
plt.show()
# train_item_property_list_2=pd.DataFrame([int(i.split(';')[1]) for i in train.item_property_list])
# train_item_property_list_3=pd.DataFrame([int(i.split(';')[2]) for i in train.item_property_list if len(i.split(';'))==3])
# test_item_property_list_1=pd.DataFrame([int(i.split(';')[0]) for i in test.item_property_list])
# test_item_property_list_2=pd.DataFrame([int(i.split(';')[1]) for i in test.item_property_list])
# test_item_property_list_3=pd.DataFrame([int(i.split(';')[2]) for i in test.item_property_list if len(i.split(';'))==3])
plt.figure()
plt.plot(train.groupby('item_brand_id').mean()['is_trade'], 'o-', label='is_trade rate')
plt.xlabel('item_brand_id')
plt.ylabel('average is_trade')
plt.legend(loc=0)
print('There are {} item_brand_id'.format(len(train.item_brand_id.unique())))
plt.figure()
plt.plot(train.groupby('item_city_id').mean()['is_trade'], 'o-', label='is_trade rate')
plt.xlabel('item_city_id')
plt.ylabel('average is_trade')
plt.legend(loc=0)
print('There are {} item_city_id'.format(len(train.item_city_id.unique())))
#list(train.item_city_id.values).count(train.groupby('item_city_id').mean()['is_trade'].index[53])
plt.figure()
plt.plot(train.groupby('item_price_level').mean()['is_trade'], 'o-', label='is_trade rate')
plt.xlabel('item_price_level')
plt.ylabel('average is_trade')
plt.legend(loc=0)
plt.figure()
plt.hist(train.item_price_level.values,bins=100)
plt.xlabel('item_price_level')
plt.ylabel('number of item_price_level')
plt.figure()
plt.plot(train.groupby('item_sales_level').mean()['is_trade'], 'o-', label='is_trade rate')
plt.xlabel('item_sales_level')
plt.ylabel('average is_trade')
plt.legend(loc=0)
plt.show()

plt.figure()
plt.plot(train.groupby('item_collected_level').mean()['is_trade'], 'o-', label='is_trade rate')
plt.xlabel('item_collected_level')
plt.ylabel('average is_trade')
plt.legend(loc=0)
plt.figure()
plt.plot(train.groupby('item_pv_level').mean()['is_trade'], 'o-', label='is_trade rate')
plt.xlabel('item_pv_level')
plt.ylabel('average is_trade')
plt.legend(loc=0)
# user information
plt.figure()
plt.plot(train.groupby('user_id').mean()['is_trade'], 'o-', label='is_trade rate')
plt.xlabel('user_id')
plt.ylabel('average is_trade')
plt.legend(loc=0)
print('There are {} users in train and {} in test'.format(len(train.user_id.unique()),len(test.user_id.unique())))
print('There are {} intersect user_id in train and test'.format(len(np.intersect1d(train.item_id.values,test.item_id.values))))
print('There are {} woman, {} man, {} family and {} missing value in train'.format(train.ix[train['user_gender_id']==0,:].shape[0],train.ix[train['user_gender_id']==1,:].shape[0],
                                                    train.ix[train['user_gender_id'] == 2, :].shape[0],train.ix[train['user_gender_id']==-1,:].shape[0]))
print('There are {} woman, {} man, {} family and {} missing value in test'.format(test.ix[test['user_gender_id']==0,:].shape[0],test.ix[test['user_gender_id']==1,:].shape[0],
                                                    test.ix[test['user_gender_id'] == 2,:].shape[0],test.ix[test['user_gender_id'] == -1,:].shape[0]))
plt.figure()
plt.plot(train.groupby('user_gender_id').mean()['is_trade'], 'o-', label='is_trade rate')
plt.xlabel('user_gender_id')
plt.ylabel('average is_trade')
plt.legend(loc=0)
plt.figure()
plt.hist(train.user_gender_id.values,bins=100)
plt.xlabel('user_gender_id')
plt.ylabel('number of user')
plt.show()

plt.figure()
plt.plot(train.groupby('user_age_level').mean()['is_trade'], 'o-', label='is_trade rate')
plt.xlabel('user_age_level')
plt.ylabel('average is_trade')
plt.xlim((1000,1010))
plt.legend(loc=0)
plt.figure()
plt.hist(train.user_age_level.values,bins=3000)
plt.xlabel('user_age_level')
plt.ylabel('number of user')
plt.xlim((1000,1010))
print('There are {} miss value in user_age_level'.format(len(train.ix[train['user_age_level']==-1,:])))
plt.figure()
plt.plot(train.groupby('user_occupation_id').mean()['is_trade'], 'o-', label='is_trade rate')
plt.xlabel('user_occupation_id')
plt.ylabel('average is_trade')
plt.xlim((2000,2010))
plt.legend(loc=0)
plt.figure()
plt.hist(train.user_occupation_id.values,bins=3000)
plt.xlim((2000,2010))
plt.xlabel('user_occupation_id')
plt.ylabel('number of user')
print('There are {} miss value in user_occupation_id'.format(len(train.ix[train['user_occupation_id']==-1,:])))
print('user_occupation_id conclude:{} in train'.format(train.user_occupation_id.unique()))
print('user_occupation_id conclude: {} in test'.format(test.user_occupation_id.unique()))
plt.figure()
plt.plot(train.groupby('user_star_level').mean()['is_trade'], 'o-', label='is_trade rate')
plt.xlabel('user_star_level')
plt.ylabel('average is_trade')
plt.xlim((3000,3010))
plt.legend(loc=0)
plt.figure()
plt.hist(train.user_star_level.values,bins=3000)
plt.xlabel('user_star_level')
plt.ylabel('number of user')
plt.xlim((3000,3010))
print('There are {} miss value in user_star_level'.format(len(train.ix[train['user_star_level']==-1,:])))
plt.show()

#Context information
print('There are {} context_id intersection in train and test'.format(len(np.intersect1d(train.context_id.values,test.context_id.values))))
plt.figure()
plt.plot(train.groupby('context_timestamp').mean()['is_trade'], '.', label='is_trade rate')
plt.xlabel('contaxt_timestamp')
plt.ylabel('average is_trade')
plt.figure()
plt.plot(train.groupby('context_page_id').mean()['is_trade'], '.-', label='is_trade rate')
plt.xlabel('context_page_id')
plt.ylabel('average is_trade')
print('context_page_id start is {} and end is {} in train'.format(train.context_page_id.min(),train.context_page_id.max()))
print('context_page_id start is {} and end is {} in test'.format(test.context_page_id.min(),test.context_page_id.max()))

train['num_predict_category_property']=[sum(map(lambda i:0 if i[-2:]=='-1' else 1,re.split(',|;',j))) for j in train.predict_category_property]
plt.figure()
plt.plot(train.groupby('num_predict_category_property').mean()['is_trade'],'.-',label='is_trade rate')
plt.xlabel('num_predict_category_property')
plt.ylabel('average is_trade')

#shop information
plt.figure()
plt.plot(train.groupby('shop_id').mean()['is_trade'],'.-',label='is_trade rate')
plt.xlabel('shop_id')
plt.ylabel('average is_trade')
print('There are {} shop_id in test'.format(len(test.shop_id.unique())))
print('There are {} shop_id intersection in train and test'.format(len(np.intersect1d(train.shop_id.values,test.shop_id.values))))
plt.figure()
plt.plot(train.groupby('shop_review_num_level').mean()['is_trade'],'.-',label='is_trade rate')
plt.xlabel('shop_review_num_level')
plt.ylabel('average is_trade')
plt.figure()
plt.plot(train.groupby('shop_review_positive_rate').mean()['is_trade'],'.-',label='is_trade rate')
plt.xlabel('shop_review_positive_rate')
plt.ylabel('average is_trade')
plt.xlim((0.8,1))
print('There are {} miss shop_review_positive_rate'.format(len(train.ix[train['shop_review_positive_rate']==-1,:])))
plt.show()
plt.figure()
plt.plot(train.groupby('shop_star_level').mean()['is_trade'],'.-',label='is_trade rate')
plt.xlim((4999,5020))
plt.xlabel('shop_star_level')
plt.ylabel('average is_trade')
plt.figure()
plt.plot(train.groupby('shop_score_service').mean()['is_trade'],'.-',label='is_trade rate')
plt.xlim((0.8,1))
plt.xlabel('shop_score_service')
plt.ylabel('average is_trade')
print('There are {} miss shop_score_service'.format(len(train.ix[train['shop_score_service']==-1,:])))
plt.figure()
plt.plot(train.groupby('shop_score_delivery').mean()['is_trade'],'.-',label='is_trade rate')
plt.xlim((0.8,1))
plt.xlabel('shop_score_delivery')
plt.ylabel('average is_trade')
print('There are {} miss shop_score_delivery'.format(len(train.ix[train['shop_score_delivery']==-1,:])))
plt.figure()
plt.plot(train.groupby('shop_score_description').mean()['is_trade'],'.-',label='is_trade rate')
plt.xlim((0.8,1))
plt.xlabel('shop_score_description')
plt.ylabel('average is_trade')
print('There are {} miss shop_score_description'.format(len(train.ix[train['shop_score_description']==-1,:])))
plt.show()
