#!/usr/bin/env
# -*- coding: utf-8 -*-
import pandas as pd
import numpy as np
import os
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold
from lightgbm import LGBMClassifier
import matplotlib.pylab as plt
import logging
from imblearn.under_sampling import RandomUnderSampler

from imblearn.over_sampling import RandomOverSampler
# log_fmt = "[%(asctime)s] %(levelname)s in %(module)s: %(message)s"
# logging.basicConfig(format=log_fmt, level=logging.INFO)

# ---------------------define a logger-----------------------
logger = logging.getLogger("execute")
logger.setLevel(logging.INFO)

fh = logging.FileHandler("./baseline2.log", mode="w")
fh.setLevel(logging.INFO)

fmt = "%(asctime)-15s %(levelname)s %(filename)s %(lineno)d %(process)d %(message)s"
datefmt = "%a %d %b %Y %H:%M:%S"
formatter = logging.Formatter(fmt, datefmt)

fh.setFormatter(formatter)
logger.addHandler(fh)
# ----------------------------- define a logger end ----------------------------------

import warnings

warnings.filterwarnings('ignore')


def extract_day(s):
    return s.apply(lambda x: int(x.split('-')[0][1:]))


def extract_hour(s):
    return s.apply(lambda x: int(x.split('-')[1][1:]))

def fea_importance_get(): #筛选特征值,把重要性小于一定值的特征提取出来
    a = pd.read_csv('feature_score_2.csv', header=0)
    #train.columns = ['feature', 'importance']
    a = a[a.importance<100].iloc[:,0]
    return a

fea_importance = fea_importance_get()

base_path = ''
# 加载邀请回答数据

train = pd.read_csv(base_path + 'invite_info_0926.txt', sep='\t', header=None)
train.columns = ['qid', 'uid', 'dt', 'label']
logging.info("invite %s", train.shape)
logger.info("invite %s", train.shape)

test = pd.read_csv(base_path + 'invite_info_evaluate_2_0926.txt', sep='\t', header=None)
test.columns = ['qid', 'uid', 'dt']
logging.info("test %s", test.shape)
logger.info("test %s", test.shape)

sub = test.copy()
#=============提取时间分解为day和hour的形式
sub_size = len(sub)

train['day'] = extract_day(train['dt'])
train['hour'] = extract_hour(train['dt'])

test['day'] = extract_day(test['dt'])
test['hour'] = extract_hour(test['dt'])

del train['dt'], test['dt']

# 加载问题
ques = pd.read_csv(base_path + 'question_info_0926.txt', header=None, sep='\t')
ques.columns = ['qid', 'q_dt', 'title_t1', 'title_t2', 'desc_t1', 'desc_t2', 'topic']
del ques['title_t1'], ques['title_t2'], ques['desc_t1'], ques['desc_t2']
logging.info("ques %s", ques.shape)
logger.info("ques %s", ques.shape)

ques['q_day'] = extract_day(ques['q_dt'])
ques['q_hour'] = extract_hour(ques['q_dt'])
del ques['q_dt']

# 加载回答
ans = pd.read_csv(base_path + 'answer_info_0926.txt', header=None, sep='\t')
ans.columns = ['aid', 'qid', 'uid', 'ans_dt', 'ans_t1', 'ans_t2', 'is_good', 'is_rec', 'is_dest', 'has_img',
               'has_video', 'word_count', 'reci_cheer', 'reci_uncheer', 'reci_comment', 'reci_mark', 'reci_tks',
               'reci_xxx', 'reci_no_help', 'reci_dis']
del ans['ans_t1'], ans['ans_t2']
logging.info("ans %s", ans.shape)
logger.info("ans %s", ans.shape)

ans['a_day'] = extract_day(ans['ans_dt'])
ans['a_hour'] = extract_hour(ans['ans_dt'])
del ans['ans_dt']

ans = pd.merge(ans, ques, on='qid')
del ques

# 回答距提问的天数
ans['diff_qa_days'] = ans['a_day'] - ans['q_day']


# 时间窗口划分
# train
# val
train_start = train['day'].min()  # train中邀请的最早时间
train_end = train['day'].max()  # train中邀请的最晚时间

val_start = test['day'].min()  # val中邀请的最早时间
val_end = test['day'].max()  # val中邀请的最晚时间

label_end = train_end  # train中邀请的最晚时间 #3867

answer_start = ans['a_day'].min()
answer_end = ans['a_day'].max()

train_label_feature_end = label_end - (val_end - val_start) - 1
train_label_feature_start = train_start

train_ans_feature_end = train_label_feature_end
train_ans_feature_start = answer_start

val_label_feature_end = val_start - 1  
val_label_feature_start = val_label_feature_end - train_start  

val_ans_feature_end = val_start - 1
val_ans_feature_start = val_ans_feature_end - answer_start


"""
# 时间窗口划分    目的是把train和ans数据集根据时间窗口划分训练测试集
# train
# val
train_start = 3838
train_end = 3867

val_start = 3868
val_end = 3874

label_end = 3867
label_start = label_end - 6

train_label_feature_end = label_end - 7
train_label_feature_start = train_label_feature_end - 22

train_ans_feature_end = label_end - 7
train_ans_feature_start = train_ans_feature_end - 50

val_label_feature_end = val_start - 1
val_label_feature_start = val_label_feature_end - 22

val_ans_feature_end = val_start - 1
val_ans_feature_start = val_ans_feature_end - 50
"""
train_label_feature = train[(train['day'] >= train_label_feature_start) & (train['day'] <= train_label_feature_end)]
logging.info("train_label_feature %s", train_label_feature.shape)
logger.info("train_label_feature %s", train_label_feature.shape)

val_label_feature = train[(train['day'] >= val_label_feature_start) & (train['day'] <= val_label_feature_end)]
logging.info("val_label_feature %s", val_label_feature.shape)
logger.info("val_label_feature %s", val_label_feature.shape)

train_label = train[(train['day'] > train_label_feature_end)]

logging.info("train feature start %s end %s, label start %s end %s", train_label_feature['day'].min(),
             train_label_feature['day'].max(), train_label['day'].min(), train_label['day'].max())

logger.info("train feature start %s end %s, label start %s end %s", train_label_feature['day'].min(),
            train_label_feature['day'].max(), train_label['day'].min(), train_label['day'].max())

logging.info("test feature start %s end %s, label start %s end %s", val_label_feature['day'].min(),
             val_label_feature['day'].max(), test['day'].min(), test['day'].max())
logger.info("test feature start %s end %s, label start %s end %s", val_label_feature['day'].min(),
            val_label_feature['day'].max(), test['day'].min(), test['day'].max())

# 确定ans的时间范围
# 3807~3874
train_ans_feature = ans[(ans['a_day'] >= train_ans_feature_start) & (ans['a_day'] <= train_ans_feature_end)]

val_ans_feature = ans[(ans['a_day'] >= val_ans_feature_start) & (ans['a_day'] <= val_ans_feature_end)]

logging.info("train ans feature %s, start %s end %s", train_ans_feature.shape, train_ans_feature['a_day'].min(),
             train_ans_feature['a_day'].max())

logger.info("train ans feature %s, start %s end %s", train_ans_feature.shape, train_ans_feature['a_day'].min(),
            train_ans_feature['a_day'].max())

logging.info("val ans feature %s, start %s end %s", val_ans_feature.shape, val_ans_feature['a_day'].min(),
             val_ans_feature['a_day'].max())
logger.info("val ans feature %s, start %s end %s", val_ans_feature.shape, val_ans_feature['a_day'].min(),
            val_ans_feature['a_day'].max())

""" 
fea_cols = ['is_good', 'is_rec', 'is_dest', 'has_img', 'has_video', 'word_count',
            'reci_cheer', 'reci_uncheer', 'reci_comment', 'reci_mark', 'reci_tks',
            'reci_xxx', 'reci_no_help', 'reci_dis', 'diff_qa_days']
"""
#这里我删去是否含有图片，我认为这个并不影响
fea_cols = ['is_good', 'is_rec', 'is_dest', 'word_count',
            'reci_cheer', 'reci_uncheer', 'reci_comment', 'reci_mark', 'reci_tks',
            'reci_xxx', 'reci_no_help', 'reci_dis', 'diff_qa_days']





def extract_feature1(target, label_feature, ans_feature):
    # 问题特征
    t1 = label_feature.groupby('qid')['label'].agg(['mean', 'sum', 'std', 'count']).reset_index() #问题被回答的总数和问题在数据集中被提到的总数,平均数可以代表回答概率
    t1.columns = ['qid', 'q_inv_mean', 'q_inv_sum', 'q_inv_std', 'q_inv_count']
    target = pd.merge(target, t1, on='qid', how='left')

    # 用户特征
    t1 = label_feature.groupby('uid')['label'].agg(['mean', 'sum', 'std', 'count']).reset_index()
    t1.columns = ['uid', 'u_inv_mean', 'u_inv_sum', 'u_inv_std', 'u_inv_count']
    target = pd.merge(target, t1, on='uid', how='left')
    #
    # train_size = len(train)
    # data = pd.concat((train, test), sort=True)

    # 回答部分特征

    t1 = ans_feature.groupby('qid')['aid'].count().reset_index()
    t1.columns = ['qid', 'q_ans_count']
    target = pd.merge(target, t1, on='qid', how='left')

    t1 = ans_feature.groupby('uid')['aid'].count().reset_index()
    t1.columns = ['uid', 'u_ans_count']
    target = pd.merge(target, t1, on='uid', how='left')

    for col in fea_cols:
        t1 = ans_feature.groupby('uid')[col].agg(['sum', 'max', 'mean','min','std']).reset_index()
        t1.columns = ['uid', 'u_' + col + '_sum', 'u_' + col + '_max', 'u_' + col + '_mean','u_' + col + '_min','u_' + col + '_std']
        target = pd.merge(target, t1, on='uid', how='left')

        t1 = ans_feature.groupby('qid')[col].agg(['sum', 'max', 'mean','min','std']).reset_index()
        t1.columns = ['qid', 'q_' + col + '_sum', 'q_' + col + '_max', 'q_' + col + '_mean','q_' + col + '_min','q_' + col + '_std']
        target = pd.merge(target, t1, on='qid', how='left')
        logging.info("extract %s", col)
        logger.info("extract %s", col)
    return target


train_label = extract_feature1(train_label, train_label_feature, train_ans_feature)
test = extract_feature1(test, val_label_feature, val_ans_feature)

# 特征提取结束
logging.info("train shape %s, test shape %s", train_label.shape, test.shape)
logger.info("train shape %s, test shape %s", train_label.shape, test.shape)
assert len(test) == sub_size

# 加载用户
user = pd.read_csv(base_path + 'member_info_0926.txt', header=None, sep='\t')
user.columns = ['uid', 'gender', 'creat_keyword', 'level', 'hot', 'reg_type', 'reg_plat', 'freq', 'uf_b1', 'uf_b2',
                'uf_b3', 'uf_b4', 'uf_b5', 'uf_c1', 'uf_c2', 'uf_c3', 'uf_c4', 'uf_c5', 'score', 'follow_topic',
                'inter_topic']
del user['follow_topic'], user['inter_topic']
logging.info("user %s", user.shape)
logger.info("user %s", user.shape)

unq = user.nunique()
logging.info("user unq %s", unq)
logger.info("user unq %s", unq)

for x in unq[unq == 1].index: #删去那些只有一个值的特征
    del user[x]
    logging.info('del unq==1 %s', x)
    logger.info('del unq==1 %s', x)

t = user.dtypes
cats = [x for x in t[t == 'object'].index if x not in ['follow_topic', 'inter_topic', 'uid']]
logging.info("user cat %s", cats)
logger.info("user cat %s", cats)

for d in cats:
    lb = LabelEncoder()  #这个的作用就是把不连续的整数变为连续的整数
    user[d] = lb.fit_transform(user[d])
    logging.info('encode %s', d)
    logger.info('encode %s', d)

q_lb = LabelEncoder()
q_lb.fit(list(train_label['qid'].astype(str).values) + list(test['qid'].astype(str).values))
train_label['qid_enc'] = q_lb.transform(train_label['qid'])
test['qid_enc'] = q_lb.transform(test['qid'])

u_lb = LabelEncoder()
u_lb.fit(user['uid'])
train_label['uid_enc'] = u_lb.transform(train_label['uid'])
test['uid_enc'] = u_lb.transform(test['uid'])

# merge user
train_label = pd.merge(train_label, user, on='uid', how='left')
test = pd.merge(test, user, on='uid', how='left')
logging.info("train shape %s, test shape %s", train_label.shape, test.shape)
logger.info("train shape %s, test shape %s", train_label.shape, test.shape)

data = pd.concat((train_label, test), axis=0, sort=True)
# del train_label, test

# count编码
count_fea = ['uid_enc', 'qid_enc', 'gender', 'freq', 'uf_c1', 'uf_c2', 'uf_c3', 'uf_c4', 'uf_c5']
for feat in count_fea:
    col_name = '{}_count'.format(feat)
    data[col_name] = data[feat].map(data[feat].value_counts().astype(int)) #将总的数值数作为该特征的值
    data.loc[data[col_name] < 2, feat] = -1#将仅有一次记录的属性值设置为-1？
    data[feat] += 1 
    data[col_name] = data[feat].map(data[feat].value_counts().astype(int))
    data[col_name] = (data[col_name] - data[col_name].min()) / (data[col_name].max() - data[col_name].min())
    #无量纲化处理，可以当做是一种标准化的处理

# 问题被回答的次数


# 压缩数据
t = data.dtypes
for x in t[t == 'int64'].index:
    data[x] = data[x].astype('int32')

for x in t[t == 'float64'].index:
    data[x] = data[x].astype('float32')

data['wk'] = data['day'] % 7

feature_cols = [x for x in data.columns if x not in ('label', 'uid', 'qid', 'dt', 'day')]
# target编码
logging.info("feature size %s", len(feature_cols))
logger.info("feature size %s", len(feature_cols))

X_train_all = data.iloc[:len(train_label)][feature_cols]
y_train_all = data.iloc[:len(train_label)]['label']
test = data.iloc[len(train_label):]
del data
assert len(test) == sub_size

logging.info("train shape %s, test shape %s", train_label.shape, test.shape)
logger.info("train shape %s, test shape %s", train_label.shape, test.shape)

fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

for index, (train_idx, val_idx) in enumerate(fold.split(X=X_train_all, y=y_train_all)):
    break



X_train, X_val, y_train, y_val = X_train_all.iloc[train_idx][feature_cols], X_train_all.iloc[val_idx][feature_cols], \
                                 y_train_all.iloc[train_idx], \
                                 y_train_all.iloc[val_idx]
del X_train_all



df = pd.DataFrame(X_train.columns.tolist(), columns=['feature'])
print(len(df))

#删除不重要特征
"""
for fea in fea_importance.values:
    print(fea)
    del X_train[fea]
    del X_val[fea]
    feature_cols.remove(fea)
    pass

df = pd.DataFrame(X_train.columns.tolist(), columns=['feature'])
print('筛选后：=====',len(df))
"""

#'''  
a = sum(y_train==1)
b = sum(y_train==0)
print('a',a,'  b',b,'rate====: ',b/a)

""" #尝试进行下采样
rus = RandomUnderSampler(random_state=0,replacement=True)
X_train, y_train = rus.fit_sample(X_train, y_train)
"""

""" #尝试上采样
ros=RandomOverSampler(random_state=0) #采用随机过采样（上采样）
X_train, y_train =ros.fit_sample(X_train, y_train)
"""

a = sum(y_train==1)
b = sum(y_train==0)
print('a',a,'  b',b,'rate_new====: ',b/a)
#'''


model_lgb = LGBMClassifier(n_estimators=2000,n_jobs=-1, objective='binary', seed=1000, silent=True)
#model_lgb = LGBMClassifier(n_estimators=2000,max_bin=100,num_leaves=20, objective='binary', seed=500, silent=True,learning_rate =0.1,n_jobs=-1)

model_lgb.fit(X_train, y_train,
              eval_metric=['logloss', 'auc'],
              eval_set=[(X_val, y_val)],
              early_stopping_rounds=50)

sub['label'] = model_lgb.predict_proba(test[feature_cols])[:, 1]


print('result===============')
sub.to_csv('result_base2_last_v2.txt', index=None, header=None, sep='\t') #2.3 调整时间窗口  2.4  调整学习率  2.5 特征筛选350


print('lgbm的特征重要性')
#print('Feature importances:', list(model_lgb.feature_importances_))

df['importance']=list(model_lgb.feature_importances_)
df = df.sort_values(by='importance',ascending=False)
df.to_csv("feature_score_last_v2.csv",index=None,encoding='gbk')

"""
print(pd.DataFrame({
        'column': feature_names,
        'importance': model_lgb.feature_importance(),
    }).sort_values(by='importance'))
"""