#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Mon May 14 10:55:14 2018

@author: llq
"""

# smote unbalance dataset
from __future__ import division
import pandas as pd
import numpy as np
import lightgbm as lgb
from sklearn.neighbors import NearestNeighbors
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import OneHotEncoder,LabelEncoder
from scipy import sparse
import os
import gc
import math


#"""
#    Parameters
#    ----------
#    data : 原始数据
#    tag_index : 因变量所在的列数，以0开始
#    max_amount : 少类别类想要达到的数据量
#    std_rate : 多类:少类想要达到的比例
#    #如果max_amount和std_rate同时定义优先考虑max_amount的定义
#    kneighbor : 生成数据依赖kneighbor个附近的同类点，建议不超过5个
#    kdistinctvalue : 认为每列不同元素大于kdistinctvalue及为连续变量，否则为class变量
#    method ： 生成方法
#"""
#
## smote unbalance dataset
#def smote(data, tag_index=None, max_amount=0, std_rate=5, kneighbor=5, kdistinctvalue=10, method='mean'):
#    try:
#        data = pd.DataFrame(data)
#    except:
#        raise ValueError
#    case_state = data.iloc[:, tag_index].groupby(data.iloc[:, tag_index]).count()
#    case_rate = max(case_state) / min(case_state)
#    location = []
#    if case_rate < 5:
#        print('不需要smote过程')
#        return data
#    else:
#        # 拆分不同大小的数据集合
#        less_data = np.array(
#            data[data.iloc[:, tag_index] == np.array(case_state[case_state == min(case_state)].index)[0]])
#        more_data = np.array(
#            data[data.iloc[:, tag_index] == np.array(case_state[case_state == max(case_state)].index)[0]])
#        # 找出每个少量数据中每条数据k个邻居
#        neighbors = NearestNeighbors(n_neighbors=kneighbor).fit(less_data)
#        for i in range(len(less_data)):
#            point = less_data[i, :]
#            location_set = neighbors.kneighbors([less_data[i]], return_distance=False)[0]
#            location.append(location_set)
#        # 确定需要将少量数据补充到上限额度
#        # 判断有没有设定生成数据个数，如果没有按照std_rate(预期正负样本比)比例生成
#        if max_amount > 0:
#            amount = max_amount
#        else:
#            amount = int(max(case_state) / std_rate)
#        # 初始化，判断连续还是分类变量采取不同的生成逻辑
#        times = 0
#        continue_index = []  # 连续变量
#        class_index = []  # 分类变量[离散变量]
#        for i in range(less_data.shape[1]):
#            if len(pd.DataFrame(less_data[:, i]).drop_duplicates()) > kdistinctvalue:
#                continue_index.append(i)
#            else:
#                class_index.append(i)
#        case_update = list()
#        location_transform = np.array(location)
#        while times < amount:
#            # 连续变量取附近k个点的重心，认为少数样本的附近也是少数样本
#            new_case = []
#            pool = np.random.permutation(len(location))[1]
#            neighbor_group = location_transform[pool]
#            if method == 'mean':
#                new_case1 = less_data[list(neighbor_group), :][:, continue_index].mean(axis=0)
#            # 连续样本的附近点向量上的点也是异常点
#            if method == 'random':
#                away_index = np.random.permutation(len(neighbor_group) - 1)[1]
#                neighbor_group_removeorigin = neighbor_group[1:][away_index]
#                new_case1 = less_data[pool][continue_index] + np.random.rand() * (
#                    less_data[pool][continue_index] - less_data[neighbor_group_removeorigin][continue_index])
#            # 分类变量取mode(众数，就是频数最高的那个)
#            new_case2 = np.array(pd.DataFrame(less_data[neighbor_group, :][:, class_index]).mode().iloc[0, :])
#            new_case = list(new_case1) + list(new_case2)
#            if times == 0:
#                case_update = new_case
#            else:
#                case_update = np.c_[case_update, new_case]
#            print('已经生成了%s条新数据，完成百分之%.2f' % (times, times * 100 / amount))
#            times = times + 1
#        less_origin_data = np.hstack((less_data[:, continue_index], less_data[:, class_index]))
#        more_origin_data = np.hstack((more_data[:, continue_index], more_data[:, class_index]))
#        data_res = np.vstack((more_origin_data, less_origin_data, np.array(case_update.T)))
#        label_columns = [0] * more_origin_data.shape[0] + [1] * (
#        less_origin_data.shape[0] + np.array(case_update.T).shape[0])
#        data_res = pd.DataFrame(data_res)
#    return data_res

def llq_smote(more_data, less_data, tag_index=None, max_amount=0, std_rate=5, kneighbor=5):
    #connect two sparse matrix
    data=sparse.vstack((more_data,less_data))
    
    case_state=pd.Series([more_data.shape[0],less_data.shape[0]])
    case_rate = max(case_state) / min(case_state)
    location = []
    if case_rate < 5:
        print('do not need smote')
        data=data.tocsr()
        #train_y
        train_y=data[0]
        #train_x
        data=data[:,1:]

        return data,train_y
        
    else:
        #find k neighbors in less_data
        neighbors = NearestNeighbors(n_neighbors=kneighbor).fit(less_data)
        for i in range(less_data.shape[0]):
            location_set = neighbors.kneighbors([less_data.getrow(i).toarray()[0]], return_distance=False)[0]
            #store the 5 neighobors in each less data.
            location.append(location_set)
            print('it processes %s in %s less_data' % (i+1,less_data.shape[0]))
            
        # if don't have max_amount,it will decide in std_rate
        if max_amount > 0:
            amount = max_amount
        else:
            amount = int(max(case_state) / std_rate)
                     
        times=0
        update_case = []
        while times<amount:
            #shuffle:choose a neighbor data
            ran=np.random.randint(1,kneighbor)
            np.random.shuffle(location)
            less_data_index=location[0][ran]
            center_index=location[0][0]
            
            #create new data. And label set to 1.
            gap=np.random.random()
            dif=less_data.getrow(center_index)-less_data.getrow(less_data_index)
            new_case=less_data.getrow(center_index)+gap*abs(dif)
            
            #transform coo to csr matrix
            new_case=new_case.tocsr()
            #set label to 1
            new_case[0,0]=1
            if times==0:
                update_case=new_case
            elif times!=0:
                update_case=sparse.vstack((update_case,new_case))

            print('it cretes %s new data，completeing %.2f' % (times+1, times * 100 / amount))
            times = times + 1              
        
        #concate
        data=sparse.vstack((data.tocsr(),update_case))
        
        #train_y
        train_y=data.getcol(0).toarray().reshape((-1,))
        #train_x
        data=data[:,1:]

        return data,train_y
              
def batch_predict(data,index):
    one_hot_feature=['LBS','age','carrier','consumptionAbility','education','gender','house','os','ct','marriageStatus','advertiserId','campaignId', 'creativeId',
           'adCategoryId', 'productId', 'productType']
    vector_feature=['appIdAction','appIdInstall','interest1','interest2','interest3','interest4','interest5','kw1','kw2','kw3','topic1','topic2','topic3']

    for feature in one_hot_feature:
        try:
            #简单来说 LabelEncoder 是对不连续的数字或者文本进行编号
            data[feature] = LabelEncoder().fit_transform(data[feature].apply(int))
        except:
            data[feature] = LabelEncoder().fit_transform(data[feature])
    
    #get the data an label
    more_data=data[data.label==0]
    less_data=data[data.label==1]
    test=data[data.label==-1]
    res=test[['aid','uid']]
    test=test.drop('label',axis=1)
    enc = OneHotEncoder()
    train_more_data=more_data[['creativeSize']]
    train_less_data=less_data[['creativeSize']]
    test_x=test[['creativeSize']]
    
    train_more_y=sparse.coo_matrix(more_data[['label']])
    train_less_y=sparse.coo_matrix(less_data[['label']])
    train_more_data=sparse.hstack((train_more_y,train_more_data))
    train_less_data=sparse.hstack((train_less_y,train_less_data))

    #concate the one-hot encode
    for feature in one_hot_feature:
        enc.fit(data[feature].values.reshape(-1, 1))
        train_more_a=enc.transform(more_data[feature].values.reshape(-1, 1))
        train_less_a=enc.transform(less_data[feature].values.reshape(-1, 1))
        test_a = enc.transform(test[feature].values.reshape(-1, 1))
        train_more_data= sparse.hstack((train_more_data, train_more_a))
        train_less_data= sparse.hstack((train_less_data, train_less_a))
        test_x = sparse.hstack((test_x, test_a))
        print(feature+' finish')
    print('one-hot prepared !')
    
    #concate the cv
    cv=CountVectorizer()
    for feature in vector_feature:
        cv.fit(data[feature])
        train_more_a = cv.transform(more_data[feature])
        train_less_a = cv.transform(less_data[feature])
        test_a = cv.transform(test[feature])
        train_more_data = sparse.hstack((train_more_data, train_more_a))
        train_less_data = sparse.hstack((train_less_data, train_less_a))
        test_x = sparse.hstack((test_x, test_a))
        print(feature + ' finish')
    print('cv prepared !')
    
    
    #Smote
    train_data, train_y = llq_smote(train_more_data,train_less_data,tag_index=0,std_rate=10, kneighbor=10)
#    return train_data,train_y,test_x
    
    del data
    return LGB_predict(train_data, train_y, test_x, res, index)

def LGB_predict(train_x,train_y,test_x,res,index):
    print("LGB test")
    clf = lgb.LGBMClassifier(
        boosting_type='gbdt', num_leaves=31, reg_alpha=0.0, reg_lambda=1,
        max_depth=-1, n_estimators=10000, objective='binary',
        subsample=0.7, colsample_bytree=0.7, subsample_freq=1,
        learning_rate=0.05, min_child_weight=50, random_state=2018, n_jobs=-1
    )
    clf.fit(train_x, train_y, eval_set=[(train_x, train_y)], eval_metric='auc',early_stopping_rounds=2000)
    res['score'+str(index)] = clf.predict_proba(test_x)[:,1]
    res['score'+str(index)] = res['score'+str(index)].apply(lambda x: float('%.6f' % x))
    print(str(index)+' predict finish!')
    gc.collect()
    res=res.reset_index(drop=True)
    return res['score'+str(index)]
   
data=pd.read_csv('./data/data.csv')

#get the train data and test data
train=data[data['label']!=-1]
test=data[data['label']==-1]
del data
predict=pd.read_csv('./data/test1.csv')
cnt=1
#返回数字的上入整数
size = math.ceil(len(train) / cnt)
result=[]
for i in range(cnt):
    start = size * i
    end = (i + 1) * size if (i + 1) * size < len(train) else len(train)
    slice = train[int(start):int(end)]
    result.append(batch_predict(pd.concat([slice,test]),i))
    gc.collect()
#train_data,train_y,test_x=batch_predict(data,0)
    
    
result=pd.concat(result,axis=1)
result['score']=np.mean(result,axis=1)
result=result.reset_index(drop=True)
result=pd.concat([predict[['aid','uid']].reset_index(drop=True),result['score']],axis=1)
result[['aid','uid','score']].to_csv('./data/submission.csv', index=False)
os.system('zip ./data/baseline.zip ./data/submission.csv')