﻿#!/usr/bin/env python

# -*- coding: UTF-8 -*-  

'''

{

"版权":"LDAE工作室",

"author":{

"1":"集体",

}

"初创时间:"2017年3月",

}

'''

#--------- 外部模块处理<<开始>> ---------#

#-----系统自带必备模块引用-----

import sys # 操作系统模块1
import os # 操作系统模块2
import types # 数据类型
import time # 时间模块
import datetime # 日期模块
import hashlib
import cgi # CGI模式 取得shell参数用
import random # 随机数

#-----系统外部需安装库模块引用-----

#-----DIY自定义库模块引用-----
from diy.inc_sys import * #自定义系统级功能模块 
from diy.inc_conn import * #自定义数据库功能模块
import config #系统配置参数
from inc_dae import * #自定义数据分析处理功能模块
import inc_vec # 词向量处理模块
import inc_user # 用户画像处理模块
import diy.inc_ml as inc_ml # 基本自定义机器学习模块
from diy.inc_hash import hash_make # 基本自定义hash模块

#--------- 外部模块处理<<结束>> ---------#

#--------- 内部模块处理<<开始>> ---------#

# ---外部参变量处理
# shell模式下参数处理
form = cgi.FieldStorage() #处理提交的参数
if form.getvalue('a'):
    args_list = form["a"].value
else:
    args_list = []

dic_args_cmd = args2dic(args_list)

model_name = os.path.basename(__file__)
skip_shell = "python " + model_name #命令行模式下递归调用命令
seed_name = "" #声明种子为全局变量
seed_hash = "" #声明种子哈希为全局变量

if ("tid" in dic_args_cmd):
    tid = int(dic_args_cmd["tid"])
else:
    tid = 0
    
if ("action" in dic_args_cmd):
    action = dic_args_cmd["action"]
else:
    action = ""
    
if ("shell_if" in dic_args_cmd):
    shell_if = int(dic_args_cmd["shell_if"])
else:
    shell_if = 1
    
if ("sleep" in dic_args_cmd):
    sleep = int(dic_args_cmd["sleep"])
else:
    sleep = 0
    
if ("after" in dic_args_cmd):
    after = dic_args_cmd["after"]
else:
    after = "00:00:00"
    
# ---全局变量处理
#rs_sqlite_file = Conn_sqlite3(config.path_main + config.dic_config["path_sqlite"],0) # 生成文件数据库实例
#rs_way_mysql = Conn_mysql(config.dic_config["host_mysql"],config.dic_config["user_mysql"],config.dic_config["pwd_mysql"], "ldae_way_" + config.dic_config["name_mysql_after"], int(config.dic_config["port_mysql"])) # 生成MYSQL数据库way方法实例
#rs_basedata_mysql = Conn_mysql(config.dic_config["host_mysql"],config.dic_config["user_mysql"],config.dic_config["pwd_mysql"], "ldae_basedata_" + config.dic_config["name_mysql_after"], int(config.dic_config["port_mysql"])) # 生成MYSQL数据库基础数据实例
#rs_index_mysql = Conn_mysql(config.dic_config["host_mysql"],config.dic_config["user_mysql"],config.dic_config["pwd_mysql"], "ldae_index_" + config.dic_config["name_mysql_after"], int(config.dic_config["port_mysql"])) # 生成MYSQL数据库索引数据实例

def run_it(*args,**kwargs):
    """
    主功能模块过程调用区
    主要功能为数据分析结果的处理
    """
    txt = ""
    sleep_start(dic_args_cmd,after_p=after) # 延迟校验
    
    if ("dim" in dic_args_cmd):
        dim = int(dic_args_cmd["dim"])
    else:
        dim = 2
    
    if ("numb_step" in dic_args_cmd):
        numb_step = int(dic_args_cmd["numb_step"])
    else:
        numb_step = 128

    if ("numb" in dic_args_cmd):
        numb_fea = int(dic_args_cmd["numb"])
    else:
        numb_fea = 16
        
    if ("fit_rate" in dic_args_cmd):
        fit_rate = float(dic_args_cmd["fit_rate"])
    else:
        fit_rate = float(config.dic_config["fit_rate"])
        
    if ("which" in dic_args_cmd):
        which = float(dic_args_cmd["which"])
    else:
        which = 1
        
    if ("do1" in dic_args_cmd):
        do1 = int(dic_args_cmd["do1"])
    else:
        do1 = 0
        
    if ("do2" in dic_args_cmd):
        do2 = int(dic_args_cmd["do2"])
    else:
        do2 = 0
        
    if ("name_model" in dic_args_cmd):
        name_model = dic_args_cmd["name_model"]
    else:
        name_model = "x"
        
    if ("rand_if" in dic_args_cmd):
        rand_if = int(dic_args_cmd["rand_if"])
    else:
        rand_if = 0
        
    if ("row_feature" in dic_args_cmd):
        row_feature = int(dic_args_cmd["row_feature"])
    else:
        row_feature = 0
        
    if ("path_model" in dic_args_cmd):
        path_model = int(dic_args_cmd["row_feature"])
    else:
        path_model = "./data/w2v/w2v.model"
        
    if ("path_save" in dic_args_cmd):
        path_save = int(dic_args_cmd["row_feature"])
    else:
        path_save = "./data/w2v/vec.txt"
        
    # 生成全监督自定义特征
    if (action == "lr_feature_make"):
        
        feature_base = inc_ml.Feature_base() # 特征基础类实例化
        # 生成特征值
        #print ("《语气词》",feature_base.dic_feeling) # 测试用
        #print ("《标准命名实体》",feature_base.dic_ne) # 测试用
        #print ("《引导词》",feature_base.dic_guide) # 测试用
        #print ("《疑问词》",feature_base.dic_query) # 测试用
        #print ("《领域动词》",feature_base.dic_verb) # 测试用
        #print ("《情感词》",feature_base.dic_em) # 测试用
        #print ("《领域经济词》",feature_base.dic_cost) # 测试用
        
        # 特征值数值化
        if (do1 == 1):
            print (feature_base.value_db_make(name_model_p=name_model,row_feature_p=row_feature,limit_p=0,dim_p=16))
        # 特征值归一化
        if (do2 == 1):
            print (feature_base.dim_db_make(name_model_p=name_model,row_feature_p=row_feature,limit_p=0,which_p=which))
        
    # 由数据库生成训练文件
    if (action == "lr_csv_train"):
        
        train_dae = inc_ml.Train_dae() # 特征对象实例化
        path_out = config.dic_config["path_lr"] + "train_lr_"
        print (train_dae.lr_csv_train(path_out_p=path_out,name_model_p=name_model,row_feature_p=row_feature,dim_p=dim))
        
    # logistic模型训练
    if (action == "train_lr"):
    
        time_start = datetime.datetime.now() # 时间初始量
        
        # 主类实例化
        feature_base = inc_ml.Feature_base() # 特征基础类实例化
        train_dae = inc_ml.Train_dae() # 特征对象实例化
        feature_dae = inc_ml.Feature_dae() # 特征引擎实例化
        
        dim_f = row_feature # 特征向量维度
        txt = ""
        row = () # 待处理元组
        rows_max = () # 最大值元组
        rows_min = () # 最小值元组
        rows_avg = () # 均值元组
        rows_std = () # 标准差元组
        dic_f = {} # 特征字典
        dic_d = {} # 归一化后的特征字典
        list_t = [] # 临时列表
        list_w = [] # 临时列表
        str_t = "" # 临时字符串
        str_t2 = "" # 临时字符串
        dic_t = {} # 训练字典
        list_test = [] # 测试队列
        rate_max = 0.0 # 最高查准率
        rate_t = 0.0 # 查准率中间值
        step_last = 1 # 最佳迭代次数
        path_dae = config.dic_config["path_lr"] # 数据分析引擎基础路径
        test_p = 0 # 调试开关参数

        if (which == 1):
            # 导入极值元组集
            with open(config.dic_config["path_f"] + "feature_min_" + name_model + ".txt", 'r') as f:
                str_t = f.read()
            rows_min = eval(str_t)
            with open(config.dic_config["path_f"] + "feature_max_" + name_model + ".txt", 'r') as f:
                str_t = f.read()
            rows_max = eval(str_t)
            #print ("rows_min",rows_min,"rows_max",rows_max) # 调试用
            
        if (which == 2):
            # 导入极值元组集
            with open(config.dic_config["path_f"] + "feature_avg_" + name_model + ".txt", 'r') as f:
                str_t = f.read()
            rows_avg = eval(str_t)
            with open(config.dic_config["path_f"] + "feature_std_" + name_model + ".txt", 'r') as f:
                str_t = f.read()
            rows_std = eval(str_t)
            #print ("rows_avg",rows_avg,"rows_std",rows_std) # 调试用

        # 导入测试集
        with open(path_dae + "train_lr_test_" + name_model + ".csv", 'r') as f:
            txt_test = f.read()
        list_test = txt_test.split("\n")
        
        i = 0
        for x in list_test:
        
            str_t = x.split("\",")[0]
            str_t = str_t.replace("\"","")
            #print (x,str_t) # 调试用
            # 计算特征原始值 
            dic_f = {}
            dic_d = {}
            list_t = []
            list_w = []
            row = ()
            
            try:
                dic_f = feature_base.value_get(str_t=str_t,dim_p=dim_f) # 注：此函数dim_p 为特征个数
            except:
                pass
                
            #print (dic_f) # 调试用
            
                
            if (len(dic_f) == dim_f):
                j = 0
                for j in range(dim_f):
                    str_t2 = "f" + str(j+1) 
                    if (str_t2 in dic_f):
                        list_t.append(dic_f[str_t2])
            row = tuple(list_t)
            
            if (which == 1):
                try:
                    dic_d = feature_base.dim_get_max_min(row_p=row,which_p=which,rows_max=rows_max,rows_min=rows_min,numb_dim=dim_f)
                except:
                    pass
                    
            if (which == 2):
                try:
                    dic_d = feature_base.dim_get_stand(row_p=row,which_p=which,rows_avg=rows_avg,rows_std=rows_std,numb_dim=dim_f)
                except:
                    pass
                    
            #print (x,dic_d,dim_f) # 调试用
            
            # 生成归一化队列
            if (len(dic_d) == dim_f):
                j = 0
                for j in range(dim_f):
                    str_t2 = "f" + str(j+1)
                    
                    if (str_t2 in dic_d):
                        list_w.append(dic_d[str_t2])
            
            # 构造测试字典
            if x:
                try:
                    dic_t[i] = [list_w,int(x.split("\",")[1])]
                    i += 1
                except:
                    pass

        numb_test = len(dic_t)
        print ("测试集容量：",numb_test)
        
        list_t = []
        # 是否进行随机多次迭代尝试
        # 顺序迭代
        if (rand_if ==0):
            for i in range(numb_step):
                list_t.append(i+1)
        # 随机迭代
        if (rand_if ==1):
            for i in range(numb_step):
                list_t.append(i+1)
            random.shuffle(list_t) # 随机排序
        # 指定迭代
        if (rand_if ==2):
            list_t = [numb_step]
        
        print ("理想拟合准确率：",fit_rate)
        if(rand_if == 0):
            print ("最大顺序迭代数：",numb_step)
        if(rand_if == 1):
            print ("最大随机迭代数：",numb_step)
        if(rand_if == 2):
            print ("单次迭代数：",numb_step)
            
        print ("分类维度：",dim)
        print ("中间数据目录：",path_dae)
        
        # 利用随机选择迭代次数
        i = 1
        for x in list_t:
            
            print ("\n")
            print ("-------- 第",i,"次处理 进行 ",x," 次迭代  --------") # 调试用
            print ("\n")
            
            j = 1
            while j<=dim:
            
                train_dae.train_lr(path_input_p=path_dae + "train_lr_" + str(j) + "_" + name_model + ".csv",path_out_p=path_dae + "result_lr_" + str(j) + "_" + name_model + ".csv",steps_p=x)
                
                # 测试查准率开始
                    
                j += 1
                
            numb_right = 0
            m = 1

            for y in dic_t:
                
                rate_t = feature_dae.what_class_lr(list_last=dic_t[y][0],path_dae_p=path_dae,dim_p=dim,test_p=test_p,name_model_p=name_model)
                #print (rate_t,y,dic_t[y][0],list_test[y]) # 调试用
                #if (rate_t == 1):
                    #print (y,dic_t[y][0],list_test[y]) # 调试用
                if (rate_t == dic_t[y][1]):
                    numb_right += 1
                    #print (m," ",type(rate_t), " - ",type(dic_t[y][1])," yes") # 调试用
                else:
                    #print (m," ",rate_t, " - ",dic_t[x][1]," no") # 调试用
                    pass
                m += 1

            rate_last = numb_right/numb_test
            print ("查准率：",rate_last) # 调试用
            
            if (rate_last > rate_max):
                rate_max = rate_last
                step_last = x
                
            #校验是否达到拟合阈值
            if (rate_max >= fit_rate):
                break
            
            i += 1
        
        # 永固化测试结果
        time_last = str(round(time_cost(time_start),2))
        print ("\n")
        print ("[ 耗时",time_last,"秒 最高查准：",rate_max,"最佳迭代次数：",step_last," ]")
        
        if (step_last > 0):
            j = 1
            while j<=dim:
            
                train_dae.train_lr(path_input_p=path_dae + "train_lr_" + str(j) + "_" + name_model + ".csv",path_out_p=path_dae + "result_lr_" + str(j) + "_" + name_model + ".csv",steps_p=step_last)
                    
                j += 1
            
        return (txt) 
    
    # 训练词向量模型
    if (action == "train_w2v"):
        txt = ""
        txt = inc_vec.run_it(str_t="vec",action_p=action,path_model_p=path_model,path_save_p=path_save)
        return txt
        
    # 保存词向量模型
    if (action == "vec_save"):
        txt = ""
        txt = inc_vec.run_it(str_t="vec",action_p=action,path_model_p=path_model,path_save_p=path_save)
        return txt
        
    # 近义词处理
    if (action == "vec_sm"):
        txt = ""
        txt = inc_vec.run_it(str_t="similar",action_p=action)
        return txt
        
    # 用户画像处理
    if (action == "ps_get"):
        txt = ""
        txt = inc_user.run_it(str_t="personas",action_p=action)
        return txt
        
    # 生成前台二级提交js码
    if (action == "class_js"):
        txt = ""
        import inc_web # web控制模块
        txt = inc_web.run_it(action_p=action)
        return txt
    
#--------- 内部模块处理<<结束>> ---------#

#---------- 主过程<<开始>> -----------#

def main():

    #1 过程一

    print (run_it())

    #2 过程二

    #3 过程三

    

if __name__ == '__main__':

    main()
    

#---------- 主过程<<结束>> -----------#