# coding=utf-8
import math
import random

spambase = []# 文件中的所有数据
all_fold = []# 所有组
testing_foldList = []# 测试组
training_foldList = []# 训练组

maillist = []# 邮件列表
spamlist = []# 训练集中的垃圾邮件
nonspamlist = []# 训练集中的正常邮件

mu = []# 各特征平均值列表
PfiBS = []# 返回垃圾邮件各特征低于平均值的概率列表blow
PfiOS = []# 返回垃圾邮件各特征高于平均值的概率列表overtop
PfiBNS = []# 返回正常邮件各特征低于平均值的概率列表
PfiONS = []# 返回正常邮件各特征高于平均值的概率列表

##############################################################
# get_maillist(string)
# 给定数据文件名路径，将数据载入maillist
def get_maillist(fname):
    feat_vects = []
    # 打开数据文件
    with open(fname) as feat_file:
        for line in feat_file:
            x = []
            # 用逗号和行拆分特征
            for w in line.strip().split(','):
                if w:
                    # 追加特征标签
                    x.append(float(w))
            maillist.append(x)
    return maillist

# partition_tenfolds(list)
# 把邮件大致分成10份。
def partition_tenfolds(database):
    for i in range(0, 10):
        for j in range(0, int(len(database) / 10)):
            all_fold[i].append(database[j * 10 + i])

# create_training_fold(list)
# 给出测试集合列表，去掉这些集合，并返回其他的集合
def create_training_fold(test_List):
    fold_list = all_fold[:]
    for i in range(testingNum):
        for j in range(10):
            # print("测试集还剩{}个集合".format(len(test_List)))
            if test_List[i] == fold_list[j]:
                del fold_list[j]
                break
    # print("训练集还剩{}个集合".format(len(fold_list)))
    return fold_list

# create_spamlist()
# 过滤所有电子邮件中的垃圾邮件，并使用列表存储所有训练集中的垃圾邮件。
def create_spamlist():
    for fold in training_foldList:
        for i in range(0, len(fold)):
            if fold[i][57] == 1:
                spamlist.append(fold[i])

# create_nonspamlist()
# 过滤所有电子邮件中的非垃圾邮件，
# 并使用一个列表来存储所有训练集中的非垃圾邮件。
def create_nonspamlist():
    for fold in training_foldList:
        for i in range(0, len(fold)):
            if fold[i][57] == 0:
                nonspamlist.append(fold[i])

# ######################################################################
# create_below_mui_spam_list(list,int,intlist)
# 给定一个垃圾邮件列表、特性序号和平均值列表，返回一个垃圾邮件列表。
# 在这个垃圾邮件列表中，所有垃圾邮件的该特征都低于平均值。
def create_below_mui_spam_list(spamlist, num, num_mu):
    BMS_helper = []
    for i in range(0, len(spamlist)):
        if spamlist[i][num] <= num_mu:
            BMS_helper.append(spamlist[i])
    below_mui_spam = BMS_helper[:]
    #print('below_mui_spam: ', len(below_mui_spam))
    return below_mui_spam

# create_below_mui_nonspam_list(list,int,intlist)
# 给定一个正常邮件列表、特性序号和平均值列表，返回一个正常邮件列表。
# 在这个正常邮件列表中，所有正常邮件的该特征都低于平均值。
def create_below_mui_nonspam_list(nonspamlist, num, num_mu):
    BMNS_helper = []
    for i in range(0, len(nonspamlist)):
        if nonspamlist[i][num] <= num_mu:
            BMNS_helper.append(nonspamlist[i])
    below_mui_nonspam = BMNS_helper[:]
    return below_mui_nonspam

# calculate_mui(int)
# 给定一个属性的数字，返回所有电子邮件的此功能的平均值。
def calculate_mui(num):
    m = 0
    counter = 0#计总数
    for fold in training_foldList:
        for i in range(0, len(fold)):
            m = m + fold[i][num]
            counter = counter + 1
    average = float(m) / counter
    # print('共有{}个数据'.format(counter))
    return average

# create_mu_list()
# 创建所有功能的平均值列表
def create_mu_list():
    for i in range(0, 58):
        mu.append(calculate_mui(i))

# get_PfiBS()
#返回垃圾邮件各特征低于平均值的概率列表
def get_PfiBS():
    #print(len(spamlist))
    for i in range(0, 58):
        # print(i) #——去掉测试打印 2019/4/14 zhangyue
        # print(mu[i])
        num_BMS = len(create_below_mui_spam_list(spamlist, i, mu[i]))#该特征低于平均值的垃圾邮件个数
        PfiBS.append(float(num_BMS + 1) / (len(spamlist) + 2))#该特征在垃圾邮件中低于平均值概率
    return PfiBS

# get_PfiBNS()
# 返回正常邮件各特征低于平均值的概率列表
def get_PfiBNS():
    for i in range(0, 58):
        num_BMNS = len(create_below_mui_nonspam_list(nonspamlist, i, mu[i]))#该特征低于平均值的正常邮件个数
        PfiBNS.append(float(num_BMNS + 1) / (len(nonspamlist) + 2))#该特征在正常邮件中低于平均值概率
    return PfiBNS

# get_PfiOS()
# 返回垃圾邮件各特征高于平均值的概率列表。
def get_PfiOS():
    for i in range(0, 58):
        num_BMS = len(create_below_mui_spam_list(spamlist, i, mu[i]))
        PfiOS.append(float((len(spamlist) - num_BMS) + 1) / (len(spamlist) + 2))#总数减去低于平均值的为大于等于平均值的
    return PfiOS

# get_PfiONS()
# 返回正常邮件各特征高于平均值的概率列表。
def get_PfiONS():
    for i in range(0, 58):
        num_BMNS = len(create_below_mui_nonspam_list(nonspamlist, i, mu[i]))
        helper = float((len(nonspamlist) - num_BMNS) + 1)
        PfiONS.append(helper / (len(nonspamlist) + 2))
    return PfiONS

# predict_spam(list)
# 给出一封电子邮件，使用伯努利模型返回它是否是垃圾邮件。
def predict_spam(mail):
    # print("计算结果为{}".format(result))
    result = get_score(mail)
    if result > 0:
        return True
    else:
        return False

# get_score(list)
# （核心）给一个电子邮件，返回用伯努利模型得到的得分
def get_score(mail):
    helper2 = 0.0
    helper3 = float(len(spamlist)) / len(nonspamlist)
    helper1 = math.log(helper3)
    for i in range(0, 57):
        if mail[i] <= mu[i]:
            helper2 = helper2 + math.log(PfiBS[i] / PfiBNS[i])
        else:
            helper2 = helper2 + math.log(PfiOS[i] / PfiONS[i])
    result = helper1 + helper2
    return result

# 大致评判过滤效果
def generalJudge():
    flag = 0
    amount = 0
    for i in range(testingNum):
        # print("zuuu:{}-len:{}".format(i,len(testing_foldList)))   ——去掉测试打印 2019/4/14 zhangyue
        # tempfold = appendixScore_fold(testing_foldList[i])
        for mail in testing_foldList[i]:
            amount += 1
            # print("第{}个,{},{}".format(flag,mail[57],predict_spam(mail)))
            if mail[57] == predict_spam(mail):
                flag += 1
    print("判断正确{}个,错误{}个，正确率为{}%".format(flag, amount - flag, float(flag) / float(amount) * 100))

# calculateParam(paramList)
# 计算需要的参数ap，at，tp
def calculateParam(paramList):
    ap = paramList[0]
    at = paramList[1]
    tp = paramList[2]
    for i in range(testingNum):
        for mail in testing_foldList[i]:
            temp = predict_spam(mail)
            if mail[57]:
                at += 1
                if mail[57] == temp:
                    tp += 1
            if temp:
                ap += 1
    paramList = [ap, at, tp]
    return paramList

############################################################################################################
# add_score(Email)
# 输入电子邮件
# 返回附加分数的新电子邮件
def add_score(mail):
    result = 0.0
    result = get_score(mail)
    mail.append(result)
    # print(result)
    return mail

# sort_fold(Fold)
# 返回排序的电子邮件得分数据集
def sort_fold(fold):
    tempmail = []
    new_fold = []
    for i in range(0, len(fold)):
        tempmail = fold[i]
        tempmail = add_score(tempmail)
        new_fold.append(tempmail)
    new_fold.sort(key=lambda afold: afold[58], reverse=True)
    # print("各邮件得分：{}".format(new_fold))
    return new_fold

# get_tp_k(Fold,Number)
# 返回第k封邮件被正确判断为正常邮件的概率
def get_tp_k(fold, k):
    counterS = 0
    thre = fold[k][58]
    correct = 0
    for mail in fold:
        if mail[57] == 1:
            counterS += 1
            helper2 = get_score(mail)
            if helper2 >= thre:
                correct += 1
    result = float(correct) / counterS
    return result

# create_tpr_k_list(Fold)
# 返回一个被正确判断为正常邮件的概率列表
def create_tp_k_list(fold):
    templist = []
    for i in range(0, len(fold)):
        helper1 = get_tp_k(fold, i)
        templist.append(helper1)
    return templist

# get_fpr_k(Fold,Number)
# 返回前k个邮件被错误判断为正常邮件的概率
def get_fp_k(fold, k):
    counterNS = 0
    thre = fold[k][58]
    wrong = 0
    for mail in fold:
        if mail[57] == 0:
            counterNS += 1
            helper2 = get_score(mail)
            if helper2 >= thre:
                wrong += 1
    result = float(wrong) / counterNS
    return result

# create_fpr_k_list(Fold)
# 返回前k个错误判断邮件概率列表
def create_fp_k_list(fold):
    templist = []
    for i in range(0, len(fold)):
        helper1 = get_fp_k(fold, i)
        templist.append(helper1)
    return templist

# calculate_auc(ListOfNumber,ListOfNumber)
# 给离开这个列表正确判断为正常邮件的概率和错误判断为非正常邮件的概率
# 返回AUC的值
def calculate_auc(fpr, tpr):
    helper1 = 0.0
    for i in range(1, len(fpr)):
        helper1 = helper1 + (fpr[i] - fpr[i - 1]) * (tpr[i] + tpr[i - 1])
    result = 0.5 * helper1
    return result

#########################################################################
#读取文件放入maillist
# readfile('spambase.data')
get_maillist('spambase.data')

fold0 = []
fold1 = []
fold2 = []
fold3 = []
fold4 = []
fold5 = []
fold6 = []
fold7 = []
fold8 = []
fold9 = []

#将所有数据大致分为十组，便于处理
all_fold = [fold0, fold1, fold2, fold3, fold4, fold5, fold6, fold7, fold8, fold9]
partition_tenfolds(maillist)

##########################################################################
#增加参数用于设置测试集组数设置为30%，便于万一改动，甚至使用者自己输入   --2019/4/10 zhangyue
testingNum = 3

#随机抽取测试集
testing_foldList = random.sample(all_fold, testingNum)
print("测试集比例为：{}%".format(len(testing_foldList)*10))

#将剩下的作为训练集
training_foldList = create_training_fold(testing_foldList)
print("训练集组数为：{}%".format(len(training_foldList)*10))

#处理训练集
create_spamlist()#将训练集中的垃圾邮件放入spamlist
create_nonspamlist()#将训练集中的正常邮件放入nonspamlist
create_mu_list()#将训练集中的所有列平均值放入mu

PfiBS = get_PfiBS()#返回垃圾邮件各特征低于平均值的概率列表blow
PfiBNS = get_PfiBNS()#返回垃圾邮件各特征高于平均的值概率列表overtop
PfiOS = get_PfiOS()#返回正常邮件各特征低于平均值的概率列表
PfiONS = get_PfiONS()#返回正常邮件各特征高于平均的值概率列表




##############################################################################################
#评判过滤效果
generalJudge()

# 判断正确与否的比例只是粗略评判，现进一步通过查准率和查全率的权衡来评判
# 加入查准率和查全率的计算 个人考虑的是垃圾邮件尽量被过滤掉，但是非垃圾邮件最好不能被错误的过滤   --2019/4/15 zhangyue
# 正确肯定True Positive TP
# 正确否定True Negative TN
# 错误肯定False Positive FP
# 错误否定False Negative FN

ap = 0.0
at = 0.0
tp = 0.0
# ap All Positive 预测为正常邮件的数量
# at All True 实际上为正常邮件的数量
# tp True Positive 正确预测为正常邮件的数量
paramList = [ap, at, tp]

# 计算ap、at、tp
paramList = calculateParam(paramList)

# 查准率 Precisio此处我定义为 判断为正常邮件中实际上为正常邮件的数量/判断为正常邮件的数量 P=TP/(TP+FP)=TP/AP
Precision = paramList[2]/paramList[1]
print("查准率的值为：{}".format(Precision))

# 查全率 Reca 此处我定义为 判断为正常邮件中实际为正常邮件的数量/实际上为正常邮件的数量 R=TP/(TP+FN)=TP/AT
Recall = paramList[2]/paramList[0]
print("查全率的值为：{}".format(Recall))

# 计算F值 F Source=PR/(P+R)    --2019/4/15 zhangyue
F = Precision*Recall/(Precision+Recall)
print("F的值为：{}".format(F))

tempfold = []
tempfold = sort_fold(fold0)
tpr_k = create_tp_k_list(tempfold)
# print ("正确判断正常邮件概率:{}".format(tpr_k))
fpr_k = create_fp_k_list(tempfold)
# print ("错误判断正常邮件概率:{}".format(fpr_k))

#计算AUC的值
auc = calculate_auc(fpr_k, tpr_k)
print ("AUC的值:{}".format(auc))

#######################################################################
# 实验数据：
# result > 0
# 判断正确1252个,错误128个，正确率为90.7246376812%
# 查准率的值为：0.85635359116
# 查全率的值为：0.902912621359
# F的值为：0.439508506616
# AUC的值:0.964730413471

# result > 0
# 判断正确1256个,错误124个，正确率为91.0144927536%
# 查准率的值为：0.844036697248
# 查全率的值为：0.921843687375
# F的值为：0.44061302682
# AUC的值:0.964473476164

# result > 0
# 判断正确1267个,错误113个，正确率为91.8115942029%
# 查准率的值为：0.856880733945
# 查全率的值为：0.930278884462
# F的值为：0.446036294174
# AUC的值:0.964078187999

# result > -0.1
# 判断正确1227个,错误153个，正确率为88.9130434783%
# 查准率的值为：0.815837937385
# 查全率的值为：0.89314516129
# F的值为：0.426371511068
# AUC的值:0.965026879595

# result > 0.1
# 判断正确1245个,错误135个，正确率为90.2173913043%
# 查准率的值为：0.819852941176
# 查全率的值为：0.923395445135
# F的值为：0.434274586173
# AUC的值:0.964592062614