# -*- coding:utf8 -*-
import os,re
import json
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.svm import SVC
import F1
class my_classifair(object):
    def begin(self,category):
        path="..\\train_data"
        word_list=[]
        need_predict=[]
        for root, dirs, files in os.walk(path):
            for file in files:
                f=open(path+"\\"+file)
                text=json.load(f)
            #url_classifair
                for result in text:
                    try:
                        div_word = re.split(r'[^a-zA-Z0-9]+', result[category])
                        for word in div_word:
                            word_list.append(word)
                    except:
                        continue
        y=[0,1]
        path2="..\\train_data_not"
        word_list_not=[]

        numlist=[]
        for root2, dirs2, files in os.walk(path2):
            for file in files:
                f=open(path2+"\\"+file)
                text=json.load(f)
            #url_classifair
                for result in text:
                    try:
                        div_word = re.split(r'[^a-zA-Z0-9]+', result[category])
                        for word in div_word:
                            word_list_not.append(word)
                    except:
                        continue
        path3 ="..\json"
        for root3, dirs3, file3 in os.walk(path3):
            for file in file3:
                f = open(path3 + "\\" + file)
                count=0
                text = json.load(f)
                # url_classifair
                for result in text:
                    try:
                        div_word = re.split(r'[^a-zA-Z0-9]+', result[category])
                        count+=1
                        need_predict.append(",".join(div_word))
                    except:
                        continue
        arr = np.array(y)
        new_word_list=[]
        new_word_list.append(",".join(word_list))
        new_word_list.append(",".join(word_list_not))
        new_word_list=new_word_list+need_predict
        vectorizer=CountVectorizer()
        csr_mat = vectorizer.fit_transform(new_word_list)
        transformer=TfidfTransformer()
        tfidf=transformer.fit_transform(csr_mat)
        model = SVC()
        model.fit(tfidf[0:2], arr)
        predicted = model.predict(tfidf[2:])
        return predicted

if __name__ == '__main__':
    test=my_classifair()
    relate_rate=[]
    for i in range(101):
        relate_rate.append(0)
    url_result=test.begin("url")
    print "finish 1"
    text_result = test.begin("text")
    print "finish 2"
  #  host_result = test.begin("host_page")
  #  print "finish 3"
    title_result = test.begin("title")
    print "finish 4"
    final=np.vstack((text_result,title_result))
    for i in range(0,len(text_result)-10,10):                   #计算相似度
        flag=final[0:2,i:i+10]
        rate=int(np.sum(flag)*100/20)
        relate_rate[100-rate]+=1
    print relate_rate
    new_list=[]
    for y in range(0, 101, 5):                                      #画图
        new_list.append((relate_rate[y]-0.00001)/np.sum(relate_rate)*100)
    F1.linePlaut(new_list)
    print 1