
### 推广类
### 第1批数据：
#  1:敏感语料（广告标题 29245条）
#  2:微博评论原文（senti100k,处理特殊字符）20000条，新闻语料9245条
#  测试集比例0.1 5849 52641

# count = 0
# with open('/Users/leo/Data/项目数据/文德数慧-文本内容审核/分类实验/数据/推广类语料_清理后.txt','r') as f:
#     for line in f.readlines():
#         line = line.strip()
#         if len(line)>5:
#             count += 1

# print(count)


import pandas as pd
import os
import re
import random
random.seed(7)

def clean(text):
    text = re.sub(r"(回复)?(//)?\s*@\S*?\s*(:| |$)", " ", text)  # 去除正文中的@和回复/转发中的用户名
    text = re.sub(r"\[\S+\]", "", text)      # 去除表情符号
    text = re.sub(r"#\S+#", "", text)      # 去除话题内容
    text = re.sub(r"【\S+】", "", text)      # 去除标题
    URL_REGEX = re.compile(
        r'(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?«»“”‘’]))',
        re.IGNORECASE)
    text = re.sub(URL_REGEX, "", text)       # 去除网址
    text = text.replace("转发微博", "")       # 去除无意义的词语
    text = text.replace("本报讯", "") 
    text = re.sub(r"\s+", " ", text) # 合并正文中过多的空格
    return text.strip()



####  非敏感语料 ####
df_0 = pd.read_csv('/Users/leo/Data/项目数据/文德数慧-文本内容审核/分类实验/数据/weibo_senti_100k.csv')
df_0 = df_0.sample(n=30000).reset_index(drop=True) # 有一些处理之后为空的，不要

# 新闻语料 388
files = []
for path,dirs,file_names in os.walk('/Users/leo/Data/项目数据/文德数慧-文本内容审核/分类实验/数据/5类'):
    for file in file_names:
        files.append(os.path.join(path,file))

lines = []
for file in files:
    if '.txt' in file:
        try:
            with open(file,'r',encoding='gb2312') as f:
                for line in f.readlines():
                    line = line.strip()
                    if len(line) >= 80 and len(line) <= 150:
                        lines.append(line)
        except:
            pass


line_select = random.sample(lines,9245)
                

data = pd.DataFrame(columns=['label','text'])

# 微博数据 49918
count = 0
for i in range(len(df_0)):
    label = 0
    text = clean(df_0.iloc[i]['review']).replace('\0','').replace('\x00', '')
    if text:
        count += 1
        data = data.append(pd.DataFrame({'label':[label],'text':[clean(text)]}),ignore_index=True)
        if count == 20000:
            break

# 加入新闻数据
for line in line_select:
    label = 0
    text = clean(line).replace('\0','').replace('\x00', '')
    data = data.append(pd.DataFrame({'label':[label],'text':[text]}),ignore_index=True)


print(len(data)) ### 1000



##### 敏感语料 #####


with open('/Users/leo/Data/项目数据/文德数慧-文本内容审核/分类实验/数据/推广类语料_清理后.txt','r') as f:
    for line in f.readlines():
        line = line.strip()
        if len(line)>5:
            label = 1
            text = line.replace('\0','').replace('\x00', '')
            data = data.append(pd.DataFrame({'label':[label],'text':[text]}),ignore_index=True)

assert len(data) == 29245*2



data['label'] = data['label'].astype(int)
data = data.sample(frac=1).reset_index(drop=True)

data_test = data[:int(len(data)*0.1)]
data_train = data[int(len(data)*0.1):]
print(len(data_test))
print(len(data_train))



data_train.to_csv(r"/Users/leo/Data/项目数据/文德数慧-文本内容审核/分类实验/数据/data_bert/广/v1/train.tsv",sep='\t',header=False,index=False)
data_test.to_csv(r"/Users/leo/Data/项目数据/文德数慧-文本内容审核/分类实验/数据/data_bert/广/v1/test.tsv",sep='\t',header=False,index=False)
data_test.to_csv(r"/Users/leo/Data/项目数据/文德数慧-文本内容审核/分类实验/数据/data_bert/广/v1/dev.tsv",sep='\t',header=False,index=False)
