from bs4 import BeautifulSoup as bs
import os
import pandas as pd
import re
import csv
import io

result = {}
new = {}
id = 0
'''
result = {id:{'title':' ', 'abstract':' ', 'key_wordsZ':{'a','b','c'}, 'key_wordsE':{'a','b','c'},'authors': {'author1'}   }}

'''
p = os.walk('./知网html')  # html文件夹路径

with open("output/中文关键词.csv", "w") as csvfile:  # 写csv文件表头
    writer = csv.writer(csvfile)
    writer.writerow(['title', 'key_words'])

with open("output/英文关键词.csv", "w") as csvfile:  # 写csv文件表头
    writer = csv.writer(csvfile)
    writer.writerow(['title', 'key_words'])
with open("output/作者.csv", "w") as csvfile:
    writer = csv.writer(csvfile)
    writer.writerow(['title', 'authors'])
with open("output/标题摘要.csv", "w") as csvfile:
    writer = csv.writer(csvfile)
    writer.writerow(['title', 'abstract'])



for path, dir_list, file_list in p:
    for file_name in file_list:
        print(file_name)

        try:
            htmlfile = io.open('./知网html/' + file_name, 'r', encoding='utf-8')
            htmlhandle = htmlfile.read()
            soup = bs(htmlhandle, 'lxml')
            result[id] = new  # 第n个文献信息

            # result = pd.DataFrame({}, index=[id]) #存储第n个文献的信息
            # author = {}
            new['author'] = set()  # {作者：研究机构} 多个作者
            new['title'] = ''      # 文章标题
            new['abstract'] = ''   # 文献摘要
            new['key_wordsZ'] = set() # 多个关键词
            new['key_wordsE'] = set()
            # author['Institute'] = ''

            content = soup.find('div', class_='top-title')  #解析标题
            # print(content)
            print(content.h1.text)
            new['title'] = content.h1.text

            data = soup.find('div', class_='data')     #解析摘要
            # print(data)
            print(data.p.text)
            new['abstract'] = data.p.text

            data = soup.find('div', id='a_keywords')    #解析关键词
            da = soup.find('div', id="a_keywordsEN")
            # print(data)
            a = data.p.text
            b = da.p.text
            a.replace(';', ',')  # 替换分号为逗号  一直不行
            #print(a.splitlines()[1])
            for i in a.splitlines():
                i.replace(';', '').replace(' ', '')

                new['key_wordsZ'].add(i)
                #new['key_words'] = new['key_words'] - set('')  #去掉空字符串 也不行
                #new['key_words'].remove(' ')
            for i in b.splitlines():
                i.replace(';', '').replace(' ', '')
                new['key_wordsE'].add(i)
            print(new['key_wordsZ'])
            print(new['key_wordsE'])


            content = soup.find('div', class_='content')   #解析作者
            # print(content)
            #print(content.h2.text.splitlines()[1])
            for i in content.h2.text.splitlines():
                i.split()
                new['author'].add(i)
                new['author'] = new['author'] - set('')
                #new['author'].remove(' ')
            print(new['author'])
            #new['author'] = content.h2.text



            with open("output/中文关键词.csv", "a") as csvfile:  #把关键词写入csv文件
                writer = csv.writer(csvfile)
                for i in result[id]['key_wordsZ']:
                    i.replace(';','')
                    list = [str(result[id]['title']),str(i)]
                    writer.writerow(list)

            with open("output/英文关键词.csv", "a") as csvfile:  #把关键词写入csv文件
                writer = csv.writer(csvfile)
                for i in result[id]['key_wordsE']:
                    i.replace(';','')
                    list = [str(result[id]['title']),str(i)]
                    writer.writerow(list)

            with open("output/作者.csv", "a") as csvfile:
                writer = csv.writer(csvfile)
                for i in result[id]['author']:
                    list = [str(result[id]['title']), str(i)]
                    writer.writerow(list)

            with open("output/标题摘要.csv", "a") as csvfile:
                writer = csv.writer(csvfile)
                list = [result[id]['title'], result[id]['abstract']]
                writer.writerow(list)

            id = id + 1
        except:
            print('异常')
        else:
            pass

