#!/usr/bin/python
# -*- coding: UTF-8 -*-

import requests #导入requests 模块
from bs4 import BeautifulSoup  #导入BeautifulSoup 模块
import re
import json
from pathlib import Path

def get_real_text(text):
    if len(text) >= 1:
        if text[0] == ':' or text[0] == '：':
            text = text[1:]
    if len(text) >= 2:
        if text[1] == ':' or text[1] == '：':
            text = text[2:]

    tmp = re.sub(r'\n*', "", text)
    tmp1 = re.sub(r'\t*', "", tmp)
    tmp2 = re.sub(r' +', " ", tmp1)
    newstr = re.sub(r' +', " ", tmp2)

    if newstr[0] == ' ' or newstr[0] == ' ':
        newstr = newstr[1:]
    if newstr[-1] == ' ' or newstr[-1] == ' ':
        newstr = newstr[0:-1]

    return newstr

def get_html_label(web_url):
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'}  # 给请求指定一个请求头来模拟chrome浏览器
    req = requests.get(web_url, headers=headers)  # 像目标url地址发送get请求，返回一个response对象
    req.encoding = 'utf-8'

    html_label = BeautifulSoup(req.text, 'lxml')
    return html_label

def get_exploit_list(labels):
    table = labels.find('table', class_="exploit_list bootstrap-wrapper")
    temp = table.find_all('td', class_="description")
    exploit_list = []
    for elem in temp:
        exploit_list.append(elem.find('a'))
    return exploit_list

def get_exploit_dict(exploit_elem):
    exploit_dict = {}
    exploit_dict["source"] = "1-0"
    exploit_dict["url"] = exploit_elem['href']
    exploit_dict["title"] = exploit_elem['title']

    labels = get_html_label(exploit_dict["url"])
    table = labels.find('table', class_="exploit_list")
    if table is None:
        print("===============================================No Table Label================================================")
        return {}

    tr_list = table.find_all('tr')
    for tr in tr_list:
        td_list = tr.find_all('td')
        for td in td_list:
            elem_string = ""
            strong_label = td.find('strong')
            try:
                key_name = strong_label.text
                strong_label.extract()
            except:
                try:
                    tdc = td['colspan']
                    continue
                except:
                    print("===============================================No Strong Label================================================")
                    return {}

            if key_name == "E-DB Verified":
                temp = td.find('img')
                if temp is not None:
                    elem_string = temp['title']
                else:
                    if td.text is not None:
                        elem_string = get_real_text(td.text)

                exploit_dict[key_name] = elem_string
                continue
            elif key_name == "Exploit":
                temp = td.find('a')
                if temp is not None:
                    elem_string = temp['href']
                else:
                    if td.text is not None:
                        elem_string = get_real_text(td.text)

                exploit_dict[key_name] = elem_string
                continue
            elif key_name == "Shellcode":
                temp = td.find('a')
                if temp is not None:
                    elem_string = temp['href']
                else:
                    if td.text is not None:
                        elem_string = get_real_text(td.text)

                exploit_dict[key_name] = elem_string
                continue
            elif key_name == "Vulnerable App":
                temp = td.find('a')
                if temp is not None:
                    elem_string = temp['href']
                else:
                    if td.text is not None:
                        elem_string = get_real_text(td.text)

                exploit_dict[key_name] = elem_string
                continue
            elif key_name == "Advisory/Source":
                temp = td.find('a')
                if temp is not None:
                    elem_string = temp['href']
                else:
                    if td.text is not None:
                        elem_string = get_real_text(td.text)

                exploit_dict[key_name] = elem_string
                continue

            a_label = td.find('a')
            if a_label is not None:
                if a_label.text is not None:
                    elem_string = get_real_text(a_label.text)
                    exploit_dict[key_name] = elem_string
                    continue
                a_label.extract()

            if td.text is not None:
                elem_string = get_real_text(td.text)

            exploit_dict[key_name] = elem_string

    related_table = labels.find('table', class_="exploit_list bootstrap-wrapper")
    if related_table is not None:
        related_tbody = related_table.find('tbody')
        related_list = related_tbody.find_all('tr')

        if len(related_list) == 1:
            td = related_list[0].find('td')
            if td.text == "No matches":
                exploit_dict['related_exploits'] = []
                return exploit_dict

        exploit_dict['related_exploits'] = []
        for tr in related_list:
            tds = tr.find_all('td')
            for td in tds:
                try:
                    test = td['class']
                except:
                    a_label = td.find('a')
                    exploit_dict['related_exploits'].append(a_label['href'])


    return exploit_dict

def get_page_list(labels):
    div_label = labels.find('div', class_="pagination")
    a_list = div_label.find_all('a')
    for elem in a_list:
        if elem.string == ">>":
            url = elem['href']
            index = url.find('&pg=')
            page_max = int(url[index+4 : ])

            prefix_url = url[0 : index+4]
            url_list = []
            for i in range(1, page_max+1):
                url_list.append(prefix_url + str(i))

            return url_list



home_url = 'https://www.exploit-db.com'
html_label = get_html_label('https://www.exploit-db.com')

finds = html_label.find_all('h2', style="text-align: justify;")

home_list = []
subject_name = []
for find in finds:
    label = find.find('a')
    if label['href'] != '/papers/':
        home_list.append(home_url + label['href'])
        tmp = label.string
        tmp = tmp.replace(" ", "_")
        tmp = tmp.replace("\xa0", "_")
        subject_name.append(tmp)

print(home_list)
print(subject_name)


index = 0
for url_elem in home_list:
    labels = get_html_label(url_elem)
    page_list = get_page_list(labels)
    page_num = 1

    for page in page_list:
        file_exist = Path('C:\\Users\\dlwog\\Pictures\\hi\\exploitdb\\' + subject_name[index] + '_' + str(page_num) + '.json')
        if file_exist.is_file():
            print('C:\\Users\\dlwog\\Pictures\\hi\\exploitdb\\' + subject_name[index] + '_' + str(page_num) + '.json' + "already exist")
            page_num = page_num + 1
            continue

        dict_list = []

        page_labels = get_html_label(page)
        exploit_list = get_exploit_list(page_labels)
        for elem in exploit_list:
            print(elem['title'])
            dict_result = get_exploit_dict(elem)
            if dict_result == {}:
                continue
            else:
                dict_list.append(get_exploit_dict(elem))

        with open('C:\\Users\\dlwog\\Pictures\\hi\\exploitdb\\' + subject_name[index] + '_' + str(page_num) + '.json', 'w',
                  encoding='utf-8') as json_file:
            for elem in dict_list:
                json.dump(elem, json_file, ensure_ascii=False, indent=4)

        page_num = page_num + 1

    index = index + 1


'''
labels = get_html_label("https://www.exploit-db.com/remote/")
page_list = get_page_list(labels)
print(page_list)
'''

'''
labels = get_html_label("https://www.exploit-db.com/remote/")
exploit_list = get_exploit_list(labels)
dict_list = []
for elem in exploit_list:
    print(elem['title'])
    dict_list.append(get_exploit_dict(elem))

with open('C:\\Users\\dlwog\\Pictures\\hi\\exploitdb\\remote.json','w',encoding='utf-8') as json_file:
    for elem in dict_list:
        json.dump(elem, json_file, ensure_ascii=False, indent=4)
'''

'''
index = 0
for elem in home_list:
    labels = get_html_label(elem)
    exploit_list = get_exploit_list(labels)
    print(exploit_list)
'''