import scrapy,re
import urlparse
import random
from bs4 import BeautifulSoup
i=0
class GithubSP(scrapy.Spider):
    global commit
    name = "XXAQJS"
    allowed_domains = ["github.com"]
    git_new_urls=[]
    f = open("All_Url.txt", "r")
    for line in f.readlines():
        line = line.strip('\n')
        if re.search(r"commit", line):
            url = re.findall(r"http[^\"]*", line)[0]
            git_new_urls.append(str(url))
    random.shuffle(git_new_urls)
    start_urls=git_new_urls


    def parse(self, response):
        self.name = []
        html = response.body
        soup = BeautifulSoup(html, 'html.parser')
        res_data = {}
        if soup.find('p', class_="commit-title"):
            title_node = soup.find('p', class_="commit-title")
            res_data['title'] = title_node.get_text()
            name_node = soup.find("strong", itemprop="name")
            res_data['name'] = name_node.get_text()
            # <div class="commit-desc"><pre>async arrow function.</pre></div>
        if soup.find('div', class_="commit-desc"):
            summary_node = soup.find('div', class_="commit-desc").find("pre")
            res_data['summary'] = summary_node.get_text()
        global i
        fout = open("D:\XXAQJS\data\\CVE-commit\\" + str(res_data["name"]) + " " + str(i) + ".txt", "w")
        self.name.append(res_data["name"])
        if res_data.has_key("title"):
            fout.write(res_data["title"].encode("utf-8"))
        if res_data.has_key("summary"):
            fout.write(res_data['summary'].encode("utf-8"))
        fout.write("\n")
        i += 1
        yield