import html.parser
import shutil
import time
import urllib.request

import os
import threading
from bs4 import BeautifulSoup
from selenium import webdriver


def main():
    driver = webdriver.Chrome('/Users/bleoo/Downloads/chromedriver')  # 打开浏览器
    driver.get("https://www.zhihu.com/question/66423632")  # 打开想要爬取的知乎页面

    # 模拟用户操作
    def execute_times(times):

        for i in range(times):
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(2)
            try:
                driver.find_element_by_css_selector('button.QuestionMainAction').click()
                print("page" + str(i))
                time.sleep(2)
            except:
                break

    execute_times(20)

    result_raw = driver.page_source  # 这是原网页 HTML 信息
    result_soup = BeautifulSoup(result_raw, 'html.parser')  # 然后将其解析
    result_bf = result_soup.prettify()  # 结构化原 HTML 文件

    SAVE_DIR_PATH = '/Users/bleoo/Desktop/urlpull/'

    shutil.rmtree(SAVE_DIR_PATH)
    os.mkdir(SAVE_DIR_PATH)

    print("爬取回答页面成功!!!")
    driver.close()

    with open(SAVE_DIR_PATH + "noscript_meta.txt", 'wb') as noscript_meta:
        noscript_nodes = result_soup.find_all('noscript')  # 找到所有<noscript>node
        noscript_inner_all = ""
        for noscript in noscript_nodes:
            noscript_inner = noscript.get_text()  # 获取<noscript>node内部内容
            noscript_inner_all += noscript_inner + "\n"

        noscript_all = html.parser.unescape(noscript_inner_all).encode('utf-8')  # 将内部内容转码并存储
        noscript_meta.write(noscript_all)

    noscript_meta.close()
    print("爬取noscript标签成功!!!")

    img_soup = BeautifulSoup(noscript_all, 'html.parser')
    img_nodes = img_soup.find_all('img')

    def download(count):
        urllib.request.urlretrieve(img_url, SAVE_DIR_PATH + str(count) + ".jpg")  # 一个一个下载图片

    with open(SAVE_DIR_PATH + "img_meta.txt", 'w') as img_meta:
        count = 0
        for img in img_nodes:
            if img.get('src') is not None:
                img_url = img.get('src')
                line = str(count) + "\t" + img_url + "\n"
                img_meta.write(line)
                t = threading.Thread(target=download, args=(count,))
                t.start()
                count += 1

    img_meta.close()
    print("图片下载成功")


if __name__ == '__main__':
    main()
