""" Uses a questions metadata file (generated by getqmeta.py) to scrape questions from Stack Exchange webpages.
    Generates several numbered "qdata.json" files which contain scraped questions.
"""

import argparse
import time
import json
import requests
from bs4 import BeautifulSoup


def scrape(html):
    soup = BeautifulSoup(html, 'html.parser')
    # qtitle = soup.find("a", attrs={"class": "question-hyperlink"}).string
    try:
        qcontent = soup.find("div", attrs={"class": "post-text"}).text
    except:
        qcontent = "FAILED"
    return qcontent


def main():
    """ Normal usage: python scrape.py --questions-metafile qmeta.json

    This program takes forever to run (several days) because of the number of questions it has to scrape,
    and also because of an artificial delay I introduced to make sure Stack Exchange doesn't kick me out.

    The program is quite robust and in case of accidental failures (network disconnection, user interruption, laptop battery dying, etc),
    it will resume scraping from where it last stopped.

    """

    parser = argparse.ArgumentParser()
    parser.add_argument("--questions-metafile", "--qf")     # this is the questions metadata file generated by scrape.py
    args = parser.parse_args()

    metadata_fp = open(args.questions_metafile)
    completed_fp = open("completed_questions.json", "r+")
    failed_fp = open("failed.txt", "a")

    # load question metadata (file with question id, link, title and tag)
    qmeta = json.load(metadata_fp)
    metadata_fp.close()

    # load file that saves progress of questions scraped so far
    try:
        qcompleted = json.load(completed_fp)
    except json.decoder.JSONDecodeError:  # if file is empty
        qcompleted = {"completed_sites": []}

    completed_sites = qcompleted["completed_sites"]

    for sitename, siteqmetadata in qmeta.items():
        if sitename in completed_sites or sitename.endswith(".meta") or sitename.startswith("meta"):
            continue

        # load file that stores the actual question text
        try:
            num_completed_sites = len(qcompleted["completed_sites"])

            # store scraped question data into multiple files so that files don't get too large (and reading/writing becomes slow)
            if num_completed_sites <= 30:
                filenum = 1
            elif num_completed_sites <= 60:
                filenum = 2
            elif num_completed_sites <= 90:
                filenum = 3
            elif num_completed_sites <= 120:
                filenum = 4
            elif num_completed_sites <= 150:
                filenum = 5
            else:
                filenum = 6

            questions_filename = "qdata" + str(filenum) + ".json"

            questions_fp = open(questions_filename, "r+")
            qdata = json.load(questions_fp)
        except json.decoder.JSONDecodeError:  # if file is empty
            qdata = {}

        try:
            qcompleted[sitename]
        except KeyError:
            qcompleted[sitename] = []

        print("Writing to " + questions_filename)

        completed_questions_for_this_site = qcompleted[sitename]
        failcount = 0
        for qid, qmetadata in siteqmetadata['questions'].items():
            if qid in completed_questions_for_this_site:
                continue
            qurl = qmetadata['link']
            r = requests.get(qurl)
            qcontent = scrape(r.text)
            if r.status_code != 200 or qcontent == "FAILED":
                failed_fp.write(qurl + "\n")
                failcount += 1
                if failcount >= 5:
                    print("Five consecutive failures occurred. Something's wrong!")
                    return
                continue
            failcount = 0
            qtitle = qmetadata['title']
            qtag = qmetadata['tag']

            try:
                qdata[sitename]
            except KeyError:
                qdata[sitename] = []

            qdata[sitename].append({
                qid: {
                    "title": qtitle,
                    "content": qcontent,
                    "tag": qtag
                }
            })
            questions_fp.seek(0)
            json.dump(qdata, questions_fp)

            qcompleted[sitename].append(qid)
            completed_fp.seek(0)
            json.dump(qcompleted, completed_fp)
            time.sleep(0.35)
        qcompleted["completed_sites"].append(sitename)
        completed_fp.seek(0)
        json.dump(qcompleted, completed_fp)
        questions_fp.close()

    completed_fp.close()


if __name__ == '__main__':
    main()
