'''
Collect requirements to source files mappings
This program gets a set of bug ids from a csv file and attaches those ids at the end of bugzilla website to
gather information about each bug ids.
'''
import pandas as pd
from bs4 import BeautifulSoup
import requests
from time import sleep
import re

URL_PREFIX = 'https://bugzilla.mozilla.org/show_bug.cgi?id={}'
input_file = pd.read_csv('scrapper3.csv')

# arrays to keep the data
open_dates = []
close_dates = []
titles = []
products = []
bug_types = []
components =[]
fixed_versions = []
assignees = []
descriptions = []
reporters = []
status_s = []

def scrape(soup):
    """
    scraping bug details from Bugzilla
    :param soup: content of each html page of bug id
    :return: list of data
    """
    # variables to store each data
    open_date = None
    close_date = None
    title = None
    product = None
    bug_type = None
    component = None
    version = None
    assignee = None
    description = None
    reporter = None
    status = None
    # contains all the characters that start with '\'. useful later to substitute such characters
    pattern = re.compile('\W')

    try:
        title = soup.find('h1', id='field-value-short_desc').text
        # substitute all the characters that are in the variable pattern with a space
        title = re.sub(pattern, ' ', title)
    except Exception:
        pass
    try:
        date = soup.find('div', id='field-status_summary')
        '''
        find all the occurrence of class 'rel-time' because both open and close dates are stored in
        class 'rel-time' in the website where opened date occurs before closed.
        There is no way to distinguish them directly
        '''
        ct = date.findAll('span', class_='rel-time')
        open_date = ct[0].get('title')[:16]  # it is limited to 16 characters because date and time is 16 characters at most
        close_date = ct[1].get('title')[:16]
        print(close_date)
    except Exception:
        pass
    try:
        product = soup.find('div', id='field-product').find('span', id='product-name').text
        product = re.sub(pattern, ' ', product)
    except Exception:
        pass
    try:
        bug_type = soup.find('div', id='field-bug_type').find('span', class_='bug-type-label iconic-text').get('data-type')
        bug_type = re.sub(pattern, '', bug_type)
    except Exception:
        pass
    try:
        component = soup.find('div', id='field-component').find('span', id='component-name').text
        component = re.sub(pattern, ' ', component)
    except Exception:
        pass
    try:
        status = soup.find(id='field-value-status-view').text
        status = re.sub(pattern, ' ', status)
    except Exception:
        pass
    try:
        version = soup.find(id='field-value-target_milestone').text.strip()
    except Exception:
        pass
    try:
        assignee = soup.find('div', id='field-assigned_to').find('a', class_='email').find('span', class_='fna').text
    except Exception:
        pass
    try:
        reporter = soup.find('div', id='field-reporter').find('a', class_='email').find('span', class_='fna').text
    except Exception:
        pass
    try:
        description = soup.find(id='ct-0').text.strip()
        description = re.sub(pattern, ' ', description)
    except Exception:
        pass

    return [title, open_date, close_date, product, bug_type, component, status, version, assignee, reporter, description]

def makerequest(url):
    """
    making requests to Bugzilla using each bug id
    :param url: bug url
    :return: return value of function scrape
    """

    print('Scraping %s' % url)
    try:
        # make requests from the website
        r = requests.get(url).content
        # save all the content from the website into variable soup
        soup = BeautifulSoup(r, 'lxml')
        s = scrape(soup)
        sleep(1)

    # in case of any network error, sleep for 8 minutes and make requests again
    except requests.ConnectionError:
        print('CONNECTION ERROR! RECONNECTING IN 8 MINUTES...\n')
        sleep(480)
        print('RECONNECTING...\n')
        r = requests.get(url).content
        soup = BeautifulSoup(r, 'lxml')
        s = scrape(soup)

    return s

j = 0     # many of the bug ids are repeating in the file. so variable to check if the id matches the previous id
for i, id in enumerate(input_file.BugID):   # get bug id numbers from input file
    print(str(i))

    if int(id) == j:     # if it matches j i.e. the previous id, the result is same as the last one
        results = results

    else:                # if it doesn't match, make a new request
        j = int(id)
        url = URL_PREFIX.format(j)      # attach each bug id's with Bugzilla url
        results = makerequest(url)

    # append extracted data to arrays
    titles.append(results[0])
    open_dates.append(results[1])
    close_dates.append(results[2])
    products.append(results[3])
    bug_types.append(results[4])
    components.append(results[5])
    status_s.append(results[6])
    fixed_versions.append(results[7])
    assignees.append(results[8])
    reporters.append(results[9])
    descriptions.append(results[10])


# export arrays to respective titles in csv file
input_file['Title'] = pd.Series(titles, index=input_file.index)
input_file['Open Date/Time'] = pd.Series(open_dates, index=input_file.index)
input_file['Close Date/Time'] = pd.Series(close_dates, index=input_file.index)
input_file['Product'] = pd.Series(products, index=input_file.index)
input_file['Type'] = pd.Series(bug_types, index=input_file.index)
input_file['Component'] = pd.Series(components, index=input_file.index)
input_file['Status'] = pd.Series(status_s, index=input_file.index)
input_file['Version'] = pd.Series(fixed_versions, index=input_file.index)
input_file['Assignee'] = pd.Series(assignees, index=input_file.index)
input_file['Reporter'] = pd.Series(reporters, index=input_file.index)
input_file['Description'] = pd.Series(descriptions, index=input_file.index)

# create and export all the data to csv file
input_file.to_csv('final_result1.csv', index=False)