import urllib.request
import re
import csv
from argparse import *

class KeywordGenerator:
    '''
    Class is dedicated to open a text file with keywords and work with them.
    '''
    def __init__(self, file_path='keywords.txt'):
        '''
        :param file_path: Path to a text file with keywords. Important: Keywords have to be separated by a new line.
        '''
        self.file_path=file_path

    def get_keywords(self):
        '''Generates keywords from opened file.'''
        try:
            with open(self.file_path, 'r') as f:
                for line in f:
                    yield line.rstrip('\n')
        except FileNotFoundError as e:
            print(e.strerror)
        except Exception as e:
            print(e)

class GoogleSearchScraper:
    '''
    Class contains methods to scrape Google Search results page.
    '''
    def __init__(self, site_url='https://www.searchenginejournal.com/', keywords_file_path='keywords.txt'):
        '''
        The required query looks like: site:{site_url} {keyword}
        :param site_url: url to a site
        :param keywords_file_path: Path to a text file with keywords. Important: Keywords have to be separated by a new line.
        '''
        self.site=site_url
        self.keyword_generator=KeywordGenerator(keywords_file_path)

    def search_and_save(self, max_number_of_results=None):
        '''
        Queries Google.com and saves required results (links pointing to a self.site and the number of results) in csv files.
        :param max_number_of_results: Max number of results generated by Google. Default = 10 (first page). It hasn't been tested with number greater than 100.
                                    Note: Real number of results could be smaller than chosen, because Google treats some blocks (e.g. a images block) like results.
        '''
        space = '%20'
        num_parameter = '' if max_number_of_results==None else '&num='+str(max_number_of_results)

        for keyword in self.keyword_generator.get_keywords():
            url = 'https://www.google.com/search?q=site:'+self.site+' '+keyword+num_parameter
            url = url.replace(' ',space)
            html=self.__get_html(url)
            if html:
                results = self.__get_results(html)
                number_of_results = self.__get_number_of_results(html) if results else 0
                self.__save_results_as_csv(results)
                self.__save_number_of_results_as_csv(keyword, number_of_results)


    def __get_html(self, url):
        '''
        :param url: site url
        :return: raw html in string or None if catched exception
        '''
        request = urllib.request.Request(url,
                                         headers={'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36'})
        try:
            page = urllib.request.urlopen(request)
            bytes_page = page.read()
            html_page = bytes_page.decode("utf8")
            page.close()
            return html_page
        except Exception as e:
            print('Url: {} Error: {}'.format(url, e))
            return None


    def __get_results(self, text):
        '''
        :param text: a text to be searched
        :return: the list of links pointing to a self.site
        '''
        pattern=r'href="'+self.site+'(.*?)"'
        results=re.findall(pattern, text)
        if not results:
            return []
        else:
            results = [self.site+x for x in results]
            return results

    def __get_number_of_results(self, text):
        '''
        :param text: a text to be searched
        :return: the number of all results, which Google shows at the top of the results page
        '''
        pattern=r'id="result-stats">(.*?)<'
        result=re.search(pattern, text).group(1)
        result=re.findall(r'\d+', result)
        number=''.join(result)
        return number

    def __save_results_as_csv(self, results, filename='output_results.csv'):
        '''
        Saves results in the csv file. Results of the one query are in the new line and has a comma as a separator.
        A number of the line is associated with a number of the line in a keywords file.
        In general it's possible that a link contains the comma, so links are in quotation marks in order to avoid errors.
        :param results: a list, which has to be saved in the csv file
        :param filename: a name of the csv file
        '''
        with open(filename, 'a', newline='') as f:
            wr = csv.writer(f, quoting=csv.QUOTE_ALL)
            wr.writerow(results)

    def __save_number_of_results_as_csv(self, keyword, number_of_results, filename='output_number_of_results.csv'):
        '''
        Saves number of results along with the associated keywords in the csv file.
        Structure of the output csv: keyword,number
        :param keyword: the keyword, which was part of the query
        :param number_of_results: the number of result, which is associated with the keyword
        :param filename: a name of the csv file
        :return:
        '''
        with open(filename, 'a', newline='') as f:
            wr = csv.writer(f)
            wr.writerow([keyword,number_of_results])

class Menu(ArgumentParser):
    '''
    Simple menu to parse arguments.
    '''
    def __init__(self):
        super().__init__()
        self.__create_and_set_options()

    def __create_and_set_options(self):
        '''
        Creating options of menu and setting the proper configuration.
        '''
        self.add_argument('--keywords-file', type=str, default='keywords.txt')
        self.add_argument('--max-number', type=int, default=None)

    def create_scraper_and_start(self, args):
        '''
        Creates GoogleSearchScraper and starts quering
        :param args: parsed args
        '''
        scraper=GoogleSearchScraper(keywords_file_path=args.keywords_file)
        scraper.search_and_save(max_number_of_results=args.max_number)


if __name__ == "__main__":
    menu = Menu()
    try:
        args = menu.parse_args()
        menu.create_scraper_and_start(args)
    except Exception as e:
        print(e)