# -----------------------------------------------------------------------------
# Name:        aggregator.py
# Purpose:     CS 21A - implement a simple general purpose aggregator
#
# Author:
# -----------------------------------------------------------------------------
"""
Implement a simple general purpose aggregator

Usage: aggregator.py filename topic
filename: input  file that contains a list of the online sources (urls).
topic:  topic to be researched and reported on
"""

import urllib.request
import urllib.error
import re
import sys


def find_match():
    """

    :return: dictionary of urls (keys) with corresponding list of match groups
    """


def matches():
    """

    :return: list of regex match groups
    """


def main():

    total = len(sys.argv)
    if total < 3 or total > 3:
        print("Error:  invalid number of arguments")
        print("Usage: aggregator.py filename topic")
        sys.exit(1)

    match_dict = {}
    with open(sys.argv[1], 'r', encoding='utf-8') as my_file:
        for line in my_file:
            try:
                url_file = urllib.request.urlopen(line)  # open URLs
                page = url_file.read()  # read http response
                # convert response from bytes to strings
                decoded_page = page.decode('UTF-8')
                # convert response to raw string
                raw_pg = decoded_page.replace("\\", "\\\\")
                regex_list = ['>((?:[^<]*\s)?', sys.argv[2], '.*?)<']
                regex_string = ''.join(regex_list)
                match_list = re.findall(regex_string, raw_pg, re.IGNORECASE |
                                        re.DOTALL | re.MULTILINE)
                match_dict[line] = match_list

            except urllib.error.URLError:
                # add output statements
                print('Error opening url: ' + line + 'HTTP Error 400:'
                                                     ' Bad Request')
                continue

            except UnicodeDecodeError:
                # add output statements
                print('Error opening url: ' + line + "'utf-8' codec can't"
                      " decode byte 0xc4 in position 10: invalid continuation"
                      " byte")
                continue

    #print(regex_string)
    #print(match_dict)
    outfile_name = ''.join([sys.argv[2], 'summary.txt'])
    with open(outfile_name, 'w', encoding='utf-8') as outfile:
        for val in match_dict:
            if match_dict[val]:
                outfile.write('Source url: ' + val + '\n\n')
                for match in match_dict[val]:
                    outfile.write(match + '\n')
                outfile.write('--------------------------------\n\n')


if __name__ == '__main__':
    main()
