#!/usr/bin/env python

"""
Module which prepares data for the database.
"""

import csv
import os
import re
import time
import json
import structlog

def get_csv_names(location, logger=structlog.get_logger()):
    """
    Get names of all csv files in given directory which was added in last 10 minutes
    :param location: path to the directory
    :param logger: logger for the method
    :return: array with csv names
    """
    list_csv = []
    for files in os.listdir(location):
        if files.endswith("masaryk_university-ip.csv"):
            if time.time() - os.path.getmtime(location + files) < 1200:
                list_csv.append(files)
                logger.info(f'Adding {files}')
    return list_csv

def get_expected_info(csv_file, location, logger=structlog.get_logger()):
    """
    Parse expected lines from file specified by name as parameter
    :param location: path to csv file
    :param csv_file: name of csv to parse
    :param logger: logger for the method
    :return: ip, timestamp and description in form of dictionary
    """
    data_dict = []
    vulnerability = get_vulnerability(csv_file, logger)
    with open(location + csv_file, 'r') as file:
        reader = csv.DictReader(file, delimiter=',')
        for row in reader:
            try:
                description = ""
                for i in row:
                    description += f'{i}: {row[i]}, '
                if vulnerability == "sinkhole":
                    result = {'ip': row["ip"], 'timestamp': change_datetime_format(row["timestamp"]),
                              'description': description[:-2], 'vulnerability': row["type"]}
                elif vulnerability == "botnet":
                    result = {'ip': row["ip"], 'timestamp': change_datetime_format(row["timestamp"]),
                              'description': description[:-2], 'vulnerability': row["infection"]}
                else:
                    result = {'ip': row["ip"], 'timestamp': change_datetime_format(row["timestamp"]),
                              'description': description[:-2], 'vulnerability': vulnerability}
                data_dict.append(result)
            except AttributeError:
                logger.warning(f'Skipping {row["ip"]} due to invalid timestamp format {row["timestamp"]}')
            except KeyError as key_error:
                logger.warning(f'Skipping  {row["ip"]}: Invalid format', error_message=key_error)
            except csv.Error as csv_error:
                logger.warning(f'Skipping  {row["ip"]}: csv.ERROR', error_message=csv_error)
    return data_dict

def change_datetime_format(timestamp):
    """
    Change format of timestamp to be in valid format for neo4j datetime function.
    :param timestamp: datetime which will be changed
    :return: Valid datetime string
    """
    pattern = r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} *$'
    if re.match(pattern, timestamp) is None:
        raise AttributeError('Invalid input for change_datetime_format() function')
    valid = timestamp.replace(" ", "T", 1)
    valid += "Z"
    valid = valid.replace(" ", "")
    pattern2 = r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$'
    if re.match(pattern2, valid) is None:
        raise AttributeError('Invalid output of change_datetime_format() function')
    return valid

def get_vulnerability(vulnerability_name_file, logger=structlog.get_logger()):
    """
    Parse name of threat from given parameter(file)
    :param vulnerability_name_file: string to parse from
    :param logger: logger for the method
    :return: name of threat
    """
    pattern = r'^\d{4}-\d{2}-\d{2}-(scan_){0,1}[a-zA-Z-_]*-masaryk_university-ip\.csv$'
    if re.match(pattern, vulnerability_name_file) is None:
        logger.error(f'Invalid name of Shadowserver file. Probably structure has been changed. Filename: {vulnerability_name_file}')
        raise AttributeError('Invalid name of Shadowserver file. Probably structure has been changed.')
    name = vulnerability_name_file
    if "sinkhole" in name:
        return "sinkhole"
    if "botnet" in name:
        return "botnet"
    name = re.sub('-masaryk_university-ip.csv', '', name)
    name = name[11:]
    if "scan_" in name:
        name = re.sub('scan_', '', name)
    return name

def update_dict(dictionary, key, values):
    """
    Adds new key and values to dictionary
    :param dictionary: dictionary which will be updated
    :param key: key for dictionary
    :param values: values which corresponds to key
    :return: None
    """
    if key not in dictionary:
        dictionary[key] = None
    for value in values:
        if isinstance(dictionary[key], list):
            dictionary[key].append(value)
        else:
            dictionary[key] = [dictionary[key], value]

def prepare_json(path, saving_path, name, logger=structlog.get_logger()):
    """
    Create json with vulnerabilities, IPs timestamps and descriptions...
    :param path: path to the location with csv files
    :param saving_path: where created json will be stored
    :param name: how the file should be called
    :param logger: logger for the method
    :return: Statistics about created json
    """
    csvlist = get_csv_names(path)
    number_of_vulnerabilities = 0
    dist = {"detection_system": "Shadowserver"}
    for csvfile in csvlist:
        try:
            info = get_expected_info(csvfile, path)
            update_dict(dist, 'results', info)
            number_of_vulnerabilities += 1
        except IOError:
            logger.warning(f'{csvfile} was not processed.')
    with open(f'{saving_path}{name}', 'w') as out_file:
        out_file.write(json.dumps(dist, indent=2, sort_keys=True))
    return get_stats(dist, number_of_vulnerabilities)

def get_stats(dist, number_of_vulnerabilities):
    """
    Get statistics about number of entries in dictionary
    Create meaningful output
    :param dist: reviewed dictionary
    :param number_of_vulnerabilities: number of reviewed vulnerabilities
    :return: Statistics in printable form
    """

    result = "Statistics: "

    number_of_cases = 0

    for values in dist.values():
        number_of_cases += len(values)

    if number_of_vulnerabilities > 1:
        result += str(number_of_vulnerabilities)
        result += " types of vulnerabilities were reviewed, "
    else:
        return result + " 0 cases discovered, nothing to do"

    if number_of_cases > 0:
        result += str(number_of_cases)
        result += " security events were written to the output file."
    else:
        result += "nothing was written to the output file."

    return result
