# -*- coding: utf-8 -*-
"""FileFactory module containing base and subclasses
TODOs:
    Finish documentation
    Build tests
    Possibly refactor for better testing

"""

from abc import ABCMeta, abstractmethod
from string import digits
import ConfigParser
import logging.config

from xlrd.xldate import xldate_as_datetime
import editdistance
import numpy as np
import pandas as pd

import constants

config = ConfigParser.RawConfigParser()
config.read('/etc/calfresh/calfresh.conf')

logging.config.fileConfig(config.get('filepaths', 'config'))
logger = logging.getLogger('file_factory')


def initialize(item):
    """Initializes the proper FileFactory object based on the source directory
    of the item

    Args:
        item (dict): object with path, source and filename keys

    Returns:
        FileFactory: subclass corresponding to the source of the item

    Raises:
        ValueError: If the source of the item is unrecognized

    """
    if item['source'] == 'tbl_cf296':
        return CF296Factory(item)
    if item['source'] == 'tbl_churn_data':
        return ChurnDataFactory(item)
    if item['filename'] == 'CFDashboard-Annual.csv':
        return DataDashboardAnnualFactory(item)
    if item['filename'] == 'CFDashboard-Quarterly.csv':
        return DataDashboardQuarterlyFactory(item)
    if item['filename'] == 'CFDashboard-Every_Mth.csv':
        return DataDashboardMonthlyFactory(item)
    if item['filename'] == 'CFDashboard-Every_3_Mth.csv':
        return DataDashboard3MthFactory(item)
    if item['filename'] == 'CFDashboard-PRI_Raw.csv':
        return DataDashboardPRIRawFactory(item)
    if item['source'] == 'tbl_dfa256':
        return DFA256Factory(item)
    if item['source'] == 'tbl_dfa296x':
        return DFA296XFactory(item)
    if item['source'] == 'tbl_dfa358f':
        return DFA358FFactory(item)
    if item['source'] == 'tbl_dfa358s':
        return DFA358SFactory(item)
    if item['source'] == 'tbl_stat47':
        return Stat47Factory(item)

    raise ValueError


class FileFactory(object):
    """Base class for all the file factories

    Attributes:
        df (pandas DataFrame): the csv file passed in converted to a \
        pd.DataFrame object
        filename (str): name of the csv file
        constants (obj constants): an object with lists of column names to \
         label the factories' df attributes and a dict of county \
         keys and values
        year (int): the year (yyyy) pertaining to the data within the factory
        quarter (int) the quarter pertaining to the data within the factory
        month (str): the month (MMM) pertaining to the data within the factory

    """
    __metaclass__ = ABCMeta

    def __init__(self, item):
        self.filename = item['filename']
        self.df = pd.read_csv(item['path'])
        if self.df.empty:
            raise ValueError
        super(FileFactory, self).__init__()

    def __str__(self):
        """Returns the first ten rows of the df attribute"""
        return str(self.df.head(10))

    def build(self):
        self.trim_bogus_rows()
        self.trim_bogus_columns()

        self.check_counties()

        self.build_specific()

        self.df = self.df.fillna(value='\N')

        self.add_quarter()
        self.add_full_date()

    @abstractmethod
    def build_specific(self):
        """Process class specific info, such as column names, year, month, etc."""
        return

    def add_full_date(self):
        """Convert the month and year to valid datetime"""
        self.df['fulldate'] = self.df.month + '/' + self.df.year.map(str)
        self.df.fulldate = pd.to_datetime(self.df.fulldate)

    def add_year(self, year):
        """Add the year to the df

        Args:
            year (str): is passed in as a slice of the filename

        Sets:
            df['year'] (pandas Series): creates the year column
             and sets its value

        Raises:
            ValueError: If the year is earlier than 2002 or later than 2019

        """
        if int('20' + year) > 2019 or int('20' + year) < 2002:
            logging.error('Bad year value: %s', year)
            raise ValueError

        self.df['year'] = int('20' + year)

    def add_month(self, month):
        """Month is passed in as a string or slice of the filename"""
        self.df['month'] = month.upper()

    def add_quarter(self):
        """Map the month to the quarter"""
        quarters = {
            'JAN': 1.0,
            'FEB': 1.0,
            'MAR': 1.0,
            'APR': 2.0,
            'MAY': 2.0,
            'JUN': 2.0,
            'JUL': 3.0,
            'AUG': 3.0,
            'SEP': 3.0,
            'OCT': 4.0,
            'NOV': 4.0,
            'DEC': 4.0,
        }

        self.df['quarter'] = self.df.month.map(quarters)

    def check_numbers(self, startCol=1, endCol=None):
        """Check the type of all values in the input columns

        Args:
            startCol (int): the first column to start looking for, enforcing numerics

        Outputs:
            A squeaky clean set of columns filled with numeric or None values

        """
        for col in self.df.columns[startCol:endCol]:
            i = 0
            for row in self.df[col]:
                self.df.loc[i, col] = self._get_valid_number(row)
                i += 1

    def _get_valid_number(self, num):
        """Extract a number from the input arg or return None

        Args:
            num (str): a value from a cell assumed to contain numeric values

        Returns:
            float or None: float if there were numeric values in the arg,
            or None if it was just junk

        """
        if num is None:
            return np.nan

        try:
            return float(num)
        except ValueError:
            return self._convert_to_number(num)

    def _convert_to_number(self, num):
        """Remove all non-numeric characters from the input

        Args:
            num (str): a string representation of a number

        Returns:
            float stripped of all non-numerics or None

        """
        if num is None:
            return np.nan

        temp = num
        for c in temp:
            if c not in digits + '.':
                num = num.replace(c, '')

        try:
            return float(num)
        except ValueError:
            return np.nan

    def check_percents(self, cols):
        """Check and standardize all percentages to fall between -1.00 and 1.00

        Args:
            cols (list of str): columns that should contain percent values

        """
        for col in cols:
            if col in self.df.columns:
                i = 0
                for row in self.df[col]:
                    if row > 1.0 or row < -1.0:
                        self.df.loc[i, col] = row / 100.0
                    i += 1

    def check_counties(self, col=0):
        """Make sure all counties are present

        This function checks that all counties are there and if any are misspelled
        it tries to identify which county it might be

        Args:
            col (int): the column to scan for county names

        Raises:
            ValueError if after its best effort a county is still missing

        """
        # get the string value of the column name
        col = self.df.columns[col]

        self._clean_county_names(col)

        self.df[col] = self.df[col].replace({'Statewide': 'California'})

        self._trim_noncounty_rows(col)
        constants.county_set.discard('Statewide')  # since we just changed it to Cali'
        # make sure all the counties are present
        observed = set(self.df[col].values)
        if not observed == constants.county_set:
            logging.error('Observed: %s', str(observed))
            logging.error(
                'Counties not found: %s',
                str(constants.county_set.difference(observed))
            )
            raise ValueError

    def _trim_noncounty_rows(self, col):
        """Remove any blank row from the county column

        Args:
            col (str): the column with observable county names

        """
        self.df = self.df.dropna(subset=[col]).reset_index(drop=True)

    def _clean_county_names(self, col):
        """Fill each cell with a valid county name or None

        Args:
            col (str): the column with observable county names

        """
        self.df[col] = self.df[col].str.strip()
        i = 0
        for county in self.df[col]:
            if county not in constants.county_set:
                if type(county) == str:
                    county = county.replace(' ', '')
                    closest = self._get_closest_spelled_county(county)
                    self.df.loc[i, col] = closest
                else:
                    self.df.loc[i, col] = np.nan
            i += 1

    def _get_nearest_spelled_counties(self, county):
        """Get county names with the shortest edit distance to the county arg

        Args:
            county (str): the invalid county name

        Returns:
            A dict of county names whose edit distance is less than 3 from the arg

        """
        vals = {}
        for key in constants.county_dict.keys():
            county = str(county)
            # Less than three is best since some counties only have four letters
            if len(county) < (len(key) + 3) and len(county) > (len(key) - 3):
                vals[key] = editdistance.eval(key, county)

        return vals

    def _get_closest_spelled_county(self, county):
        """Get the county with the smallest edit distance or None

        Args:
            county (str): invalid county name to replace

        Returns:
            A valid county name or None if there were none within 3 edits

        """
        potentials = self._get_nearest_spelled_counties(county)
        if not potentials:
            return np.nan

        closest = min(potentials, key=potentials.get)
        # Needs to have fewer than three edits made
        if potentials[closest] < 3L:
            return constants.county_dict[closest]
        else:
            return np.nan

    def trim_bogus_columns(self):
        """Drop columns off the end of the table with more than a quarter empty rows"""
        rowcount = self.df.shape[0] / 4
        while self.df[self.df.columns[-1]].isnull().sum() > rowcount:
            self.df.drop(self.df.columns[-1], axis=1, inplace=True)

    def trim_bogus_rows(self):
        """Drop rows off the bottom of the table with more than half empty columns"""
        colcount = self.df.shape[1] / 2
        while self.df.iloc[-1].isnull().sum() > colcount:
            self.df.drop(self.df.index[-1], inplace=True)


class CF296Factory(FileFactory):

    def build(self):
        # drop columns with data we don't need
        self.df.drop(
            [
                self.df.columns[0],
                self.df.columns[2],
                self.df.columns[3],
                self.df.columns[4],
                self.df.columns[5],
            ],
            axis=1,
            inplace=True,
        )
        super(CF296Factory, self).build()

    def build_specific(self):
        self.check_numbers()
        # dates in this column come in excel number format
        date_info = [
            xldate_as_datetime(xldate, 0) for xldate in self.df[self.df.columns[1]]
        ]

        self.df['year'] = [pydate.strftime('%Y').upper() for pydate in date_info]
        self.df['month'] = [pydate.strftime('%b').upper() for pydate in date_info]

        self.df.drop(self.df.columns[1], axis=1, inplace=True)
        self.df.columns = constants.CF296Columns


class ChurnDataFactory(FileFactory):

    def build_specific(self):
        self.check_numbers()
        if self.filename[0] in digits:
            self.add_year(self.filename[2:4])
        else:
            self.add_year(self.filename[4:6])

        Q1 = ['Q1', 'JAN']
        Q2 = ['Q2', 'APRIL']
        Q3 = ['Q3', 'JUL']
        Q4 = ['Q4', 'OCT']

        if any(indicator in self.filename for indicator in Q1):
            self.add_month('MAR')
        elif any(indicator in self.filename for indicator in Q2):
            self.add_month('JUN')
        elif any(indicator in self.filename for indicator in Q3):
            self.add_month('SEP')
        elif any(indicator in self.filename for indicator in Q4):
            self.add_month('DEC')

        self.df.columns = constants.ChurnDataColumns

        self.check_percents(constants.ChurnDataPercentColumns)

        self.add_additional_percentages()

    def add_additional_percentages(self):
        """These are figures we precompute for a better user experience"""
        self.df['pct_apps_rcvd_from_this_county'] = 0.0
        total = self.df['snap_apps_rcvd'].where(self.df['county'] == 'California')[0]
        self.df['pct_apps_rcvd_from_this_county'] = self.df['snap_apps_rcvd'] / total

        self.df['pct_cases_sched_recert_this_county'] = 0.0
        total = self.df['cases_sched_recert'].where(self.df['county'] == 'California')[0]
        self.df['pct_cases_sched_recert_this_county'] = \
            self.df['cases_sched_recert'] / total


class DataDashboardAnnualFactory(FileFactory):

    def build_specific(self):
        self.check_numbers(startCol=4)

        self.add_month('DEC')

        self.df.columns = constants.DataDashboardAnnualColumns

        self.df.year = pd.to_numeric(self.df.year, downcast='integer')

        self.check_percents(constants.DataDashboardPercentColumns)


class DataDashboardQuarterlyFactory(FileFactory):

    def build_specific(self):
        self.check_numbers(startCol=3)

        self.df.columns = constants.DataDashboardQuarterlyColumns

        self.df.year = pd.to_numeric(self.df.year, downcast='integer')
        self.df['month'] = self.df.quarter.str[:3].str.upper()

        self.check_percents(constants.DataDashboardPercentColumns)


class DataDashboardMonthlyFactory(FileFactory):

    def build_specific(self):
        self.check_numbers(startCol=6)

        self.df.columns = constants.DataDashboardMonthlyColumns

        self.df.year = pd.to_numeric(self.df.year, downcast='integer')
        self.df.month = self.df.month.str[:3].str.upper()

        self.check_percents(constants.DataDashboardPercentColumns)


class DataDashboard3MthFactory(FileFactory):

    def build_specific(self):
        self.check_numbers(startCol=3)

        self.df.columns = constants.DataDashboard3MthColumns

        self.df.year = pd.to_numeric(self.df.year, downcast='integer')
        self.df.month = self.df.month.str[:3].str.upper()

        self.check_percents(constants.DataDashboardPercentColumns)


class DataDashboardPRIRawFactory(FileFactory):

    def build_specific(self):
        self.check_numbers(startCol=7)
        # This file is only updated at the end of the year
        self.add_month('DEC')

        self.df.columns = constants.DataDashboardPRIRawColumns

        self.df.year = pd.to_numeric(self.df.year, downcast='integer')

        self.check_percents(constants.DataDashboardPercentColumns)


class DFA256Factory(FileFactory):

    def build(self):
        # drop columns with data we don't need
        self.df.drop(
            [
                self.df.columns[0],
                self.df.columns[2],
                self.df.columns[3],
                self.df.columns[4],
                self.df.columns[5],
            ],
            axis=1,
            inplace=True,
        )
        super(DFA256Factory, self).build()

    def build_specific(self):
        self.check_numbers()
        # dates in this column come in excel number format
        date_info = [
            xldate_as_datetime(xldate, 0) for xldate in self.df[self.df.columns[1]]
        ]

        self.df['year'] = [pydate.strftime('%Y').upper() for pydate in date_info]
        self.df['month'] = [pydate.strftime('%b').upper() for pydate in date_info]

        self.df.drop(self.df.columns[1], axis=1, inplace=True)
        # some logic for determining columns in the file based on date follows...
        if self.df.year.unique()[0] == 2002 or \
                (self.df.year.unique()[0] == 2003 and
                    self.df.month.unique()[0] in ['JAN', 'FEB', 'MAR']):
            self.df.columns = constants.DFA256Columns1

        elif self.df.year.unique()[0] == 2003 and \
                self.df.month.unique()[0] in \
                ['APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT']:
            self.df.columns = constants.DFA256Columns2

        else:
            self.df.columns = constants.DFA256Columns3
        # we precompute this for ease of user analysis
        self.df['total_households'] = (
            self.df.num_hh_pub_asst_fed +
            self.df.num_hh_pub_asst_fed_st +
            self.df.num_hh_pub_asst_st +
            self.df.num_hh_nonpub_asst_fed +
            self.df.num_hh_nonpub_asst_fed_st +
            self.df.num_hh_nonpub_asst_st
        )
        # this is also something we precompute for users
        self.df['big_six'] = 0
        self.df = self.df.sort_values(by='total_households', ascending=False)
        self.df = self.df.reset_index(drop=True)
        i = 1
        while i < 7:
            self.df.at[i, 'big_six'] = 1
            i += 1


class DFA296XFactory(FileFactory):

    def build_specific(self):
        self.check_numbers()
        self.add_year(self.filename[-6:-4])
        self.add_month(self.filename[-13:-10])
        # some logic for determining columns in the file based on date follows...
        if self.df.year.unique()[0] < 2004 or \
                (self.df.year.unique()[0] == 2004 and
                    self.df.month.unique()[0] in ['JAN', 'APR', 'JUL']):
            self.df.columns = constants.DFA296XColumns1

        elif (self.df.year.unique()[0] == 2004 and
                self.df.month.unique()[0] == 'OCT') or \
                (self.df.year.unique()[0] in
                    [2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012])\
                or (self.df.year.unique()[0] == 2013 and
                    self.df.month.unique()[0] in ['JAN', 'APR']):
            self.df.columns = constants.DFA296XColumns2

        else:
            self.df.columns = constants.DFA296XColumns3


class DFA358FFactory(FileFactory):

    def build_specific(self):
        self.check_numbers()
        self.add_year(self.filename[-6:-4])
        self.add_month(self.filename[-9:-6])

        if self.df.year.unique()[0] < 2007:
            self.df.columns = constants.DFA358Columns1
        else:
            self.df.columns = constants.DFA358Columns2


class DFA358SFactory(DFA358FFactory):
    def __init__(self, item):
        super(DFA358SFactory, self).__init__(item)


class Stat47Factory(FileFactory):

    def build_specific(self):
        self.check_numbers()
        self.add_year(self.filename[25:27])
        self.add_month(self.filename[18:21])

        if '(Items 1-14)' in self.filename:
            self.df.columns = constants.Stat47Columns1
        else:
            self.df.columns = constants.Stat47Columns2
