import sys
import pandas as pd
from sqlalchemy import create_engine


def load_data(messages_filepath, categories_filepath):
    '''
    Description:
        Loads data from csv files in specified locations and merges the datasets
        
    Params:
        messages_filepath (String): Filepath to messages csv file
        categories_filepath (String): Filepath to categorical variable csv file
    Returns:
        df (Dataframe): Dataframe of merged messages data with categorical encodings
     '''
    messages = pd.read_csv(messages_filepath)
    categories = pd.read_csv(categories_filepath)
    
    # merge datasets
    df = messages.merge(categories, on = 'id')    
    
    return df


def clean_data(df):
    '''
    Description:
        Cleans up dataframe for use. Transforms categories data into clean integer representations expanded into columns, and names columns appropriately.
        
    Params:
        df (Dataframe): Original dataframe
        
    Returns:
        df (Dataframe): Dataframe with categorical variables expanded into columns
    '''
    
    # create a dataframe of the 36 individual category columns
    categories = df.categories.str.split(';', expand = True)
    
    # select the first row of the categories dataframe
    row = categories.iloc[0]

    # use this row to extract a list of new column names for categories.
    # one way is to apply a lambda function that takes everything 
    # up to the second to last character of each string with slicing
    category_colnames = row.apply(lambda x: x[:-2])
    
    # rename the columns of `categories`
    categories.columns = category_colnames
    
    for column in categories:
        # set each value to be the last character of the string
        categories[column] = categories[column].apply(lambda x: x[-1])

        # convert column from string to numeric
        categories[column] = categories[column].astype(int)
        
    # drop the original categories column from `df`
    df.drop(columns = ['categories'], inplace = True)
    
    # concatenate the original dataframe with the new `categories` dataframe
    df = pd.concat([df, categories], axis = 1)

    
    # drop duplicates
    df.drop_duplicates(inplace = True)

    # Only keeping columns necessary for modeling
    df.drop(columns = ['id', 'original','genre'], inplace = True)

    # Only keeping rows where related values are either 0 or 1
    df = df[df.related != 2]
    
    return df


def save_data(df, database_filename):
    '''
    Description:
        Saves data into database file using sqlalchemy and pandas functionality
    
    Params:
        df (Dataframe): Dataframe to be saved
        database_filename (String): File path and name where database file will be stored
    
    Returns:
        None
    '''
    engine = create_engine('sqlite:///{}'.format(database_filename))
    df.to_sql(database_filename, engine, index=False, if_exists = 'replace') 
    print(df.shape)
    print(engine.table_names())


def main():
    if len(sys.argv) == 4:

        messages_filepath, categories_filepath, database_filepath = sys.argv[1:]

        print('Loading data...\n    MESSAGES: {}\n    CATEGORIES: {}'
              .format(messages_filepath, categories_filepath))
        df = load_data(messages_filepath, categories_filepath)

        print('Cleaning data...')
        df = clean_data(df)
        
        print('Saving data...\n    DATABASE: {}'.format(database_filepath))
        save_data(df, database_filepath)
        
        print('Cleaned data saved to database!')
        
    
    else:
        print('Please provide the filepaths of the messages and categories '\
              'datasets as the first and second argument respectively, as '\
              'well as the filepath of the database to save the cleaned data '\
              'to as the third argument. \n\nExample: python process_data.py '\
              'disaster_messages.csv disaster_categories.csv '\
              'DisasterResponse.db')


if __name__ == '__main__':
    main()