USAGE="""

  Given a hybrid configuration index, generates UrbanSim inpput files.

  Input:  p10_plu_boc_allAttrs.csv, p10 combined with pba40 and basis boc data output by 1_PLU_BOC_data_combine.py
          hybrid configuration index indicating which variable/jurisdiction will use BASIS data verus PBA40 data

  Output: 

"""

import pandas as pd
import numpy as np
import argparse, os, logging, sys, time
import dev_capacity_calculation_module

today = time.strftime('%Y_%m_%d')

if os.getenv('USERNAME')    =='ywang':
    BOX_DIR                 = 'C:\\Users\\{}\\Box\\Modeling and Surveys\\Urban Modeling\\Bay Area UrbanSim\\PBA50'.format(os.getenv('USERNAME'))
    M_DIR                   = 'M:\\Data\\Urban\\BAUS\\PBA50'
    GITHUB_PETRALE_DIR      = 'C:\\Users\\{}\\Documents\\GitHub\\petrale\\'.format(os.getenv('USERNAME'))
elif os.getenv('USERNAME')  =='lzorn':
    BOX_DIR                 = 'C:\\Users\\{}\\Box\\Modeling and Surveys\\Urban Modeling\\Bay Area UrbanSim\\PBA50'.format(os.getenv('USERNAME'))
    M_DIR                   = 'M:\\Data\\Urban\\BAUS\\PBA50'
    GITHUB_PETRALE_DIR      = 'X:\\petrale'


# input file locations
PREV_ZONING_PARCELS_FILE    = os.path.join(M_DIR, 'Horizon', 'Large General Input Data', '2015_12_21_zoning_parcels.csv')
PLU_BOC_M_DIR               = os.path.join(M_DIR, 'Final_Blueprint', 'Base zoning', 'output')
PLU_BOC_FILE                = os.path.join(PLU_BOC_M_DIR, '2020_10_20_p10_plu_boc_allAttrs.csv')
HYBRID_INDEX_DIR            = os.path.join(GITHUB_PETRALE_DIR, 'policies\\plu\\hybrid_index')
PARCEL_UGB_TYPE_FILE        = os.path.join(M_DIR, 'Final_Blueprint', 'Large General Input Data', '2020_11_10_parcels_geography.csv')
# TODO: change to idx_urbansim.csv when we have one
HYBRID_INDEX_FILE           = os.path.join(HYBRID_INDEX_DIR, "idx_urbansim_fb_11052020.csv")

# output file locations
HYBRID_ZONING_OUTPUT_DIR    = os.path.join(BOX_DIR, 'Policies\\Base zoning\\outputs\\hybrid_base_zoning')
# In test mode (specified by --test), outputs to cwd and without date prefix; otherwise, outputs to HYBRID_ZONING_OUTPUT_DIR with date prefix
HYBRID_PARCELS_FILE         = 'p10_plu_boc_hybrid.csv'
LOG_FILE                    = "create_hybrid_urbansim_input.log"
# UrbanSim Inputs
ZONING_PARCELS_FILE         = 'zoning_parcels_hybrid_pba50.csv'
ZONING_LOOKUP_FILE          = 'zoning_lookup_hybrid_pba50.csv'


if __name__ == '__main__':

    parser = argparse.ArgumentParser(description=USAGE, formatter_class=argparse.RawDescriptionHelpFormatter,)
    parser.add_argument("--test", action="store_true", help="Test mode")
    args = parser.parse_args()

    if args.test == False:
        LOG_FILE            = os.path.join(HYBRID_ZONING_OUTPUT_DIR, "{}_{}".format(today, LOG_FILE))
        HYBRID_PARCELS_FILE = os.path.join(HYBRID_ZONING_OUTPUT_DIR, "{}_{}".format(today, HYBRID_PARCELS_FILE))
        ZONING_PARCELS_FILE = os.path.join(HYBRID_ZONING_OUTPUT_DIR, "{}_{}".format(today, ZONING_PARCELS_FILE))
        ZONING_LOOKUP_FILE  = os.path.join(HYBRID_ZONING_OUTPUT_DIR, "{}_{}".format(today, ZONING_LOOKUP_FILE))

    pd.set_option('max_columns',   200)
    pd.set_option('display.width', 200)

   # create logger
    logger = logging.getLogger(__name__)
    logger.setLevel('DEBUG')

    # console handler
    ch = logging.StreamHandler()
    ch.setLevel('INFO')
    ch.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p'))
    logger.addHandler(ch)
    # file handler
    fh = logging.FileHandler(LOG_FILE, mode='w')
    fh.setLevel('DEBUG')
    fh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p'))
    logger.addHandler(fh)

    plu_boc = pd.read_csv(PLU_BOC_FILE)
    logger.info("Read {} lines from {}".format(len(plu_boc), PLU_BOC_FILE))
    logger.debug("head:\n{}".format(plu_boc.head()))

    hybrid_idx = pd.read_csv(HYBRID_INDEX_FILE)
    logger.info("Read {} lines from {}".format(len(hybrid_idx), HYBRID_INDEX_FILE))
    logger.debug("head:\n{}".format(hybrid_idx.head()))

    # create the parcel data set with hybrid index attached
    plu_boc_hybrid = dev_capacity_calculation_module.create_hybrid_parcel_data_from_juris_idx(logger, plu_boc, hybrid_idx)
    plu_boc_hybrid = pd.merge(left=plu_boc, right=plu_boc_hybrid, how="left", on=['PARCEL_ID', 'juris_zmod'])
            
    # calculate 'allow_res' and 'allow_nonres' based on the allowable development type
    allowed_urbansim = dev_capacity_calculation_module.set_allow_dev_type(plu_boc_hybrid,'urbansim')
        
    plu_boc_hybrid = pd.merge(left = plu_boc_hybrid,
                              right= allowed_urbansim, 
                              on   = 'PARCEL_ID', 
                              how  = 'left')

    # force max_dua_urbansim to 0 to avoid over-estimating max_dua during imputation
    # two options: 
    #   option 1: for parcels outside of UGB and don't allow residential development, force max_dua_urbansim to 0
    #   option 2: for parcels that don't allow residential development, force max_dua_urbansim to 0
    parcel_UBG_type = pd.read_csv(PARCEL_UGB_TYPE_FILE,
                                  usecols = ['PARCEL_ID', 'fbp_exp2020_id'])
    logger.info('Read {} lines from {}'.format(len(parcel_UBG_type), PARCEL_UGB_TYPE_FILE))

    parcel_UBG_type['UGB_type'] = 'outside UGB'
    parcel_UBG_type.loc[(parcel_UBG_type.fbp_exp2020_id == 'in') | (parcel_UBG_type.fbp_exp2020_id == 'inun'), 
                        'UGB_type'] = 'inside UGB'
    logger.info(parcel_UBG_type.UGB_type.value_counts())

    plu_boc_hybrid = pd.merge(left = plu_boc_hybrid,
                              right= parcel_UBG_type,
                              on   = 'PARCEL_ID',
                              how  = 'left')
    # option 1
    # plu_boc_hybrid.loc[(plu_boc_hybrid.UGB_type == 'outside UGB') & (plu_boc_hybrid.allow_res_urbansim == 0),
    #                    'max_dua_urbansim'] = 0

    # option 2
    plu_boc_hybrid.loc[plu_boc_hybrid.allow_res_urbansim == 0,
                       'max_dua_urbansim'] = 0    

    logger.info('Saving hybrid zoning for {} parcels to {}'.format(len(plu_boc_hybrid),HYBRID_PARCELS_FILE))
    logger.debug('plu_boc_hybrid dtypes:\n{}'.format(plu_boc_hybrid.dtypes))
    plu_boc_hybrid.to_csv(HYBRID_PARCELS_FILE,index = False)

    logger.info('Create BAUS base zoning input files:')

    # select hybrid fields
    plu_boc_urbansim_cols = ['PARCEL_ID','geom_id','county_id','county_name', 'juris_zmod', 'jurisdiction_id', 'ACRES',
                             'fbpzoningm_zmod','nodev_zmod','name_pba40','plu_code_basis'] + \
                            ['{}_urbansim'.format(devType)       for devType   in dev_capacity_calculation_module.ALLOWED_BUILDING_TYPE_CODES] + \
                            ['max_{}_urbansim'.format(intensity) for intensity in dev_capacity_calculation_module.INTENSITY_CODES]

    plu_boc_urbansim = plu_boc_hybrid[plu_boc_urbansim_cols].copy()

    # rename the fields to remove '_urbansim'
    rename_cols = {}
    for col in plu_boc_urbansim_cols:
        if col.endswith('_urbansim'):
            rename_cols[col] = col[:-9]
    plu_boc_urbansim.rename(columns=rename_cols, inplace=True)

    # convert allowed types to integer
    for attr in dev_capacity_calculation_module.ALLOWED_BUILDING_TYPE_CODES:
        plu_boc_urbansim[attr] = plu_boc_urbansim[attr].fillna(-1).astype(int)
    plu_boc_urbansim.replace({-1: None}, inplace = True)

    # create zoning_lookup table with unique jurisdiction and zoning attributes
    zoning_lookup_pba50 = plu_boc_urbansim[['county_name','juris_zmod'] + \
      dev_capacity_calculation_module.ALLOWED_BUILDING_TYPE_CODES + \
      ['max_dua','max_far','max_height']].drop_duplicates()     

    # sort zoning type by county and jurisdiction and assign zoning_id
    zoning_lookup_pba50.sort_values(by=['county_name', 'juris_zmod'], inplace = True)
    zoning_lookup_pba50['zoning_id_pba50'] = range(1,len(zoning_lookup_pba50) + 1)
    logger.info('Zoning lookup table has {} unique types (juris + zoning attributes), header:'.format(len(zoning_lookup_pba50)))
    logger.info(zoning_lookup_pba50.head())
    
    # create zoning_parcels file and attach zoning_id 
    plu_boc_urbansim_ID = plu_boc_urbansim.merge(zoning_lookup_pba50,
                                                 on = list(zoning_lookup_pba50)[:-1],
                                                 how = 'left')
    zoning_parcels_pba50 = plu_boc_urbansim_ID[['PARCEL_ID','geom_id','juris_zmod','jurisdiction_id','zoning_id_pba50','nodev_zmod']]

    # bring into other attributes from previous parcel zoning file
    zoning_parcels_prev  = pd.read_csv(PREV_ZONING_PARCELS_FILE, usecols = ['geom_id','prop'])
    zoning_parcels_pba50 = zoning_parcels_pba50.merge(zoning_parcels_prev, on = 'geom_id', how = 'left')          


    ## assign zoning name to each zoning_id based on the most frequent occurance of zoning name among all the parcels with the same zoning_id
    zoning_names = plu_boc_urbansim[['PARCEL_ID','name_pba40','plu_code_basis']]

    # merge zoning names of pba40 and BASIS into zoning_parcels
    zoning_names['name_pba40'] = zoning_names['name_pba40'].apply(lambda x: str(x)+'_pba40')
    zoning_names['plu_code_basis'] = zoning_names['plu_code_basis'].apply(lambda x: str(x)+'_basis')
    parcel_zoning_names = zoning_parcels_pba50[['PARCEL_ID','zoning_id_pba50']].merge(zoning_names,
                                                                                      on = 'PARCEL_ID',
                                                                                      how = 'left')
    # use name_pba40 as the default for pab50 zoning name, unless it is null, then use basis zoning name
    parcel_zoning_names['zoning_name_pba50'] = zoning_names['name_pba40']
    name_null_index = parcel_zoning_names.name_pba40.isnull()
    parcel_zoning_names.loc[name_null_index,'zoning_name_pba50'] = parcel_zoning_names.loc[name_null_index,'plu_code_basis']

    # find the most frenquent zoning name of each zoning_id
    name_by_zone = parcel_zoning_names[['zoning_id_pba50','zoning_name_pba50']].groupby(['zoning_id_pba50']).agg(lambda x:x.value_counts().index[0]).reset_index()
    zoning_lookup_pba50 = zoning_lookup_pba50.merge(name_by_zone,
                                                    on = 'zoning_id_pba50',
                                                    how = 'left')
    # attach zoning name to the zoning lookup table
    zoning_lookup_pba50 = zoning_lookup_pba50[['zoning_id_pba50','juris_zmod','zoning_name_pba50','max_dua','max_far','max_height'] + \
                                               dev_capacity_calculation_module.ALLOWED_BUILDING_TYPE_CODES]

    # change field names to be consistent with the previous version
    zoning_lookup_pba50.rename(columns = {'zoning_id_pba50'  :'id',
                                          'juris_zmod'       :'juris',
                                          'zoning_name_pba50':'name'}, inplace = True)
    logger.info('zoning_lookup has {} unique zoning_ids; zoning_lookup table header:'.format(len(zoning_lookup_pba50)))
    logger.info(zoning_lookup_pba50.head())         

    # lastly, append zone name to zoning_parcel
    zoning_parcels_pba50 = zoning_parcels_pba50.merge(zoning_lookup_pba50[['id','name']], 
                                                      left_on = 'zoning_id_pba50', 
                                                      right_on = 'id',
                                                      how = 'left')
    # rename fields to be consistent with the model
    zoning_parcels_pba50.rename(columns = {'juris_zmod'     : 'juris_id',
                                           'zoning_id_pba50': 'zoning_id',
                                           'jurisdiction_id': 'juris',
                                           'nodev_zmod'     : 'nodev',
                                           'name'           : 'zoning'}, inplace = True)
    # remove lines with null geom_id
    logger.info('zoning_parcels_pba50 is length {} but has {} rows with null geom_id; removing'.format(
                len(zoning_parcels_pba50),
                len(zoning_parcels_pba50.loc[pd.isnull(zoning_parcels_pba50['geom_id'])])))
    zoning_parcels_pba50 = zoning_parcels_pba50.loc[pd.notnull(zoning_parcels_pba50['geom_id']),]
    logger.info('zoning_parcels_pba50 is length {}'.format(len(zoning_parcels_pba50)))
    
    # so we can convert geom_id to int
    zoning_parcels_pba50['geom_id'] = zoning_parcels_pba50['geom_id'].round(0).astype(np.int64)

    logger.info('zoning_parcels_pba50 has {} records; table head:\n{}'.format(len(zoning_parcels_pba50), zoning_parcels_pba50.head()))

    # export
    logger.info('Export zoning_lookup table with the following attributes: {}'.format(zoning_lookup_pba50.dtypes))
    zoning_parcels_pba50.to_csv(ZONING_PARCELS_FILE,index = False)   

    logger.info('Export zoning_parcels table with the following attributes: {}'.format(zoning_parcels_pba50.dtypes))
    zoning_lookup_pba50.to_csv(ZONING_LOOKUP_FILE,index = False)
