import pandas as pd
import os
import data_paths
import numpy as np
from collections import Counter
import jenkspy
import statsmodels.api as sm

from content_analysis.traffic_weibo import accident_traffic_word_set, congestion_traffic_word_set
from utils import get_traffic_dataframes, create_weibo_count_table, count_positive_neutral_negative
from visualizations import kde_plot, sentiment_against_density_plot


def create_hotspot_dataframe(path: str, search_string: str) -> pd.DataFrame:
    """
    Create the Weibo dataframe for one type of traffic hotspots
    :param path: a local path saving the filtered Weibos posted in hotspots
    :param search_string: a string you want to use to find the txt file
    :return: a pandas dataframe containing all the Weibos posted in one type of traffic hotspot
    """
    dataframe_list = []
    for file in os.listdir(path):
        if file.endswith('.csv') and search_string in file:
            print("Coping with the file: {}".format(file))
            dataframe = pd.read_csv(os.path.join(path, file), index_col=0, encoding='utf-8')
            dataframe_list.append(dataframe)
    concat_data = pd.concat(dataframe_list, axis=0)
    # Reformat the column names generated by ArcGIS
    if 'traffic_weibo' in concat_data:
        renamed_data = concat_data.copy()
    else:
        select_columns = ['index_val', 'author_id', 'weibo_id', 'created_at', 'text', 'lat', 'lon', 'retweeters',
                          'retweete_1', 'local_time', 'year', 'month', 'traffic_we', 'traffic_re', 'sent_weibo',
                          'sent_repos', 'Name', 'datatype', 'traffic_ty', 'hotspot_id']
        rename_dict = {'traffic_we': 'traffic_weibo', 'traffic_re': 'traffic_repost', 'retweete_1': 'retweeters_text'}
        renamed_data = concat_data[select_columns].rename(columns=rename_dict).reset_index(drop=True)
    return renamed_data


def count_traffic_weibos_in_districts(dataframe: pd.DataFrame, district_name_column: str):
    """
    Count the number of traffic Weibos across districts
    :param dataframe: a traffic Weibo dataframe, which has a column saving the district location of each row
    :param district_name_column: the column name saving the district name
    :return: the dataframe saving the number of traffic Weibos found in each district
    """
    # Reformat the column names generated by ArcGIS
    if 'traffic_weibo' in dataframe:
        renamed_data = dataframe.copy()
    else:
        select_columns = ['index_val', 'author_id', 'weibo_id', 'created_at', 'text', 'lat', 'lon', 'retweeters',
                          'retweete_1', 'local_time', 'year', 'month', 'traffic_we', 'traffic_re', 'sent_weibo',
                          'sent_repos', 'Name', 'datatype', 'traffic_ty']
        rename_dict = {'traffic_we': 'traffic_weibo', 'traffic_re': 'traffic_repost', 'retweete_1': 'retweeters_text'}
        renamed_data = dataframe[select_columns].rename(columns=rename_dict)
    # Count the number of Weibos in each district
    district_counter = {}
    for district_name, district_dataframe in renamed_data.groupby(district_name_column):
        pos_count, neutral_count, neg_count = count_positive_neutral_negative(district_dataframe,
                                                                              repost_column='retweeters_text')
        district_counter[district_name] = pos_count + neutral_count + neg_count
    print(district_counter)
    district_name_list = list(district_counter.keys())
    result_dataframe = pd.DataFrame()
    result_dataframe['districts'] = district_name_list
    result_dataframe['count'] = [district_counter[name] for name in district_name_list]
    result_dataframe_sorted = result_dataframe.sort_values(by='count', ascending=False).reset_index(drop=True)
    return result_dataframe_sorted


def process_geocoded_data(data: pd.DataFrame, datatype_label: str, saved_filename: str) -> pd.DataFrame:
    """
    Get the cleaned geocoded data after ArcMap process
    :param data: a geocoded Weibo dataframe after ArcMap processing
    :param datatype_label: a string label used to label some tweet data
    :param saved_filename: the filename used to save to the local directory
    :return: a cleaned geocoded dataframe
    """
    # Reformat the column names generated by ArcGIS
    select_columns = ['index_val', 'author_id', 'weibo_id', 'created_at', 'text', 'lat', 'lon', 'retweeters',
                      'retweete_1', 'retweets_i', 'local_time', 'year', 'month', 'traffic_we', 'traffic_re', 'sent_weibo',
                      'sent_repos', 'Name']
    rename_dict = {'traffic_we': 'traffic_weibo', 'traffic_re': 'traffic_repost', 'retweete_1': 'retweeters_text',
                   'retweets_i': 'retweet_ids'}
    renamed_data = data[select_columns].rename(columns=rename_dict)
    renamed_data_copy = renamed_data.copy()
    renamed_data_copy['datatype'] = [datatype_label]*renamed_data_copy.shape[0]
    renamed_data_copy.to_csv(os.path.join(data_paths.weibo_data_path, saved_filename), encoding='utf-8')
    return renamed_data_copy


def process_nongeocoded_data(data: pd.DataFrame, datatype_label: str, saved_filename: str) -> pd.DataFrame:
    """
    Get the cleaned nongeocoded data after ArcMap process
    :param data: a nongeocoded Weibo dataframe after ArcMap processing
    :param datatype_label: a string label some of the datatype
    :param saved_filename: the filename used to save to the local directory
    :return: a cleaned nongeocoded dataframe
    """
    # Reformat the column names generated by ArcGIS
    select_columns = ['index_val', 'author_id', 'weibo_id', 'created_at', 'text', 'loc_lat', 'loc_lon', 'retweeters',
                      'retweete_1', 'retweets_i', 'local_time', 'year', 'month', 'traffic_we', 'traffic_re', 'sent_weibo',
                      'sent_repos', 'Name']
    rename_dict = {'traffic_we': 'traffic_weibo', 'traffic_re': 'traffic_repost', 'retweete_1': 'retweeters_text',
                   'loc_lat': 'lat', 'loc_lon': 'lon', 'retweets_i': 'retweet_ids'}
    renamed_data = data[select_columns].rename(columns=rename_dict)
    renamed_data_copy = renamed_data.copy()
    renamed_data_copy['datatype'] = [datatype_label]*renamed_data_copy.shape[0]
    renamed_data_copy['retweet_ids'] = renamed_data_copy['retweet_ids'].astype(str)
    renamed_data_copy.to_csv(os.path.join(data_paths.weibo_data_path, saved_filename), encoding='utf-8')
    return renamed_data_copy


def process_data_with_density_classes(dataframe: pd.DataFrame, traffic_type: str):
    """
    Get the weibo dataframe with different density values
    :param dataframe: a pandas dataframe saving traffic relevant Weibos in Shanghai with density classes
    :param traffic_type: the type of traffic information considered in this function
    :return: a python dict saving the pandas dataframe for each density class
    """
    assert 'GRIDCODE' in dataframe, "The dataframe should contain a column named GRIDCODE, saving the density class"
    for density_val, weibo_data in dataframe.groupby('GRIDCODE'):
        if density_val != 0:
            weibo_data.to_csv(os.path.join(data_paths.hotspot_text_path,
                                           '{}_density_{}.csv'.format(traffic_type, density_val)))


def count_traffic_in_dataframe(dataframe: pd.DataFrame):
    """
    Count the number of Weibos in each type
    :param dataframe: a Weibo pandas dataframe
    :return: None. The number of tweets we have got for each type of traffic relevant information
    """
    traffic_geo_count = 0
    traffic_geo_have_loc = 0
    traffic_geo_not_have_loc = 0
    traffic_non_geo_count = 0
    traffic_non_geo_have_loc = 0
    traffic_non_geo_not_have_loc = 0

    geocoded_weibo, geocoded_repost, nongeocoded_weibo, nongeocoded_repost = get_traffic_dataframes(dataframe)
    geocoded_weibo_counter = Counter(geocoded_weibo['traffic_weibo'])
    geocoded_repost_counter = Counter(geocoded_repost['traffic_repost'])
    nongeocoded_weibo_counter = Counter(nongeocoded_weibo['traffic_weibo'])
    nongeocoded_repost_counter = Counter(nongeocoded_repost['traffic_repost'])
    traffic_geo_count += (geocoded_weibo.shape[0] + geocoded_repost.shape[0])
    traffic_geo_have_loc += (geocoded_weibo_counter.get(2, 0) + geocoded_repost_counter.get(2, 0))
    traffic_geo_not_have_loc += (geocoded_weibo_counter.get(1, 0) + geocoded_weibo_counter.get(0, 0))
    traffic_geo_not_have_loc += (geocoded_repost_counter.get(1, 0) + geocoded_repost_counter.get(0, 0))
    traffic_non_geo_count += (nongeocoded_weibo.shape[0] + nongeocoded_repost.shape[0])
    traffic_non_geo_have_loc += (nongeocoded_weibo_counter.get(2, 0) + nongeocoded_repost_counter.get(2, 0))
    traffic_non_geo_not_have_loc += (nongeocoded_weibo_counter.get(1, 0) + nongeocoded_weibo_counter.get(0, 0))
    traffic_non_geo_not_have_loc += (nongeocoded_repost_counter.get(1, 0) + nongeocoded_repost_counter.get(0, 0))
    geocoded_considered_count = geocoded_weibo.shape[0] + geocoded_repost.shape[0]
    nongeocoded_considered_count = traffic_non_geo_have_loc

    print('In total, we have {} traffic relevant Weibos'.format(geocoded_considered_count+nongeocoded_considered_count))
    print(traffic_non_geo_not_have_loc)


def get_kde_weibo_for_arcmap(geo_data, nongeo_weibo_data, nongeo_repost_data) -> pd.DataFrame:
    """
    Get the combined weibo dataframe for social media based kde processing (find traffic hotspots)
    :param geo_data: the Weibo data with geo information (latitude, longitude)
    :param nongeo_weibo_data: the nongeocoded traffic Weibo data
    :param nongeo_repost_data: the nongeocoded traffic Repost data
    :return: a combined traffic Weibo dataframe
    """
    processed_geo = process_geocoded_data(geo_data, datatype_label='geocoded',
                                          saved_filename='geocoded_traffic_shanghai.csv')
    processed_nongeo_weibo = process_nongeocoded_data(nongeo_weibo_data, datatype_label='nongeo_weibo',
                                                      saved_filename='nongeocoded_weibo_traffic_shanghai.csv')
    processed_nongeo_repost = process_nongeocoded_data(nongeo_repost_data, datatype_label='nongeo_repost',
                                                       saved_filename='nongeocoded_repost_traffic_shanghai.csv')
    combined_traffic_shanghai = pd.concat([processed_geo, processed_nongeo_weibo, processed_nongeo_repost], axis=0)
    combined_traffic_shanghai['weibo_id'] = combined_traffic_shanghai['weibo_id'].astype(np.int64)
    combined_traffic_shanghai_copy = combined_traffic_shanghai.copy()
    count_traffic_in_dataframe(combined_traffic_shanghai_copy)
    print('Derive the traffic information type...')
    # Get the traffic information type based on keywords
    # If a weibo contains only congestion keywords, we regard this Weibo as congestion-related Weibo
    # If a weibo contains accident-related keywords, we regard this Weibo as accident-related Weibo
    traffic_type_list = []
    for index, row in combined_traffic_shanghai_copy.iterrows():
        traffic_type = ''
        if row['datatype'] != 'nongeo_repost':  # geocoded data and nongeocoded weibo
            considered_text = row['text']
        else:  # nongeocoded repost
            considered_text = row['retweeters_text']
        decision_conges = any(traffic_word in considered_text for traffic_word in congestion_traffic_word_set)
        decision_accident = any(traffic_word in considered_text for traffic_word in accident_traffic_word_set)
        if decision_conges and (not decision_accident):
            traffic_type = 'congestion'
        if decision_accident:
            traffic_type = 'accident'
        if (not decision_conges) and (not decision_accident):
            traffic_type = 'condition'
        traffic_type_list.append(traffic_type)
    combined_traffic_shanghai_copy['traffic_type'] = traffic_type_list
    combined_traffic_shanghai_copy['lat'] = combined_traffic_shanghai_copy['lat'].astype(np.float64)
    combined_traffic_shanghai_copy['lon'] = combined_traffic_shanghai_copy['lon'].astype(np.float64)
    create_weibo_count_table(combined_traffic_shanghai_copy)
    combined_traffic_shanghai_copy.to_csv(os.path.join(data_paths.weibo_data_path,
                                                       'combined_traffic_weibo_shanghai.csv'), encoding='utf-8')
    return combined_traffic_shanghai_copy


def create_official_fishnet_dataframe(data: pd.DataFrame, considered_column: str) -> pd.DataFrame:
    """
    Count the number of real-world traffic events in each fishnet cell
    :param data: dataframe saving the location of real-world traffic event in which fishnet cell
    :param considered_column: the column saving the fishnet cell num that a real-world traffic event belongs to
    :return: dataframe storing the number of real-world traffic events in each fishnet cell
    """
    cell_counter = Counter(data[considered_column])
    cell_list = list(cell_counter.keys())
    value_list = [cell_counter[cell_key] for cell_key in cell_list]
    result_dataframe = pd.DataFrame(columns=['cell_num', 'count'])
    result_dataframe['cell_num'] = cell_list
    result_dataframe['count'] = value_list
    result_dataframe_sorted = result_dataframe.sort_values(by='count', ascending=False)
    result_dataframe_reindex = result_dataframe_sorted.reset_index(drop=True)
    return result_dataframe_reindex


def create_weibo_fishnet_dataframe(data: pd.DataFrame, considered_column: str) -> pd.DataFrame:
    """
    Count the number of traffic Weibos in each fishnet cell
    :param data: dataframe saving the location of traffic Weibo in which fishnet cell
    :param considered_column: the column saving the fishnet cell num that a Weibo belongs to
    :return: dataframe storing the number of traffic Weibos in each fishnet cell
    """
    if 'retweeters_text' not in data:
        data_renamed = data.rename(columns={'retweete_1': 'retweeters_text'})
    else:
        data_renamed = data.copy()
    cell_counter = Counter(data_renamed[considered_column])
    cell_list = list(cell_counter.keys())
    accident_count_list, congestion_count_list, condition_count_list = [], [], []
    pos_sent_list, neg_sent_list = [], []
    for cell in cell_list:
        cell_data = data_renamed.loc[data_renamed[considered_column] == cell]
        traffic_type_counter = Counter(cell_data['traffic_ty'])
        accident_count_list.append(traffic_type_counter['accident'])
        congestion_count_list.append(traffic_type_counter['congestion'])
        condition_count_list.append(traffic_type_counter['condition'])
        sent_result, sent_counter = [], Counter()
        for index, row in cell_data.iterrows():
            weibo_sent = eval(row['sent_weibo'])[0]
            sent_result.append(weibo_sent)
            if row['retweeters_text'] != 'no retweeters':
                repost_sent = eval(row['sent_repos'])[0]
                sent_result.append(repost_sent)
        sent_counter = Counter(sent_result)
        pos_sent_list.append(sent_counter['positive'])
        neg_sent_list.append(sent_counter['negative'])
    sent_index_array = (np.array(neg_sent_list) - np.array(pos_sent_list)) / (
                np.array(neg_sent_list) + np.array(pos_sent_list))
    result_dataframe = pd.DataFrame(columns=['cell_num', 'acc_count', 'conges_count', 'condition_count', 'sent_index'])
    result_dataframe['cell_num'] = cell_list
    result_dataframe['acc_count'] = accident_count_list
    result_dataframe['conges_count'] = congestion_count_list
    result_dataframe['condition_count'] = condition_count_list
    result_dataframe['sent_index'] = sent_index_array
    result_dataframe['total'] = result_dataframe.apply(
        lambda row: row['acc_count'] + row['conges_count'] + row['condition_count'], axis=1)
    result_dataframe_sorted = result_dataframe.sort_values(by='total', ascending=False)
    result_dataframe_reindex = result_dataframe_sorted.reset_index(drop=True)
    return result_dataframe_reindex


def compute_tweets_in_hotspot_district(dataframe, area_dataframe, district_column: str,
                                       in_hotspot_column: str) -> pd.DataFrame:
    """
    Compute the PAI index for Weibo and actual traffic events across considered districts
    :param dataframe: the pandas dataframe saving the official traffic events or Weibo traffic events
    :param area_dataframe: the pandas dataframe saving the hotspot area and district area
    :param district_column: the column name saving the district information
    :param in_hotspot_column: the column name saving whether an event is in hotspot information
    :return: a pandas dataframe saving the PAI index across districts
    """
    pai_district_dict, area_dict = {}, {}
    # Save the intersect area and total area of a district
    for district_name, district_dataframe in area_dataframe.groupby(district_column):
        reindex_dataframe = district_dataframe.reset_index(drop=True)
        area_dict[district_name] = (reindex_dataframe.at[0, 'Intersect'], reindex_dataframe.at[0, 'District'])
    # Save the number of actual or Weibo traffic events in hotspot & in district
    for district_name, district_dataframe in dataframe.groupby(district_column):
        pai_district_dict[district_name] = (
            district_dataframe.loc[district_dataframe[in_hotspot_column] != 0].shape[0], district_dataframe.shape[0])
    district_name_list = list(area_dict.keys())
    result_pai_dataframe = pd.DataFrame(columns=['district', 'hotspot_area', 'district_area', 'count_in_hotspot',
                                                 'count_total'])
    result_pai_dataframe['district'] = district_name_list
    result_pai_dataframe['hotspot_area'] = [area_dict.get(name, (0, 0))[0] for name in district_name_list]
    result_pai_dataframe['district_area'] = [area_dict.get(name, (0, 0))[1] for name in district_name_list]
    result_pai_dataframe['count_in_hotspot'] = [pai_district_dict.get(name, (0, 0))[0] for name in district_name_list]
    result_pai_dataframe['count_total'] = [pai_district_dict.get(name, (0, 0))[1] for name in district_name_list]
    result_pai_dataframe['pai'] = (result_pai_dataframe['count_in_hotspot']/result_pai_dataframe['count_total'])/(
            result_pai_dataframe['hotspot_area']/result_pai_dataframe['district_area'])
    return result_pai_dataframe


def compare_pai_with_or_without_sentiment(check_type: str) -> None:
    """
    Compute the PAI for the 'with sentiment' and 'without sentiment' settings
    :param check_type: either 'with_sentiment' or 'without_sentiment'
    :return: None. The PAI values are saved to local
    """
    assert check_type in ['with_sentiment', 'without_sentiment'], 'The check_type argument is wrong. Check this line!'
    shanghai_total_area = 7015.05  # in km^2
    shanghai_weibo_total_acc, shanghai_weibo_total_cgs = 1809, 1520
    shanghai_actual_total_acc, shanghai_actual_total_cgs = 60, 1823
    if check_type == 'with_sentiment':  # Specify the path for PAI computation: with or without sentiment
        print('Computing the PAI index {}'.format(check_type))
        path = os.path.join(data_paths.kde_analysis, 'text_sent')
    else:
        print('Computing the PAI index {}'.format(check_type))
        path = os.path.join(data_paths.kde_analysis, 'text_default')
    # Load the actual & Weibo accident, congestion records
    official_acc_shanghai_hotspot_join = pd.read_csv(os.path.join(path, 'official_acc_hotspot_district_join.txt'),
                                                     encoding='utf-8', index_col=0)
    official_cgs_shanghai_hotspot_join = pd.read_csv(os.path.join(path, 'official_cgs_hotspot_district_join.txt'),
                                                     encoding='utf-8', index_col=0)
    weibo_acc_shanghai_hotspot_join = pd.read_csv(os.path.join(path, 'weibo_acc_hotspot_district_join.txt'),
                                                  encoding='utf-8', index_col=0)
    weibo_cgs_shanghai_hotspot_join = pd.read_csv(os.path.join(path, 'weibo_cgs_hotspot_district_join.txt'),
                                                  encoding='utf-8', index_col=0)
    # Load the area information
    acc_area_data = pd.read_csv(os.path.join(path, 'weibo_acc_hotspot_areas.txt'), encoding='utf-8', index_col=0)
    cgs_area_data = pd.read_csv(os.path.join(path, 'weibo_cgs_hotspot_areas.txt'), encoding='utf-8', index_col=0)
    # Create the PAI actual dataframe
    pai_acc_actual_dataframe = compute_tweets_in_hotspot_district(dataframe=official_acc_shanghai_hotspot_join,
                                                                  area_dataframe=acc_area_data, district_column='Name',
                                                                  in_hotspot_column='hotspot_id')
    pai_cgs_actual_dataframe = compute_tweets_in_hotspot_district(dataframe=official_cgs_shanghai_hotspot_join,
                                                                  area_dataframe=cgs_area_data,
                                                                  district_column='Name',
                                                                  in_hotspot_column='hotspot_id')
    # Create the PAI Weibo dataframe
    pai_acc_weibo_dataframe = compute_tweets_in_hotspot_district(dataframe=weibo_acc_shanghai_hotspot_join,
                                                                 area_dataframe=acc_area_data, district_column='Name',
                                                                 in_hotspot_column='hotspot_id')
    pai_cgs_weibo_dataframe = compute_tweets_in_hotspot_district(dataframe=weibo_cgs_shanghai_hotspot_join,
                                                                 area_dataframe=cgs_area_data,
                                                                 district_column='Name', in_hotspot_column='hotspot_id')
    # Compute the PAI weibo and PAI actual
    total_acc_area = sum(list(pai_acc_weibo_dataframe['hotspot_area']))
    total_cgs_area = sum(list(pai_cgs_weibo_dataframe['hotspot_area']))
    print('The total accident hotspot area: {} km^2'.format(round(total_acc_area, 4)))
    print('The total congestion hotspot area: {} km^2'.format(round(total_cgs_area, 4)))
    percent_acc_actual = sum(list(pai_acc_actual_dataframe['count_in_hotspot']))/shanghai_actual_total_acc
    percent_acc_weibo = sum(list(pai_acc_weibo_dataframe['count_in_hotspot']))/shanghai_weibo_total_acc
    percent_cgs_actual = sum(list(pai_cgs_actual_dataframe['count_in_hotspot'])) / shanghai_actual_total_cgs
    percent_cgs_weibo = sum(list(pai_cgs_weibo_dataframe['count_in_hotspot'])) / shanghai_weibo_total_cgs
    print('For accident: n1/N1: {}'.format(percent_acc_actual))
    print('For accident: n2/N2: {}'.format(percent_acc_weibo))
    print('For congestion: n1/N1: {}'.format(percent_cgs_actual))
    print('For congestion: n2/N2: {}'.format(percent_cgs_weibo))

    pai_actual_acc = (sum(list(pai_acc_actual_dataframe['count_in_hotspot']))/shanghai_actual_total_acc)/(
            total_acc_area/shanghai_total_area)
    pai_weibo_acc = (sum(list(pai_acc_weibo_dataframe['count_in_hotspot']))/shanghai_weibo_total_acc)/(
            total_acc_area/shanghai_total_area)
    pai_actual_cgs = (sum(list(pai_cgs_actual_dataframe['count_in_hotspot'])) / shanghai_actual_total_cgs) / (
            total_cgs_area / shanghai_total_area)
    pai_weibo_cgs = (sum(list(pai_cgs_weibo_dataframe['count_in_hotspot'])) / shanghai_weibo_total_cgs) / (
            total_cgs_area / shanghai_total_area)
    # Output the results
    print('For the setting {}: \n Accident: the PAI actual is: {}; the PAI weibo is: {}; actual/weibo: {}'.format(
        check_type, pai_actual_acc, pai_weibo_acc, pai_actual_acc/pai_weibo_acc))
    print('Number of actual accident in hotspot: {}'.format(sum(list(pai_acc_actual_dataframe['count_in_hotspot']))))
    print('Number of weibo accident in hotspot: {}'.format(sum(list(pai_acc_weibo_dataframe['count_in_hotspot']))))
    print('For the setting {}: \n Congestion: the PAI actual is: {}; the PAI weibo is: {}; actual/weibo: {}'.format(
        check_type, pai_actual_cgs, pai_weibo_cgs, pai_actual_cgs/pai_weibo_cgs))
    print('Number of actual congestion in hotspot: {}'.format(sum(list(pai_cgs_actual_dataframe['count_in_hotspot']))))
    print('Number of weibo congestion in hotspot: {}'.format(sum(list(pai_cgs_weibo_dataframe['count_in_hotspot']))))
    pai_acc_actual_dataframe.to_excel(os.path.join(path, 'pai_actual_acc_across_districts.xlsx'))
    pai_cgs_actual_dataframe.to_excel(os.path.join(path, 'pai_actual_cgs_across_districts.xlsx'))
    pai_acc_weibo_dataframe.to_excel(os.path.join(path, 'pai_weibo_acc_across_districts.xlsx'))
    pai_cgs_weibo_dataframe.to_excel(os.path.join(path, 'pai_weibo_cgs_across_districts.xlsx'))


def create_sentiment_against_density_data(weibo_dataframe: pd.DataFrame, hotspot_id_colname: str,
                                          repost_colname: str) -> pd.DataFrame:
    """
    Create a pandas dataframe used to save the density class and sentiment index of each hotspot
    :param weibo_dataframe: a traffic Weibo dataframe with hotspot id information and hotspot density information
    :param hotspot_id_colname: the colname of the hotspot id
    :param repost_colname: the colname of the retweeters' text
    :return: a pandas dataframe saving the density and sentiment index of each traffic hotspot
    """
    weibo_dataframe_select = weibo_dataframe.loc[weibo_dataframe['hotspot_id'] != 0]
    print("The column used to save the hotspot id is: {}".format(hotspot_id_colname))

    if 'traffic_weibo' in weibo_dataframe_select:
        renamed_data = weibo_dataframe_select.copy()
    elif hotspot_id_colname == 'hotspot_id':
        select_columns = ['index_val', 'author_id', 'weibo_id', 'created_at', 'text', 'lat', 'lon', 'retweeters',
                          'retweete_1', 'local_time', 'year', 'month', 'traffic_we', 'traffic_re', 'sent_weibo',
                          'sent_repos', 'Name', 'datatype', 'traffic_ty', 'hotspot_id']
        rename_dict = {'traffic_we': 'traffic_weibo', 'traffic_re': 'traffic_repost', 'retweete_1': 'retweeters_text'}
        renamed_data = weibo_dataframe_select[select_columns].rename(columns=rename_dict)
    elif hotspot_id_colname == 'gmm_labels':
        select_columns = ['index_val', 'author_id', 'weibo_id', 'created_at', 'text', 'lat', 'lon', 'retweeters',
                          'retweete_1', 'local_time', 'year', 'month', 'traffic_we', 'traffic_re', 'sent_weibo',
                          'sent_repos', 'Name', 'datatype', 'traffic_ty', 'gmm_labels']
        rename_dict = {'traffic_we': 'traffic_weibo', 'traffic_re': 'traffic_repost', 'retweete_1': 'retweeters_text'}
        renamed_data = weibo_dataframe_select[select_columns].rename(columns=rename_dict)
    else:
        raise ValueError('Please specify a right hotspot id column name')

    hotspot_id_list, hotspot_weibo_count_list = [], []
    sentiment_indices = []
    for hotspot_id, dataframe in renamed_data.groupby(hotspot_id_colname):
        hotspot_id_list.append(hotspot_id)
        pos_count, neutral_count, neg_count = count_positive_neutral_negative(dataframe, repost_column=repost_colname)
        hotspot_weibo_count_list.append(pos_count + neutral_count + neg_count)
        sentiment_index = (pos_count - neg_count)/(pos_count + neutral_count + neg_count)
        sentiment_indices.append(sentiment_index)

    result_dataframe = pd.DataFrame()
    result_dataframe['hotspot_id'] = hotspot_id_list
    result_dataframe['sentiment'] = sentiment_indices
    result_dataframe['density'] = hotspot_weibo_count_list  # Use Weibo count as the density
    result_dataframe_sorted = result_dataframe.sort_values(by='density', ascending=False).reset_index(drop=True)
    return result_dataframe_sorted


def get_priority_ranges(density_dataframe, setting: str):
    """
    Compute the density value range to get the hotspots with different priority
    :param density_dataframe: a pandas datafame saving the density of each cell
    :param setting: a description about the density values in the density_dataframe
    :return: a list saving the lists of density range for hotspots with different priority (from lower to higher)
    """
    assert 'VALUE' in density_dataframe, "The dataframe should contain columns saving the density valuess."
    assert 'COUNT' in density_dataframe, "The dataframe should contain columns saving the number of cells " \
                                    "equalling a density val"
    density_list = []
    for _, row in density_dataframe.iterrows():
        density_list += [row['VALUE']] * np.int(row['COUNT'])
    final_dense_vals = [dense_val / 100 for dense_val in density_list]

    # Compute the threshold value
    threshold_value = round(np.mean(final_dense_vals) + 3 * np.std(final_dense_vals), 2)
    print('The threshold value is set to: {}'.format(threshold_value))

    # Compute the natural break vals list
    natural_breaks_val_list = jenkspy.jenks_breaks(final_dense_vals, nb_class=5)

    # Output the result natural break ranges
    result_list = [threshold_value]
    for break_val in natural_breaks_val_list:
        if break_val > threshold_value:
            result_list.append(break_val)
    result = [[x, y] for x, y in zip(result_list, result_list[1:])]
    print('*' * 15)
    print('For {}, the threshold value is: {}'.format(setting, threshold_value))
    print('For {}, the priority range is: {}'.format(setting, result))
    print('*' * 15)
    return result


def create_data_for_regres(official_traffic_data, traffic_weibo_data, considered_count: int):
    """
    Create the data for regression based on the spatial join result based on traffic data from official traffic
    account and traffic relevant Weibo
    :param official_traffic_data: dataframe saving the official traffic data spatial joined on fishnet data
    :param traffic_weibo_data: dataframe saving the traffic relevant data spatial joined on fishnet data
    :param considered_count: the least number of real-world traffic events in a fishnet cell
    :return: dataframe for regression
    """
    considered_cells = set(official_traffic_data['FID_shan_1'])
    traffic_weibo_fishnet_select = traffic_weibo_data.loc[traffic_weibo_data['FID_2'].isin(considered_cells)]
    weibo_fishnet_count = create_weibo_fishnet_dataframe(traffic_weibo_fishnet_select, considered_column='FID_2')
    official_fishnet_count = create_official_fishnet_dataframe(official_traffic_data, considered_column='FID_shan_1')
    assert 'cell_num' in official_fishnet_count and 'cell_num' in weibo_fishnet_count, \
        'The fishnet cell column is not prepared!'
    merged_data = pd.merge(weibo_fishnet_count, official_fishnet_count, how='inner', on=['cell_num'])
    merged_data_select = merged_data.loc[merged_data['count'] >= considered_count]
    merged_data_select_sorted = merged_data_select.sort_values(by='count', ascending=False)
    merged_data_select_reindex = merged_data_select_sorted.reset_index(drop=True)
    return merged_data_select_reindex


def regres_analysis(dataframe: pd.DataFrame, feature_columns: list, predict_column: list,
                    output_dataframe: bool) -> None:
    """
    Conduct the regression analysis given dataframe containing features and y values
    :param dataframe: the dataframe for regression analysis
    :param feature_columns: the columns saving the independent variables
    :param predict_column: the column saving the dependent variable
    :param output_dataframe: whether we save the pandas dataframe regression result to local
    """
    feature_values = dataframe[feature_columns].values
    y_values = dataframe[predict_column]
    mod = sm.OLS(y_values, feature_values)
    res = mod.fit()
    if output_dataframe:
        res_summary = res.summary(xname=['# of Accidents', '# of Congestions', '# of Conditions', 'Sentiment Index'])
        results_as_html = res_summary.tables[1].as_html()
        regression_result = pd.read_html(results_as_html, header=0, index_col=0)[0]
        regression_result.to_csv(os.path.join(data_paths.weibo_data_path, 'regression_result.csv'), encoding='utf-8')
    else:
        print(res.summary())
    return


if __name__ == '__main__':

    # Count the number of traffic Weibos across districts
    combined_dataframe = pd.read_csv(os.path.join(
        data_paths.weibo_data_path, 'combined_traffic_weibo_shanghai.csv'), encoding='utf-8', index_col=0)
    district_count = count_traffic_weibos_in_districts(combined_dataframe, district_name_column='Name')
    district_count.to_csv(os.path.join(data_paths.weibo_data_path, 'district_count.csv'), encoding='utf-8')
    district_count.to_excel(os.path.join(data_paths.weibo_data_path, 'district_count.xlsx'), encoding='utf-8')
	
    # Create the sentiment against density plots
    weibo_acc_hotspot_join = pd.read_csv(os.path.join(hotspot_text_path, 'weibo_acc_hotspot_join.txt'),
                                         encoding='utf-8')
    weibo_cgs_hotspot_join = pd.read_csv(os.path.join(hotspot_text_path, 'weibo_cgs_hotspot_join.txt'),
                                         encoding='utf-8')
    acc_sent_density = create_sentiment_against_density_data(weibo_acc_hotspot_join,
                                                             hotspot_id_colname='hotspot_id',
                                                             repost_colname='retweeters_text')
    cgs_sent_density = create_sentiment_against_density_data(weibo_cgs_hotspot_join,
                                                             hotspot_id_colname='hotspot_id',
                                                             repost_colname='retweeters_text')
    plot_sent_dense(sent_dense_data=acc_sent_density, traffic_type='acc',
                    save_filename='acc_hotspot_priority.png')
    plot_sent_dense(sent_dense_data=cgs_sent_density, traffic_type='cgs',
                    save_filename='cgs_hotspot_priority.png')
    acc_sent_density.to_csv(os.path.join(hotspot_text_path, 'acc_sent_density.csv'), encoding='utf-8')
    cgs_sent_density.to_csv(os.path.join(hotspot_text_path, 'cgs_sent_density.csv'), encoding='utf-8')
