#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 14 00:27:51 2019

@author: kazuki.onodera
"""

import numpy as np
import pandas as pd
import utils


SUBMIT_FILE_PATH = '../output/0313-3.csv.gz'

COMMENT = 'mean(ireko*8) + mean(tosh*7) + mean(onodera*2) + mean(nyanp*3) + mean(nejumi*2)'

EXE_SUBMIT = True

usecols = ['HasDetections']

# =============================================================================
# def
# =============================================================================
sub_best = pd.read_csv(utils.SUB_BEST)

def get_rank(preds:list):
    
    rank = pd.Series(0, index=sub_best.index)
    
    for i,pred in enumerate(preds):
        corr = sub_best['HasDetections'].corr(pred)
        if corr < 0:
            raise Exception(corr, i)
        rank += pred.rank()
    
    return rank.rank()

# =============================================================================
# ireko
# =============================================================================
ireko707    = pd.Series(np.load('../external/sub/ireko707/test_preds.npy'), name='ireko707')
ireko708    = pd.Series(np.load('../external/sub/ireko708/test_preds.npy'), name='ireko708')
ireko708_v2 = pd.Series(np.load('../external/sub/ireko708_v2/test_preds.npy'), name='ireko708_v2')
ireko709    = pd.Series(np.load('../external/sub/ireko709/test_preds.npy'), name='ireko709')
ireko710    = pd.Series(np.load('../external/sub/ireko710/test_preds.npy'), name='ireko710')
ireko708_lovasz    = pd.Series(np.load('../external/sub/ireko708_lovasz/test_preds.npy'), name='ireko708_lovasz')
ireko710_v2    = pd.Series(np.load('../external/sub/ireko710_v2/test_preds.npy'), name='ireko710_v2')
ireko_split    = pd.Series(np.load('../external/sub/ireko_split/test_preds.npy'))

ireko = get_rank([ireko707, ireko708, ireko708_v2, ireko709, ireko710, ireko708_lovasz, ireko710_v2, ireko_split])

del ireko707, ireko708, ireko708_v2, ireko709, ireko710, ireko708_lovasz, ireko710_v2

# =============================================================================
# tosh
# =============================================================================
tosh708    = pd.read_csv('../external/sub/subNN_v1722.csv.gz', usecols=usecols)['HasDetections']
tosh709    = 1-pd.read_csv('../external/sub/subNN_v176.csv.gz', usecols=usecols)['HasDetections']
tosh709_v2 = 1-pd.read_csv('../external/sub/subNN_v1763.csv.gz', usecols=usecols)['HasDetections']
tosh707    = 1-pd.read_csv('../external/sub/sub_inv_NN_v178.csv.gz', usecols=usecols)['HasDetections']
tosh708_v2 = 1-pd.read_csv('../external/sub/sub_inv_NN_v1764.csv.gz', usecols=usecols)['HasDetections']
tosh709_v3 = pd.read_csv('../external/sub/subNN_v182.csv.gz', usecols=usecols)['HasDetections']
tosh_split    = pd.read_csv('../external/sub/subNN_v1863.csv.gz', usecols=usecols)['HasDetections']


tosh = get_rank([tosh708, tosh709, tosh709_v2, tosh707, tosh708_v2, tosh709_v3, tosh_split])

del tosh708, tosh709, tosh709_v2, tosh707, tosh708_v2, tosh709_v3

# =============================================================================
# onodera
# =============================================================================
onodera709 = 1-pd.read_csv('../output/0308-2.csv.gz', usecols=usecols)['HasDetections']
onodera_noised = 1-pd.read_csv('../output/0310-1.csv.gz', usecols=usecols)['HasDetections']

onodera = get_rank([onodera709, onodera_noised])

del onodera709, onodera_noised

# =============================================================================
# nyanp
# =============================================================================
nyanp709_wo = pd.read_csv('../external/sub/sub_wo1_x24.csv.gz', usecols=usecols)['HasDetections']
nyanp709_w = pd.read_csv('../external/sub/sub_w1_x24.csv.gz', usecols=usecols)['HasDetections']
nyanp709_v2 = pd.read_csv('../external/sub/sub_lgbm_te_01100101102103173613614615701_auc0.745.csv.csv.gz', usecols=usecols)['HasDetections']


nyanp = get_rank([nyanp709_wo, nyanp709_w, nyanp709_v2])

del nyanp709_wo, nyanp709_w, nyanp709_v2

# =============================================================================
# nejumi
# =============================================================================
nejumi712 = 1-pd.read_csv('../external/sub/1_m_nejumi_0098_seed71_test.csv.gz', usecols=usecols)['HasDetections']
nejumi714 = 1-pd.read_csv('../external/sub/1_m_bagging_lgbm_on_nn.csv.gz', usecols=usecols)['HasDetections']


nejumi = get_rank([nejumi712, nejumi714])

del nejumi712, nejumi714

# =============================================================================
# 謎のkernel
# =============================================================================
#kernel = pd.read_csv('')
#
#kernel = get_rank([kernel])

# =============================================================================
# 
# =============================================================================

#y_pred = ireko + tosh + onodera + nyanp + nejumi + kernel
y_pred = ireko + tosh + onodera + nyanp + nejumi
y_pred /= y_pred.max()

sub = pd.read_csv('../input/sample_submission.csv.zip')
sub['HasDetections'] = y_pred.values

#sub['HasDetections'] = 1 - sub['HasDetections']

pri_id = pd.read_pickle('../data/pri_id.pkl')['MachineIdentifier']

sub_pri = sub[sub.MachineIdentifier.isin(pri_id)]
sub_pub = sub[~sub.MachineIdentifier.isin(pri_id)]

sub_best_pri = sub_best[sub_best.MachineIdentifier.isin(pri_id)]
sub_best_pub = sub_best[~sub_best.MachineIdentifier.isin(pri_id)]



print(f'corr with {utils.SUB_BEST}')
print('with best(ALL):', sub['HasDetections'].corr( sub_best['HasDetections'], method='spearman') )
print('with best(pub):', sub_pub['HasDetections'].corr( sub_best_pub['HasDetections'], method='spearman') )
print('with best(pri):', sub_pri['HasDetections'].corr( sub_best_pri['HasDetections'], method='spearman') )

# save
sub.to_csv(SUBMIT_FILE_PATH, index=False, compression='gzip')

# =============================================================================
# submission
# =============================================================================
if EXE_SUBMIT:
    print('submit')
    utils.submit(SUBMIT_FILE_PATH, COMMENT)

