import pandas as pd
import numpy
import pickle
import sklearn
import math
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from keras.layers import Input, Dense, Activation, LeakyReLU, BatchNormalization
from keras.layers.core import Layer, Dense, Dropout, Activation, Flatten, Reshape,Permute
from keras.layers import Input, merge, Convolution2D, MaxPooling2D, UpSampling2D, Reshape, core, Dropout,Lambda
from keras.layers.normalization import BatchNormalization
from keras.layers.convolutional import Convolution3D, MaxPooling3D, ZeroPadding3D , ZeroPadding3D , UpSampling3D
from keras.layers.convolutional import Convolution2D, MaxPooling2D, UpSampling2D, ZeroPadding2D,Conv2DTranspose
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from keras.layers import Dense, Conv2D, MaxPooling2D, Dropout, Flatten,Conv2DTranspose,concatenate
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D,UpSampling2D
from keras.models import Sequential
from keras.optimizers import Adam,Adamax
import keras
from keras.utils import np_utils
import h5py
from sklearn.preprocessing import Imputer

chunksize = 12687
features = ['MachineIdentifier', 'ProductName', 'EngineVersion', 'AppVersion', 'AvSigVersion',
            'IsBeta', 'RtpStateBitfield', 'IsSxsPassiveMode', 'DefaultBrowsersIdentifier', 'AVProductStatesIdentifier',
            'AVProductsInstalled', 'AVProductsEnabled', 'HasTpm', 'CountryIdentifier', 'CityIdentifier', 'OrganizationIdentifier',
            'GeoNameIdentifier', 'LocaleEnglishNameIdentifier', 'Platform', 'Processor', 'OsVer', 'OsBuild', 'OsSuite', 'OsPlatformSubRelease',
            'OsBuildLab', 'SkuEdition', 'IsProtected', 'AutoSampleOptIn', 'PuaMode', 'SMode', 'IeVerIdentifier', 'SmartScreen', 'Firewall',
            'UacLuaenable', 'Census_MDC2FormFactor', 'Census_DeviceFamily', 'Census_OEMNameIdentifier', 'Census_OEMModelIdentifier', 'Census_ProcessorCoreCount',
            'Census_ProcessorManufacturerIdentifier', 'Census_ProcessorModelIdentifier', 'Census_ProcessorClass', 'Census_PrimaryDiskTotalCapacity',
            'Census_PrimaryDiskTypeName', 'Census_SystemVolumeTotalCapacity', 'Census_HasOpticalDiskDrive', 'Census_TotalPhysicalRAM', 'Census_ChassisTypeName',
            'Census_InternalPrimaryDiagonalDisplaySizeInInches', 'Census_InternalPrimaryDisplayResolutionHorizontal', 'Census_InternalPrimaryDisplayResolutionVertical',
            'Census_PowerPlatformRoleName', 'Census_InternalBatteryType', 'Census_InternalBatteryNumberOfCharges', 'Census_OSVersion', 'Census_OSArchitecture',
            'Census_OSBranch', 'Census_OSBuildNumber', 'Census_OSBuildRevision', 'Census_OSEdition', 'Census_OSSkuName',
            'Census_OSInstallTypeName', 'Census_OSInstallLanguageIdentifier', 'Census_OSUILocaleIdentifier', 'Census_OSWUAutoUpdateOptionsName',
            'Census_IsPortableOperatingSystem', 'Census_GenuineStateName', 'Census_ActivationChannel', 'Census_IsFlightingInternal',
            'Census_IsFlightsDisabled', 'Census_FlightRing', 'Census_ThresholdOptIn', 'Census_FirmwareManufacturerIdentifier',
            'Census_FirmwareVersionIdentifier', 'Census_IsSecureBootEnabled', 'Census_IsWIMBootEnabled', 'Census_IsVirtualDevice',
            'Census_IsTouchEnabled', 'Census_IsPenCapable', 'Census_IsAlwaysOnAlwaysConnectedCapable', 'Wdft_IsGamer',
            'Wdft_RegionIdentifier']
c = 0

y = []
def to_int(n) :
    n = list(n)
    sum = 0
    for i in n:
        sum += ord(i)
    return sum

d = 0
x = []

with open('x1', 'rb') as f:
    x = pickle.load(f)
x_mean = x.mean(axis=0)
x_std = x.std(axis=0)

x_max = x.max(axis=0)
for ab in range(6):
    x_test = []
    y_test = []
    d = 0
    c = 0
    for chunk in pd.read_csv("test.csv", chunksize=chunksize):
        print(chunk)
        c += 1
        if c % 6 != ab:
            continue
        x_test.append(chunk.loc[:,features].values)
        #print(x_test)
        y_test.append(chunk.loc[:,['MachineIdentifier']].values)
        #print(chunk.loc[:,['HasDetections']].values)
    #y = numpy.array(y)
    print(len(x_test))
    x_test = numpy.array(x_test)
    #x_test = x_test.reshape(40,82)
    #print(x_test)
    y_test = numpy.array(y_test)

    #pickle_out = open("x_test1","wb")
    #pickle.dump(x_test,pickle_out)
    #pickle_out = open("y1","wb")
    #pickle.dump(y,pickle_out)
    print(x_test.shape)
    #exit()
    #print(y.shape)
    #with open('x_test1','rb') as f:
    #    x_test = pickle.load(f)
    #with open('y1','rb') as f:
    #    y = pickle.load(f)
    print(x_test.shape)
    print(y_test.shape)
    #print(y.shape)
    x_test = x_test.reshape(x_test.shape[0] * x_test.shape[1],x_test.shape[2])
    y_test = y_test.reshape(y_test.shape[0] * y_test.shape[1], y_test.shape[2])
    #y = y.reshape(y.shape[0] * y.shape[1],y.shape[2])
    print(y_test.shape)
    print(x_test.shape)
    print(y_test)
    #print(y.shape)
    #pickle_out = open("x_test1","wb")
    #pickle.dump(x_test,pickle_out)
    #pickle_out = open("y1","wb")
    #pickle.dump(y,pickle_out)

    """with open('x_test1','rb') as f:
        x_test = pickle.load(f)
    with open('y1','rb') as f:
        y = pickle.load(f)
        """
    m, n = x_test.shape
    #m1,n1 = y.shape
    print(x_test.shape)

    #z = x_test.max(axis = 0)
    first = []
    second = []
    c = 0
    for i in range(x_test.shape[0]):
        c += 1
        #print(c)
        for j in range(x_test.shape[1]):
            if isinstance(x_test[i][j],float) and math.isnan(x_test[i][j]) :
                #print("Nan")
                first.append(i)
                second.append(j)
            elif isinstance(x_test[i][j],int) or isinstance(x_test[i][j],float) :
                continue
            else :
                x_test[i][j] = to_int(x_test[i][j])
    x_test =  Imputer().fit_transform(x_test)

    """x_mean = x_test.mean(axis=0)
    for i in range(len(first)):
        x_test[first[i]][second[i]] = x_mean[second[i]]"""
    for i in range(x_test.shape[0]):
        # print(c)
        for j in range(x_test.shape[1]):
            if math.isnan(x_test[i][j]):
                print("Nan")
                x_test[i][j] = 0
    #pickle_out = open("x_test1","wb")
    #pickle.dump(x_test,pickle_out)
    #pickle_out = open("y1","wb")
    #pickle.dump(y,pickle_out)
    #with open('x_test1','rb') as f:
    #    x_test = pickle.load(f)
    #with open('y1','rb') as f:
    #    y = pickle.load(f)
    print(x_test)
    #print(y)
    with open('x_mean'+ str(ab + 1), 'rb') as f:
        x_mean = pickle.load(f)
    with open('x_std' + str(ab + 1), 'rb') as f:
        x_std = pickle.load(f)
    x_test = numpy.array(x_test,dtype=numpy.float32)
    x_test -= x_mean
    x_test /= x_std

    for i in range(x_test.shape[0]) :
        #print(c)
        for j in range(x_test.shape[1]):
            if math.isnan(x_test[i][j]) or not numpy.isfinite(x_test[i][j]) :
                print("Nan")
                x_test[i][j] = 0

    pickle_out = open("x_test"+str(ab+1),"wb")
    pickle.dump(x_test,pickle_out)
    pickle_out = open("label" + str(ab + 1), "wb")
    pickle.dump(y_test, pickle_out)

    #pickle_out = open("x_mean","wb")
    #pickle.dump(x_mean,pickle_out)

    #pickle_out = open("x_std","wb")
    #pickle.dump(x_std,pickle_out)

    print(x_test)



