#!/usr/bin/env python
# coding: utf-8

# In[1]:


import torch
import pickle
import numpy as np


# In[2]:


with open('trainval.pkl', 'rb') as handle:
    DB = pickle.load(handle)


# In[3]:


# train
batchSize = 5000
totalNumber = 8921483
iterPerEpoch = totalNumber/ batchSize
print(iterPerEpoch)


# In[6]:


totalNumber = len(DB['X_train'])
print(totalNumber, DB.keys())


# In[7]:


def row2vec(row):
    vec = []
    for i in range(len(row[1])):
        vec += [row[1][i]]
    return vec


# In[29]:


trainX = []
trainY = []

count = 0
iterObj = DB['X_train'].iterrows()
for row in iterObj:
    vec = row2vec(row)
    trainX += [vec]
    count +=1
    if count%10000==0:
        print('{}/{}'.format(count, totalNumber))

        
count = 0
for i, v in DB['Y_train'].items():
    count += 1
    trainY += [v]    
    if count%10000==0:
        print('{}/{}'.format(count, totalNumber))    
    


# In[30]:


valX = []
valY = []

count = 0
iterObj = DB['X_validation'].iterrows()
for row in iterObj:
    vec = row2vec(row)
    valX += [vec]
    count +=1
    if count%10000==0:
        print('{}/{}'.format(count, totalNumber))
    if count==100:
        break

count = 0
for i, v in DB['Y_validation'].items():
    count += 1
    valY += [v]    
    if count%10000==0:
        print('{}/{}'.format(count, totalNumber))    
    


# In[31]:


trainX = np.asarray(trainX)
print(trainX.shape)
valX = np.asarray(valX)
print(valX.shape)

minPerAttribute = np.min(trainX, axis=0).reshape((1, -1))
trainX = trainX - minPerAttribute
valX = valX - minPerAttribute

maxPerAttribute = np.max(trainX, axis=0).reshape((1, -1))
trainX = trainX / (maxPerAttribute+0.001)
valX = valX / (maxPerAttribute+0.001)


# In[32]:


### save
DB = {}
DB['trainX'] = trainX
DB['valX'] = valX
DB['trainY'] = trainY
DB['valY'] = valY


with open('DB_trainval.pkl', 'wb') as handle:
    pickle.dump(DB, handle, protocol=pickle.HIGHEST_PROTOCOL)


# In[ ]:





# In[ ]:





# In[ ]:





# In[ ]:


# with open('db.pkl', 'rb') as handle:
#     DB = pickle.load(handle)    


# In[ ]:





# # leaving blank (backup)
# 
#     1. dataloader, batch size
#     2. network architecture
#     3. optimizer
#     4. loss functioin, binary classification, BCEloss
#     5. training function
#     6. evaluation
#     
#     
# jupyter nbconvert --to script step001_prepare_dataset.ipynb    
# nohup python step001_prepare_dataset.py 1>step001_prepare_dataset.log &

# In[ ]:





# In[ ]:




