#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Dec  3 20:07:51 2019

@author: gkalstn
"""

# -*- coding: utf-8 -*-
"""AI-PBL 3.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1nT0Mm48fk9iuPwES8dP4peh7KuJJ6Q29
"""
import os
import struct
import numpy as np
import math
import sys
import time
import argparse

#============================파일 불러오기===========================
absol_rou = "/Users/gkalstn/my_project/MNIST"

def read(dataset = "training", path = "."):
    """
    Python function for importing the MNIST data set.  It returns an iterator
    of 2-tuples with the first element being the label and the second element
    being a numpy.uint8 2D array of pixel data for the given image.
    """

    if dataset is "training":
        fname_img = os.path.join(path, 'newtrain-images-idx3-ubyte')
        fname_lbl = os.path.join(path, 'newtrain-labels-idx1-ubyte')
    elif dataset is "testing":
        fname_img = os.path.join(path, 'new1k-images-idx3-ubyte')
        fname_lbl = os.path.join(path, 'new1k-labels-idx1-ubyte')
    else:
        raise Exception("dataset must be 'testing' or 'training'")

    # Load everything in some numpy arrays
    with open(fname_lbl, 'rb') as flbl:
        magic, num = struct.unpack(">II", flbl.read(8))
        lbl = np.fromfile(flbl, dtype=np.int8)

    with open(fname_img, 'rb') as fimg:
        magic, num, rows, cols = struct.unpack(">IIII", fimg.read(16))
        img = np.fromfile(fimg, dtype=np.uint8).reshape(len(lbl), 784)
        img = ((img / 255.) - .5) * 2
        
    return img, lbl

def read_test(dataset = "testall", path = "."):    
    fname_img = os.path.join(path, 'testall-images-idx3-ubyte')    
    with open(fname_img, 'rb') as fimg:
        magic, num, rows, cols = struct.unpack(">IIII", fimg.read(16))
        img = np.fromfile(fimg, dtype=np.uint8).reshape(60000, 784)
        img = ((img / 255.) - .5) * 2
        
    return img
#============================파일 불러오기===========================

"""
#============================cupy gradientGPU===========================

import cupy as cp

def gradientGPU(self, X_batch,  y_batch):
    grad_w = cp.zeros(28*28)
    grad_b = 0
    mask = cp.less_equal(cp.multiply(y_batch, cp.dot(X_batch, self.w_[1:])) + self.w_[0], 1)
    Xy = cp.multiply(X_batch.T, y_batch)
    masked_Xy = cp.multiply(Xy,mask)
    grad_w = cp.sum(-masked_Xy, 1)
    grad_w = grad_w / self.batch_size
    grad_w += self.lamda * self.w_[1:]
    
    masked_y = cp.multiply(y_batch, mask)
    grad_b = cp.sum(-masked_y, 0)
    grad_b = grad_b / self.batch_size
    
    return grad_w, grad_b
#============================cupy gradientGPU===========================


#============================SVC class===========================
from sklearn.base import BaseEstimator, ClassifierMixin
class CustomSVM (BaseEstimator, ClassifierMixin):

    def __init__(self, eta= 0.01, epochs=50, random_state=1, batch_size=32, lamda=0.01):
        self.eta = eta
        self.epochs = epochs   
        self.random_state = random_state
        self.batch_size = batch_size
        self.lamda = lamda

    def fit(self, X, y):

        # X_n = count of records
        X_n = X.shape[0]  
        # 랜덤 generator 생성. numpy random RandomState 사용   
        rgen = np.random.RandomState(self.random_state)
        # Weight는 평균이 0, 표준편차가 0.01, 사이즈는 train의 열 + 1 형태로 무작위 정규분포 표본 추출,  
        self.w_ = cp.array(rgen.normal(loc = 0.0, scale = 0.01, size = 1 + X.shape[1]))
        # Weight의 0번째 인덱스는 0 
        self.w_[0] = 0   
  
    

        #Mini-batch SGD
        for epoch in range (0, self.epochs):
      
            # 트레이닝 셋 크기만큼 증가하는 배열 randomize 생성 ex.) [0, 1, ... 59999]
            randomize = np.array(np.arange(X_n))     
            # randomize 배열 shuffle
            np.random.shuffle(randomize)
    
            # training image, label suffle
            X = X[randomize]
            y = y[randomize]
    
            # batchs = 전체 데이터셋을 batch_size로 나눈 값. 
            batches = math.ceil(X_n / self.batch_size)
    
          # Minibatch
            for j in range(batches):    
    
                # batch_size만큼 추출
                X_batch = cp.array(X[self.batch_size * j : self.batch_size * (j + 1)])
                y_batch = cp.array(y[self.batch_size * j : self.batch_size * (j + 1)])
              
                # Gradient GPU
                grad_w, grad_b = gradientGPU(self, X_batch, y_batch)
                                            
            #SVM SGD weight update        
            self.w_[1:] -= (self.eta * ((grad_w / self.batch_size) + self.lamda * self.w_[1:]))
            self.w_[0] -= self.eta * grad_b / self.batch_size

            if epoch % 10 == 0:
                print(f'Iteration {epoch + 1} / {self.epochs}')

        return self     

    def net_input(self, X):
        return np.dot(X, self.w_[1:]) + self.w_[0]

    def predict(self, X):
        # 양수면 1, 음수면 -1 반환
        return np.where(self.net_input(X) >= 0.0, 1, -1)

class CustomOvrClassifier(BaseEstimator, ClassifierMixin):
    def __init__(self, eta=0.01, epochs=50, random_state=1, batch_size=32, lamda=0.01):
        self.eta = eta
        self.epochs = epochs
        self.random_state = random_state
        self.batch_size = batch_size
        self.lamda = lamda


    def fit(self, X, y):
        print("eta :", self.eta, 
              "epoch :", self.epochs, 
              "batch size :", self.batch_size,
              "lamda :", self.lamda)
        # unique 한 타겟 찾아서 담는 배열
        self.classes_, y = np.unique(y, return_inverse=True)
        # classifer들을 담는 배열
        self.clfs_ = []

        # unique한 타겟의 개수 MNIST 경우 0~9까지 10개
        class_num = len(self.classes_)     
        
        # 클래스 개수 만큼 순회하며 classifier을 self.clfs_에 담는다.
        for target in range(0, class_num):
            clf = CustomSVM(self.eta, self.epochs, self.random_state, self.batch_size, self.lamda) 
            # y_transfomred = target을 제외하고 모두 -1로 만들어버림
            # ex.) target이 1이면 [1, 4, 3, 6, 9...] => [1, -1, -1, -1, -1 ...]
            y_transformed = np.where(y == target, 1, -1)
            # binary SVM에 fit
            clf.fit(X, y_transformed)    
            # fit한 classifier들을 배열에 append
            # ex.) [binary SVM(), binary SVM(), binary SVM()…]   
            self.clfs_.append(clf)
            print(target, "fitting is done")

        return self

    def predict(self, X):
        
        prediction = []
        # Classifer들을 순회하며 net_input값을 prediction 배열에 담는다.  
        # prediction = [-1.5244, -1,2424, 1.232424….]      
        for clf in self.clfs_:
            prediction.append(clf.net_input(X))
               
        # final_pred = prediction 배열에서 최대값을 가지는 인덱스를 담는다.
        # ex.) [-1.5244, -1,2424, 1.232424….] 이면 2로 예측한 것.        
        final_pred = np.argmax(prediction, axis=0)       
        # 클래스 배열에서 final_pred 담긴 인덱스 값을 리턴.  
        
        return self.classes_[final_pred]
#============================SVC class===========================

"""
#============================old===========================
from sklearn.base import BaseEstimator, ClassifierMixin

class CustomSVM (BaseEstimator, ClassifierMixin):

    def __init__(self, eta= 0.01, epochs=50, random_state=1, batch_size=32, lamda=0.01):
        self.eta = eta
        self.epochs = epochs   
        self.random_state = random_state
        self.batch_size = batch_size
        self.lamda = lamda

    def fit(self, X, y):

        # X_n = count of records
        X_n = X.shape[0]  
        # 랜덤 generator 생성. numpy random RandomState 사용
        rgen = np.random.RandomState(self.random_state)
        # Weight는 평균이 0, 표준편차가 0.01, 사이즈는 train의 열 + 1 형태로 무작위 정규분포 표본 추출,  
        self.w_ = rgen.normal(loc = 0.0, scale = 0.01, size = 1 + X.shape[1])
        # Weight의 0번째 인덱스는 0 
        self.w_[0] = 0   
  
    

        #Mini-batch SGD
        for epoch in range (0, self.epochs):
      
            # 트레이닝 셋 크기만큼 증가하는 배열 randomize 생성 ex.) [0, 1, ... 59999]
            randomize = np.arange(X_n)     
            # randomize 배열 shuffle
            np.random.shuffle(randomize)
    
            # training image, label suffle
            X = X[randomize]
            y = y[randomize]
    
            # batchs = 전체 데이터셋을 batch_size로 나눈 값. 
            batches = math.ceil(X_n / self.batch_size)
    
          # Minibatch
            for j in range(batches):    
    
                # batch_size만큼 추출
                X_batch = X[self.batch_size * j : self.batch_size * (j + 1)]
                y_batch = y[self.batch_size * j : self.batch_size * (j + 1)]
              
                # Gradient Weight, bias Initialize
                grad_w = np.zeros(X.shape[1])          
                grad_b = 0.0
              
    
                #Weight, bias Update
                for Xi_batch, yi_batch in zip(X_batch, y_batch):            
                    if yi_batch * self.net_input(Xi_batch) < 1:                
                        grad_w += (- yi_batch * Xi_batch)
                        grad_b += (- yi_batch)
                              
                #SVM SGD weight update        
                self.w_[1:] -= (self.eta * ((grad_w / self.batch_size) + self.lamda * self.w_[1:]))
                self.w_[0] -= self.eta * grad_b / self.batch_size  
            
            if epoch % 10 == 0:
                print(f'Iteration {epoch + 1} / {self.epochs}')

        return self     

    def net_input(self, X):
        return np.dot(X, self.w_[1:]) + self.w_[0]

    def predict(self, X):
        # 양수면 1, 음수면 -1 반환
        return np.where(self.net_input(X) >= 0.0, 1, -1)

class CustomOvrClassifier(BaseEstimator, ClassifierMixin):
    def __init__(self, eta=0.01, epochs=50, random_state=1, batch_size=32, lamda=0.01):
        self.eta = eta
        self.epochs = epochs
        self.random_state = random_state
        self.batch_size = batch_size
        self.lamda = lamda


    def fit(self, X, y):
        print("eta :", self.eta, 
              "epoch :", self.epochs, 
              "batch size :", self.batch_size,
              "lamda :", self.lamda)
        # unique 한 타겟 찾아서 담는 배열
        self.classes_, y = np.unique(y, return_inverse=True)
        # classifer들을 담는 배열
        self.clfs_ = []

        # unique한 타겟의 개수 MNIST 경우 0~9까지 10개
        class_num = len(self.classes_)     
        
        # 클래스 개수 만큼 순회하며 classifier을 self.clfs_에 담는다.
        for target in range(0, class_num):
            clf = CustomSVM(self.eta, self.epochs, self.random_state, self.batch_size, self.lamda) 
            # y_transfomred = target을 제외하고 모두 -1로 만들어버림
            # ex.) target이 1이면 [1, 4, 3, 6, 9...] => [1, -1, -1, -1, -1 ...]
            y_transformed = np.where(y == target, 1, -1)
            # binary SVM에 fit
            clf.fit(X, y_transformed)    
            # fit한 classifier들을 배열에 append
            # ex.) [binary SVM(), binary SVM(), binary SVM()...]   
            self.clfs_.append(clf)
            print(target, "fitting is done")

        return self

    def predict(self, X):
        
        prediction = []
        # Classifer들을 순회하며 net_input값을 prediction 배열에 담는다.  
        # prediction = [-1.5244, -1,2424, 1.232424....]      
        for clf in self.clfs_:
            prediction.append(clf.net_input(X))
               
        # final_pred = prediction 배열에서 최대값을 가지는 인덱스를 담는다.
        # ex.) [-1.5244, -1,2424, 1.232424....] 이면 2로 예측한 것.        
        final_pred = np.argmax(prediction, axis=0)       
        # 클래스 배열에서 final_pred 담긴 인덱스 값을 리턴.  
        
        return self.classes_[final_pred]
#============================old===========================


#============================Deskew===========================

from scipy.ndimage import interpolation
def moments(image):
    c0,c1 = np.mgrid[:image.shape[0],:image.shape[1]] # create a mesh grid
    totalImage = np.sum(image) #sum of pixels
    m0 = np.sum(c0*image)/totalImage #mu_x
    m1 = np.sum(c1*image)/totalImage #mu_y
    m00 = np.sum((c0-m0)**2*image)/totalImage #var(x)
    m11 = np.sum((c1-m1)**2*image)/totalImage #var(y)
    m01 = np.sum((c0-m0)*(c1-m1)*image)/totalImage #covariance(x,y)
    mu_vector = np.array([m0,m1]) # Notice that these are \mu_x, \mu_y respectively
    covariance_matrix = np.array([[m00,m01],[m01,m11]])
    return mu_vector, covariance_matrix

def deskew(image):
    c,v = moments(image)
    alpha = v[0,1]/v[0,0]
    affine = np.array([[1,0],[alpha,1]])
    ocenter = np.array(image.shape)/2.0
    offset = c-np.dot(affine,ocenter)
    img = interpolation.affine_transform(image,affine,offset=offset)
    return (img - img.min()) / (img.max() - img.min())
#============================SVC class===========================


#============================Poly Feature Extraction===========================

from itertools import combinations_with_replacement
import math

def col_mul(X, comb) :
    x1 = X[:, comb[0]]
    x2 = X[:, comb[1]]
    return np.multiply(x1, x2)



def poly(X) :
    t = [] #빈 리스트
    n = np.arange(0, X.shape[1], 1) #0~783이 들어간 np리스트 만듬
    comb = list(combinations_with_replacement(n, 2))
    
    for i in comb : #comb길이만큼(nC2 만큼 반복)
        t.append(col_mul(X, i)) #tmp를 t에 추가
    X_poly = np.array(t).T #t를 np배열로 바꾸고 Transpose
    #return np.hstack((X,X_poly)) # 원본 데이터셋과 Poly Feature들을 열결합 
    return X_poly


def poly_(X, factor) :#factor은 28의 약수
    #(60000,784) 배열을 (60000, 596) 으로 바꾸기
    n = int(math.sqrt(X.shape[1]))
    t = np.arange(0, X.shape[1], 1)
    index_mat = t.reshape(n, n)
    cut_mat = index_mat[2:26, 2:26]
    flat_ = cut_mat.reshape(1, 576)[0]
    X = X[:, flat_]
    
    
    n = int(math.sqrt(X.shape[1]))

    X_poly = X
    t = np.arange(0, X.shape[1], 1)
    index_mat = t.reshape(n, n)
    
    for i in range(int(n / factor)) : #0~factor
        for j in range(int(n / factor)) : #0~factor
            list_ = []
            for k in range(factor) :
                for q in range(factor) :
                    list_.append(index_mat[i*factor + k][j*factor + q])
            X_ = X[:, list_]
            X_poly = np.hstack((X_poly, poly(X_)))
    return X_poly[:, :5200]      

#============================Poly Feature Extraction===========================
    
#============================Convolution Feature Extraction np===========================
def matmul(mat1, mat2) :
    a = 0
    for i in range(mat1.shape[0]) :
        a += np.dot(mat1[:,i], mat2[:, i])    
    return a

def Convolution(X) :
    
    filters = []
    filter0 = np.array([[0.7, 0.8, 0.7, 0.8, 0.9, 0.8, 0.7, 0.0],
                        [0.0, 0.8, 0.8, 0.9, 0.9, 0.8, 0.6, 0.0],
                        [0.0, 0.7, 0.9, 0.9, 0.9, 0.7, 0.5, 0.0],
                        [0.0, 0.7, 1.0, 0.9, 0.9, 0.7, 0.4, 0.0],
                        [0.0, 0.7, 1.0, 1.0, 0.9, 0.7, 0.4, 0.0],
                        [0.0, 0.7, 0.9, 1.0, 0.9, 0.7, 0.4, 0.0],
                        [0.0, 0.8, 0.7, 0.9, 1.0, 0.8, 0.5, 0.0],
                        [0.7, 0.9, 1.0, 0.8, 1.0, 0.9, 0.7, 0.0]])
    filters.append(filter0)
    filter1 = np.array([[0.0, 0.8, 0.9, 1. , 1. , 0.9, 0.0, 0. ],
                        [0.0, 0.8, 0.9, 1. , 0.9, 0.9, 0.0, 0. ],
                        [0.0, 0.8, 1. , 1. , 0.9, 0.9, 0.0, 0. ],
                        [0.0, 0.9, 1. , 1. , 0.9, 0.8, 0.0, 0. ],
                        [0.0, 0.9, 1. , 1. , 0.9, 0.8, 0.0, 0. ],
                        [0.0, 1. , 1. , 1. , 0.9, 0.7, 0.0, 0. ],
                        [0.0, 1. , 1. , 1. , 0.9, 0.7, 0.0, 0. ],
                        [0.0, 1. , 1. , 1. , 0.9, 0.8, 0.0, 0. ]])
    filters.append(filter1)
    filter2 = np.array([[0.5, 0.5, 0.6, 0.7, 0.8, 0.9, 0.9, 0.9],
                        [0.7, 0.7, 0.7, 0.8, 0.9, 0.9, 0.9, 0.9],
                        [0.9, 0.9, 0.8, 0.8, 0.9, 0.9, 0.9, 0.9],
                        [1. , 1. , 0.9, 0.9, 0.9, 0.9, 0.9, 0.9],
                        [0.9, 1. , 1. , 1. , 1. , 1. , 0.9, 0.8],
                        [0.6, 0.8, 0.9, 1. , 1. , 0.9, 0.8, 0.7],
                        [0.3, 0.5, 0.6, 0.7, 0.7, 0.6, 0.5, 0.5],
                        [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]])
    filters.append(filter2)
    filter3 = np.array([[0.2, 0.2, 0.4, 0.5, 0.7, 0.9, 0.8, 0.6],
                        [0.3, 0.5, 0.7, 0.8, 0.9, 0.9, 0.7, 0.5],
                        [0.6, 0.8, 0.9, 1. , 0.9, 0.9, 0.7, 0.4],
                        [0.8, 0.9, 1. , 1. , 0.9, 0.8, 0.7, 0.5],
                        [0.7, 0.8, 0.8, 0.8, 0.7, 0.7, 0.7, 0.7],
                        [0.5, 0.5, 0.5, 0.5, 0.5, 0.6, 0.7, 0.7],
                        [0.2, 0.2, 0.2, 0.2, 0.3, 0.4, 0.6, 0.8],
                        [0.1, 0.1, 0.1, 0.1, 0.2, 0.3, 0.5, 0.8]])
    filters.append(filter3)
    filter4 = np.array([[0.2, 0.3, 0.6, 0.8, 0.9, 0.9, 0.6, 0.3],
                        [0.4, 0.5, 0.7, 0.9, 0.9, 0.9, 0.6, 0.3],
                        [0.7, 0.8, 0.9, 1. , 1. , 1. , 0.7, 0.4],
                        [1. , 1. , 1. , 1. , 1. , 1. , 0.7, 0.4],
                        [1. , 1. , 1. , 0.9, 0.9, 0.8, 0.6, 0.3],
                        [0.8, 0.9, 1. , 0.8, 0.6, 0.5, 0.3, 0.2],
                        [0.6, 0.9, 0.9, 0.7, 0.4, 0.3, 0.1, 0.1],
                        [0.6, 0.9, 0.8, 0.5, 0.3, 0.2, 0.1, 0. ]])
    filters.append(filter4)
    filter5 = np.array([[0.1, 0.2, 0.4, 0.6, 0.8, 1. , 1. , 0.9],
                        [0.2, 0.4, 0.6, 0.8, 1. , 1. , 0.9, 0.8],
                        [0.3, 0.6, 0.8, 1. , 1. , 0.9, 0.7, 0.6],
                        [0.4, 0.8, 0.9, 1. , 0.9, 0.7, 0.5, 0.4],
                        [0.5, 1. , 1. , 1. , 0.9, 0.7, 0.5, 0.4],
                        [0.6, 1. , 1. , 1. , 0.9, 0.8, 0.7, 0.5],
                        [0.5, 0.9, 0.9, 1. , 0.9, 0.9, 0.8, 0.7],
                        [0.4, 0.7, 0.7, 0.8, 0.8, 0.8, 0.8, 0.7]])
    filters.append(filter5)
    filter6 = np.array([[0.6, 0.7, 0.8, 0.8, 0.7, 0.7, 0.8, 0.8],
                        [0.7, 0.8, 0.7, 0.7, 0.7, 0.7, 0.9, 1. ],
                        [0.7, 0. , 0. , 0. , 0. , 0.8, 1. , 1. ],
                        [0.6, 0. , 0. , 0. , 0.7, 0.9, 1. , 0.9],
                        [0.6, 0. , 0.5, 0.7, 0.9, 1. , 0.9, 0.7],
                        [0.7, 0.8, 0.8, 1. , 1. , 0.8, 0. , 0. ],
                        [1. , 1. , 1. , 1. , 0.7, 0.5, 0. , 0. ],
                        [1. , 0.9, 0.7, 0. , 0. , 0. , 0. , 0. ]])
    filters.append(filter6)
    filter7 = np.array([[0.5, 0.6, 1. , 1. , 1. , 1. , 1. , 0.5],
                        [1. , 1. , 1. , 1. , 1. , 0.8, 1. , 0.7],
                        [0.8, 0.7, 0.7, 0.7, 0.7, 1. , 1. , 0.7],
                        [0.6, 0.5, 0.5, 0.6, 0.8, 1. , 1. , 0.7],
                        [0.5, 0.4, 0.5, 0.7, 0.9, 1. , 0.9, 0.5],
                        [0.5, 0.5, 0.7, 0.9, 1. , 0.9, 0.7, 0.3],
                        [0.8, 0.8, 1. , 1. , 0.8, 0.6, 0.3, 0.1],
                        [1. , 1. , 1. , 0.7, 0.5, 0.2, 0.1, 0. ]])
    filters.append(filter7)
    filter8 = np.array([[0.7, 0.5, 0.3, 0.3, 0.5, 0.7, 0.8, 0.9],
                        [0.7, 0.7, 0.6, 0.6, 0.7, 0.8, 0.8, 0.8],
                        [0.7, 0.9, 0.8, 0.8, 0.9, 0.9, 0.7, 0.6],
                        [0.7, 0.9, 1. , 1. , 1. , 0.8, 0.6, 0.4],
                        [0.6, 0.9, 1. , 1. , 1. , 0.8, 0.5, 0.4],
                        [0.7, 0.9, 0.9, 0.8, 0.8, 0.7, 0.6, 0.4],
                        [0.8, 0.9, 0.7, 0.6, 0.6, 0.7, 0.6, 0.5],
                        [0.8, 0.8, 0.5, 0.4, 0.5, 0.7, 0.7, 0.7]])
    filters.append(filter8)
    filter9 = np.array([[1. , 1. , 1. , 1. , 1. , 1. , 1. , 0.9],
                        [0.9, 0.9, 0.9, 0.9, 0.9, 1. , 1. , 0.7],
                        [0.6, 0.5, 0.5, 0.5, 0.7, 1. , 0.9, 0.6],
                        [0.3, 0.3, 0.3, 0.4, 0.7, 1. , 0.8, 0.4],
                        [0.2, 0.1, 0.2, 0.4, 0.8, 0.9, 0.6, 0.3],
                        [0.1, 0.1, 0.2, 0.5, 0.9, 0.9, 0.5, 0.2],
                        [0. , 0. , 0.3, 0.7, 0.9, 0.8, 0.3, 0.1],
                        [0. , 0.1, 0.4, 0.8, 0.9, 0.6, 0.2, 0.1]])
    filters.append(filter9)    
    i = 0
    X_conv = []
    for xi in X : #X의 한행씩 가져오기
        tmp = list()
        mat = xi.reshape(28, 28) # 한 행을 28x28매트릭스로 변환
        tmp_mat = np.zeros(21*21).reshape(21, 21) #conv 매트릭스 행렬
        
        for p in range(10): 
            for j in range(mat.shape[0] - 8 + 1) :
                for k in range(mat.shape[1] - 8 + 1) : #다끝나고나면 21x21행렬
                    tmp_mat[j][k] = matmul(mat[j:j+8, k:k+8], filters[p])
        
        
            #22x22행렬 -> 1x484
            flat = tmp_mat.reshape(1, tmp_mat.shape[0]*tmp_mat.shape[0]) #1x441
            tmp += flat.tolist()[0]    
            
            
        X_conv.append(tmp)
        
        print(i)  
        i += 1
    return np.hstack((X,np.array(X_conv)))

#============================Convolution Feature Extraction===========================


"""
#============================Convolution Feature Extraction===========================
def make_grad(x):
    size=x.shape[0]
    x_filter=np.array([[-1, 0, 1],
                       [-1, 0, 1],
                       [-1, 0, 1]])
    y_filter=np.array([[-1, -1, -1],
                       [ 0,  0,  0],
                       [ 1,  1,  1]])
    x_reshaped=np.reshape(x,(size, 28, 28))
    bucket=[]
    for i in range(size):
        x_padded=np.lib.pad(x_reshaped[i], (1,1), 'edge')
        if(i==1): debug(x_padded.shape)
        shapes=x_padded.shape
        item=[]
        #x_filter=x_filter.reshape(3,3)
        x1=x_padded.reshape((30,30))
        xgrads=np.zeros_like([],shape=(28,28))
        ygrads=np.zeros_like([],shape=(28,28))

        width, height=x1.shape
        for w in range(1, width-1):
            for h in range(1, height-1):
                x_grad_slice=np.multiply(x1[w-1:w+2, h-1:h+2], x_filter)
                y_grad_slice=np.multiply(x1[w-1:w+2, h-1:h+2], y_filter)

                xgrad=np.sum(x_grad_slice)
                ygrad=np.sum(y_grad_slice)
                xgrads[w-1][h-1]=xgrad
                ygrads[w-1][h-1]=ygrad
            
        #if(i==1):
        bucket.append([xgrads, ygrads])
        
          #for x2 in range(shapes[1]):
            #item.append(x_padded[x1]*x_padded[x2])
    
        return np.array(bucket)
#============================Convolution Feature Extraction===========================
"""

#============================Load data===========================
from sklearn.metrics import accuracy_score
from sklearn.metrics import roc_auc_score
from sklearn.metrics import recall_score
from sklearn.metrics import precision_score
from sklearn.metrics import f1_score

X_train, y_train = read("training", absol_rou)
X_test, y_test = read("testing", absol_rou)
X_test_all = read_test("testall", absol_rou)

X_train += 1
X_test += 1
X_test_all += 1

#============================Load data===========================


#============================Deskew X===========================
deskewed_X_train = np.array([np.reshape(deskew(np.reshape(x, (28,28))), 784) for x in X_train])
deskewed_X_test = np.array([np.reshape(deskew(np.reshape(x, (28,28))), 784) for x in X_test])
deskewed_X_test_all = np.array([np.reshape(deskew(np.reshape(x, (28,28))), 784) for x in X_test_all])
#============================Deskew X===========================


#============================MinMaxScaler===========================
from sklearn.preprocessing import MinMaxScaler

mm = MinMaxScaler()
mm.fit(deskewed_X_train)

X_train_mm = mm.transform(deskewed_X_train)
X_test_mm = mm.transform(deskewed_X_test)
X_test_all_mm = mm.transform(deskewed_X_test_all)

immg = X_train_mm[0].reshape(28, 28)
#============================Standard Scaler===========================
from sklearn.preprocessing import StandardScaler

sc = StandardScaler()
sc.fit(deskewed_X_train)

X_train_std = sc.transform(deskewed_X_train)
X_test_std = sc.transform(deskewed_X_test)
X_test_all_std = sc.transform(deskewed_X_test_all)


immg = X_train_mm[0].reshape(28, 28)
a1 = X_train[12].reshape(28,28)
a2 = deskewed_X_train[12].reshape(28,28)
a3 = X_train_std[12].reshape(28,28)
a4 = X_train_mm[12].reshape(28,28)

for i in X_train :
    summ += i


dis = summ.reshape(28, 28)
cut = dis[2:26, 2:26]

summ = np.zeros(784)
for xi, yi in zip(X_train, y_train) :
    if yi == 5 :
        summ += xi
num = summ.reshape(28, 28)

"""
2가지 Poly & Convolution 을 Deskew된 데이터에 실행
"""


#============================Poly and Convolution===========================
#poly
X_train_poly = poly_(X_train_mm, 4)
X_test_poly = poly_(X_test_mm, 4)
X_test_all_poly = poly(X_test_all_mm, 4)
#Convolution
X_train_conv = Convolution(X_train_mm)
X_train_conv.shape
X_test_conv = Convolution(X_test_mm)

#============================Poly and Convolution===========================



#============================PCA===========================

from mlxtend.feature_extraction import PrincipalComponentAnalysis
#pca to poly
pca_poly = PrincipalComponentAnalysis()
pca_poly.fit(X_train_poly)
X_train_pca = pca_poly.transform(X_train_poly)
X_test_pca = pca_poly.transform(X_test_poly)

pca_conv = PrincipalComponentAnalysis()
pca_conv.fit(X_train_conv)
X_train_pca = pca_conv.transform(X_train_conv)
X_test_pca = pca_conv.transform(X_test_conv)
#pca to convolution
#============================PCA===========================
from mlxtend.preprocessing import standardize
import matplotlib.pyplot as plt

tot = sum(pca_conv.e_vals_)
var_exp = [(i / tot)*100 for i in sorted(pca_conv.e_vals_, reverse=True)]
cum_var_exp = np.cumsum(pca_conv.e_vals_normalized_*100)

a = pca_conv.e_vals_.shape[0]

with plt.style.context('seaborn-whitegrid'):
    fig, ax = plt.subplots(figsize=(400, 10))
    plt.bar(range(a), var_exp, alpha=0.5, align='center',
            label='individual explained variance')
    plt.step(range(a), cum_var_exp, where='mid',
             label='cumulative explained variance')
    plt.ylabel('Explained variance ratio')
    plt.xlabel('Principal components')
    plt.xticks(range(a))
    ax.set_xticklabels(np.arange(1, X_train_conv.shape[1] + 1))
    plt.legend(loc='best')
    plt.tight_layout()


from sklearn.metrics import accuracy_score
from sklearn.metrics import roc_auc_score
from sklearn.metrics import recall_score
from sklearn.metrics import precision_score
from sklearn.metrics import f1_score
from sklearn.metrics import make_scorer


model = CustomOvrClassifier()
model.fit(X_train_pca, y_train)
pred = model.predict(X_test_pca)
score = make_scorer(accuracy_score) #F1 score
accuracy_score(y_test[:500], pred)
import pandas as pd


X_train_conv_df = pd.DataFrame(X_train_conv)
X_test_conv_df = pd.DataFrame(X_test_conv)

X_train_conv_df.to_excel('X_train_conv.xlsx')
X_test_conv_df.to_excel('X_test_conv.xlsx')
