from __future__ import print_function
import numpy as np 
from keras.datasets import mnist 
from keras.models import Sequential 
from keras.layers.core import Dense, Activation, Dropout 
from keras.optimizers import SGD,RMSprop, Adam 
from keras.utils import np_utils 
from keras import regularizers
np.random.seed(1671) # for reproducibility 
# network and training 
NB_EPOCH = 200 
BATCH_SIZE = 128 
VERBOSE = 1 
NB_CLASSES = 10 # number of outputs = number of digits 
#Model can be tested against various optimizer funcitons RMSprop(), Adam() and the one given below
OPTIMIZER = SGD() # SGD optimizer, explained later in this chapter 
N_HIDDEN = 128 
VALIDATION_SPLIT=0.2 # how much TRAIN is reserved for VALIDATION 
DROPOUT = 0.3
# data: shuffled and split between train and test sets 
# 
(X_train, y_train), (X_test, y_test) = mnist.load_data() 
#X_train is 60000 rows of 28x28 values --> reshaped in 60000 x 784 
RESHAPED = 784 
# 
X_train = X_train.reshape(60000, RESHAPED) 
X_test = X_test.reshape(10000, RESHAPED) 
X_train = X_train.astype('float32') 
X_test = X_test.astype('float32') 
# normalize 
# 
X_train /= 255 
X_test /= 255 
print(X_train.shape[0], 'train samples') 
print(X_test.shape[0], 'test samples') 
# convert class vectors to binary class matrices 
Y_train = np_utils.to_categorical(y_train, NB_CLASSES) 
Y_test = np_utils.to_categorical(y_test, NB_CLASSES) 
# M_HIDDEN hidden layer
# 10 outputs 
# final stage is softmax 
model = Sequential() 
model.add(Dense(N_HIDDEN, input_shape=(RESHAPED,), kernel_regularizer=regularizers.l2(0.01)))
model.add(Activation('relu'))
model.add(Dropout(DROPOUT))
model.add(Dense(N_HIDDEN))
model.add(Activation('relu'))
model.add(Dropout(DROPOUT))
model.add(Dense(NB_CLASSES))
model.add(Activation('softmax')) 
model.summary()
model.compile(loss='categorical_crossentropy', optimizer=OPTIMIZER, metrics=['accuracy']) 
#-------------------------------------------------------------
# save best model 
checkpoint = ModelCheckpoint(filepath=os.path.join(MODEL_DIR, "model-{epoch:02d}.h5", save_best_only = True)) 
#---------------------------------------------------------------
history = model.fit(X_train, Y_train, batch_size=BATCH_SIZE, epochs=NB_EPOCH, verbose=VERBOSE, validation_split=VALIDATION_SPLIT) 
score = model.evaluate(X_test, Y_test, verbose=VERBOSE) 
print("Test score:", score[0]) 
print('Test accuracy:', score[1]) 
