import pandas as pd
import numpy  as np

from sklearn.neighbors 		  import KNeighborsClassifier
from sklearn.model_selection  import KFold
from sklearn.metrics 		  import accuracy_score
from sklearn.svm 			  import LinearSVC
from sklearn.ensemble 		  import RandomForestClassifier
from sklearn 				  import svm, grid_search
from sklearn 				  import tree
from time 					  import time
from sklearn.metrics          import precision_score
from sklearn.metrics          import recall_score
from sklearn 				  import cross_validation


print "Loading data..."
# load data into workable form
training_set = pd.read_csv('train.csv')
# this data is used for the generated file at the end
test_final  = pd.read_csv('test.csv')

# set labels and samples
y = training_set['label']
X = training_set[training_set.columns[1:]]

# remove some samples to speed up testing
#y = y.drop(y.index[[np.arange(31000).tolist()]])
#X = X.drop(X.index[[np.arange(31000).tolist()]])

# Knn turns out to be good for digit classification
# Parameters selected with grid search
clf = KNeighborsClassifier(weights='distance' ,algorithm='auto', p=2, n_jobs=6)
#clf = RandomForestClassifier(n_estimators=10)
#clf = LinearSVC()

# remove unimportant features using a decision tree and its methods
'''
clft = tree.DecisionTreeClassifier()
features = clft.feature_importances_

unimportant_feature_indexes = [
	index for index in range(len(features))
	if features[index] <= 0
]
'''

# this indices belong to unnecesary features generated by
# a decision tree with less than 42000 samples
unimportant_feature_indexes = [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42,43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58,
59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74,75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90,
91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105,106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
120, 121, 122, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134,135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147,
148, 149, 150, 151, 152, 153, 155, 156, 158, 159, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177,
179, 180, 181, 182, 183, 185, 186, 188, 190, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 205, 206, 207, 208, 211, 212,
213, 214, 215, 216, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 235, 237, 238, 240, 242, 243, 245, 246,
247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 265, 267, 269, 270, 271, 273, 274, 275, 277, 278,
279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 294, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309,
310, 311, 312, 313, 314, 315, 316, 319, 320, 321, 324, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341,
342, 343, 344, 345, 346, 351, 352, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373,
375, 376, 378, 379, 381, 382, 383, 384, 385, 386, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 399, 401, 403, 404, 406, 407, 408,
409, 410, 411, 412, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 431, 432, 433, 435, 436, 437, 438, 439,
440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 463, 465, 466, 468, 469, 470,
471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 485, 486, 487, 488, 490, 491, 492, 493, 494, 495, 496, 497, 498, 500,
501, 502, 503, 504, 505, 506, 507, 508, 510, 511, 512, 513, 514, 516, 519, 520, 524, 525, 527, 528, 529, 530, 531, 532, 533, 534, 535,
536, 537, 538, 539, 540, 541, 542, 544, 545, 546, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565,
566, 567, 569, 570, 571, 572, 573, 574, 575, 576, 577, 579, 580, 581, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595,
596, 598, 600, 602, 603, 604, 605, 606, 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624, 625, 626,
628, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 653, 654, 655, 657,
659, 660, 661, 662, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685, 686,
687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713,
714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740,
741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, 767,
768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 780, 781, 782, 783
]

# drop unnecesary features
num_features = len(X.columns) - len(unimportant_feature_indexes) 
X = X.drop(X.columns[unimportant_feature_indexes], axis=1) 
print "Feature space size:", num_features
print "Original feature space size:", len(X.columns) + len(unimportant_feature_indexes)

# grid search algorithm for knn tuning
'''
parameters = {
				'weights':('uniform', 'distance'), 'algorithm':['ball_tree', 'kd_tree'],
				'p': [1, 2], 
			}
knn = KNeighborsClassifier()
clf = grid_search.GridSearchCV(knn, parameters)
clf.fit(X, y)

print clf.best_params_
'''

# we have to extract the values only, if not then
# X and y will not work in KFold because it will include
# data frame index names. iloc[] could have also been used
X_v = X.values
y_v = y.values

# we meassure time for both training and prediction
t0 = time()

# do cross validation with K folds
kf = KFold(n_splits=2)

print "Training and testing..."
scores = [
	clf.fit(X_v[train], y_v[train]).score(X_v[test],y_v[test]) 
	for train, test in kf.split(X_v)
]
print np.array(scores).mean()

print "total time:", round(time()-t0, 3), "s"

# these metrics cannot be used because we have non-binary labels.
# unless labels=[pos_label] and average != 'binary', for each label
'''
X_train, X_test, y_train, y_test = cross_validation.train_test_split(X/255.,y,test_size=0.1,random_state=0)
pred = clf.predict(X_test)
print 'precision = ', precision_score(y_test,pred)
print 'recall = ', recall_score(y_test,pred)
'''	
print "Generating predictions..."
test_final = test_final.drop(test_final.columns[unimportant_feature_indexes], axis=1) 
predictions = clf.predict(test_final)
predictions = predictions.astype(int)

print "Generating CSV"
submission = pd.DataFrame({
        "ImageId": range(1, len(predictions)+1),
        "Label": predictions
    })
 
submission.to_csv('mnistSubmission1.csv', index=False)

print "Done!"

# ------------ usefull notes ------------
'''
Kaggle test set score: 0.94043

data = df.drop('label',axis = 1) # insted of doing transpose

import pydotplus 
dot_data = tree.export_graphviz(clf, out_file=None) 
graph = pydotplus.graph_from_dot_data(dot_data) 
graph.write_pdf("iris.pdf")

---times---
discarting features (removing 703 features):
accuracy: 0.935119047619
or 0.937214285714 after parameter tuning with gridSearch
total time: 239.369 s

without discarting features:
accuracy: 0.959904761905
total time: 1358.581 s
'''





