from copy import deepcopy
import numpy as np
import random
import operator
from itertools import combinations
import csv
import time
import os
import gc
import sys
import inspect

from keras.callbacks import EarlyStopping
from sklearn.metrics import f1_score, accuracy_score, confusion_matrix

from .RNN import generate_model
from .useful import *

random.seed(12)
np.random.seed(12)

class Experiment():
	"""docstring for Experiment"""
	def __init__(self, parameters, search_algorithm="grid",
		x_test=None, y_test=None,
		x_train=None, y_train=None,
		data=None, folds=10,
		folder_name=str(time.time()),
		thresholding=False, threshold=0.5, model_type="rnn"):

		assert (type(folds) is int) or (data == None), "folds must be an integer if data tuple is provided"
		assert (search_algorithm.lower() == "grid") or (search_algorithm.lower() == "random"), "Only 'grid' and 'random' permissible values for search_algorithm"

		self.headers = [
			'bytes.common_bytes1',
            'bytes.common_bytes2',
            'bytes.common_bytes3',
            'bytes.common_bytes4',
            'bytes.common_bytes5',
            'bytes.common_bytes6',
            'bytes.entropy',
            'bytes.longest_sequence.byte',
            'bytes.longest_sequence.length',
            'bytes.longest_sequence.offset',
            'bytes.max_entropy',
            'bytes.min_entropy',
            'bytes.null_bytes',
            'bytes.printable',
            'bytes.rarest_bytes1',
            'bytes.rarest_bytes2',
            'bytes.rarest_bytes3',
            'bytes.rarest_bytes4',
            'bytes.rarest_bytes5',
            'bytes.rarest_bytes6',
            'bytes.unique_bytes',
            'bytes.white_spaces',
            'elf.e_phentsize',
            'elf.e_phnum',
            'elf.e_phoff',
            'elf.e_shentsize',
            'elf.e_shnum',
            'elf.e_shoff',
            'elf.e_shstrndx',
            'elf.nsections',
            'elf.nsegments',
            'elf.stripped',
            'elf.stripped_sections',
            'funcover.findcrypt.total',
            'funcover.idapro.average_bytes_func',
            'funcover.idapro.avg_basic_blocks',
            'funcover.idapro.avg_cyclomatic_complexity',
            'funcover.idapro.avg_loc',
            'funcover.idapro.badstack',
            'funcover.idapro.branch_instr',
            'funcover.idapro.bytes_func',
            'funcover.idapro.call_instr',
            'funcover.idapro.func_loc',
            'funcover.idapro.indirect_branch_instr',
            'funcover.idapro.indirect_call_instr',
            'funcover.idapro.loc',
            'funcover.idapro.max_basic_blocks',
            'funcover.idapro.max_cyclomatic_complexity',
            'funcover.idapro.nfuncs',
            'funcover.idapro.overlapped_instr',
            'funcover.idapro.percent_load_covered',
            'funcover.idapro.percent_text_covered',
            'funcover.idapro.syscall_instr',
            'libide.lstrings.libc.matches',
            'unpacker.packed']

		#hyperparameters
		self.original_h_params = parameters
		self.h_params = parameters

		# set up parameter search space depending on algorithm
		self.search_algorithm = search_algorithm
		self.current_params = {}

		if self.search_algorithm == "grid":
			self.h_params = dict([(key, list(self.h_params[key])) for key in self.h_params])
			self.original_h_params = deepcopy(self.h_params)
			self.current_params = dict([(key, self.h_params[key][0]) for key in self.h_params])
		if self.search_algorithm == "random":
			self.__list_to_dict_params() # list instances to even probability dictionary instances
			self.__map_to_0_1() # re-adjust any instances in which sum(probabilities) > 1

		# metrics and writer objects, to be assigned when first experiment written up
		self.folder_name = check_filename(folder_name)
		self.experiment_id = 0
		self.metrics_headers = None
		self.metrics_writer = None

		#Model type
		self.model_type = model_type

		# Thresholding set up
		self.thresholding = thresholding
		if self.thresholding:
			self.min_threshold = threshold + K.epsilon()
			self.temp_min_threshold = threshold + K.epsilon()

		# test-train experiment - on all dataset
		if (data == None) or (folds == None):
			self.folds = None
			self.X_TRAIN = x_train
			self.Y_TRAIN = y_train
			self.X_TEST = x_test
			self.Y_TEST = y_test
			print("Test-train experiment")
		# k-fold cross-validation experiment - just on training set
		else:
			assert folds != None, "Supply number of folds for k-fold cross validation or supply x_train, y_train, x_test, y_test"
			self.folds = folds
			self.x = data[0]
			self.y = data[1]
			print(self.folds, "- fold cross validation experiment")



	def __list_to_dict_params(self):
			for key in self.h_params:
				if type(self.h_params[key]) is list:
					self.h_params[key] = dict([(x, 1/(len(self.h_params[key]) + K.epsilon() )) for x in self.h_params[key]])

	def __map_to_0_1(self):
		"""maps input probabilities to values between 0 and 1, preserving scalar relationships"""
		for key in self.h_params:
			running_total = 0
			scalar = 1/(sum(self.h_params[key].values()))
			for possible_value in self.h_params[key]:
				if self.h_params[key][possible_value] < 0:
					raise ValueError("Negative hyperparameter probabilities are not allowed ({} for {})").format(self.h_params[key][possible_value], possible_value)
				new_value = self.h_params[key][possible_value] * scalar
				self.h_params[key][possible_value] = new_value + running_total
				running_total += new_value

	def __random_config(self):
		"""randomly generate a configuration of hyperparameters from dictionary self.h_params"""
		for key in self.h_params:
			choice = random.random()
			sorted_options = sorted(self.h_params[key].items(), key=operator.itemgetter(1))
			for option in sorted_options:
				if choice < option[1]:
					self.current_params[key] = option[0]
					break
		if self.current_params["optimiser"] == "adam":
			self.current_params["learning_rate"] = 0.001
		print()

	def run_one_experiment(self):
		print("run one experiment - orig")
		#Get new configuration if random search
		if self.search_algorithm == "random":
			self.__random_config()

		self.experiment_id += 1
		print("running expt", self.experiment_id, "of", self.num_experiments)
		print(self.current_params)

		self.metrics = {}
		self.current_fold = 1
		self.accuracy_scores = []

		# k-fold cross-validation
		if self.folds != None:
			y = deepcopy(self.y)
			x = deepcopy(self.x)

			#remove short seqeunces and store indicies of kept items
			x, y, identifiers = remove_short_idx(x, y, list(range(len(y))), self.current_params["sequence_length"])
			labels = {}
			temp_y = deepcopy(y).flatten() # get labels as flat array to find stratified folds

			for i, class_label in zip(identifiers, temp_y):
				if class_label in labels:
					labels[class_label].append(i)
				else:
					labels[class_label] = [i]

			#split into number of folds
			fold_indicies = [[] for x in range(self.folds)]
			# Want to represent class distribution in each set (stratified split)
			# Divide indicies list in chunks of size folds
			for key in labels:
				labels[key] = to_chunks(labels[key], self.folds)
				for i, fold_ids in enumerate(labels[key]):
					fold_indicies[i] += fold_ids

			fold_indicies = np.array([[int(x) for x in index_set] for index_set in fold_indicies])
			#take new copies of the original to use indicies from the original sets
			x, y = deepcopy(self.x), deepcopy(self.y)

			for i in list(range(self.folds)):
				test = np.array([(i) % self.folds]) # one fold is test set
				train = np.array([i for i in range(self.folds) if i not in [test]]) # remaining folds are training set ( not in [test, val])

				test_idxs = np.concatenate(tuple([fold_indicies[i] for i in test]))
				train_idxs = np.concatenate(tuple([fold_indicies[i] for i in train]))

				self.x_train, self.y_train = truncate_and_tensor(x[train_idxs], y[train_idxs], self.current_params["sequence_length"])
				self.x_test, self.y_test = truncate_and_tensor(x[test_idxs], y[test_idxs], self.current_params["sequence_length"])

				self.test_idxs = test_idxs

				stop = self.set_up_model()
				if stop:
					return

				self.current_fold += 1

		# test-train
		else:
			self.x_train = deepcopy(self.X_TRAIN)
			self.y_train = deepcopy(self.Y_TRAIN)
			self.x_test = deepcopy(self.X_TEST)
			self.y_test = deepcopy(self.Y_TEST)

			self.test_idxs = np.array(range(1, len(self.y_test) + 1)) / 10 #divide by 10 to distinguish from training/10-fold data

			# remove short sequences - store indicies for test data
			self.x_train, self.y_train = remove_short(self.x_train, self.y_train, self.current_params["sequence_length"])
			self.x_test, self.y_test, self.test_idxs = remove_short_idx(self.x_test, self.y_test, self.test_idxs, self.current_params["sequence_length"])

			self.set_up_model()


	def set_up_model(self):
		# Leave out feature if specified in dictionary
		if "leave_out_feature" in self.current_params:
			print("Omitting feature:", self.headers[self.current_params["leave_out_feature"]])
			self.x_train = np.delete(self.x_train, self.current_params["leave_out_feature"], 2)
			self.x_test = np.delete(self.x_test, self.current_params["leave_out_feature"], 2)

		#Shuffle data
		self.x_train, self.y_train = unison_shuffled_copies([self.x_train, self.y_train])
		self.x_test, self.y_test, self.test_idxs = unison_shuffled_copies([self.x_test, self.y_test, self.test_idxs])

		#scale data by test data mean and variance
		means, stdvs = get_mean_and_stdv(self.x_train)
		self.x_train = scale_array(self.x_train, means, stdvs)
		self.x_test = scale_array(self.x_test, means, stdvs)

		#Output size - in future delete any cols for categorical which are all zero
		print("train, test set size (x):", self.x_train.shape, self.x_test.shape)
		model = generate_model(self.x_train, self.y_train, self.current_params, model_type=self.model_type)

		#if self.current_fold == 1:
		#	print(model.summary())

		return self.train_model(model) #Returns TRUE if accuracy below threshold


	def train_model(self, model):
		"""run one fold and write up results"""
		print("		fold ", self.current_fold, "of", self.folds)
		metrics = self.metrics
		reset_states = ResetStatesCallback()
		early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=8, verbose=0, mode='auto')

		start_train = time.time()
		h = model.fit(
			self.x_train, self.y_train,
			batch_size=self.current_params["batch_size"],
			epochs=self.current_params["epochs"],
			shuffle=True,
			verbose=0,
			callbacks=[reset_states])

		end_train = time.time()

		metrics["train_acc"] = h.history["acc"]

		start_test = time.time()
		pred_Y = model.predict(self.x_test, batch_size=self.current_params["batch_size"])
		metrics["preds"] = [x[0] for x in pred_Y]
		end_test = time.time()
		metrics["truth"] = self.y_test.flatten().tolist()
		metrics["categorical_preds"] = [np.round(x) for x in metrics["preds"]]
		metrics["fscore"] = f1_score(metrics["truth"], metrics["categorical_preds"])
		metrics["accuracy"] = accuracy_score(metrics["truth"], metrics["categorical_preds"])
		metrics["experiment_id"] = self.experiment_id
		metrics["training_size"] = len(self.y_train)
		metrics["test_size"] = len(self.y_test)
		metrics["train_time"] = end_train - start_train
		metrics["test_time"] = end_test - start_test
		metrics["fold_id"] = self.current_fold
		metrics["test_idxs"] = self.test_idxs

		tn, fp, fn, tp =  confusion_matrix(metrics["truth"], metrics["categorical_preds"]).ravel()
		self.metrics["tp"] = tp/metrics["truth"].count(1)
		self.metrics["tn"] = tn/metrics["truth"].count(0)
		self.metrics["fp"] = fp/metrics["truth"].count(0)
		self.metrics["fn"] = fn/metrics["truth"].count(1)

		if not self.metrics_headers:
			#Create files and Write file headers
			os.mkdir(self.folder_name)
			self.metrics_headers = list(metrics.keys()) + list(self.current_params.keys())
			self.metrics_file = open("{}/results.csv".format(self.folder_name), "w")
			self.metrics_writer = csv.DictWriter(self.metrics_file, fieldnames=self.metrics_headers)
			self.metrics_writer.writeheader()

		#Write up metric results

		self.metrics_writer.writerow(merge_two_dicts(self.current_params, self.metrics))

		#Search type changes
		print("acc:", metrics["accuracy"], "fscore:", metrics["fscore"])
		for x in ['tn', 'fp', 'fn', 'tp']:
			print("{}: {}".format(x, self.metrics[x]), end=" ")
		print()
		""",
			"mal%:", np.round(
				metrics["truth"].count(1)/len(metrics["truth"]), decimals=2)
			,"tp", tp/metrics["truth"].count(1), "tn", tn/metrics["truth"].count(0)
			)
		"""
		#make space in memory
		del model
		gc.collect()

		self.accuracy_scores.append(metrics["accuracy"])
		if self.current_fold == self.folds:
			average_acc = sum(self.accuracy_scores) / len(self.accuracy_scores)
			print("average acc:", average_acc)

		if self.thresholding:
			if metrics["accuracy"] < self.temp_min_threshold:
				return True
			#On last fold check if average accuracy > current threshold, update temporary minimum to smallest from folds accuracy
			elif (self.current_fold == self.folds) and (average_acc > self.min_threshold):
				self.temp_min_threshold = min(self.accuracy_scores)
				self.min_threshold = average_acc
				print("* * * NEW RECORD avg acc:", average_acc, "min acc:", self.temp_min_threshold)

		return False # Only return true to stop models running


	def run_experiments(self, num_experiments=100):
		# GRID SEARCH
		#Find total possible configurations from options
		self.total = 1
		for key in self.original_h_params:
			self.total *= len(self.original_h_params[key])

		if self.search_algorithm == "grid":
			header_list = list(self.h_params.keys()) #Fixed keys list to loop in order
			countdown = len(self.h_params) - 1
			self.num_experiments = self.total
			print("grid search of ", self.total, "configurations...")
			self.loop_values(header_list, countdown)

		# RANDOM SEARCH
		elif self.search_algorithm == "random":
			self.num_experiments = num_experiments
			print("random search of ", self.num_experiments, "configurations of a possible", self.total, "configurations")
			while(self.experiment_id <= self.num_experiments):
				self.run_one_experiment()


		# Experiments run - close data files
		print(self.experiment_id, " models run.")

		self.metrics_file.close()


	def loop_values(self, header_list, countdown):
		# loop through all possible configurations in original parameter dictionary
		# http://stackoverflow.com/questions/7186518/function-with-varying-number-of-for-loops-python
		if (countdown > 0):
			for i in self.original_h_params[header_list[countdown]]:
				self.current_params[header_list[countdown]] = i
				self.loop_values(header_list, countdown - 1)
		else:
			for i in self.original_h_params[header_list[countdown]]:
				self.current_params[header_list[countdown]] = i
				self.run_one_experiment()



class Increase_Snaphot_Experiment(Experiment):
	"""Experiment to look at change in data snapshot intervals"""
	def __init__(self, parameters, search_algorithm="grid",
		x_test=None, y_test=None,
		x_train=None, y_train=None,
		data=None, folds=10,
		folder_name=str(time.time()),
		thresholding=False, threshold=0.5, run_on_factors=True, model_type="rnn"):

		super(Increase_Snaphot_Experiment, self).__init__(parameters, search_algorithm="grid",
		x_test=x_test, y_test=y_test,
		x_train=x_train, y_train=y_train,
		data=data, folds=folds,
		folder_name=folder_name,
		thresholding=thresholding, threshold=threshold)

		self.run_on_factors = run_on_factors

	def set_up_model(self):
		# Keep only every step-th data snapshot, do not run unless new data involved
		if (self.run_on_factors and ((self.current_params["sequence_length"] - 1) % self.current_params["step"] == 0)) or (self.run_on_factors == False):
			self.x_train = self.x_train[:,::self.current_params["step"]]
			self.x_test = self.x_test[:,::self.current_params["step"]]

			# Shuffle data
			self.x_train, self.y_train = unison_shuffled_copies([self.x_train, self.y_train])
			self.x_test, self.y_test, self.test_idxs = unison_shuffled_copies([self.x_test, self.y_test, self.test_idxs])

			# scale data by test data mean and variance
			means, stdvs = get_mean_and_stdv(self.x_train)
			self.x_train = scale_array(self.x_train, means, stdvs)
			self.x_test = scale_array(self.x_test, means, stdvs)

			# Output size - in future delete any cols for categorical which are all zero
			model = generate_model(self.x_train, self.y_train,
				self.current_params, model_type=self.model_type)

			if self.current_fold == 1:
				print(model.summary())

			#UNINDENT line BELOW
			return self.train_model(model) #Returns TRUE if accuracy below threshold


class Ensemble(Experiment):
	def write_up_models(self, models, preds):
		self.metrics["truth"] = self.y_test.flatten()
		self.metrics["preds"] = np.array(preds).max(axis=0)
		self.metrics["categorical_preds"] = [np.round(x) for x in self.metrics["preds"]]
		self.metrics["fscore"] = f1_score(self.metrics["truth"], self.metrics["categorical_preds"])
		self.metrics["accuracy"] = accuracy_score(self.metrics["truth"], self.metrics["categorical_preds"])

		writeable = merge_two_dicts(self.current_params, self.metrics)

		if not self.metrics_headers:
			#Create files and Write file headers
			os.mkdir(self.folder_name)
			self.metrics_headers = writeable.keys()
			self.metrics_file = open("{}/results.csv".format(self.folder_name), "w")
			self.metrics_writer = csv.DictWriter(self.metrics_file, fieldnames=self.metrics_headers)
			self.metrics_writer.writeheader()
		self.metrics_writer.writerow(merge_two_dicts(self.current_params, self.metrics))

		#Search type changes
		print("acc:", self.metrics["accuracy"])

		#make space in memory
		for model in models:
			del model
		gc.collect()


class  Ensemble_configurations(Ensemble):
	"""Average the predictions of multiple models passed as a list as configurations"""
	#setup this class adding our own custom member variables
	def __init__(self, parameters, search_algorithm="grid",
		x_test=None, y_test=None,
		x_train=None, y_train=None,
		data=None, folds=10,
		folder_name=str(time.time()),
		thresholding=False, threshold=0.5, run_on_factors=True,
		model_type="rnn", batch_size=64):

		#baseclass-Experiment, Ensemble-childclass of Experiment, Ensemble_confg-child of Ensemble
		#setup baseclass- Experiment with variable from setting up Ensemble_configurations class
		super(Ensemble_configurations, self).__init__(parameters[0],
			search_algorithm="grid",
			x_test=x_test, y_test=y_test, x_train=x_train, y_train=y_train,
			data=data, folds=folds, folder_name=folder_name,
			thresholding=thresholding, threshold=threshold, model_type=model_type)
		self.search_algorithm = None

		# Sequence length is the only shared variable - get distinct values
		self.sequence_lengths = set([s for seq_lens in [config["sequence_length"] for config in parameters] for s in seq_lens])

		#Only one parameter considered from each config, not a search experiment
		if not(all([all([len(config[key]) == 1 for key in config]) for config in parameters])):
			"not a search experiment, only one parameter will be used from each configuration"

		# dictionary values to list
		self.configurations = []
		for config in parameters:
			config = dict([(key, list(config[key])[0]) for key in config])
			self.configurations.append(config)
			#Can set all batch sizes the same if specified
			if batch_size:
				config["batch_size"] = batch_size


	def set_up_model(self):
		# Shuffle data
		#self.x_train, self.y_train, self.x_val, self.y_val = extract_val_set_binary(self.x_train, self.y_train, 0.1)
		self.x_train, self.y_train = unison_shuffled_copies([self.x_train, self.y_train])
		self.x_test, self.y_test, self.test_idxs = unison_shuffled_copies([self.x_test, self.y_test, self.test_idxs])
		#self.x_val, self.y_val = unison_shuffled_copies([self.x_val, self.y_val])

		# scale data by test data mean and variance
		means, stdvs = get_mean_and_stdv(self.x_train)
		self.x_train = scale_array(self.x_train, means, stdvs)
		self.x_test = scale_array(self.x_test, means, stdvs)
		#self.x_val = scale_array(self.x_val, means, stdvs)

		# Output size - in future delete any cols for categorical which are all zero
		models = [generate_model(
			self.x_train, self.y_train, config, model_type=self.model_type
			) for config in self.configurations]

		reset_states = ResetStatesCallback()
		preds = []
		#early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.01, patience=, verbose=0, mode='auto')

		print("train, test set size (x):", self.x_train.shape, self.x_test.shape)
		print("train mal:",self.y_train.flatten().tolist().count(1) , "ben:",self.y_train.flatten().tolist().count(0) , "test mal:", self.y_test.flatten().tolist().count(1), "ben:", self.y_test.flatten().tolist().count(0))
		for model, config in zip(models, self.configurations):
			h = model.fit(
				self.x_train, self.y_train,
				batch_size=config["batch_size"],
				epochs=config["epochs"],
				shuffle=True,
				verbose=0,
				callbacks=[reset_states]
			)

			d = config["description"]

			self.metrics["train_acc_{}".format(d)] = h.history["acc".format(d)]
			pred_Y = model.predict(self.x_test, batch_size=config["batch_size"])
			preds.append(pred_Y)
			#self.metrics["val_preds_{}"] = model.predict(self.x_val, batch_size=config["batch_size"]).flatten().tolist()
			#self.metrics["val_truth"] = self.y_val.flatten().tolist()
			self.metrics["test_idxs"] = self.test_idxs
			self.metrics["preds_{}".format(d)] = pred_Y.flatten()

			self.metrics["acc_{}".format(d)] = accuracy_score(
				self.y_test.flatten().round(),
				self.metrics["preds_{}".format(d)].round()
			)

			print(d, "acc :", self.metrics["acc_{}".format(d)], "f1", f1_score(self.y_test.flatten().round(),
				self.metrics["preds_{}".format(d)].round()))

		self.write_up_models(models, preds)
		return False

	def run_experiments(self):
		# Only changeable parameter is sequence length
		self.num_experiments = len(self.sequence_lengths)

		for s in self.sequence_lengths:
			self.current_params = {"sequence_length": s}
			self.run_one_experiment()

		# Experiments run - close data files
		print(self.experiment_id, " models run.")

		self.metrics_file.close()


class Ensemble_sub_sequences(Ensemble):
	"""Ensemble models for sub-seqeunces of data"""
	def __init__(self, parameters, search_algorithm="grid",
		x_test=None, y_test=None,
		x_train=None, y_train=None,
		data=None, folds=10,
		folder_name=str(time.time()),
		thresholding=False, threshold=0.5):

		super(Ensemble_sub_sequences, self).__init__(parameters, search_algorithm="grid", x_test=x_test, y_test=y_test, x_train=x_train, y_train=y_train, data=data, folds=folds, folder_name=folder_name, thresholding=thresholding, threshold=threshold)
		self.search_algorithm = None

		# Sequence length is the only shared variable - get distinct values
		self.sequence_lengths = self.h_params["sequence_length"]

	def set_up_model(self):
		# Shuffle data
		self.x_train, self.y_train = unison_shuffled_copies([self.x_train, self.y_train])
		self.x_test, self.y_test, self.test_idxs = unison_shuffled_copies([self.x_test, self.y_test, self.test_idxs])

		# scale data by test data mean and variance
		means, stdvs = get_mean_and_stdv(self.x_train)
		self.x_train = scale_array(self.x_train, means, stdvs)
		self.x_test = scale_array(self.x_test, means, stdvs)

		# Output size - in future delete any cols for categorical which are all zero
		models = []
		reset_states = ResetStatesCallback()
		metrics = self.metrics
		preds = []

		training_sets = []
		testing_sets = []
		idx_len_tuples = []

		#Create multiple training sets
		for length in list(range(1, self.current_params["sequence_length"])):
			mini_train_X, b = into_sliding_chunk_arrays(self.x_train, length)
			mini_test_X, b = into_sliding_chunk_arrays(self.x_test, length)
			idx_len_tuples += b
			training_sets += mini_train_X #append for merge2
			testing_sets += mini_test_X

		#Finally add whole sets
		training_sets.append(self.x_train)
		testing_sets.append(self.x_test)
		idx_len_tuples.append((0, self.current_params["sequence_length"]))

		for i, train_set in enumerate(training_sets):
			d = idx_len_tuples[i]
			self.x_train = train_set
			self.x_test =  testing_sets[i]

			#Output size - in future delete any cols for categorical which are all zero
			model = generate_model(self.x_train, self.y_train, self.current_params)

			h = model.fit(self.x_train, self.y_train,
				batch_size=self.current_params["batch_size"],
				epochs=self.current_params["epochs"],
				verbose=0, shuffle=True, callbacks=[reset_states])

			self.metrics["train_acc_{}".format(d)] = h.history["acc"]
			pred_Y = model.predict(self.x_test, batch_size=self.current_params["batch_size"])
			preds.append(pred_Y)
			self.metrics["preds_{}".format(d)] = [x[0] for x in pred_Y]
			self.metrics["acc_{}".format(d)] = accuracy_score(self.y_test.flatten(), [np.round(x) for x in self.metrics["preds_{}".format(d)]])
			print(d, "acc :", self.metrics["acc_{}".format(d)])

		self.write_up_models(models, preds)
		return False

	def run_experiments(self):
		# Only changeable parameter is sequence length
		self.num_experiments = len(self.sequence_lengths)

		for s in self.sequence_lengths:
			self.run_one_experiment()

		# Experiments run - close data files
		print(self.experiment_id, " models run.")

		self.metrics_file.close()


class SlidingWindow(Experiment):
	"""Ensemble models for sub-seqeunces of data"""
	def __init__(self, parameters, search_algorithm="grid",
		x_test=None, y_test=None,
		x_train=None, y_train=None,
		data=None, folds=10,
		folder_name=str(time.time()),
		thresholding=False, threshold=0.5):

		super(SlidingWindow, self).__init__(parameters, search_algorithm="grid", x_test=x_test, y_test=y_test, x_train=x_train, y_train=y_train, data=data, folds=folds, folder_name=folder_name, thresholding=thresholding, threshold=threshold)


	def run_one_experiment(self):
		print("run one experiment - sliding window")
		#Get new configuration if random search
		if self.search_algorithm == "random":
			self.__random_config()

		self.experiment_id += 1
		print("running expt", self.experiment_id, "of", self.num_experiments)
		print(self.current_params)

		self.metrics = {}
		self.current_fold = 1
		self.accuracy_scores = []

		# k-fold cross-validation
		if self.folds != None:
			raise NotImplementedError("Sliding Window has not been implemented for k-fold validation")
		# test-train
		else:
			self.x_train = deepcopy(self.X_TRAIN)
			self.y_train = deepcopy(self.Y_TRAIN)
			self.x_test = deepcopy(self.X_TEST)
			self.y_test = deepcopy(self.Y_TEST)

			print("1---", self.x_train.shape)

			self.test_idxs = np.array(range(1, len(self.y_test) + 1)) / 10 #divide by 10 to distinguish from training/10-fold data

			self.set_up_model()

	def set_up_model(self):

		print("---", self.x_train.shape)
		# Output size - in future delete any cols for categorical which are all zero
		models = []
		reset_states = ResetStatesCallback()
		metrics = self.metrics
		preds = []

		all_x_train = []
		all_y_train = []
		testing_x = []
		testing_y = []
		test_idxs = []

		#Get sliding window data
		seq_len = self.current_params["sequence_length"]
		for i in list(range(max([len(x) for x in self.x_train]) - seq_len)):
			for x, y, store_x, store_y in zip(
				[self.x_train, self.x_test],
				[self.y_train, self.y_test],
				[all_x_train, testing_x],
				[all_y_train, testing_y],
				):
				temp_x = []
				temp_y = []
				idxs = []

				for n in range(len(x)):
					if len(x[n][i:seq_len+i]) == seq_len:
						new = np.array(x[n][i:seq_len+i])
						print(new.shape, np.array([range(i, seq_len+i)]).T.shape)
						temp_x.append(
							np.concatenate((new, np.array([range(i, seq_len+i)]).T), axis=1)
						)
						temp_y.append(np.array(y[n]))
						if store_x is testing_x:
							idxs.append(self.test_idxs[n])
				if len(temp_y):
					store_x.append(np.array(temp_x))
					store_y.append(np.array(temp_y))
					if len(idxs):
						test_idxs.append(idxs)


		# Concatenate training data
		self.x_train = np.concatenate(all_x_train)
		self.y_train = np.concatenate(all_y_train)
		self.x_test = testing_x
		self.y_test = testing_y
		self.test_idxs = test_idxs

		# scale data by test data mean and variance
		means, stdvs = get_mean_and_stdv(self.x_train)
		self.x_train = scale_array(self.x_train, means, stdvs)
		self.x_test = [scale_array(x, means, stdvs) for x in self.x_test]


		print(2, self.x_train.shape, self.y_train.shape)

		# Shuffle trainig data
		self.x_train, self.y_train = unison_shuffled_copies([self.x_train, self.y_train])

		#Output size - in future delete any cols for categorical which are all zero
		print("train, test set size (x):", self.x_train.shape, [x.shape for x in self.x_test])
		model = generate_model(self.x_train, self.y_train, self.current_params, model_type=self.model_type)

		#if self.current_fold == 1:
		#	print(model.summary())

		return self.train_model(model) #Returns TRUE if accuracy below threshold


	def train_model(self, model):
		# Train model
		model = generate_model(self.x_train, self.y_train, self.current_params)
		reset_states = ResetStatesCallback()
		start_train = time.time()
		h = model.fit(
			self.x_train, self.y_train,
			batch_size=self.current_params["batch_size"],
			epochs=self.current_params["epochs"],
			shuffle=True,
			verbose=1,
			callbacks=[reset_states])
		end_train = time.time()

		# test model

		for i, test_set in enumerate(self.x_test):
			self.metrics["train_acc"] = h.history["acc"]
			self.metrics["slide_start"] = i

			pred_Y = model.predict(test_set, batch_size=self.current_params["batch_size"])
			self.metrics["preds"] = [x[0] for x in pred_Y]

			metrics = self.metrics

			metrics["train_acc"] = h.history["acc"]

			metrics["truth"] = self.y_test[i].flatten().tolist()
			metrics["categorical_preds"] = [np.round(x) for x in metrics["preds"]]
			metrics["fscore"] = f1_score(metrics["truth"], metrics["categorical_preds"])
			metrics["accuracy"] = accuracy_score(metrics["truth"], metrics["categorical_preds"])
			metrics["experiment_id"] = self.experiment_id
			metrics["training_size"] = len(self.y_train)
			metrics["test_size"] = len(self.y_test[i])
			metrics["train_time"] = end_train - start_train
			metrics["test_idxs"] = self.test_idxs

			if not self.metrics_headers:
				#Create files and Write file headers
				os.mkdir(self.folder_name)
				self.metrics_headers = list(metrics.keys()) + list(self.current_params.keys())
				self.metrics_file = open("{}/results.csv".format(self.folder_name), "w")
				self.metrics_writer = csv.DictWriter(self.metrics_file, fieldnames=self.metrics_headers)
				self.metrics_writer.writeheader()

			#Write up metric results
			self.metrics_writer.writerow(merge_two_dicts(self.current_params, self.metrics))

			print("acc:", metrics["accuracy"], "fscore:", metrics["fscore"], "start", i, test_set.shape)

		del model
		gc.collect()

		return False

class SlidingWindow_Support(SlidingWindow):
	def __init__(self, parameters, search_algorithm="grid",
		x_test=None, y_test=None,
		x_train=None, y_train=None,
		data=None, folds=10,
		folder_name=str(time.time()),
		thresholding=False, threshold=0.5):

		super(SlidingWindow, self).__init__(parameters, search_algorithm="grid", x_test=x_test, y_test=y_test, x_train=x_train, y_train=y_train, data=data, folds=folds, folder_name=folder_name, thresholding=thresholding, threshold=threshold)


	def set_up_model(self):

		print("---", self.x_train.shape)
		# Output size - in future delete any cols for categorical which are all zero
		models = []
		reset_states = ResetStatesCallback()
		metrics = self.metrics
		preds = []

		testing_x = []
		testing_y = []
		test_idxs = []

		#Get sliding window data
		seq_len = self.current_params["sequence_length"]
		for i in list(range(max([len(x) for x in self.x_train]) - seq_len)):
			for x, y, store_x, store_y in zip(
				[self.x_test],
				[self.y_test],
				[testing_x],
				[testing_y],
				):
				temp_x = []
				temp_y = []
				idxs = []

				for n in range(len(x)):
					if len(x[n][i:seq_len+i]) == seq_len:
						temp_x.append(np.array(x[n][i:seq_len+i]))
						temp_y.append(np.array(y[n]))
						idxs.append(self.test_idxs[n])
				if len(temp_y):
					store_x.append(np.array(temp_x))
					store_y.append(np.array(temp_y))
					test_idxs.append(idxs)


		# Concatenate training data
		self.x_train, self.y_train = remove_short(self.x_train, self.y_train, self.current_params["sequence_length"])
		self.x_test = testing_x
		self.y_test = testing_y
		self.test_idxs = test_idxs

		# scale data by test data mean and variance
		means, stdvs = get_mean_and_stdv(self.x_train)
		self.x_train = scale_array(self.x_train, means, stdvs)
		self.x_test = [scale_array(x, means, stdvs) for x in self.x_test]


		print(2, self.x_train.shape, self.y_train.shape)

		# Shuffle trainig data
		self.x_train, self.y_train = unison_shuffled_copies([self.x_train, self.y_train])

		#Output size - in future delete any cols for categorical which are all zero
		print("train, test set size (x):", self.x_train.shape, [x.shape for x in self.x_test])
		model = generate_model(self.x_train, self.y_train, self.current_params, model_type=self.model_type)

		#if self.current_fold == 1:
		#	print(model.summary())

		return self.train_model(model) #Returns TRUE if accuracy below threshold

class Omit_test_data(Experiment):
	def train_model(self, model):
		"""run one fold and write up results"""
		print("		fold ", self.current_fold, "of", self.folds)
		metrics = self.metrics
		reset_states = ResetStatesCallback()

		if not len(self.y_test):
			return

		model.fit(
		self.x_train, self.y_train,
		batch_size=self.current_params["batch_size"],
		epochs=self.current_params["epochs"],
		verbose=0,
		shuffle=True,
		callbacks=[reset_states])

		headers = [x + "_ON" for x in self.headers[:len(self.x_test[0][0])]]

		indicies = list(range(len(headers)))

		for num_missing in range(1, len(indicies)+1):
			for subset in combinations(indicies, num_missing):
				temp_test_X = deepcopy(self.x_test)
				states = [1 for x in range(len(indicies))]

				for feature_index in subset:
					states[feature_index] = 0
					temp_test_X[:,:,feature_index] = 0 # because 0 is the mean of the training data
				print(states)

				pred_Y = model.predict(temp_test_X, batch_size=self.current_params["batch_size"]) #Predicted value of Y
				pred_Y = [x[0] for x in pred_Y]
				Y_classes =  [np.round(x) for x in pred_Y] #Choose a class
				metrics["preds"] = pred_Y
				metrics["categorical_preds"] = Y_classes
				metrics["truths"] = [int(x[0]) for x in self.y_test]
				metrics["variance"] = np.var(pred_Y)
				metrics["accuracy"] = accuracy_score(self.metrics["truths"], self.metrics["categorical_preds"])
				metrics["fmeasure"] = accuracy_score(self.metrics["truths"], self.metrics["categorical_preds"])
				for h in range(len(headers)):
					metrics[headers[h]] = states[h]

				if not self.metrics_headers:
					#Create files and Write file headers
					os.mkdir(self.folder_name)
					self.metrics_headers = list(metrics.keys()) + list(self.current_params.keys())
					self.metrics_file = open("{}/results.csv".format(self.folder_name), "w")
					self.metrics_writer = csv.DictWriter(self.metrics_file, fieldnames=self.metrics_headers)
					self.metrics_writer.writeheader()

				#Write up metric results
				self.metrics_writer.writerow(merge_two_dicts(self.current_params, self.metrics))

				#Search type changes
				print("acc:", metrics["accuracy"], "features on:", str(states))

		#make space in memory
		del model
		gc.collect()
