import os
from sqlite3 import dbapi2 as sqlite3
from flask import Flask, request, session, g, redirect, url_for, abort, render_template, flash
import pandas as pd
from werkzeug import secure_filename
import re
import numpy as np
from io import StringIO
import matplotlib.pyplot as plt
from datetime import datetime
import pickle 
from regression import *
from scipy.stats import ttest_ind, levene
import spc_fun
from process_raw_data import processdata, clean_line
from batchclass import Batch


def dig_data_t():
	raw_data = "static/raw_data/temp_files.txt"
	fr = open(raw_data)
	data = fr.readlines()
	fr.close()
	data2 = processdata(data, year = '2014')
	teststr = ""
	pclist = []
	for ele in data2:
		temp_batch = Batch(ele)
		init_pressure = 't' + temp_batch.outpressure
		temp_records = temp_batch.records
		temp_records.insert(0, init_pressure)
		if ele[4] == 1:
			temp_records.insert(0, 'switch')
		pclist.append(temp_records)
	gather_list = []
	templist = []
	for ele in pclist:
		if 'switch' in ele:
			gather_list.append(templist)
			templist = []
			templist.extend(ele)
			templist.remove('switch')
		else:
			templist.extend(ele)
	gather_list.pop(0)

	raw_data = "static/raw_data/15.4-3.3(mod).txt"
	fr = open(raw_data)
	data = fr.readlines()
	fr.close()
	cleanls = []
	temp_store_check = []
	for line in data:
		line = line.strip().split()
		doneline = []
		for ele in line:
			if ('t' not in ele) and ('j' not in ele) and '/' not in ele and ele not in ['m', 'l', 'switch']:
				doneline.append('3.' + ele)
				if len(temp_store_check) > 0:
					doneline.extend(temp_store_check)
					temp_store_check = []
			elif '/' in ele:
				temp_store_check.append('/' + '3.' + ele[1: ])
			else:
				doneline.append(ele)
		if 'switch' in doneline:
			doneline.remove('switch')
		cleanls.append(doneline)
	
	gather_list.extend(cleanls)

	new_key = []
	for line in gather_list:
		last_t = ''
		temp_line = []
		new_list = []
		for ele in line:
			if 't' in ele:
				if ele == last_t:
					pass
				else:
					if 'switch' in temp_line:
						temp_line.remove('switch')
					if len(temp_line) > 0:
						new_list.append(temp_line)
					temp_line = []
					temp_line.append(ele)
					last_t = ele
					
			else:
				temp_line.append(ele)
		new_list.append(temp_line)
		new_key.append(new_list)

  	td1 = output_df_t(new_key)
	
	return td1
	
def output_df_t(gather_m):

	data_dic = {'step_before20': [], 
				#'step_after20' : [], 
				'step_before10' : [], 
				#'step_after10' : [], 
				'step_before5' : [], 
				#'step_after5' : [],
				'step_before3':[],
				'mean_before20' : [], 
				#'mean_after20':[], 
				'mean_before10' : [], 
				#'mean_after10' : [], 
				'mean_before5' : [], 
				#'mean_after5' : [],
				'std_before20' : [],
				#'std_after20' : [],
				'std_before10' : [],
				#'std_after10' : [],
				'std_before5' : [],
				#'std_after5' : [],
				'range_before' : [],
				#'range_after' : [],
				'max_before' : [],
				#'max_after' : [],
				'min_before' :[],
				#'min_after' : [],
				#'p_value20' : [],
				#'p_value10' : [],
				'p_value5' : [],
				#'levene20' : [],
				#'levene10' : [],
				'levene5' : [],
				#'amount_l20' : [],
				#'amount_l10' : [],
				'amount_l5' : [],
				'len_1' : [],
				'len_2' : [],
				'mean_ahead_15' : [],
				'mean_ahead_10' : [],
				'step_ahead_10' : [],
				'step_ahead_15' : []}
				
				
				
	spc_dic =  {'xCL' : [],
				'RsCL' : [],
				'err1' : [],
				'zone4_dist': [],
				'zone3_dist': [],	
				#'zone2_dist': [],	
				#'zone1_dist': [],
				'err2': [],
				'err3':[],
				#'err4':[],
				'err5': [],
				'err6' : [], 
				'err8' : [],
				'Rs_err1' : [],
				'Rsd4_dist':[],
				'Rsd3_dist':[],
				'last_record':[],
				#'Rsd2_dist':[],
				#'Rsd1_dist':[],
				#'Rsd0_dist':[],
						'mean_dev19_1':[],
						'mean_dev9_1':[],
						'mean_dev5_1':[],
						'mean_dev17_3':[],
						'mean_dev7_3':[],
						'mean_dev15_5':[],
						'mean_dev10_5':[],
						'step_dev17_3':[],
						'step_dev10_3':[],
						'step_dev15_5':[],
						'step_dev10_5':[],
						'step_before5':[],
						'step_before3':[],
						'max_before5':[],
						'max_before20':[],
						'max_dev20_5':[],
						'max_dev20_20':[],
						'min_before5':[],
						'min_before20':[],
						'min_dev20_5':[],
						'min_dev20_20':[]
				
				}
				
								
	for ele in gather_m:
		if len(ele) == 0:
			print "!!!!", ele
			pass
		else:
			for indx in range(1, len(ele)):
				tt = T_process(ele[indx-1], ele[indx])
				if tt.reason_valid(5) and tt.effect_valid():
					if len(tt.get_records2()) >= 15 and len(tt.get_records1()) >=10 and tt.count_l(3) == 0:
						data_dic['step_before20'].append(tt.get_steps_means()[0])
						#data_dic['step_after20'].append(tt.get_steps_means()[1])
						data_dic['step_before10'].append(tt.get_steps_means(10)[0])
						#data_dic['step_after10'].append(tt.get_steps_means(10)[1]) 
						data_dic['step_before5'].append(tt.get_steps_means(5)[0]) 
						#data_dic['step_after5'].append(tt.get_steps_means(5)[1])
						data_dic['step_before3'].append(tt.get_steps_means(3)[0])
						data_dic['mean_before20'].append(tt.get_means()[0])
						#data_dic['mean_after20'].append(tt.get_means()[1])
						data_dic['mean_before10'].append(tt.get_means(10)[0])
						#data_dic['mean_after10'].append(tt.get_means(10)[1])
						data_dic['mean_before5'].append(tt.get_means(5)[0])
						#data_dic['mean_after5'].append(tt.get_means(5)[1])
						data_dic['std_before20'].append(tt.get_stds()[0])
						#data_dic['std_after20'].append(tt.get_stds()[1])
						data_dic['std_before10'].append(tt.get_stds(10)[0])
						#data_dic['std_after10'].append(tt.get_stds(10)[1])
						data_dic['std_before5'].append(tt.get_stds(5)[0])
						#data_dic['std_after5'].append(tt.get_stds(5)[1])
						data_dic['range_before'].append(tt.get_ranges(5)[0])
						#data_dic['range_after'].append(tt.get_ranges(5)[1])
						data_dic['max_before'].append(tt.get_max()[0])
						#data_dic['max_after'].append(tt.get_max()[1])
						data_dic['min_before'].append(tt.get_min()[0])
						#data_dic['min_after'].append(tt.get_min()[1])
						#data_dic['p_value20'].append(tt.caluate_p_value())
						#data_dic['p_value10'].append(tt.caluate_p_value(10))
						data_dic['p_value5'].append(tt.caluate_p_value(5))
						#data_dic['levene20'].append(tt.get_levene())
						#data_dic['levene10'].append(tt.get_levene(10))
						data_dic['levene5'].append(tt.get_levene(5))
						#data_dic['amount_l20'].append(tt.count_l())					
						#data_dic['amount_l10'].append(tt.count_l(10))
						data_dic['amount_l5'].append(tt.count_l(5))
						data_dic['len_1'].append(len(tt.get_records1()))
						data_dic['len_2'].append(len(tt.get_records2()))
						data_dic['mean_ahead_10'].append(tt.mean_before(10, 20))
						data_dic['mean_ahead_15'].append(tt.mean_before(5, 20))
						data_dic['step_ahead_10'].append(tt.step_before(10, 20))
						data_dic['step_ahead_15'].append(tt.step_before(5, 20))
						
						spc_dic['xCL'].append(tt.set_xCL())
						spc_dic['RsCL'].append(tt.set_RsCL())		
						spc_dic['err1'].append(tt.amount_error1())	
						zd = tt.zone_distribution(10)
						spc_dic['zone4_dist'].append(zd[0])
						spc_dic['zone3_dist'].append(zd[1])
						#spc_dic['zone2_dist'].append(zd[2])
						#spc_dic['zone1_dist'].append(zd[3])
						spc_dic['err2'].append(tt.amount_error2(20))	
						spc_dic['Rs_err1'].append(tt.Rs_error1())
						Rsd = tt.Rs_zone_distribution(10)
						spc_dic['Rsd4_dist'].append(Rsd[0])
						spc_dic['Rsd3_dist'].append(Rsd[1])
						#spc_dic['Rsd2_dist'].append(Rsd[2])
						#spc_dic['Rsd1_dist'].append(Rsd[3])
						#spc_dic['Rsd0_dist'].append(Rsd[4])	
						spc_dic['err3'].append(tt.amount_error3(10))			
						#spc_dic['err4'].append(tt.amount_error4())
						spc_dic['err5'].append(tt.amount_error5(10))
						spc_dic['err6'].append(tt.amount_error6(10))
						spc_dic['err8'].append(tt.amount_error8(10))
						spc_dic['last_record'].append(tt.get_records1()[-1])	
						spc_dic['mean_dev19_1'].append(tt.mean_before(1, 20) - tt.get_records1()[-1])
						spc_dic['mean_dev9_1'].append(tt.mean_before(1, 10) - tt.get_records1()[-1])
						spc_dic['mean_dev5_1'].append(tt.mean_before(1, 5) - tt.get_records1()[-1])
						spc_dic['mean_dev17_3'].append(tt.mean_before(3, 20) - tt.get_means(3)[0])
						spc_dic['mean_dev7_3'].append(tt.mean_before(3, 10) - tt.get_means(3)[0])
						spc_dic['mean_dev15_5'].append(tt.mean_before(5, 20) - tt.get_means(5)[0])
						spc_dic['mean_dev10_5'].append(tt.mean_before(5, 10) - tt.get_means(5)[0])
						spc_dic['step_dev17_3'].append(tt.step_before(3, 20) - tt.get_steps_means(3)[0])
						spc_dic['step_dev10_3'].append(tt.step_before(3, 10) - tt.get_steps_means(3)[0])
						spc_dic['step_dev15_5'].append(tt.step_before(5, 20) - tt.get_steps_means(5)[0])
						spc_dic['step_dev10_5'].append(tt.step_before(5, 20) - tt.get_steps_means(5)[0])
						spc_dic['step_before5'].append(tt.get_steps_means(5)[0])
						spc_dic['step_before3'].append(tt.get_steps_means(3)[0])
						spc_dic['max_before5'].append(tt.get_max(5)[0])
						spc_dic['max_before20'].append(tt.get_max()[0])
						spc_dic['max_dev20_5'].append(tt.get_max(5)[0] - tt.get_means(20)[0])
						spc_dic['max_dev20_20'].append(tt.get_max(20)[0] - tt.get_means(20)[0])
						spc_dic['min_before5'].append(tt.get_min(5)[0])
						spc_dic['min_before20'].append(tt.get_min()[0])
						spc_dic['min_dev20_5'].append(tt.get_means(20)[0] - tt.get_min(5)[0])
						spc_dic['min_dev20_20'].append(tt.get_means(20)[0] - tt.get_min(20)[0])
						
						
	td1 = pd.DataFrame(data_dic)    #  columns = ['initial_pressure', 'pressure_dev', 'best_show', '5s', '8s', '10s', 'p_value', 't_value', 'levene', 'step_before', 'step_after', 'sample_size', 'reason_mean', 'wt_index'])	
	tspc1 = pd.DataFrame(spc_dic)
	
	#tspc1['effect_mean'] = abs(td1.mean_after10 - td1.mean_before10)
	#mspc1['effect_mean'] = abs(md1.mean_after10 - md1.mean_before10)
	tspc1['reason_mean'] = td1.mean_ahead_15 - td1.mean_before5
	#mspc1['reason_mean'] = md1.mean_ahead_15 - md1.mean_before5
	tspc1['reason_step'] = abs(td1.step_ahead_15 - td1.step_before5)
	#mspc1['reason_step'] = abs(md1.step_ahead_15 - md1.step_before5)
	#tspc1['effect_step'] = abs(td1.step_after20 - td1.step_before20)
	#mspc1['effect_step'] = abs(md1.step_after20 - md1.step_before20)
	
	return td1, tspc1


class T_process:
	def __init__(self, line1, line2):
		self.records_before = line1[1:]
		self.records_after = line2[1:]
		self.pressure1 = float(line1[0][1:])
		self.pressure2 = float(line2[0][1:])

	def get_pressure_dev(self):
		return self.pressure2 - self.pressure1

	def get_records1(self):
		# the record before m
		positive_records = []
		for indx in range(len(self.records_before)):
			if self.records_before[indx] == 'l':
				#if len(positive_records) > 0:
				positive_records.pop()
				positive_records.append(self.records_before[indx])		
			elif self.records_before[indx] == 'm':
				positive_records = []          # if m occur, drop the data before
			elif '/' in self.records_before[indx]:
				pass
				#if not '/' in self.records_before[indx - 1]:
				#	last_one = positive_records.pop()
				#	positive_records.append((last_one + float(self.records_before[indx][1:]))/2.0)
				#else:
				#	last_one = positive_records.pop()
				#	positive_records.append(last_one * (2.0/3) + float(self.records_before[indx][1:]) * (1.0/3))							
			else: 
				positive_records.append(float(self.records_before[indx]))
		#if len(positive_records) >= 20:
		#	return positive_records[-20:]
		#else:
		return positive_records
		
		
	def get_records2(self):
		positive_records = []
		for indx in range(len(self.records_after)):
			if self.records_after[indx] == 'l':
				positive_records.pop()
				positive_records.append(self.records_after[indx])		
			elif self.records_after[indx] == 'm':
				positive_records.pop()
				positive_records.pop()
				positive_records.pop()
				positive_records.pop()
				positive_records.pop()
				return positive_records          # if m occur, drop the data before
			elif '/' in self.records_after[indx]:
				pass
				#if not '/' in self.records_after[indx - 1]:
				#	last_one = positive_records.pop()
				#	positive_records.append((last_one + float(self.records_after[indx][1:]))/2.0)
				#else:
				#	last_one = positive_records.pop()
				#	positive_records.append(last_one * (2.0/3) + float(self.records_after[indx][1:]) * (1.0/3))							
			else: 
				positive_records.append(float(self.records_after[indx]))
		return positive_records		
		
	def get_means(self, n = 20):
		try:
			data1 = [ele for ele in self.get_records1() if type(ele) == float][-n:]
			data2 = [ele for ele in self.get_records2() if type(ele) == float][:n]
		except:
			data1 = [ele for ele in self.get_records1() if type(ele) == float]
			data2 = [ele for ele in self.get_records2() if type(ele) == float]		
		return np.mean(data1), np.mean(data2)
		
	def get_stds(self, n = 20):
		try:
			data1 = [ele for ele in self.get_records1() if type(ele) == float][-n:]
			data2 = [ele for ele in self.get_records2() if type(ele) == float][:n]
		except:
			data1 = [ele for ele in self.get_records1() if type(ele) == float]
			data2 = [ele for ele in self.get_records2() if type(ele) == float]
		return np.std(data1), np.std(data2)
		
	def get_step_pool(self, records):
		test_sample = records
		step_pool = []
		for indx in range(1, len(test_sample)):
			single_step = abs(test_sample[indx] - test_sample[indx - 1])
			step_pool.append(single_step)
		return step_pool
		
	def get_steps_means(self, n = 20):
		try:
			data1 = [ele for ele in self.get_records1() if type(ele) == float][-n:]
			data2 = [ele for ele in self.get_records2() if type(ele) == float][:n]
		except:
			data1 = [ele for ele in self.get_records1() if type(ele) == float]
			data2 = [ele for ele in self.get_records2() if type(ele) == float]
		step_pool1 = self.get_step_pool(data1)
		step_pool2 = self.get_step_pool(data2)
		return np.mean(step_pool1), np.mean(step_pool2)

	def count_l(self, n = 20):
		try:
			data = self.get_records1()[-n:]
		except:
			data = self.get_records1()
		l = 0
		for ele in data:
			if ele == 'l':
				l += 1
		return l
		
	def get_max(self, n = 20):
		try:
			data1 = [ele for ele in self.get_records1() if type(ele) == float][-n:]
			data2 = [ele for ele in self.get_records2() if type(ele) == float][:n]
		except:
			data1 = [ele for ele in self.get_records1() if type(ele) == float]
			data2 = [ele for ele in self.get_records2() if type(ele) == float]
		return np.max(data1), np.max(data2)		

	def get_min(self, n = 20):
		try:
			data1 = [ele for ele in self.get_records1() if type(ele) == float][-n:]
			data2 = [ele for ele in self.get_records2() if type(ele) == float][:n]
		except:
			data1 = [ele for ele in self.get_records1() if type(ele) == float]
			data2 = [ele for ele in self.get_records2() if type(ele) == float]
		return np.min(data1), np.min(data2)

	def get_ranges(self, n = 20):		
		### before and after 20 samples
		return np.array(self.get_max(n)) - np.array(self.get_min(n))

	def get_levene(self, n = 20):
		try:
			data1 = [ele for ele in self.get_records1() if type(ele) == float][-n:]
			data2 = [ele for ele in self.get_records2() if type(ele) == float][:n]
		except:
			data1 = [ele for ele in self.get_records1() if type(ele) == float]
			data2 = [ele for ele in self.get_records2() if type(ele) == float]
		p = levene(data1, data2)[1]
		return p

	def caluate_p_value(self, n = 20):
		try:
			data1 = [ele for ele in self.get_records1() if type(ele) == float][-n:]
			data2 = [ele for ele in self.get_records2() if type(ele) == float][:n]
		except:
			data1 = [ele for ele in self.get_records1() if type(ele) == float]
			data2 = [ele for ele in self.get_records2() if type(ele) == float]
		p_value = ttest_ind(data1, data2)[1]		
		return p_value 
		
	def reason_valid(self, n = 5):
		data1 = [ele for ele in self.get_records1() if type(ele) == float][-20:]
		mean_near_before = np.mean(data1[-5:])
		mean_before = np.mean(data1[:-5])
		#sample_near = self.get_sample(n)[0]
		#mean_near_before = self.get_sample_means(n)[0]
		#mean_before = np.mean(self.get_pure_float_record1()[:-len_sample])
		if (mean_near_before - mean_before) * self.get_pressure_dev() > 0:
			return True
		else:
			return False		

	def get_best_result(self):
		data1 = [ele for ele in self.get_records1() if type(ele) == float][-5:]
		mean_before = np.mean(data1)
		data2 = [ele for ele in self.get_records2() if type(ele) == float]
		best_result = 0
		for n in [5, 8, 10, 15]:
			mean_after = np.mean(data2[: n + 1])
			if abs(mean_after - mean_before) > best_result:
				best_result = mean_after - mean_before
		return best_result
		
	def effect_valid(self):
		best_result = self.get_best_result()
		if best_result * self.get_pressure_dev() > 0:
			return False
		else:
			return True

	def mean_before(self, n1 = 10, n2 = 20):
		data1 = [ele for ele in self.get_records1() if type(ele) == float]
		if len(data1) < 20:
			#return np.nan
			return np.mean(data1[: -3])
		else:
			return np.mean(data1[-n2: -n1])
		
	def set_xCL(self):
		return self.mean_before(5, 20)
		mean_before = self.mean_before(10, 20)
		data1 = [ele for ele in self.get_records1() if type(ele) == float]
		data2 = [ele for ele in self.get_records2() if type(ele) == float]
		if  len(data1) >= 20 and len(data2) >= 25:
			return (2.0/3)*self.get_means(20)[1] + (1.0/3)*mean_before
		elif len(data1) < 20 and len(data2) >= 25:
			return self.get_means(20)[1]
		elif len(data1) >= 20 and len(data2) < 25:
			return (.5)*self.get_means(10)[1] + (.5)*mean_before
		elif len(data1) < 20 and len(data2) < 25:
			return self.get_means(10)[1]
		

	def step_before(self, n1 = 10, n2 = 20):
		data1 = [ele for ele in self.get_records1() if type(ele) == float]
		step_pool1 = self.get_step_pool(data1)
		if len(data1) < 20:
			return np.mean(step_pool1[:-5])
		else:
			step_ahead1 = np.mean(step_pool1[-n2 :-n1])	
			return step_ahead1
				
	def set_RsCL(self):
		return self.get_steps_means(20)[0]
		data1 = [ele for ele in self.get_records1() if type(ele) == float]
		n = len(self.get_records2()) - 5		
		RsCL2 = self.get_steps_means(n)[1]
		
		if len(data1) > 25:
			step_pool1 = self.get_step_pool(data1[-25 :-10])
			RsCL1 = np.mean(step_pool1)
			if RsCL1 < RsCL2:
				RsCL =  RsCL1		
		RsCL = RsCL2
		if RsCL > 0.08:
			return 0.08
		elif RsCL < 0.045:
			return 0.045
		return RsCL			
		
		
		#if RsCL >= 0.08 or RsCL <= 0.04:
		#	return 0.06
		#return RsCL
		
		
	def x_zone_list(self):
		data1 = [ele for ele in self.get_records1() if type(ele) == float]
		xCL = self.set_xCL()
		RsCL = self.set_RsCL()
		zone_ls = spc_fun.x_zone_ls(data1, xCL, RsCL)
		return zone_ls
		
	def amount_error1(self):
		return spc_fun.count_error1(self.x_zone_list()[-10:])
		
	def zone_distribution(self, n = 10):
		zone_list10 = self.x_zone_list()[-n : ]
		zone4, zone3, zone2, zone1 = spc_fun.summary_distribution(zone_list10)
		return zone4, zone3, zone2, zone1

	def amount_error2(self, n = 20):
		zone_list_n = self.x_zone_list()[-n:]
		amount_err2 = 0
		for indx in range(len(zone_list_n) - 7):
			tt_list = zone_list_n[indx : indx + 8]
			amount_err2 += spc_fun.verify_error2(tt_list)
		return amount_err2

	def amount_error3(self, n = 20):
		data1 = [ele for ele in self.get_records1() if type(ele) == float]
		step_pool = self.get_step_pool(data1)[-n: ]
		amount_err3 = 0
		for indx in range(len(step_pool) - 3):
			tt_list = step_pool[indx : indx + 4]
			amount_err3 += spc_fun.verify_error3(tt_list)
		return amount_err3
		
	def amount_error4(self, n = 20):
		data1 = [ele for ele in self.get_records1() if type(ele) == float]
		step_pool = self.get_step_pool(data1)[-n: ]
		amount_err4 = 0
		for indx in range(len(step_pool) - 5):
			tt_list = step_pool[indx : indx + 6]
			amount_err4 += spc_fun.verify_error4(tt_list)
		return amount_err4

	def amount_error5(self, n = 20):
		zone_list_n = self.x_zone_list()[-n:]
		amount_err5 = 0
		for indx in range(len(zone_list_n) - 2):
			tt_list = zone_list_n[indx : indx + 3]
			amount_err5 += spc_fun.verify_error5(tt_list)
		return amount_err5

	def amount_error6(self, n = 20):
		zone_list_n = self.x_zone_list()[-n:]
		amount_err6 = 0
		for indx in range(len(zone_list_n) - 4):
			tt_list = zone_list_n[indx : indx + 5]
			amount_err6 += spc_fun.verify_error6(tt_list)
		return amount_err6

	def amount_error8(self, n = 20):
		zone_list_n = self.x_zone_list()[-n:]
		amount_err8 = 0
		for indx in range(len(zone_list_n) - 4):
			tt_list = zone_list_n[indx : indx + 5]
			amount_err8 += spc_fun.verify_error8(tt_list)
		return amount_err8

	def Rs_zone_list(self):
		data1 = [ele for ele in self.get_records1() if type(ele) == float]
		step_pool = self.get_step_pool(data1)
		RsCL = self.set_RsCL()
		zone_ls = spc_fun.Rs_zone_ls(step_pool, RsCL)
		return zone_ls
		
	def Rs_error1(self):
		return spc_fun.Rs_error1(self.Rs_zone_list()[-10:])		

	def Rs_zone_distribution(self, n=10):
		zone_list10 = self.Rs_zone_list()[-n : ]
		zone4, zone3, zone2, zone1, zone0 = spc_fun.Rs_distribution(zone_list10)
		return zone4, zone3, zone2, zone1, zone0







