import adddeps
import datetime
import argparse
import opentuner
import logging
import os
import re
import sys
import pandas as pd
import time
import threading
import paramiko

from opentuner.resultsdb.models import Result, TuningRun
from opentuner.search import manipulator
import subprocess

log = logging.getLogger('jvmflags')

def get_max(default):
    max = default
    if default>=0:
        max= default + (default/2)
    else:
        max = default - (default/2)
    return max

def get_min(default):
    min = default
    if default>=0:
        min= default - (default/2)
    else:
        min = default + (default/2)
    return min

def convert_to_int(string):
    value = 0
    try:
        value = int(string)
    except TypeError:
        value = float(string)
    finally:
        return value
    
def assign_default_max(value):
    if(value == 0):
        return 5
    return value

argparser = argparse.ArgumentParser(parents=opentuner.argparsers())
argparser.add_argument('--source',
                       help='source file to compile (only give name e.g: MatrixMultiply)')
argparser.add_argument(
  '--bytecode-compile-template', default='javac {source}.java',
  help='command to compile to java byte code')
argparser.add_argument(
  '--jit-compile-template', default='ab -k -n 100000 -c 149 http://localhost:8080',
  help='command to jit compilation (optimization occurs here)')
argparser.add_argument('--clients',help='Specifies the number of clients generated to the tomcat server default value=50')
argparser.add_argument('--requests',help='Specifies the number of requests generated by each client default value=100000')

class JvmFlagsTuner(opentuner.measurement.MeasurementInterface):
    lock = threading.Lock()
    def __init__(self, *pargs, **kwargs):
        super(JvmFlagsTuner, self).__init__(program_name=args.source, *pargs,
                                        **kwargs)
        self.ssh  = paramiko.SSHClient()
        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        self.ssh.connect('10.8.106.245',username='cse',password='cse')
        self.tomcat_bin_location = '~/SapientS/hotspottuner/TomcatTuner/src/apps/tomcat/bin/'
        self.remote_ip = '10.8.106.245'
        self.numberof_requests = 10000
        self.ab_command = 'ab -k -n '+str(self.numberof_requests)+' -c 149 http://'+self.remote_ip+':8080/sample/hello'
        self.extract_jvm_flags()
        self.get_default_runtime()
        self.improvement_best=0
        self.num_clients=50 #specifies the number of clients
        self.num_requests=100000 #Specifies the number of requests per client
        self.sleep_time=20
        
    def open_ssh_connection(self):
        self.ssh  = paramiko.SSHClient()
        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        self.ssh.connect('10.8.106.245',username='cse',password='cse')
        
    def get_default_runtime(self):
        
        self.set_catalinash('')
        run_command = 'sh '+self.tomcat_bin_location+'startup.sh'
        stdin, stdout, stderr = self.ssh.exec_command(run_command)
        time.sleep(10)
    
#        run_command = 'ab -k -n 1000000 -c 149 http://'+self.remote_ip+':8080/sample/hello'
#        print self.ab_command
        run_result = self.call_program(self.ab_command)
        ab_bench_output = run_result['stdout']
        print 'ab_bench_output defaultruntime = ',ab_bench_output
        if len(ab_bench_output)>0:
            m = re.search('Time per request(.*)\[ms\] \(mean\)', ab_bench_output,flags=re.DOTALL)
            ab_bench_time = '10000'
        else:
            print 'Error running ab. In the get_default_runtime'
            sys.exit()
        if m:
            ab_bench_time = m.group(1)
        ab_bench_time = re.sub(':', '', ab_bench_time,flags=re.DOTALL) 
        try:
            time_per_request = float(ab_bench_time)
        except:
            time_per_request=10000
            print 'default configuration benchmarking was failed.. Tuning Aborting'
            sys.exit()   

        #print 'run_result',run_result
        self.default_run_time= time_per_request
        print 'self.default_run_time = ',time_per_request
        time.sleep(10)
        
        run_command = 'sh '+self.tomcat_bin_location+'shutdown.sh'
        stdin, stdout, stderr =self.ssh.exec_command(run_command)
        time.sleep(10)
        
        f = open('configuration.txt',"w")
        f.write(run_command)
        f.write('self.default_run_time = '+str(self.default_run_time)+'\n')
        f.close()
        
           
    
    def extract_jvm_flags(self):
        """
        print all available flags and write to csv
        """
        if(os.path.isfile('configuration.txt')):
            f = open('configuration.txt',"w")
            time_date = datetime.datetime.now()
            f.write('Run Commands for test on' + time_date.strftime("%I:%M%p on %B %d, %Y") +'\n')
            f.close()
        
        if(not(os.path.isfile('flags.csv'))):
            subprocess.call("java -XX:+PrintFlagsFinal > flags.txt", shell=True)
            f = open("flags.txt", "r");
            flags = f.read()
            flags = re.sub('\[.*?\]','',flags )
            flags = re.sub('{.*?}','',flags)
            flags = re.sub(':* *=','',flags)
            flags = re.sub(' *\n *', '\n', flags)
            flags = re.sub(' +', ',', flags)
            if flags[0] == '\n':
                flags = 'type,flagname,default' + flags[0:]
            else:
                flags = 'type,flagname,default\n' + flags[0:]
        
            f = open("flags.csv", "w");
            f.write(flags)
            f.close()  
        flags = pd.read_csv('flags.csv')

        self.bool_flags = flags[flags.type == 'bool']
        self.bool_flags.to_csv('jvm_bool_flags.csv',index=False)
        
        self.param_flags = flags[flags.type != 'bool']
        self.param_flags.default = self.param_flags.default.apply(convert_to_int)
        self.param_flags['min'] = self.param_flags.default.apply(get_min)
        self.param_flags['max'] = self.param_flags.default.apply(get_max)
        #self.param_flags['max'] = self.param_flags['max'].apply(assign_default_max)
        self.param_flags.to_csv('jvm_param_flags.csv', index=False)
        """
        Read specific flags with dependencies
        """
        
        self.gc_select_flags = pd.read_csv('flags/gc_select.csv').flagname
        
        self.gc_common_bool,self.gc_common_param = self.prepare_flags('flags/gc_common.csv')
        self.gc_troughput_bool,self.gc_throughput_param = self.prepare_flags('flags/throughput_collector.csv')
        self.gc_cms_bool,self.gc_cms_param = self.prepare_flags('flags/cms_collector.csv')
        self.g1_gc_bool,self.g1_gc_param=self.prepare_flags('flags/g1_collector.csv')
        self.compiler_flags_bool,self.compiler_flags_param=self.prepare_flags('flags/jit.csv')
        
        self.all_bool_flags = [self.gc_select_flags,self.gc_common_bool,self.gc_troughput_bool,
                               self.gc_cms_bool,self.g1_gc_bool,self.compiler_flags_bool]
        
        self.all_param_flags = [self.gc_common_param,self.gc_throughput_param,self.gc_cms_param,
                                self.g1_gc_param,self.compiler_flags_param] 
        # First compile source to byte code
       # args.source = 'Matrix/MatrixMultiplyMain'
        #java_bytecode_compile = args.bytecode_compile_template.format(source=args.source)
#        print 'compiling ',java_bytecode_compile
        #subprocess.call(java_bytecode_compile,shell=True)
                   
    def set_catalinash(self,flags):
        #text = # OPENTUNER JVM SETTINGS STARTED.......# OPENTUNER JVM SETTINGS END
        self.open_ssh_connection()
        if(os.path.isfile('apps/tomcat/bin/catalina.sh')):
            cat_sh = open('apps/tomcat/bin/catalina.sh',"r")
            cat_text = cat_sh.read()
            if len(flags)>0:
                cat_text = re.sub('# OPENTUNER JVM SETTINGS STARTED.*# OPENTUNER JVM SETTINGS END', '# OPENTUNER JVM SETTINGS STARTED\nCATALINA_OPTS="'+flags+'"\n# OPENTUNER JVM SETTINGS END', cat_text,flags=re.DOTALL)       
                cat_text = re.sub('#OPENTUNER JVM SETTINGS END.*# OS specific support.  $var _must_ be set to either true or false.', '', cat_text,flags=re.DOTALL)
            else:
                print 'in set_catalinash setting with empty flag.'
                cat_text = re.sub('# OPENTUNER JVM SETTINGS STARTED.*# OPENTUNER JVM SETTINGS END', '# OPENTUNER JVM SETTINGS STARTED\n# OPENTUNER JVM SETTINGS END', cat_text,flags=re.DOTALL)
                cat_text = re.sub('#OPENTUNER JVM SETTINGS END.*# OS specific support.  $var _must_ be set to either true or false.', '', cat_text,flags=re.DOTALL)
            cat_sh.close()
            cat_sh = open('apps/tomcat/bin/catalina.sh',"w")
            cat_sh.write(cat_text)
            cat_sh.close()
            sftp = self.ssh.open_sftp()
            sftp.put('apps/tomcat/bin/catalina.sh', '/home/cse/SapientS/hotspottuner/TomcatTuner/src/apps/tomcat/bin/catalina.sh')
    
    def prepare_flags(self,filename):
        temp = pd.read_csv(filename)
        bool_out = pd.merge(temp,self.bool_flags,on='flagname').flagname
        
        param_out = dict()
        df = pd.merge(temp,self.param_flags,on='flagname')
        for index,row in df.iterrows():
            param_out[index] = {'flagname': row['flagname'],
                                          'min': row['min'],
                                          'max': row['max']}
        return bool_out,param_out
       
    def manipulator(self):
  #      print 'manipulator started'
        m = manipulator.ConfigurationManipulator()
        for flag_set in self.all_bool_flags:
            for flag in flag_set:
                m.add_parameter(manipulator.EnumParameter(flag, ['on', 'off', 'default']))
        for flag_set in self.all_param_flags:
#         print flag_set
            for flag in flag_set:
                value = flag_set[flag]
#                print value
                if(value['min'] >= value['max']):
                    m.add_parameter(manipulator.IntegerParameter(value['flagname'],value['max'],value['min']))
                else:
                    m.add_parameter(manipulator.IntegerParameter(value['flagname'],value['min'],value['max']))
        
        return m
    
    def run(self, desired_result, input, limit):
#        print 'run function started'
        
        #self.set_tomcat()
        cfg = desired_result.configuration.data
        JvmFlagsTuner.lock.acquire()
        
        try:
            flags = ''
            for flag in self.gc_select_flags:
                if cfg[flag] == 'on':
                    flags += ' -XX:+{0}'.format(flag)
                    if(flag == 'UseSerialGC'):
                        break
                    elif(flag == 'UseParallelOldGC' or flag == 'UseParallelGC'):
                        for gc_through_flag in self.gc_troughput_bool:
                            if cfg[gc_through_flag] == 'on':
                                flags += ' -XX:+{0}'.format(flag)
                            elif cfg[gc_through_flag] == 'off':
                                flags += ' -XX:-{0}'.format(flag)
                        for gc_through_flag in self.gc_throughput_param:
                            value = self.gc_throughput_param[gc_through_flag]
                            param_flag = value['flagname']
                            flags += ' -XX:'+param_flag+"="+str(cfg[param_flag])
                        break
                            
                    elif(flag == 'UseConcMarkSweepGC' or flag == 'UseParNewGC'):
                        for gc_flag in self.gc_cms_bool:
                            if cfg[gc_flag] == 'on':
                                flags += ' -XX:+{0}'.format(flag)
                            elif cfg[gc_flag] == 'off':
                                flags += ' -XX:-{0}'.format(flag)
                        for gc_flag in self.gc_cms_param:
                            value = self.gc_cms_param[gc_flag]
                            param_flag = value['flagname']
                            flags += ' -XX:'+param_flag+"="+str(cfg[param_flag])
                        break
                            
                    elif(flag == 'UseG1GC'):
                        for gc_flag in self.g1_gc_bool:
                            if cfg[gc_flag] == 'on':
                                flags += ' -XX:+{0}'.format(flag)
                            elif cfg[gc_flag] == 'off':
                                flags += ' -XX:-{0}'.format(flag)
                        for gc_flag in self.g1_gc_param:
                            value = self.g1_gc_param[gc_flag]
                            param_flag = value['flagname']
                            flags += ' -XX:'+param_flag+"="+str(cfg[param_flag])
                        break
                            
                elif cfg[flag] == 'off':
                    flags += ' -XX:-{0}'.format(flag)
            
            #set common gc flags
            for flag in self.gc_common_bool:
                if cfg[flag] == 'on':
                    flags += ' -XX:+{0}'.format(flag)
                elif cfg[flag] == 'off':
                    flags += ' -XX:-{0}'.format(flag)
            
            for flag_dict in self.gc_common_param:
                flag = self.gc_common_param[flag_dict]
                param_flag = flag['flagname']
                flags += ' -XX:'+param_flag+"="+str(cfg[param_flag])
            
            #set jit compiler flags
            for flag in self.compiler_flags_bool:
                if cfg[flag] == 'on':
                    flags += ' -XX:+{0}'.format(flag)
                elif cfg[flag] == 'off':
                    flags += ' -XX:-{0}'.format(flag)
                    
            for flag_dict in self.compiler_flags_param:
                flag = self.compiler_flags_param[flag_dict]
                param_flag = flag['flagname']
                flags += ' -XX:'+param_flag+"="+str(cfg[param_flag])
            
            
            self.set_catalinash(flags)
            print 'Catalina sh is set with flags.'
            run_command = 'sh '+self.tomcat_bin_location+'startup.sh'
            stdin, stdout, stderr = self.ssh.exec_command(run_command)
            time.sleep(self.sleep_time)
            
#            run_command = 'ab -k -n 1000000 -c 149 http://'+self.remote_ip+':8080/sample/hello'
            run_result = self.call_program(self.ab_command)
            ab_bench_output = run_result['stdout']
            print 'ab_bench_output defaultruntime = ',ab_bench_output
            if len(ab_bench_output)>0:
                m = re.search('Time per request(.*)\[ms\] \(mean\)', ab_bench_output,flags=re.DOTALL)
                ab_bench_time = '10000'
            else:
                print 'Error running ab. In the run function.'
                sys.exit()
            if m:
                ab_bench_time = m.group(1)
            ab_bench_time = re.sub(':', '', ab_bench_time,flags=re.DOTALL) 

            try:
                time_per_request = float(ab_bench_time)
            except:
                time_per_request=10000
            print 'time_per_request = ' , time_per_request
            time.sleep(self.sleep_time)

            if time_per_request==10000:
                self.set_catalinash('')
            
            run_command = 'sh '+self.tomcat_bin_location+'shutdown.sh'
            stdin, stdout, stderr =self.ssh.exec_command(run_command)
            time.sleep(self.sleep_time)
            
        finally:
            JvmFlagsTuner.lock.release()
            
        improvement=(self.default_run_time-time_per_request)*100.0/self.default_run_time
        if improvement>=self.improvement_best :
            self.improvement_best=improvement
            commandfile = open('configuration.txt',"a")
            commandfile.write('\n'+ run_command)
            commandfile.write(flags)
            commandfile.write('Improvement:'+str(self.improvement_best)+'\n')
            commandfile.close() 
        print 'Run function returned successfully. Time per sequest:=>'+str(time_per_request)
        return Result(time=time_per_request)
     
if __name__ == '__main__':
    opentuner.init_logging()
    args = argparser.parse_args()
    JvmFlagsTuner.main(args)
    
    
      
