#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" Executables to manage project with REDCap
    by generating the settings file automatically

    dax_manager SHOULD run as a crontab job every hour
"""

from __future__ import print_function
from __future__ import division

from builtins import filter
from builtins import str
from builtins import range
from builtins import object
from past.builtins import basestring

import os
import sys
import redcap
import smtplib
import traceback
import subprocess
import multiprocessing
from string import Template
from datetime import datetime
from email.mime.text import MIMEText
from subprocess import CalledProcessError

import dax
from dax import DAX_Settings
from dax.launcher import BUILD_SUFFIX, UPDATE_SUFFIX, LAUNCH_SUFFIX


try:
    basestring
except NameError:
    basestring = str

__copyright__ = 'Copyright 2013 Vanderbilt University. All Rights Reserved'
__exe__ = os.path.basename(__file__)
__author__ = 'byvernault'
__purpose__ = "dax_manager to manage project from redcap database or \
lists of settings files"
DAX_SETTINGS = DAX_Settings()
ADMIN_EMAIL = DAX_SETTINGS.get_admin_email()
RESULTS_DIR = DAX_SETTINGS.get_results_dir()
DEFAULT_GATEWAY = DAX_SETTINGS.get_gateway()
REDCAP_VAR = DAX_SETTINGS.get_dax_manager_config()
API_URL = DAX_SETTINGS.get_api_url()
API_KEY_DAX = DAX_SETTINGS.get_api_key_dax()
SMTP_FROM = DAX_SETTINGS.get_smtp_from()
SMTP_HOST = DAX_SETTINGS.get_smtp_host()
SMTP_PASS = DAX_SETTINGS.get_smtp_pass()
DEFAULT_EMAIL_OPTS = DAX_SETTINGS.get_email_opts()
DEFAULT_MAX_AGE = DAX_SETTINGS.get_max_age()
SKIP_LASTUPDATE = DAX_SETTINGS.get_skip_lastupdate()
USER = os.environ['USER']
try:
    GATEWAY = subprocess.check_output(
        'hostname', stderr=subprocess.STDOUT, shell=True).strip()
except (CalledProcessError, ValueError):
    GATEWAY = DEFAULT_GATEWAY

# Template for Module:
MOD_TEMPLATE = Template("${PROJECT}_${MODNAME}=${MODNAME}(${ARGUMENTS})")
# Template for Process:
PROC_TEMPLATE = Template("${PROJECT}_${PROCNAME}=${PROCNAME}(${ARGUMENTS})")
# Template for settings files:
SETTINGS_TEMPLATE = Template("""#import packages:
import os
from dax import Launcher
from Xnat_process_importer import *
from Xnat_module_importer import *

#Email & tmp folder:
email = "${EMAIL}"
tmpdir = "${TMPDIR}"

#Create TEMP if doesn't exist
if not os.path.exists(tmpdir):
    os.mkdir(tmpdir)

## Init proc and mod ##
#Mod
${MOD}
#Proc
${PROC}

#Set up modules for projects
proj_mod = ${PROJ_MOD}
#Set up processors for projects
proj_proc = ${PROJ_PROC}

#Order project:
p_order = ${PROJ_ORDER}

#Launch jobs:
myLauncher = Launcher(proj_proc, proj_mod, priority_project=p_order, \
job_email=email, job_email_options="${EMAIL_OPTS}", \
queue_limit=${QUEUE_LIMIT}, max_age=${MAX_AGE}, \
skip_lastupdate="${SKIP_LASTUPDATE}")
""")

EMAIL_HEADER = """Long updates running on <{gateway}> for the user <{user}>:
----------------------------------------------------------------------------------
"""

UPDATE_EMAIL_DICT = {'dax_build still running after 2 days': [],
                     'dax_build still running after 7 days': [],
                     'dax_launch still running after 2 days': [],
                     'dax_launch still running after 7 days': [],
                     'dax_update_tasks still running after 2 days': [],
                     'dax_update_tasks still running after 7 days': []}


class Settings_File(object):
    """ Class to generate settings file """
    def __init__(self, filepath, tmpdir='/tmp/', email=None,
                 email_opts=DEFAULT_EMAIL_OPTS, dax_logs_path='/tmp/',
                 queue_limit=200, max_age=DEFAULT_MAX_AGE,
                 admin_email=ADMIN_EMAIL,
                 skip_lastupdate=SKIP_LASTUPDATE):
        """
        Entry point for the filter_variable class

        :param filepath: path for the settings file
        :param tmpdir: temp directory to write the settings file
        :param email: email for the jobs in the settings file
        :param email_opts: email options for the jobs in the settings file
        :param dax_logs_path: path for the logs
        :param queue_limit: queue limit to submit to the cluster for the
                            settings file
        :param max_age: maximum age before restarting a session and update it
                        again
        :param admin_email: admin email address for this settings file
        :return: None
        """
        self.filepath = filepath
        # email
        if isinstance(email, list):
            self.email = email
        else:
            self.email = email.split(',')
        # email_opts
        self.email_opts = email_opts
        # tmpdir
        self.tmpdir = tmpdir
        # Project order
        self.proj_order = list()
        # String with all modules definitions
        self.mod_str = ''
        # dict with project: [] list of modules
        self.pm_dict = dict()
        # String with all processes definitions
        self.proc_str = ''
        # dict with project: [] list of processes
        self.pp_dict = dict()
        # logs for dax path
        self.dax_logs_path = dax_logs_path
        # Maximum Number of job that can be submitted
        if not queue_limit:
            self.queue_limit = 400
        elif isinstance(queue_limit, basestring):    
            self.queue_limit = int(queue_limit)
        else:
            self.queue_limit = 400
        # Admin email to receve error from dax
        if isinstance(admin_email, list):
            self.admin_email = admin_email
        else:
            self.admin_email = admin_email.split(',')
        # max age for sessions:
        if not max_age:
            self.max_age = DEFAULT_MAX_AGE
        else:
            self.max_age = max_age

        if not skip_lastupdate:
            self.skip_lastupdate = SKIP_LASTUPDATE
        else:
            self.skip_lastupdate = skip_lastupdate

    def add_mod_str(self, project, mod_str):
        """
        Add a module define by the string to a project

        :param project: project id of XNAT to add the module
        :param mod_str: module string to add
        :return: None
        """
        self.mod_str += mod_str + '\n'
        if project in list(self.pm_dict.keys()):
            self.pm_dict[project].append(mod_str.split('=')[0])
        else:
            self.pm_dict[project] = [mod_str.split('=')[0]]

    def add_proc_str(self, project, proc_str):
        """
        Add a processor define by the string to a project

        :param project: project id of XNAT to add the processor
        :param proc_str: processor string to add
        :return: None
        """
        self.proc_str += proc_str + '\n'
        if project in list(self.pp_dict.keys()):
            self.pp_dict[project].append(proc_str.split('=')[0])
        else:
            self.pp_dict[project] = [proc_str.split('=')[0]]

    def get_project_list(self):
        """
        Get the list of projects for the settings file

        :return: a list of projects ID
        """
        return set(list(self.pm_dict.keys()) + list(self.pp_dict.keys()))

    def merge_settings(self, settings):
        """
        Merge an other settings class object to this class

        :param settings: settings class object to add to this class
        :return: None
        """
        # Email:
        if list(set(self.email) - set(settings.email)):
            self.email += list(set(self.email) - set(settings.email))
        if list(set(self.admin_email) - set(settings.admin_email)):
            self.admin_email += list(
                set(self.admin_email) - set(settings.admin_email))
        # queue_limit (difference divide by 2):
        self.queue_limit = (self.queue_limit + settings.queue_limit) / 2
        # Add mod_str and proc_str
        self.mod_str += settings.mod_str
        self.proc_str += settings.proc_str
        # Add project to dict:
        self.pm_dict.update(settings.pm_dict)
        self.pp_dict.update(settings.pp_dict)

    def set_order(self, proj_order):
        """
        Set the order for the project

        :param proj_order: list of projects ID ordered
        :return: None
        """
        self.proj_order = proj_order

    def get_order(self):
        """
        Get the order for the projects

        :return: list of projects ordered in a string
        """
        order_str = '['
        for project in self.proj_order:
            order_str += """'{project}',""".format(project=project)
        if order_str[-1] == ',':
            order_str = order_str[:-1]
        order_str += ']'
        return order_str

    def get_admin_email(self):
        """
        Get the list of the administrator email addresses for the settings file

        :return: list of email addresses from admins
        """
        email_list = list()
        if '' in self.email:
            self.email.remove('')

        if not self.admin_email:
            return email_list
        else:
            for email in self.admin_email:
                if email.find('@') == -1:
                    e_addr = email + '@vanderbilt.edu'
                else:
                    e_addr = email
                email_list.append(e_addr)
            return email_list

    def get_email_list(self):
        """
        Get the list of the email addresses for the settings file dedicated
         to the cluster jobs

        :return: list of email addresses for jobs
        """
        email_list = list()
        if '' in self.email:
            self.email.remove('')

        if not self.email:
            return email_list
        else:
            for email in self.email:
                if email.find('@') == -1:
                    e_addr = email + '@vanderbilt.edu'
                else:
                    e_addr = email
                email_list.append(e_addr)
            return email_list

    def get_email_opts(self):
        """
        Get the email options for the settings file dedicated
         to the cluster jobs

        :return: email options
        """
        if self.email_opts == '':
            return DEFAULT_EMAIL_OPTS
        else:
            return self.email_opts

    def get_pm_dict(self):
        """
        Get the dictionary representing the project/modules as strings
         to run for the settings file

        :return: dictionary of project/modules strings
        """
        pm_dict_str = '{'
        first_elem = True
        spacing = 12 * " "
        for project in sorted(self.pm_dict):
            # Generate the string:
            dict_elem = """'{project}':[""".format(project=project)
            for mod in self.pm_dict[project]:
                dict_elem += mod + ','
            if dict_elem[-1] == ',':
                dict_elem = dict_elem[:-1]
            dict_elem += '],\n'
            # Keep a nice format:
            if not first_elem:
                pm_dict_str += spacing + dict_elem
            else:
                pm_dict_str += dict_elem
            first_elem = False
        # Remove the last comma
        if pm_dict_str.strip().endswith(','):
            pm_dict_str = pm_dict_str[:-2]  # for the \n
        pm_dict_str += '}'
        return pm_dict_str

    def get_pp_dict(self):
        """
        Get the dictionary representing the project/processors as strings
         to run for the settings file

        :return: dictionary of project/processors strings
        """
        pp_dict_str = '{'
        first_elem = True
        spacing = 13 * " "
        for project in sorted(self.pp_dict):
            # Generate the string:
            dict_elem = """'{project}':[""".format(project=project)
            for proc in self.pp_dict[project]:
                dict_elem += proc + ','
            if dict_elem[-1] == ',':
                dict_elem = dict_elem[:-1]
            dict_elem += '],\n'
            # Keep a nice format:
            if not first_elem:
                pp_dict_str += spacing + dict_elem
            else:
                pp_dict_str += dict_elem
            first_elem = False
        # Remove the last comma
        if pp_dict_str.strip().endswith(','):
            pp_dict_str = pp_dict_str[:-2]  # for the \n
        pp_dict_str += '}'
        return pp_dict_str


def get_list_modproc(redcap_project):
    """
    Get the list of modules and processors from redcap project

    :param redcap_project: pycap project object
    :return: list of modules, list of processors
    """
    # Getting modules/processors
    all_forms = list()
    fnames, _ = redcap_project.names_labels()
    k = ''
    for k in fnames:
        try:
            field = list(filter(lambda x: x['field_name'] == k, redcap_project.metadata))[0]
            all_forms.append(field['form_name'])
        except IndexError:
            LOGGER.error('IndexError when checking the libraries.',
                         exc_info=True)
            sys.exit()
    # split module from process:
    list_forms = set(all_forms)
    mod = [name for name in list_forms if name[:6] == 'module']
    proc = [name for name in list_forms if name[:7] == 'process']
    return mod, proc


def get_field_for_modproc(redcap_project, form_name):
    """
    Get the list of fields for the form requested from redcap project

    :param redcap_project: pycap project object
    :param form_name: name of the form
    :return: list of fields
    """
    avoid_labels = [form_name[7:] + '_inputs',
                    form_name[7:] + '_on',
                    form_name[8:] + '_inputs',
                    form_name[8:] + '_on']
    fields_list = list()
    fnames, _ = redcap_project.names_labels()
    k = ''
    for k in fnames:
        try:
            field = list(filter(lambda x: x['field_name'] == k, redcap_project.metadata))[0]
            if field['form_name'] == form_name and k not in avoid_labels:
                fields_list.append(field)
        except IndexError:
            LOGGER.error('IndexError when checking the libraries.',
                         exc_info=True)
            sys.exit()
    return fields_list


def read_redcap_db(redcap_project):
    """
    Read the project on redcap and extract the records for the user and
    gateway requested

    :param redcap_project: pycap project object
    :return: list of records
    """
    # Extract all attributes for the records that belong to the gateway/user
    # we are on:
    fields_search = [redcap_project.def_field, REDCAP_VAR['user'],
                     REDCAP_VAR['gateway']]
    record_list = redcap_project.export_records(fields=fields_search)
    records = [record[redcap_project.def_field] for record in record_list
               if (record[REDCAP_VAR['user']] == USER and
                   record[REDCAP_VAR['gateway']] == GATEWAY)]
    if records:
        return redcap_project.export_records(records=records)
    else:
        return list()


def generate_settings(redcap_project, settings_info):
    """
    Generate the settings file for the redcap project selected

    :param redcap_project: pycap project object
    :param settings_info: dictionary representing the settings information
    :return: dictionary of settings
             (key = filepath, value = Settings_File Class object)
    """
    # Variable:
    settings_dict = dict()
    modules_list, processes_list = get_list_modproc(redcap_project)
    LOGGER.info('Projects found on Redcap: ')
    if not settings_info:
        LOGGER.info('  - No project found')
    # Organize the data to in complex dictionaries:
    for project_settings in settings_info:
#        if project_settings[REDCAP_VAR['project']] != 'BLSA': - SHUNXING: for testing BLSA only
#            continue
        print(project_settings[REDCAP_VAR['project']])
        user = project_settings[REDCAP_VAR['user']]
        gateway = project_settings[REDCAP_VAR['gateway']]
        mess = """  - {project} for settings {settings}"""
        message = mess.format(
            project=project_settings[REDCAP_VAR['project']],
            settings=project_settings[REDCAP_VAR['settingsfile']])
        LOGGER.info(message)
        if project_settings['general_complete'] == '2':
            # Check the process:
            check_executables(project_settings)
            # New PSettings
            psettings = Settings_File(
                filepath=project_settings[REDCAP_VAR['settingsfile']],
                tmpdir=project_settings[REDCAP_VAR['tmp']],
                email=project_settings[REDCAP_VAR['email']],
                email_opts=project_settings[REDCAP_VAR['email_opts']],
                dax_logs_path=project_settings[REDCAP_VAR['logsdir']],
                queue_limit=project_settings[REDCAP_VAR['queue']],
                max_age=project_settings[REDCAP_VAR['max_age']],
                admin_email=project_settings[REDCAP_VAR['admin_email']])

            # Look for modules
            for mod_name in modules_list:
                args_mod_dict = dict()
                # If module on:
                if project_settings[mod_name[7:] + '_on'] == '1' and \
                   project_settings[mod_name + '_complete'] == '2':
                    # for variables from the module:
                    for field in get_field_for_modproc(redcap_project,
                                                       mod_name):
                        ff = field['field_name']
                        field_name = ff.split(mod_name[7:] + '_')[1]
                        args_mod_dict[field_name] = project_settings[ff]
                        # Get ModName:
                        if field_name == 'mod_name':
                            if project_settings[ff] == '':
                                spt = 'Default: '
                                modname = field['field_note'].strip()\
                                                             .split(spt)[1]
                            else:
                                modname = project_settings[field['field_name']]
                    # Add tmpdir for modules:
                    folder_name = (project_settings[REDCAP_VAR['project']] +
                                   '_' + mod_name[7:])
                    args_mod_dict['directory'] = os.path.join(
                        project_settings[REDCAP_VAR['tmp']], folder_name)
                    module = generate_mod(
                        project_settings[REDCAP_VAR['project']],
                        modname, args_mod_dict)
                    psettings.add_mod_str(
                        project_settings[REDCAP_VAR['project']], module)

            # Look for processes
            for proc_name in processes_list:
                args_proc_dict = dict()
                # Number of process of this type: if ';' in version or
                # processName or spiderpath
                nb_proc = 1
                # If process on:
                if project_settings[proc_name[8:] + '_on'] == '1' and \
                   project_settings[proc_name + '_complete'] == '2':
                    # Check the number of process (fiels version,
                    # processname,spiderpath
                    nb_proc = check_number_process(project_settings, proc_name)
                    # For each proc define by the tag (version or processName)
                    for index in range(nb_proc):
                        # for variables from the process:
                        for field in get_field_for_modproc(redcap_project,
                                                           proc_name):
                            # Get ProcName:
                            ff = field['field_name']
                            if ff == proc_name[8:] + '_proc_name':
                                procname = read_procname(project_settings,
                                                         index, ff,
                                                         field['field_note'])
                            else:
                                field_name = ff.split(proc_name[8:] + '_')[1]
                                args_proc_dict[field_name] = read_value(
                                    project_settings, index, ff)
                        processors = generate_proc(
                            project_settings[REDCAP_VAR['project']],
                            procname, args_proc_dict, str(index))
                        psettings.add_proc_str(
                            project_settings[REDCAP_VAR['project']],
                            processors)

            if psettings.filepath in list(settings_dict.keys()):
                settings_dict[psettings.filepath].merge_settings(psettings)
            else:
                settings_dict[psettings.filepath] = psettings
    LOGGER.info('\n')

    email_text = ''
    for header, proj_list in list(UPDATE_EMAIL_DICT.items()):
        if proj_list:
            email_text += '\t{} --> {}\n'.format(header, ' , '.join(proj_list))
            email_text += '-----------------------------------------'
            email_text += '-----------------------------------------\n'

    if ADMIN_EMAIL and email_text:
        email_text = EMAIL_HEADER.format(gateway=gateway,
                                         user=user) + email_text
        subject = 'Dax_manager ADMIN: updates running for 2/7days '
        send_email(email_text, ADMIN_EMAIL, subject)

    return settings_dict


def get_status(settings_info):
    """
    Get the status of the executables that are running

    :param settings_info: dictionary representing the settings information
    :return: dictionary of settings
             (key = filepath, value = Settings_File Class object)
    """
    LOGGER.info('Status on each settings updates:')
    separator = '-' * 160
    LOGGER.info(' ' + separator)
    header = ('| %*s | %*s | %*s | %*s | %*s | %*s | %*s | %*s |'
              % (-23, 'Project',
                 -3, 'R',
                 -17, 'Last Build',
                 -20, 'Build Info',
                 -17, 'Last Launch',
                 -20, 'Launch Info',
                 -17, 'Last Update Tasks',
                 -20, 'Update Tasks Info'))
    LOGGER.info(header)
    LOGGER.info(' ' + separator)
    for pset in sorted(settings_info, key=lambda k: k['general_complete']):
        if pset:
            projectstr = smaller_str(str(pset[REDCAP_VAR['project']]), size=23)
            status = get_status_string(pset['general_complete'])
            build_start = str(pset[REDCAP_VAR['dax_build_start_date']][:-3])
            launch_start = str(pset[REDCAP_VAR['dax_launch_start_date']][:-3])
            up_tasks_start = str(
                pset[REDCAP_VAR['dax_update_tasks_start_date']][:-3])
            build_info = get_update_info(
                pset[REDCAP_VAR['dax_build_start_date']],
                pset[REDCAP_VAR['dax_build_end_date']],
                pset[REDCAP_VAR['dax_build_pid']])
            launch_info = get_update_info(
                pset[REDCAP_VAR['dax_launch_start_date']],
                pset[REDCAP_VAR['dax_launch_end_date']],
                pset[REDCAP_VAR['dax_launch_pid']])
            up_tasks_info = get_update_info(
                pset[REDCAP_VAR['dax_update_tasks_start_date']],
                pset[REDCAP_VAR['dax_update_tasks_end_date']],
                pset[REDCAP_VAR['dax_update_tasks_pid']])
            val = ('| %*s | %*s | %*s | %*s | %*s | %*s | %*s | %*s |'
                   % (-23, projectstr,
                      -3, status,
                      -17, build_start,
                      -20, build_info,
                      -17, launch_start,
                      -20, launch_info,
                      -17, up_tasks_start,
                      -20, up_tasks_info))
            LOGGER.info(val)
    LOGGER.info(' ' + separator)


def get_status_string(status):
    """
    Get the status of the project on REDCap from the value (0,1,2)

    :param status: status int value
    :return: Y if project is ON, N otherwise
    """
    if str(status) == '2':
        return 'Y'
    else:
        return 'N'


def get_update_info(update_start, update_end, update_pid):
    """
    Get the string update information

    :param update_start: update start date
    :param update_end: update end date
    :param update_pid: update PID
    :return: string
    """
    if update_end == 'In Process':
        return '{} PID: {}'.format(update_end, update_pid)
    else:  # both are datetimes
        date_format = '%Y-%m-%d %H:%M:%S'
        try:
            time_delta = (datetime.strptime(update_end, date_format) -
                          datetime.strptime(update_start, date_format))
            secs = time_delta.total_seconds()
            hours = int(secs // 3600)
            mins = int((secs % 3600) // 60)
            if hours > 0:
                elapsed_time = '{} hrs {} mins'.format(hours, mins)
            else:
                elapsed_time = '{} mins'.format(mins)

        except ValueError:
            elapsed_time = update_end[:-3]

        return elapsed_time


def smaller_str(str_option, size, end=False):
    """
    Method to shorten a string into the smaller size

    :param str_option: string to shorten
    :param size: size of the string to keep (default: 10 characters)
    :param end: keep the end of the string visible (default beginning)
    :return: shortened string
    """
    if len(str_option) > size:
        if end:
            return '...' + str_option[-size - 3:]
        else:
            return str_option[:size - 3] + '...'
    else:
        return str_option


def check_number_process(project_settings, proc_name):
    """
    Shorten a string into the smaller size

    :param str_option: string to shorten
    :param size: size of the string to keep (default: 10 characters)
    :param end: keep the end of the string visible (default beginning)
    :return: shortened string
    """
    proc_name_tags = project_settings[proc_name[8:] + '_proc_name'].split(';')
    spider_tags = project_settings[proc_name[8:] + '_spider_path'].split(';')
    suffix_tags = project_settings[proc_name[8:] + '_suffix_proc'].split(';')
    version_tags = project_settings[proc_name[8:] + '_version'].split(';')
    return max(len(l) for l in [proc_name_tags, spider_tags,
                                suffix_tags, version_tags])


def read_procname(project_settings, index, field_name, field_note):
    """
    Read the procname from the settings by giving the field name and note on
    REDCap

    :param project_settings: settings for a project
    :param index: index of the process
    :param field_name: name of the field on REDCap to retrieve
    :param field_note: note for the field on REDCap
    :return: value from the project_settings
    """
    if ';' in project_settings[field_name]:
        value = project_settings[field_name].split(';')
        if len(value) <= index:
            mess = """wrong procname with ";" in {record_id} and field \
{field}: using default"""
            message = mess.format(
                record_id=project_settings[REDCAP_VAR['project']],
                field=field_name)
            LOGGER.warn(message)
            return field_note.strip().split('Default: ')[1]
        else:
            if value[index] == '':
                return field_note.strip().split('Default: ')[1]
            else:
                return value[index]
    else:
        if project_settings[field_name] == '':
            return field_note.strip().split('Default: ')[1]
        else:
            return project_settings[field_name]


def read_value(project_settings, index, field_name):
    """
    Read the value from the settings by giving the field name on REDCap

    :param project_settings: settings for a project
    :param index: index of the process
    :param field_name: name of the field on REDCap to retrieve
    :return: value for a field
    """
    if ';' in project_settings[field_name]:
        value = project_settings[field_name].split(';')
        if len(value) <= index:
            mess = """wrong procname with ";" in {record_id} and field \
{field}: using default"""
            message = mess.format(
                record_id=project_settings[REDCAP_VAR['project']],
                field=field_name)
            LOGGER.warn(message)
            return ''
        else:
            if value[index] == '':
                return ''
            else:
                return value[index]
    else:
        return project_settings[field_name]


def generate_proc(project, procname, args_dict, tag):
    """
    Generate the processor call to write in the settings

    :param project: project ID on XNAT
    :param procname: processors to run for the project
    :param args_dict: arguments for the processor
    :param tag: tag added to the project for the procname of the processor
    :return: string representing the processor
    """
    # Get the arguments
    args = dict2str(args_dict)
    # Get the string for a processor
    proc_data = {'PROJECT': project.replace('-', '_') + '_' + tag,
                 'PROCNAME': procname,
                 'ARGUMENTS': args}

    return PROC_TEMPLATE.safe_substitute(**proc_data)


def generate_mod(project, modname, args_dict):
    """
    Generate the module call to write in the settings

    :param project: project ID on XNAT
    :param modname: modules to run
    :param args_dict: arguments for the module
    :return: string representing the module
    """
    # Get the arguments
    args = dict2str(args_dict)
    # Get the string for a processor
    mod_data = {'PROJECT': project.replace('-', '_'),
                'MODNAME': modname,
                'ARGUMENTS': args}

    return MOD_TEMPLATE.safe_substitute(**mod_data)


def dict2str(arg_dict):
    """
    Convert a dictionary into a string for the template

    :param arg_dict: argument dictionary
    :return: string of the arguments
    """
    arg_str = ''
    for key, value in list(arg_dict.items()):
        if value != '':
            arg_str += '{}="{}",'.format(key, value)
    if arg_str:
        return arg_str[:-1]  # remove last comma
    else:
        return ''


def ordering_project(setting, settings_info):
    """
    Order the projects in a list from REDCap priority settings

    :param setting: Settings_File Class Object
    :param settings_info: list of information for settings
    :return: list of ordered projects
    """
    list_priority = dict()
    # Organize the data to in complex dictionaries:
    for project in setting.get_project_list():
        for project_settings in settings_info:
            if project_settings[REDCAP_VAR['project']] == project and \
               project_settings[REDCAP_VAR['priority']] != '':
                list_priority[project] = int(
                    project_settings[REDCAP_VAR['priority']])
    return sorted(list_priority, key=list_priority.get)


def write_settings(settings):
    """
    Write the settings file

    :param settings: Settings_File Class Object
    :return: None
    """
    # Check the Settings path:
    if '.py' not in os.path.basename(settings.filepath):
        LOGGER.error('Settings filepath is not a proper file. It needs the \
extension .py .')
    else:
        if not os.path.exists(os.path.dirname(settings.filepath)):
            os.makedirs(os.path.dirname(settings.filepath))
        # Write the Settings file
        settings_data = {'EMAIL': ','.join(settings.get_email_list()),
                         'EMAIL_OPTS': settings.get_email_opts(),
                         'MAX_AGE': settings.max_age,
                         'TMPDIR': settings.tmpdir,
                         'PROJ_ORDER': settings.get_order(),
                         'QUEUE_LIMIT': settings.queue_limit,
                         'MOD': settings.mod_str,
                         'PROC': settings.proc_str,
                         'PROJ_MOD': settings.get_pm_dict(),
                         'PROJ_PROC': settings.get_pp_dict(),
                         'SKIP_LASTUPDATE': settings.skip_lastupdate}

        with open(settings.filepath, 'w') as f:
            f.write(SETTINGS_TEMPLATE.safe_substitute(**settings_data))


def send_email(content, to_addr, subject):
    """
    Send an email to the address

    :param content: content of the email
    :param to_addr: address to send the email to
    :param subject: subject of the email
    :return: None
    """
    if SMTP_FROM and SMTP_HOST and SMTP_PASS:
        # Create the container (outer) email message.
        msg = MIMEText(content)
        msg['Subject'] = subject
        # me == the sender's email address
        # family = the list of all recipients' email addresses
        msg['From'] = SMTP_FROM
        msg['To'] = ','.join(to_addr)
        # Send the email via our own SMTP server.
        smtp = smtplib.SMTP(SMTP_HOST)
        smtp.starttls()
        smtp.login(SMTP_FROM, SMTP_PASS)
        smtp.sendmail(SMTP_FROM, to_addr, msg.as_string())
        smtp.quit()


def unlock_executables(flagfile, suffix):
    """
    Unlock the executables by removing the FlagFiles

    :param flagfile: flagfile path
    :param suffix: suffix added to the flagfile
    :return: None
    """
    lockfile_prefix = os.path.splitext(os.path.basename(flagfile))[0]
    flagfile = os.path.join(RESULTS_DIR, 'FlagFiles',
                            '{}_{}'.format(lockfile_prefix, suffix))
    if os.path.exists(flagfile):
        os.remove(flagfile)


def check_executables(project_settings):
    """
    Check the date on REDCap to see if any dax executables ran
     for more than few days

    :param project_settings: list of project settings
    :return: None
    """
    today_date = int('{:%Y%m%d%H%M%S}'.format(datetime.now()))
    # dax_build:
    if 'In Process' in project_settings['dax_build_end_date']:
        check_date(project_settings[REDCAP_VAR['project']],
                   project_settings['dax_build_start_date'],
                   today_date, 'dax_build')

    # dax_launch
    if 'In Process' in project_settings['dax_launch_end_date']:
        check_date(project_settings[REDCAP_VAR['project']],
                   project_settings['dax_launch_start_date'],
                   today_date, 'dax_launch')

    # dax_update_tasks
    if 'In Process' in project_settings['dax_launch_end_date']:
        check_date(project_settings[REDCAP_VAR['project']],
                   project_settings['dax_update_tasks_start_date'],
                   today_date, 'dax_update_tasks')


def check_date(project, start_date, now, exec_name):
    """
    Check the date for an executable

    :param project: project ID
    :param start_date: date when the dax executable started
    :param now: date at the time of today
    :param exec_name: name of the dax executable (dax_build/dax_launch/...)
    :return: None
    """
    if start_date:
        start_date = int(start_date.replace('-', '')
                                   .replace(':', '')
                                   .replace(' ', ''))
        start_date_2days = start_date + 2000000  # adding 2 days
        start_date_2days2hours = start_date + 2020000  # add 2 days - 2 hours
        start_date_7days = start_date + 7000000  # adding 7 days
        if start_date_2days <= now <= start_date_2days2hours:
            tag = '{} still running after 2 days'.format(exec_name)
            UPDATE_EMAIL_DICT[tag].append(project)
        if start_date_7days <= now:
            tag = '{} still running after 7 days'.format(exec_name)
            UPDATE_EMAIL_DICT[tag].append(project)


def get_logfile_name(logdir, prefix, filepath):
    """
    Get the path of the logfile

    :param logdir: directory for the logs
    :param prefix: prefix for the log file
    :param filepath: filepath of the settings file
    :return: path of the log file
    """
    if not os.path.exists(logdir):
        os.makedirs(logdir)
    project = os.path.basename(filepath).split('.')[0]
    now = '{:%Y%m%d_%H:%M:%S}'.format(datetime.now())
    filename = """{prefix}_{project}_{date}.log""".format(prefix=prefix,
                                                          project=project,
                                                          date=now)
    return os.path.join(logdir, filename)


def email_error(exec_name, email_add, filepath):
    """
    Send an email if an error is caught in the dax executables

    :param exec_name: dax executable failing
    :param email_add: email address
    :param filepath: filepath of the settings file
    :return: None
    """
    temp = """Caught exception in worker "{exec_name}" {file} : \n{trace}"""
    content = temp.format(exec_name=exec_name,
                          file=filepath,
                          trace=traceback.format_exc())
    subject = """Error in {exec_name} for {file}""".format(exec_name=exec_name,
                                                           file=filepath)
    send_email(content, email_add, subject)


def generate_settings_REDCap():
    """
    Extract information from REDCap and generate the settings file

    :return: list of path settings generated, dict of logs for dax executables,
              dictionaries of email addresses for dax executables
    """
    settings_path_list = list()
    daxlogsdir_dict = dict()
    email_dict = dict()
    # get the information for xnat from redcap for the projects:
    LOGGER.info('Loading RedCap to extract information to run dax_updates...')
    try:
        redcap_project = redcap.Project(API_URL, API_KEY)
    except Exception:
        LOGGER.error('Could not access redcap. Either wrong API_URL/API_KEY \
or redcap down.', exc_info=True)
        sys.exit()

    # Extract the information about the settings from redcap
    settings_info = read_redcap_db(redcap_project)
    # Status:
    if PARGS.status:
        get_status(settings_info)
    else:
        # Generate information for the settings
        settings_dict = generate_settings(redcap_project,
                                          settings_info)
        # For each settings in the dict:
        for filepath, setting in list(settings_dict.items()):
            # Ordering project:
            setting.set_order(ordering_project(setting, settings_info))
            # Write the settings:
            LOGGER.debug(' -> Writing settings : {}'.format(filepath))
            write_settings(setting)
            settings_path_list.append(setting.filepath)
            daxlogsdir_dict[filepath] = setting.dax_logs_path
            email_dict[filepath] = setting.get_admin_email()

    return settings_path_list, daxlogsdir_dict, email_dict, settings_info


def run_dax_build(filepath, logdir, debug, cachedir, email_add, proj_lastrun=None):
    """
    Function to submit dax_build to the cluster and redirect output to the
    logfile

    :param filepath: filepath of the settings file
    :param logdir: directory for the logs
    :param debug: debug mode or not
    :param email_add: email address
    :return: None
    """
    try:
        logfile = get_logfile_name(logdir, "buildlog", filepath)
        dax.bin.build(filepath, logfile, debug, cachedir, proj_lastrun=proj_lastrun)
    except Exception:
        err = 'Caught exception in worker "dax_build {}" : \n{}'
        LOGGER.error(err.format(filepath, traceback.format_exc()))
        # Remove the flagfile:
        unlock_executables(filepath, BUILD_SUFFIX)
        if email_add:
            email_error('dax_build', email_add, filepath)


def run_dax_launch(filepath, logdir, debug, email_add):
    """
    Function to submit dax_launch to the cluster and redirect output to the
    logfile

    :param filepath: filepath of the settings file
    :param logdir: directory for the logs
    :param debug: debug mode or not
    :param email_add: email address
    :return: None
    """
    try:
        logfile = get_logfile_name(logdir, "launchlog", filepath)
        dax.bin.launch_jobs(filepath, logfile, debug)
    except Exception:
        err = 'Caught exception in worker "dax_launch {}" : \n{}'
        LOGGER.error(err.format(filepath, traceback.format_exc()))
        # Remove the flagfile:
        unlock_executables(filepath, LAUNCH_SUFFIX)
        if email_add:
            email_error('dax_launch', email_add, filepath)


def run_dax_update_tasks(filepath, logdir, debug, email_add):
    """
    Function to submit dax_update_tasks to the cluster and redirect output to
    the logfile

    :param filepath: filepath of the settings file
    :param logdir: directory for the logs
    :param debug: debug mode or not
    :param email_add: email address
    :return: None
    """
    try:
        logfile = get_logfile_name(logdir, "update_taskslog", filepath)
        dax.bin.update_tasks(filepath, logfile, debug)
    except Exception:
        err = 'Caught exception in worker "dax_update_open_tasks {}" : \n{}'
        LOGGER.error(err.format(filepath, traceback.format_exc()))
        # Remove the flagfile:
        unlock_executables(filepath, UPDATE_SUFFIX)
        if email_add:
            email_error('dax_update_tasks', email_add, filepath)


def submit_exec(target, args, filepath, exec_name):
    """
    Function to submit one executable using multiprocessing on the gateway

    :param target: one of the function define above
                   (run_dax_build/run_dax_launch/run_dax_update_tasks)
    :param args: arguments for the function
    :param filepath: filepath of the settings file
    :param exec_name: name of the dax executable
    :return: PID for the executable
    """
    LOGGER.debug("""new process {execn} on {file}""".format(
        execn=exec_name, file=filepath))
    proc = multiprocessing.Process(target=target, args=args)
    proc.start()
    return proc


def parse_args():
    """
    Method to parse arguments base on argparse

    :return: parser object parsed
    """
    from argparse import ArgumentParser
    argp = ArgumentParser(prog=__exe__, description=__purpose__)
    argp.add_argument(
        '--key', dest='key', default=None,
        help='API Key for redcap project containing dax informations.')
    argp.add_argument('--settings', dest='settings',
                      help='List of Settings Path', default=None)
    argp.add_argument('--daxlogsdir', dest='daxlogsdir', default='/tmp',
                      help='Directory for dax logs file.Default: /tmp')
    argp.add_argument('--writeonly', dest='writeonly', action='store_true',
                      help='Write the settings from REDCap only')
    argp.add_argument('--status', dest='status', action='store_true',
                      help='Status on the updates')
    argp.add_argument('--email', dest='email', default=None,
                      help='Email address to send errors.')
    argp.add_argument('--logfile', dest='logfile', default=None,
                      help='Logs file path if needed.')
    argp.add_argument('--nodebug', dest='debug', action='store_false',
                      help='Avoid printing DEBUG information.')
    argp.add_argument('--uselastrun', dest='uselastrun', action='store_true',
                      help='Use last run to skip sessions.')
    argp.add_argument('--cachedir', dest='cachedir',
                      help='Run build if need caching', default=None)
    return argp.parse_args()


if __name__ == '__main__':
    if not DAX_SETTINGS.is_dax_manager_valid():
        sys.stdout.write('Please edit your settings via dax_setup for the \
[dax_manager] section\n')
        sys.exit()
    # Arguments:
    PARGS = parse_args()
    
    #print('SHUNXING EDIT #### %s' % PARGS.cachedir)
    if PARGS.email:
        elist = PARGS.email.split(',')
    else:
        elist = list()
    API_KEY = PARGS.key
    if not API_KEY and not PARGS.settings and API_KEY_DAX:
        API_KEY = API_KEY_DAX
    if PARGS.settings:
        SETTINGS_PATHS = PARGS.settings.split(',')
    else:
        SETTINGS_PATHS = list()
    # hour right now:
    NOW_HOUR = datetime.now().hour

    # Logger for logs
    if PARGS.debug and not PARGS.status:
        LOGGER = dax.log.setup_debug_logger('manager', PARGS.logfile)
    else:
        LOGGER = dax.log.setup_info_logger('manager', PARGS.logfile)

    dax_logs_dir = PARGS.daxlogsdir
    DAX_LOGS_DIRS_DICT = dict()
    EMAIL_DICT = dict()
    msg = 'Time at the beginning of the DAX Manager: {}'
    LOGGER.info(msg.format(str(datetime.now())))
    LOGGER.info('Current Process ID: %s' % (str(os.getpid())))
    LOGGER.info('Current Process Name: %s' % (os.path.basename(__file__)))
    LOGGER.info('Current User: %s' % (USER))
    if GATEWAY:
        print(GATEWAY)
        LOGGER.info('Current Gateway/Computer: %s\n' % (GATEWAY))
    else:
        LOGGER.debug('Current Gateway/Computer: not found...\n')
    print(API_KEY)
    # If api_key given, use the redcap db:
    if API_KEY:
        SETTINGS_PATHS, DAX_LOGS_DIRS_DICT, EMAIL_DICT, SETTINGS_INFO = \
            generate_settings_REDCap()

    if PARGS.writeonly or PARGS.status:
        pass
    else:
        if PARGS.debug:
            LOGGER.debug('Settings file found: ')
            for fpath in SETTINGS_PATHS:
                LOGGER.debug('  - %s' % (os.path.basename(fpath)))
            LOGGER.debug('\n')

        # Workers:
        WORKERS = []
        proj_lastrun = {}
        if PARGS.uselastrun:
            LOGGER.info('running with uselastrun')
            # Build a dictionary mapping project to the lastrun start time for
            # each project, but only if there's also a valid end time,
            # otherwise set to None
            for pset in SETTINGS_INFO:
                # TODO: limit the projects to only those we care about at this
                # point
                proj = pset[REDCAP_VAR['project']]
                try:
                    date_format = '%Y-%m-%d %H:%M:%S'
                    build_start = datetime.strptime(
                        pset[REDCAP_VAR['dax_build_start_date']], date_format)
                    build_end = datetime.strptime(
                        pset[REDCAP_VAR['dax_build_end_date']], date_format)
                    proj_lastrun[proj] = build_start
                    LOGGER.debug(proj + ': last run time ' + str(build_start))
                except ValueError:
                    err = '{}: could not load build dates for project'
                    LOGGER.debug(err.format(proj))
                    proj_lastrun[proj] = None
        else:
            # When to run dax_build / dax_update_tasks / dax_launch
            # dax_build -> 6am / 8pm
            # dax_update_tasks -> every odd hours (1-3-5...)
            # dax_launch -> every hour
            # Submit executables
            LOGGER.info('dax_build will run at 6am and 8pm...')
            LOGGER.info('dax_update_tasks will run on odd hours...')
            LOGGER.info('dax_launcher will run every hour...')

        LOGGER.info('Submitting dax executables for each settings file...')
        for fpath in SETTINGS_PATHS:
            if fpath in list(DAX_LOGS_DIRS_DICT.keys()):
                dax_logs_dir = DAX_LOGS_DIRS_DICT[fpath]
            if fpath in list(EMAIL_DICT.keys()):
                elist = EMAIL_DICT[fpath]

            if PARGS.uselastrun:
                # start processes
                arguments_build = (fpath, dax_logs_dir, PARGS.debug, PARGS.cachedir,elist)
                process = submit_exec(
                    run_dax_build, arguments_build + (proj_lastrun,),
                    fpath, 'dax_build')
                WORKERS.append(process)

                arguments = (fpath, dax_logs_dir, PARGS.debug,elist)
                process = submit_exec(
                    run_dax_update_tasks, arguments, fpath, 'dax_update_tasks')
                WORKERS.append(process)
            else:
                # start processes
                arguments_build = (fpath, dax_logs_dir, PARGS.debug, PARGS.cachedir,elist)
                if NOW_HOUR == 6 or NOW_HOUR == 20:  # 6am-8pm
                    process = submit_exec(
                        run_dax_build, arguments_build, fpath, 'dax_build')
                    WORKERS.append(process)

                arguments = (fpath, dax_logs_dir, PARGS.debug, elist)
                if NOW_HOUR % 2 == 1:  # odd hours
                    process = submit_exec(
                        run_dax_update_tasks, arguments, fpath,
                        'dax_update_tasks')
                    WORKERS.append(process)

            # Run every time that dax_manager run
            process = submit_exec(
                run_dax_launch, arguments, fpath, 'dax_launch')
            WORKERS.append(process)

        # If ctrl-c, kill all children
        try:
            for worker in WORKERS:
                worker.join()
        except KeyboardInterrupt:
            LOGGER.warn('Dax_manager received ctrc-c/kill. \
All workers are going to be terminated.')
            worker.terminate()
            worker.join()

    LOGGER.info('Time at the end of the DAX Manager: %s'
                % (str(datetime.now())))
