#!/usr/bin/python

""" GLOBUS UPLOADER SERVICE
    This will query Postgres for Globus tasks that are marked as
    'SUCCEEDED' but not yet 'PROCESSED' and notify Clowder to
    process the contents.
"""

import os, shutil, json, time, datetime, thread, copy, atexit, collections, fcntl
import logging, logging.config, logstash
import requests
import signal
import psycopg2
import socket
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
from io import BlockingIOError
from urllib3.filepost import encode_multipart_formdata

from pyclowder.datasets import download_metadata
from terrautils.metadata import clean_metadata
from terrautils.extractors import build_dataset_hierarchy

rootPath = "/home/globusmonitor"

"""
Config file has 2 important entries which do not have default values:
{
    "globus": {
*** (1) The valid_users sub-object describes which users can submit jobs ***
        "valid_users": {
            <globus username>: {
                "password": <globus password>,
                "endpoint_id": <globus endpoint ID corresponding to user>
            }
        }
    },
    "clowder": {
        "user_map": {
*** (2) The user_map sub-object maps a Globus user to the Clowder credentials that upload that user's files. ***
            <globus username>: {
                "clowder_user": <clowder username>
                "clowder_pass": <clowder password>
            }
        }
    }
"""
config = {}

# Store any PENDING task ID here
current_task = None


# ----------------------------------------------------------
# SHARED UTILS
# ----------------------------------------------------------
"""Create copy of dict in safe manner for multi-thread access (won't change during copy iteration)"""
def safeCopy(obj):
    # Iterate across a copy since we'll be changing object
    copied = False
    while not copied:
        try:
            newObj = copy.deepcopy(obj)
            copied = True
        except RuntimeError:
            # This can occur on the deepcopy step if another thread is accessing object
            time.sleep(0.1)

    return newObj

"""Attempt to lock a file so API and monitor don't write at once, and wait if unable"""
def lockFile(f):
    # From http://tilde.town/~cristo/file-locking-in-python.html
    while True:
        try:
            fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
            return
        except (BlockingIOError, IOError) as e:
            # Try again in 1/10th of a second
            time.sleep(0.1)

"""Nested update of python dictionaries for config parsing"""
def updateNestedDict(existing, new):
    # Adapted from http://stackoverflow.com/questions/3232943/update-value-of-a-nested-dictionary-of-varying-depth
    for k, v in new.iteritems():
        if isinstance(existing, collections.Mapping):
            if isinstance(v, collections.Mapping):
                r = updateNestedDict(existing.get(k, {}), v)
                existing[k] = r
            else:
                existing[k] = new[k]
        else:
            existing = {k: new[k]}
    return existing

"""Load contents of .json file into a JSON object"""
def loadJsonFile(filename):
    try:
        f = open(filename)
        jsonObj = json.load(f)
        f.close()
        return jsonObj
    except IOError:
        logger.error("- unable to open %s" % filename)
        return {}

"""Find list of file objects in a given dataset"""
def fetchDatasetFileList(datasetId, requestsSession):
    clowkey = config['clowder']['secret_key']
    filelist = requests.get(config['clowder']['host']+"/api/datasets/%s/listFiles?key=%s" % (datasetId, clowkey),
                            headers={"Content-Type": "application/json"})

    if filelist.status_code == 200:
        return filelist.json()
    else:
        logger.error("- cannot find file list for dataset %s" % datasetId)
        return []


# ----------------------------------------------------------
# POSTGRES LOGGING COMPONENTS
# ----------------------------------------------------------
"""Return a connection to the PostgreSQL database"""
def connectToPostgres():
    """
    If globusmonitor database does not exist yet:
        $ initdb /home/globusmonitor/postgres/data
        $ pg_ctl -D /home/globusmonitor/postgres/data -l /home/globusmonitor/postgres/log
        $   createdb globusmonitor
    """
    psql_db = os.getenv("POSTGRES_DATABASE", config['postgres']['database'])
    psql_host = os.getenv("POSTGRES_HOST", config['postgres']['host'])
    psql_user = os.getenv("POSTGRES_USER", config['postgres']['username'])
    psql_pass = os.getenv("POSTGRES_PASSWORD", config['postgres']['password'])

    connected = False
    total_retry = 0
    while not connected:
        try:
            conn = psycopg2.connect(dbname=psql_db, user=psql_user, password=psql_pass, host=psql_host)
            connected = True
        except Exception as e:
            """Attempt to create database if not found
            conn = psycopg2.connect(dbname='postgres', host=psql_host)
            conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
            curs = conn.cursor()
            curs.execute('CREATE DATABASE %s;', (psql_db))
            curs.close()
            conn.commit()
            conn.close()

            conn = psycopg2.connect(dbname=psql_db, user=psql_user, password=psql_pass, host=psql_host)
            initializeDatabase(conn)
            """
            logger.error("Could not connect to PSQL: %s" % e.message)
            if total_retry >= 600:
                logger.error("Exceeded maximum number of retries")
                raise
            time.sleep(30)
            total_retry += 30

    logger.info("Connected to Postgres")
    return conn

"""Safer way to call psql_conn.cursor()"""
def getPostgresCursor():
    global psql_conn

    try:
        curs = psql_conn.cursor()
    except Exception as e:
        logger.error("PSQL reconnecting; cursor error: %s" % e.message)
        psql_conn = connectToPostgres()
        curs = psql_conn.cursor()

    return curs

"""Create PostgreSQL database tables"""
def initializeDatabase(db_connection):
    # Table creation queries
    ct_tasks = "CREATE TABLE globus_tasks (globus_id TEXT PRIMARY KEY NOT NULL, status TEXT NOT NULL, received TEXT NOT NULL, completed TEXT, globus_user TEXT, contents JSON);"
    ct_dsets = "CREATE TABLE datasets (name TEXT PRIMARY KEY NOT NULL, clowder_id TEXT NOT NULL);"
    ct_colls = "CREATE TABLE collections (name TEXT PRIMARY KEY NOT NULL, clowder_id TEXT NOT NULL);"

    # Index creation queries
    ix_tasks = "CREATE UNIQUE INDEX globus_idx ON globus_tasks (globus_id);"
    ix_dsets = "CREATE UNIQUE INDEX dset_idx ON datasets (name);"
    ix_colls = "CREATE UNIQUE INDEX coll_idx ON collections (name);"

    # Execute each query
    curs = db_connection.cursor()
    logger.info("Creating PostgreSQL tables...")
    curs.execute(ct_tasks)
    curs.execute(ct_dsets)
    curs.execute(ct_colls)
    logger.info("Creating PostgreSQL indexes...")
    curs.execute(ix_tasks)
    curs.execute(ix_dsets)
    curs.execute(ix_colls)
    curs.close()
    db_connection.commit()

    logger.info("PostgreSQL initialization complete.")

"""Write a Globus task into PostgreSQL, insert/update as needed"""
def writeTaskToDatabase(task):
    gid = task['globus_id']
    stat = task['status']
    recv = task['received']
    comp = task['completed']
    guser = task['user']
    jbody = json.dumps(task['contents'])

    # Attempt to insert, update if globus ID already exists
    q_insert = "INSERT INTO globus_tasks (globus_id, status, received, completed, globus_user, contents) " \
               "VALUES (%s, %s, %s, %s, %s, %s) " \
               "ON CONFLICT (globus_id) DO UPDATE " \
               "SET status=%s, received=%s, completed=%s, globus_user=%s, contents=%s;"

    curs = getPostgresCursor()
    #logger.debug("Writing task %s to PostgreSQL..." % gid)
    curs.execute(q_insert, (gid, stat, recv, comp, guser, jbody, stat, recv, comp, guser, jbody))
    psql_conn.commit()
    curs.close()

"""Fetch all Globus tasks with a particular status"""
def getNextUnprocessedTask(status="SUCCEEDED", reverse=False):

    # Use Common Table Expression to update status to PENDING and return row at the same time
    if reverse:
        q_fetch = "WITH cte AS (SELECT globus_id FROM globus_tasks where STATUS = %s ORDER BY completed DESC LIMIT 1 FOR UPDATE SKIP LOCKED) UPDATE globus_tasks gt  SET status = 'PENDING' FROM cte WHERE gt.globus_id = cte.globus_id RETURNING *"

    else:
        q_fetch = "WITH cte AS (SELECT globus_id FROM globus_tasks where STATUS = %s ORDER BY completed ASC LIMIT 1 FOR UPDATE SKIP LOCKED) UPDATE globus_tasks gt  SET status = 'PENDING' FROM cte WHERE gt.globus_id = cte.globus_id RETURNING *"

    nextTask = None

    try:
        curs = getPostgresCursor()
        logger.debug("Fetching next %s task from PostgreSQL..." % status)
        logger.debug(q_fetch % status)
        curs.execute(q_fetch, (status,))
        for result in curs:
            nextTask = {
                "globus_id": result[0],
                "status": result[1],
                "received": result[2],
                "completed": result[3],
                "user": result[4],
                "contents": result[5]
            }
        curs.close()
    except Exception as e:
        logger.error("Exception fetching task: %s" % str(e))

    global current_task
    if nextTask:
        current_task = nextTask['globus_id']
        logger.debug("Found task %s [%s]" % (nextTask['globus_id'], nextTask['completed']))
    else:
        current_task = None
        logger.debug("No task found.")
    return nextTask

"""Remove PENDING status from any ongoing processing if this is killed"""
def gracefulExit(signum, frame):
    if current_task:
        curs = getPostgresCursor()
        query = "update globus_tasks set STATUS='SUCCEEDED' where globus_id = %s;"
        logger.debug("Gracefully resolving PENDING for %s" % current_task)
        curs.execute(query, (current_task,))
        psql_conn.commit()
        curs.close()


# ----------------------------------------------------------
# SERVICE COMPONENTS
# ----------------------------------------------------------
"""Send Clowder necessary details to load local file after Globus transfer complete"""
def notifyClowderOfCompletedTask(task):
    # Verify that globus user has a mapping to clowder credentials in config file
    globUser = task['user']
    userMap = config['clowder']['user_map']

    if globUser in userMap:
        logger.info("%s task complete; notifying Clowder" % task['globus_id'], extra={
            "globus_id": task['globus_id'],
            "action": "NOTIFYING CLOWDER OF COMPLETION"
        })
        clowder_host = config['clowder']['host']
        clowder_key  = config['clowder']['secret_key']
        clowder_user = userMap[globUser]['clowder_user']
        clowder_pass = userMap[globUser]['clowder_pass']
        clowder_id = userMap[globUser]['clowder_id']
        clowder_context = userMap[globUser]['context']

        sess = requests.Session()
        sess.auth = (clowder_user, clowder_pass)

        # Response can be OK, RETRY or ERROR
        response = "OK"

        # Prepare upload object with all file(s) found
        updatedTask = safeCopy(task)

        space_id = task['contents']['space_id'] if 'space_id' in task['contents'] else config['clowder']['primary_space']
        for ds in task['contents']:
            # Skip any unexpected files at root level, e.g.
            #   /home/clowder/sites/ua-mac/raw_data/GetFluorescenceValues.m
            #   /home/clowder/sites/ua-mac/raw_data/irrigation/2017-06-04/@Recycle/flowmetertotals_March-2017.csv",
            if ds in ["LemnaTec - MovingSensor"] or ds.find("@Recycle") > -1:
                continue
                
            filesQueued = []
            fileFormData = []
            datasetMD = None
            datasetMDFile = False
            lastFile = None
            lastFileKey = None
            sensorname = ds.split(" - ")[0]

            logger.info("%s -- Processing [%s]" % (task['globus_id'], ds))

            # Assign dataset-level metadata if provided
            if "md" in task['contents'][ds]:
                datasetMD = task['contents'][ds]['md']

            # Add local files to dataset by path
            if 'files' in task['contents'][ds]:
                for fkey in task['contents'][ds]['files']:
                    fobj = task['contents'][ds]['files'][fkey]
                    if 'clowder_id' not in fobj or fobj['clowder_id'] == "":
                        if os.path.exists(fobj['path']):
                            if fobj['name'].find("metadata.json") == -1:
                                if 'md' in fobj:
                                    # Use [1,-1] to avoid json.dumps wrapping quotes
                                    # Replace \" with " to avoid json.dumps escaping quotes
                                    mdstr = ', "md":' + json.dumps(fobj['md'])[1:-1].replace('\\"', '"')
                                else:
                                    mdstr = ""
                                filesQueued.append((fobj['path'], mdstr))
                                lastFile = fobj['name']
                                lastFileKey = fkey
                            else:
                                try:
                                    datasetMD = loadJsonFile(fobj['path'])
                                    datasetMDFile = fkey
                                except:
                                    logger.error("[%s] could not decode JSON from %s" % (ds, fobj['path']))
                                    updatedTask['contents'][ds]['files'][fkey]['clowder_id'] = "FILE NOT FOUND"
                                    updatedTask['contents'][ds]['files'][fkey]['error'] = "Failed to load JSON"
                                    writeTaskToDatabase(updatedTask)
                                    if response == "OK": response = "ERROR" # Don't overwrite a RETRY
                        else:
                            logger.error("[%s] file not found: %s" % (ds, fobj['path']))
                            updatedTask['contents'][ds]['files'][fkey]['clowder_id'] = "FILE NOT FOUND"
                            updatedTask['contents'][ds]['files'][fkey]['error'] = "File not found"
                            writeTaskToDatabase(updatedTask)
                            if response == "OK": response = "ERROR" # Don't overwrite a RETRY

            if len(filesQueued)>0 or datasetMD:
                # Try to clean metadata first
                if datasetMD:
                    cleaned_dsmd = None
                    try:
                        cleaned_dsmd = clean_metadata(datasetMD, sensorname)
                    except Exception as e:
                        logger.error("[%s] could not clean md: %s" % (ds, str(e)))
                        task['contents'][ds]['error'] = "Could not clean metadata: %s" % str(e)
                        # TODO: possible this could be recoverable with more info from clean_metadata
                        if response == "OK": response = "ERROR" # Don't overwrite a RETRY

                if ds.find(" - ") > -1:
                    # e.g. "co2Sensor - 2016-12-25" or "VNIR - 2016-12-25__12-32-42-123"
                    c_sensor = ds.split(" - ")[0]
                    c_date  = ds.split(" - ")[1]
                    c_year  = c_date.split('-')[0]
                    c_month = c_date.split('-')[1]
                    if c_date.find("__") == -1:
                        # If we only have a date and not a timestamp, don't create date collection
                        c_date = None
                    else:
                        c_date = c_date.split("__")[0].split("-")[2]
                else:
                    c_sensor, c_date, c_year, c_month = ds, None, None, None


                # Get dataset from clowder, or create & associate with collections
                try:
                    hierarchy_host = clowder_host + ("/" if not clowder_host.endswith("/") else "")
                    dsid = build_dataset_hierarchy(hierarchy_host, clowder_key, clowder_user, clowder_pass, space_id,
                                                   c_sensor, c_year, c_month, c_date, ds)
                    logger.info("   [%s] id: %s" % (ds, dsid))
                except Exception as e:
                    logger.error("[%s] could not build hierarchy: %s" % (ds, str(e)))
                    task['contents'][ds]['retry'] = "Could not build dataset hierarchy: %s" % str(e)
                    response = "RETRY"
                    continue

                if dsid:
                    dsFileList = fetchDatasetFileList(dsid, sess)
                    # Only send files not already present in dataset by path
                    for queued in filesQueued:
                        alreadyStored = False
                        for storedFile in dsFileList:
                            if queued[0] == storedFile['filepath']:
                                logger.info("   skipping file %s (already uploaded)" % queued[0])
                                alreadyStored = True
                                break
                        if not alreadyStored:
                            fileFormData.append(("file",'{"path":"%s"%s}' % (queued[0], queued[1])))

                    if datasetMD and cleaned_dsmd:
                        # Check for existing metadata from the site user
                        alreadyAttached = False
                        md_existing = download_metadata(None, hierarchy_host, clowder_key, dsid)
                        for mdobj in md_existing:
                            if 'agent' in mdobj and 'user_id' in mdobj['agent']:
                                if mdobj['agent']['user_id'] == "https://terraref.ncsa.illinois.edu/clowder/api/users/%s" % clowder_id:
                                    logger.info("   skipping metadata (already attached)")
                                    alreadyAttached = True
                                    break
                        if not alreadyAttached:
                            md = {
                                "@context": ["https://clowder.ncsa.illinois.edu/contexts/metadata.jsonld",
                                             {"@vocab": clowder_context}],
                                "content": cleaned_dsmd,
                                "agent": {
                                    "@type": "cat:user",
                                    "user_id": "https://terraref.ncsa.illinois.edu/clowder/api/users/%s" % clowder_id
                                }
                            }

                            dsmd = sess.post(clowder_host+"/api/datasets/"+dsid+"/metadata.jsonld",
                                             headers={'Content-Type':'application/json'},
                                             data=json.dumps(md))

                            if dsmd.status_code in [500, 502, 504]:
                                logger.error("[%s] failed to attach metadata (%s: %s)" % (ds, dsmd.status_code, dsmd.text))
                                updatedTask['contents'][ds]['files'][datasetMDFile]['retry'] = "%s: %s" % (dsmd.status_code, dsmd.text)
                                response = "RETRY"
                            elif dsmd.status_code != 200:
                                logger.error("[%s] failed to attach metadata (%s: %s)" % (ds, dsmd.status_code, dsmd.text))
                                updatedTask['contents'][ds]['files'][datasetMDFile]['error'] = "%s: %s" % (dsmd.status_code, dsmd.text)
                                response = "ERROR"
                            else:
                                if datasetMDFile:
                                    logger.info("   [%s] added metadata from .json file" % ds, extra={
                                        "dataset_name": ds,
                                        "dataset_id": dsid,
                                        "action": "METADATA ADDED",
                                        "metadata": datasetMD
                                    })
                                    updatedTask['contents'][ds]['files'][datasetMDFile]['metadata_loaded'] = True
                                    updatedTask['contents'][ds]['files'][datasetMDFile]['clowder_id'] = "attached to dataset"
                                    writeTaskToDatabase(updatedTask)
                                else:
                                    # Remove metadata from activeTasks on success even if file upload fails in next step, so we don't repeat md
                                    logger.info("   [%s] added metadata" % ds, extra={
                                        "dataset_name": ds,
                                        "dataset_id": dsid,
                                        "action": "METADATA ADDED",
                                        "metadata": datasetMD
                                    })
                                    del updatedTask['contents'][ds]['md']
                                    writeTaskToDatabase(updatedTask)

                    if len(fileFormData)>0:
                        # Upload collected files for this dataset
                        # Boundary encoding from http://stackoverflow.com/questions/17982741/python-using-reuests-library-for-multipart-form-data
                        logger.info("   [%s] uploading unprocessed files" % ds, extra={
                            "dataset_id": dsid,
                            "dataset_name": ds,
                            "action": "UPLOADING FILES",
                            "filelist": fileFormData
                        })

                        (content, header) = encode_multipart_formdata(fileFormData)
                        fi = sess.post(clowder_host+"/api/uploadToDataset/"+dsid,
                                       headers={'Content-Type':header},
                                       data=content)

                        if fi.status_code in [104, 500, 502, 504]:
                            logger.error("[%s] failed to attach files (%s: %s)" % (ds, fi.status_code, fi.text))
                            updatedTask['contents'][ds]['files'][datasetMDFile]['retry'] = "%s: %s" % (fi.status_code, fi.text)
                            response = "RETRY"
                        elif fi.status_code != 200:
                            logger.error("[%s] failed to attach files (%s: %s)" % (ds, fi.status_code, fi.text))
                            updatedTask['contents'][ds]['files'][datasetMDFile]['error'] = "%s: %s" % (fi.status_code, fi.text)
                            response = "ERROR"
                        else:
                            loaded = fi.json()
                            if 'ids' in loaded:
                                for fobj in loaded['ids']:
                                    logger.info("   [%s] added file %s" % (ds, fobj['name']))
                                    for fkey in updatedTask['contents'][ds]['files']:
                                        if updatedTask['contents'][ds]['files'][fkey]['name'] == fobj['name']:
                                            updatedTask['contents'][ds]['files'][fkey]['clowder_id'] = fobj['id']
                                            # remove any previous retry/error messages
                                            if 'retry' in updatedTask['contents'][ds]['files'][fkey]:
                                                del(updatedTask['contents'][ds]['files'][fkey]['retry'])
                                            if 'error' in updatedTask['contents'][ds]['files'][fkey]:
                                                del(updatedTask['contents'][ds]['files'][fkey]['error'])
                                            break
                                    writeTaskToDatabase(updatedTask)
                            else:
                                logger.info("   [%s] added file %s" % (ds, lastFile))
                                updatedTask['contents'][ds]['files'][lastFileKey]['clowder_id'] = loaded['id']
                                # remove any previous retry/error messages
                                if 'retry' in updatedTask['contents'][ds]['files'][lastFileKey]:
                                    del(updatedTask['contents'][ds]['files'][lastFileKey]['retry'])
                                if 'error' in updatedTask['contents'][ds]['files'][lastFileKey]:
                                    del(updatedTask['contents'][ds]['files'][lastFileKey]['error'])
                                writeTaskToDatabase(updatedTask)

        return response
    else:
        logger.error("%s task: no credentials for Globus user %s" % (task['globus_id'], globUser))
        return "ERROR"

"""Work on completed Globus transfers to process them into Clowder"""
def clowderSubmissionLoop():
    clowderWait = config['clowder']['globus_processing_frequency'] - 1
    while True:
        time.sleep(1)
        clowderWait += 1

        # Check with Globus for any status updates on monitored tasks
        if clowderWait >= config['clowder']['globus_processing_frequency']:
            # First handle all regular tasks
            task = getNextUnprocessedTask()
            while task:
                globusID = task['globus_id']
                try: 
                    clowderDone = notifyClowderOfCompletedTask(task)
                    if clowderDone == "OK":
                        logger.info("%s task successfully processed!" % globusID, extra={
                            "globus_id": globusID,
                            "action": "PROCESSING COMPLETE"
                        })
                        task['status'] = 'PROCESSED'
                        writeTaskToDatabase(task)
                    else:
                        logger.error("%s not successfully processed; marking %s" % (globusID, clowderDone))
                        task['status'] = clowderDone
                        writeTaskToDatabase(task)
                except socket.error as e:
                    if e.errno != errno.ECONNRESET:
                        logger.error("Exception processing task %s; marking ERROR (%s)" % (globusID, str(e)))
                        task['status'] = 'ERROR'
                        writeTaskToDatabase(task)
                    else:
                        logger.error("Connection reset on %s; marking RETRY (%s)" % (globusID, str(e)))
                        task['status'] = 'RETRY'
                        writeTaskToDatabase(task)
                except requests.ConnectionError as e:
                    logger.error("Connection error on %s; marking RETRY (%s)" % (globusID, str(e)))
                    task['status'] = 'RETRY'
                    writeTaskToDatabase(task)
                except Exception as e:
                    logger.error("Exception processing task %s; marking ERROR (%s)" % (globusID, str(e)))
                    task['status'] = 'ERROR'
                    writeTaskToDatabase(task)
   
                task = getNextUnprocessedTask()
  
            # Next attempt to handle any ERROR tasks a second time
            task = getNextUnprocessedTask("RETRY", reverse=True)
            while task:
                globusID = task['globus_id']
                try: 
                    clowderDone = notifyClowderOfCompletedTask(task)
                    if clowderDone == "OK":
                        logger.info("%s task successfully processed!" % globusID, extra={
                            "globus_id": globusID,
                            "action": "PROCESSING COMPLETE"
                        })
                        task['status'] = 'PROCESSED'
                        writeTaskToDatabase(task)
                    else:
                        logger.error("%s not successfully processed; marking %s" % (globusID, clowderDone))
                        task['status'] = clowderDone
                        writeTaskToDatabase(task)
                except socket.error as e:
                    if e.errno != errno.ECONNRESET:
                        logger.error("Exception processing task %s; marking ERROR (%s)" % (globusID, str(e)))
                        task['status'] = 'ERROR'
                        writeTaskToDatabase(task)
                    else:
                        logger.error("Connection reset on %s; marking RETRY (%s)" % (globusID, str(e)))
                        task['status'] = 'RETRY'
                        writeTaskToDatabase(task)
                except requests.ConnectionError as e:
                    logger.error("Connection error on %s; marking RETRY (%s)" % (globusID, str(e)))
                    task['status'] = 'RETRY'
                    writeTaskToDatabase(task)
                except Exception as e:
                    logger.error("Exception processing task %s; marking ERROR" % globusID, str(e))
                    task['status'] = 'ERROR'
                    writeTaskToDatabase(task)
 
                task = getNextUnprocessedTask("RETRY", reverse=True)

            clowderWait = 0

if __name__ == '__main__':

    signal.signal(signal.SIGINT, gracefulExit)
    signal.signal(signal.SIGTERM, gracefulExit)
    

    # Try to load custom config file, falling back to default values where not overridden
    config = loadJsonFile(os.path.join(rootPath, "config_default.json"))
    if os.path.exists(os.path.join(rootPath, "data/config_custom.json")):
        print("...loading configuration from config_custom.json")
        config = updateNestedDict(config, loadJsonFile(os.path.join(rootPath, "data/config_custom.json")))
    else:
        print("...no custom configuration file found. using default values")
                    
    # Initialize logger handlers
    with open(os.path.join(rootPath,"config_logging.json"), 'r') as f:
        log_config = json.load(f)
        logging.config.dictConfig(log_config)
    logger = logging.getLogger('gantry')

    psql_conn = connectToPostgres()

    logger.info("- initializing service")
    # Create thread for service to begin monitoring
    clowderSubmissionLoop()

