#! /usr/bin/python2
# coding:utf-8
#
# Copyright (c) 2017 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
#

"""
Test Target Broker Daemon

.. admonition:: FIXME

   How do we get the help screen auto-extracted?

Authentication
--------------

FIXME: write up in progress

"""

import commonl.requirements

# Verify that we can run TTBD
commonl.requirements.verify_python_version()
commonl.requirements.verify_flask_version()

# FIXME: diagnostics returning can be significant to load in memory; a
# generator must be put in place; we must log to a temporary file and
# then stream it see
# https://blog.al4.co.nz/2016/01/streaming-json-with-flask/

import base64
import codecs
import contextlib
import cPickle
import datetime
import errno
import fnmatch
import json
import signal
import threading
import ttblc
import ttbl.user_control
import os
import multiprocessing
import hashlib
import ssl
import werkzeug
import werkzeug.serving
import flask
# Change from flask 0.10 -> 0.11
try:
    import flask_login
    import flask_principal
except ImportError as e:
    import flask.ext.login as flask_login
    import flask.ext.principal as flask_principal
import StringIO
import ttbl
import traceback
import argparse
import re
import pprint
import tempfile
import time
import commonl
import ttbl.config
import requests
import shutil
import sys
reload(sys)
# change the default string encoding to ASCII to remove the mess of encodings
# this is currently a controversial hack and with Py3 it will dissapear
sys.setdefaultencoding('utf-8')
import urlparse
import glob

#import prctl

try:
    import sdnotify
    sd_notify = sdnotify.SystemdNotifier()
    systemd_available = True
except ImportError as e:
    class fake_SystemdNotifier(object):
        def notify(self, state):
            pass
    sd_notify = fake_SystemdNotifier()
    systemd_available = False

# I bet there is a better way to do this...but we need the symbol to
# be in the logging module so that it is not included in the "function
# that called this" by the logging's internals.
# For debugging, levels are D2: 9, D3: 8, D4:7 ...
import logging
# FIXME: trick -- change function.f_code to match logging.root.critical.f_code
# FIXME: log* functions not working right, no output printed. tf?
setattr(logging, "logc", logging.root.critical)
setattr(logging, "logx", logging.root.exception)
setattr(logging, "loge", logging.root.error)
setattr(logging, "logw", logging.root.warning)
setattr(logging, "logi", logging.root.info)
setattr(logging, "logd", logging.root.debug)
setattr(logging, "logdl", logging.root.log)
from logging import logc, loge, logx, logw, logi, logd, logdl

app = flask.Flask(__name__)
API_VERSION = 1
API_PATH = "/ttb-v"
API_PREFIX = API_PATH + str(API_VERSION) + "/"

codec_utf8 = codecs.lookup("utf8")

def flask_loge_abort(http_code, *args, **kwargs):
    loge(*args, **kwargs)
    flask.abort(http_code, *args)

def flask_logw_abort(http_code, *args, **kwargs):
    logw(*args, **kwargs)
    flask.abort(http_code, *args)

def flask_logi_abort(http_code, *args, **kwargs):
    logi(*args, **kwargs)
    flask.abort(http_code, *args)

def flask_logd_abort(http_code, *args, **kwargs):
    logd(*args, **kwargs)
    flask.abort(http_code, *args)

def flask_log_abort(l, http_code, *args, **kwargs):
    logdl(l, *args, **kwargs)
    flask.abort(http_code, *args)


#
# FIXME: Can't figure out a way how to just handle all with this one,
# so I need to link each
#
@app.errorhandler(400)
@app.errorhandler(404)
#@app.errorhandler(423)
def _error_handler(error):
    r = flask.make_response(flask.jsonify({ 'message': error.description }))
    r.status_code = error.code
    return r

@contextlib.contextmanager
def log_to_str_too(logger, io):
    """
    Context manager to add a handler to a logger and then remove it upon exit

    This is used to add a temporary string handler that makes a copy
    of the logs generated by an operation in a target and then send it
    as diagnostics with the HTML response to the client.
    """

    # logger.logger? Yup, we are passed a logadapter, we want to  the
    # actual logger object to set handlers
    lh = logging.StreamHandler(io)
    try:
        logger.logger.addHandler(lh)
        yield
    finally:
        logger.logger.removeHandler(lh)

def who_make(ticket = None, user = None):
    # current user is going to be a ttbl.user_control.User object; the
    # convention across this daemon (see
    # ttbl.user_control.who_is_admin) is that we name owners of
    # resources USERNAME[:TICKET].
    #
    # see ttbl.who_split()
    if user == None:
        user = flask_login.current_user
    else:
        assert isinstance(user, ttbl.user_control.User)
    return ttbl.who_create(user.get_id(), ticket)

def current_user_is_admin():
    """
    Evaluate if the current user, is an admin
    """
    user = flask_login.current_user
    if user:
        return user.is_admin()
    return False

@app.route(API_PREFIX + 'login', methods = ['PUT'])
def login():
    """
    Given a User ID token and a password, go over all the
    authentication mechanisms that we have defined and see who can
    authenticate it.
    """
    while True:
        username = flask.request.form.get('username', None)
        if username:
            break
        # backwards compat
        username = flask.request.form.get('email', None)
        if username:
            break
        return flask.abort(404, "missing user ID 'username' field")

    password = flask.request.form.get('password', None)

    user = flask_login.current_user
    user_id = user.get_id()

    if user == ttbl.who_daemon():
        flask.abort(404, "user %s: not allowed (reserved)" % username)

    logd("user %s: trying to authenticate with token '%s'", user_id, username)

    if isinstance(user, ttbl.user_control.User) \
       and user.get_id() == username \
       and user.is_authenticated():
        logi("user %s: authenticated, has a valid session" % user_id)
        return True

    # Go over all the configured authentication mechanisms and try to
    # obtain roles that this combination of username/password allows us
    # Each authenticator might plug us to LDAP, or to a local
    # database...who knows
    token_roles = set()
    for authenticator in ttbl.config._authenticators:
        try:
            logd("user %s: authenticating with %s", username, authenticator)
            _token_roles = authenticator.login(
                username, password,
                remote_address = flask.request.remote_addr)
            logi("user %s: authenticated with %s: roles: %s",
                 username, authenticator, " ".join(_token_roles))
            token_roles.update(_token_roles)
            break
        except ttbl.authenticator_c.invalid_credentials_e as e:
            logi("user %s: invalid credentials from %s: %s",
                 username, authenticator, e)
            pass
        except ttbl.authenticator_c.unknown_user_e as e:
            logi("user %s: unknown user to '%s'", username, authenticator)
            pass
        except Exception as e:
            logi("user %s: auth error with %s: %s", username, authenticator, e)
            logx(e)
            pass
            # Try next authenticator

    if not 'user' in token_roles:
        message = "user %s: not allowed, no 'user' role (roles: %s)" % (
            username,
            " ".join(token_roles) if token_roles
            else "<empty, meaning no permissions>")
        logging.info(message)
        flask.abort(404, message)

    # Now let's go ahead and make sure there is a user instance that
    # Flask understands
    if not user or not isinstance(user, ttbl.user_control.User):
        user = ttbl.user_control.User(username)
        flask_login.login_user(user, remember = True)
        flask_principal.identity_changed.send(
            flask.current_app._get_current_object(),
            identity = flask_principal.Identity(user.get_id()))
    for role in token_roles:
        user.role_add(role)
    logi("user %s: authenticated with roles: %s",
         username, " ".join(token_roles))
    return flask.jsonify({
        'login': "user %s: authenticated with roles: %s" \
        % (username, " ".join(token_roles))
    })

@app.route(API_PREFIX + 'logout', methods = [ 'PUT' ])
@flask_login.login_required
def logout():
    flask_login.current_user.wipe()
    flask_login.logout_user()
    for key in ('identity.name', 'identity.auth_type'):
        flask.session.pop(key, None)
        flask_principal.identity_changed.send(
            flask.current_app._get_current_object(),
            identity = flask_principal.AnonymousIdentity())
    return flask.jsonify({'ok': "Session closed"})


@app.route(API_PREFIX + "validate_session", methods = ['GET'])
@flask_login.login_required
def validate_session():
    return flask.jsonify({'status': "You have a valid session"})


def _flask_arg_projections():
    projections_json = flask.request.form.get('projection', None)
    if projections_json:
        projections = json.loads(projections_json)
        if not isinstance(projections, list):
            raise ValueError("parameter projections: expected JSON-encoded "
                             "list of strings naming fields")
        return projections
    else:
        return []

@app.route(API_PREFIX + 'users/', methods = [ 'GET' ])
@app.route(API_PREFIX + 'users/<userid>', methods = [ 'GET' ])
@flask_login.login_required
def _users(userid = None):
    r = {}
    calling_user = flask_login.current_user._get_current_object()
    # FIXME: move to ttbl.check_user_is_admin(calling_user), no ifs,
    # it raises exceptions on its own?
    if calling_user.is_admin():
        if userid == None:
            userl = ttbl.user_control.known_user_list()
        else:
            userl = [ userid ]
    else:
        if userid == None:
            userl = [ calling_user.userid ]
        elif userid != calling_user.userid:
            # admins can list'em all FIXME: check user-elevate
            r[userid]  = {
                "error": "user '%s' needs admin role to query"
                         " users other than themselves" %
                         calling_user.userid
            }
            userl = []
        else:
            userl = [ calling_user.userid ]

    for userid in userl:
        try:
            user = ttbl.user_control.User(userid, fail_if_new = True)
            r[user.userid] = user.to_dict()
        except ttbl.user_control.User.user_not_existant_e as e:
            logging.warning("user '%s' does not exist: %s", userid, e)
    return flask.jsonify(r)

@app.route(API_PREFIX + 'users/', methods = [ 'DELETE' ])
@app.route(API_PREFIX + 'users/<userid>', methods = [ 'DELETE' ])
@flask_login.login_required
def _user_delete(userid = None):
    if userid == None:	# remove current user
        user = flask_login.current_user._get_current_object()
    elif userid == flask_login.current_user.get_id():
        # if specified and it is the current user, proceed
        user = flask_login.current_user._get_current_object()
    elif flask_login.current_user.is_admin():
        # if it is admin, can remove any existing user ID
        try:
            user = ttbl.user_control.User(userid, fail_if_new = True)
        except ttbl.user_control.User.user_not_existant_e as e:
            flask_logi_abort(
                403, "user '%s' does not exist: %s" % (userid, e))
    else:
        flask_logi_abort(
            403, "not enough privilege to remove user %s" % userid)
    user.wipe()
    return flask.jsonify({'result': "session closed"})

@app.route(
    API_PREFIX + 'users/<userid>/<any("drop","gain"):action>/<string:role>',
    methods = [ 'PUT' ])
@flask_login.login_required
def _user_role(userid, action, role):
    if userid == "self":
        # if specified and it is the current user, proceed
        user = flask_login.current_user._get_current_object()
    elif userid == flask_login.current_user.get_id():
        # if specified and it is the current user, proceed
        user = flask_login.current_user._get_current_object()
    elif flask_login.current_user.is_admin():
        # if it is admin, can toggle roles for any existing user ID
        try:
            user = ttbl.user_control.User(userid, fail_if_new = True)
        except ttbl.user_control.User.user_not_existant_e as e:
            flask_logi_abort(
                403, "user '%s' does not exist: %s" % (userid, e))
    else:
        flask_logi_abort(
            403, "not enough privilege to remove user %s" % userid)
    if action == "drop":
        if not user.role_present(role):
            flask_logi_abort(
                403, "user '%s' has no access to role %s" % (userid, role))
        user.role_drop(role)
        r = { "result": "user '%s' dropped role '%s'" % (userid, role) }
    elif action == "gain":
        if not user.role_present(role):
            flask_logi_abort(
                403, "user '%s' has no access to role %s" % (userid, role))
        user.role_gain(role)
        r = { "result": "user '%s' gained role '%s'" % (userid, role) }
    else:
        r = { "result": "user '%s' role '%s' invalid action %s"
                        % (userid, role, action) }
    return flask.jsonify(r)

@app.route(API_PREFIX + 'targets/', methods = ['GET'])
@flask_login.login_required
def _targets_gets():
    projections = _flask_arg_projections()
    _list = []
    calling_user = flask_login.current_user._get_current_object()
    for target_name in ttbl.config.targets:
        target = ttbl.test_target.get_for_user(target_name, calling_user)
        if target:
            _list.append(target.to_dict(projections))
    return flask.jsonify({'targets': _list})

@app.route(API_PREFIX + 'targets/<string:target_id>', methods = ['GET'])
@flask_login.login_required
def _target_get(target_id):
    projections = _flask_arg_projections()
    calling_user = flask_login.current_user._get_current_object()
    target = ttbl.test_target.get_for_user(target_id, calling_user)
    if target == None:
        flask.abort(404, "%s: unknown target" % target_id)
    return flask.jsonify(target.to_dict(projections))

_admin_required = [
    'disabled',
    'instrumentation',
    'instrumentation.*',
    'interfaces',
    'interconnects',
    'interconnects.*',
    'pos_capable',
    'pos_capable.*',
    'pos_partsizes',
    'pos_boot_*',
    'fixture_*',
    'fixture_*',
    'support',
    'support.*',
    'versions',
    'versions.*',
]

def _target_property_set_policy_check(target, props, user_is_admin):
    for prop in props:
        for check_prop in _admin_required:
            if fnmatch.fnmatchcase(prop, check_prop) and not user_is_admin:
                flask_logi_abort(
                    403, "not enough privilege to change property %s" % prop)

def _target_property_set(target, prop, value, user_is_admin, ticket):
    if value != None \
       and not isinstance(value, ( basestring, int, float, bool)):
        flask_logi_abort(
            400, "property %s: invalid data type %s;"
            " only null/string/number/boolean allowed"
            % (prop, type(value).__name__))
    if user_is_admin:
        target.property_set(prop, value)
    else:
        # A non-admin must own the target before setting anything
        target.property_set_locked(who_make(ticket), prop, value)


@app.route(API_PREFIX + 'targets/<string:target_id>',
           methods = [ 'PATCH', 'PUT' ])
@flask_login.login_required
def _target_patch(target_id):
    calling_user = flask_login.current_user._get_current_object()
    target = ttbl.test_target.get_for_user(target_id, calling_user)
    if target == None:
        flask.abort(404, "%s: unknown target" % target_id)
    ticket = flask.request.form.get('ticket', "")
    user_is_admin = current_user_is_admin()
    try:
        # check policy setting first
        _target_property_set_policy_check(
            target, flask.request.form.keys(), user_is_admin)
        data = flask.request.get_json()
        if data:
            if 'ticket' in data:
                if not ticket:
                    ticket = data.pop('ticket', "")
                else:
                    data.pop('ticket')
            data_flat = commonl.dict_to_flat(data)
            _target_property_set_policy_check(
                target, [ x[0] for x in data_flat ],
                user_is_admin)

        # Set parameter via form parameters
        for prop in flask.request.form.keys():
            # we pop 'ticket' because we don't want to allow it as a
            # parameter, since we use it as a descriptor until we move away
            # from it
            if prop == 'ticket':
                continue
            _target_property_set(target, prop, flask.request.form[prop],
                                 user_is_admin, ticket)

        if data:			# Set parameter via JSON data body
            for prop, val in data_flat:	# convert to our interal flat key space
                _target_property_set(target, prop, val, user_is_admin, ticket)
    except ttbl.test_target_e as e:
        flask_logi_abort(400, "%s" % e, exc_info = True)
    except Exception as e:
        flask_logi_abort(400, "%s: %s" % (target_id, e), exc_info = True)
    return flask.jsonify({ })


@app.route(API_PREFIX + 'targets/<string:target_id>/acquire', methods = ['PUT'])
@flask_login.login_required
def _target_acquire(target_id):
    # a valid user can try to acquire multiple times (it's a PUT request after all) and it
    # would succeed if already acquired.
    # FIXME: some security checks on this ticket?
    ticket = flask.request.form.get('ticket', "")
    if flask.request.form.get('force', 'false').lower() == 'true':
        force = True
        if not current_user_is_admin():
            flask_logd_abort(400, "forced acquisition requires admin rights")
    else:
        force = False

    calling_user = flask_login.current_user._get_current_object()
    target = ttbl.test_target.get_for_user(target_id, calling_user)
    if target == None:
        flask.abort(404, "%s: unknown target" % target_id)
    try:
        target.acquire(who_make(ticket), force)
        target.timestamp()
    except ttbl.test_target_busy_e as e:
        flask_log_abort(3, 400, "%s" % e, exc_info = False)
    except ttbl.test_target_e as e:
        # Not considered as errors, no traces printed -- as these are
        # normal during operation.
        flask_logd_abort(400, "%s" % e, exc_info = False)
    except Exception as e:
        flask_logi_abort(400, "%s: %s" % (target_id, e), exc_info = True)
    return flask.jsonify({})


@app.route(API_PREFIX + 'targets/<string:target_id>/active', methods = ['PUT'])
@flask_login.login_required
def _target_active(target_id):
    """
    If this target is owned by the caller, mark it as active
    """
    calling_user = flask_login.current_user._get_current_object()
    target = ttbl.test_target.get_for_user(target_id, calling_user)
    if target == None:
        flask.abort(404, "%s: unknown target" % target_id)
    ticket = flask.request.form.get('ticket', "")
    try:
        if who_make(ticket) == target.owner_get():
            # when this is the owner, this marks the target as active
            target.timestamp()
    except ttbl.test_target_e as e:
        flask_logi_abort(400, "%s" % e, exc_info = True)
    except Exception as e:
        flask_logi_abort(400, "%s: %s" % (target_id, e), exc_info = True)
    return flask.jsonify({})


@app.route(API_PREFIX + 'targets/<string:target_id>/release', methods = ['PUT'])
@flask_login.login_required
def _target_release(target_id):
    calling_user = flask_login.current_user._get_current_object()
    target = ttbl.test_target.get_for_user(target_id, calling_user)
    if target == None:
        flask.abort(404, "%s: unknown target" % target_id)
    if flask.request.form.get('force', 'false').lower() == 'true':
        force = True
    else:
        force = False

    # FIXME: some security checks on this ticket?
    ticket = flask.request.form.get('ticket', "")
    try:
        who = who_make(ticket)
        if force and not current_user_is_admin():
            raise ttbl.test_target_not_admin_e(target)
        target.release(who, force)
    except ttbl.test_target_e as e:
        flask_logi_abort(400, "%s" % e, exc_info = True)
    except Exception as e:
        flask_logi_abort(400, "%s: %s" % (target_id, e), exc_info = True)
    return flask.jsonify({})

def tb_file_local_path_get(username, filepath):
    """
    Verify the *image_name* is available in the target broker's
    storage for the current *username* and return it's local file
    path.
    """
    local_filepath = os.path.normpath("%s/%s/%s" % (
        app.config['files_path'], username, filepath))
    if not os.path.isfile(local_filepath):
        return None
    else:
        return local_filepath

@app.route(API_PREFIX + 'targets/<string:target_id>/' \
           + '<string:interface>/<string:call>',
           methods = [ 'PUT', 'POST', 'DELETE', 'GET' ])
@flask_login.login_required
def _target_interface(target_id, interface, call):
    calling_user = flask_login.current_user._get_current_object()
    target = ttbl.test_target.get_for_user(target_id, calling_user)
    if target == None:
        flask.abort(404, "%s: unknown target" % target_id)
    ticket = flask.request.form.get('ticket', "")
    iostr = StringIO.StringIO()
    iostr_wrapper = codecs.StreamReaderWriter(
        iostr, codec_utf8.streamreader,
        codec_utf8.streamwriter, errors = 'replace')
    try:
        if not interface in target.tags['interfaces']:
            flask_logi_abort(400, "%s: unavailable interface" % interface)
        iface = getattr(target, interface, None)
        if iface == None:
            flask_logi_abort(400, "%s: interface broken" % interface)
        assert isinstance(iface, ttbl.tt_interface)
        username = flask_login.current_user.get_id()
        user_path = os.path.join(app.config['files_path'], username)
        with log_to_str_too(target.log, iostr_wrapper):
            method_name = flask.request.method.lower() + "_" + call
            method = getattr(iface, method_name, None)
            if method:
                result = method(
                    target, who_make(ticket),
                    # https://flask.palletsprojects.com/en/1.1.x/patterns/fileuploads/
                    flask.request.form, flask.request.files,
                    user_path)
                iface.assert_return_type(
                    result, dict, target,
                    flask.request.form.get('component', None),
                    method_name, none_ok = False)
            else:
                result = iface.request_process(
                    target, who_make(ticket),
                    flask.request.method,
                    call,
                    # https://flask.palletsprojects.com/en/1.1.x/patterns/fileuploads/
                    flask.request.form, flask.request.files,
                    user_path)
        assert isinstance(result, dict), \
            "BUG: %s: request_process() did not return a dictionary" \
            " but a %s" \
            % (interface, type(result).__name__)
        if 'diagnostics' in result:
            target.log.error("BUG: %s: request_process() added a "
                             "'diagnostics' field that will be overriden"
                             % interface)
        result['diagnostics'] = iostr.getvalue()
    except ttbl.test_target_e as e:
        flask_logi_abort(400, "%s" % e, exc_info = True)
    except Exception as e:
        flask_logi_abort(400, "%s: %s" % (target_id, e), exc_info = True)
    if 'stream_file' in result:
        filepath = result['stream_file']
        generation = result.get('stream_generation', 0)
        offset = result.get('stream_offset', 0)
        try:
            fd = open(filepath, 'rb')	# flask.Response closes it
            s = os.fstat(fd.fileno())
            if offset >= s.st_size:	# cap offset to max size so when...
                offset = s.st_size	# ...we report it it goes right
            if offset > 0:
                fd.seek(offset)
            response = flask.Response(fd, direct_passthrough = True)
            # attach a header indicating the offset; this allows the
            # client to calculate how big the stream is at precisely
            # the time the last byte was sent, by adding the
            # Content-Length. This is used for example, by the console
            # code, to know the offset at which to read the next time.
            # stream-size = X-stream-offset + Content-length
            response.headers['X-stream-gen-offset'] = \
                str(generation) + " " + str(offset)
            return response
        except Exception as e:
            flask_logi_abort(400, "%s: can't stream file: %s" % (filepath, e),
                             exc_info = True)
    else:
        return flask.jsonify(result)


@contextlib.contextmanager
def target_acquired(target, who):
    try:
        target.acquire(who, False)
        yield
    except:
        # do not force releasing it if we don't own it
        target.release(who, False)
        raise
    target.release(who, False)

def _do_cleanup_released(target, who):
    if not hasattr(target, "power"):	# Only with power control
        return
    idle_poweroff = target.tags.get('idle_poweroff',
                                    ttbl.config.target_max_idle)
    if idle_poweroff == 0:
        target.log.debug("skiping powering off, idle_poweroff is 0")
        return
    # yeah, we do without acquiring; technically it is legal since it
    # does not modify state...almost (except for 'powered' property)
    any_powered_on = target.power._get_any(target)
    if any_powered_on == False:		# Is it off? pass
        return
    with target_acquired(target, who):	# Acquire, check idleness
        # calculate the idle time once we own the target, to be sure
        # none came in the middle and we are killing it for them...
        idle_time = time.time() - target.timestamp_get()
        if idle_time > idle_poweroff:
            target.log.info("powering off, idle for %.2fs (max %s)",
                            idle_time, idle_poweroff)
            target.power.put_off(target, who, {}, {}, None)

def _do_cleanup_acquired(target, owner, who):
    idle_time = time.time() - target.timestamp_get()
    if idle_time > ttbl.config.target_owned_max_idle:
        target.log.warning("releasing from %s, idle for %.2fs (max %s)",
                           owner, idle_time, ttbl.config.target_owned_max_idle)
        target.release(who, True)

def cleanup_files():
    for f in glob.iglob(app.config['files_path'] + "/*/*"):
       if (time.time() - os.stat(f).st_mtime ) > ttbl.config.cleanup_files_maxage:
               os.remove(f)

# Notify systemd we are alive
#
# Note we do this from two places: the beginning of the cleanup
# process in _cleanup_process_fn() and then inside _do_cleanup() for
# each thread.
#
# Why? because querying each target might be lengthy as we have to
# access the hardware and it might take time. So on each iteration, we
# do a keepalive if we have to.

_systemd_watchdog_period = None

def _systemd_keepalive(t):
    global _systemd_watchdog_period
    if _systemd_watchdog_period == None:
        return
    # We keepalive whenever we can
    sd_notify.notify("WATCHDOG=1")

def _do_cleanup():
    """
    """
    daemon_user = ttbl.user_control.User('local', roles = [ "admin" ])
    target_list = ttbl.config.targets.keys()
    who = who_make("cleanup", user = daemon_user)
    for target_id in target_list:
        with ttbl.config.targets_lock:
            target = ttbl.config.targets[target_id]
        try:
            now = time.time()
            if target.tags.get('skip_cleanup', False) == True:
                # Some targets are used for manual power control of
                # infrastructure
                continue
            _systemd_keepalive(now)	# keepalive, this loop takes time
            owner = target.owner_get()
            if owner == None:
                _do_cleanup_released(target, who)
            else:
                _do_cleanup_acquired(target, owner, who)
        except ttbl.test_target_e as e:
            loge("%s" % e)
        except Exception as e:
            loge("%s: %s" % (target_id, e))
            traceback.print_exc()

# Another kludge: disable security warnings from request's urllib3
# vendored version. We know, we know. And probably there is a way to
# extract the ad-hoc SSL certificate info from within the Flask app.
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

# This is one of the dirtiest kludges ever done
def cleanup_process_fn():
    global _systemd_watchdog_period
    # Cleanup tasks we can run in a separate process
    #
    # We also run a systemd watchdog (if systemd available and
    # configured) so systemd will restart the service if the cleanup
    # thread dies--happens sometimes, we don't know why
    # http://0pointer.de/blog/projects/watchdog.html
    if systemd_available and 'WATCHDOG_USEC' in os.environ:
        _systemd_watchdog_period =  float(os.environ['WATCHDOG_USEC']) / 1000000
        sleep_period = min(_systemd_watchdog_period / 2,
                           ttbl.config.target_max_idle / 2)
    else:
        _systemd_watchdog_period =  None
        sleep_period = ttbl.config.target_max_idle / 2
    logi("Clean up process [period %.2fs]" % sleep_period)
    cleanup_files_last = time.time()
    cleanup_targets_last = time.time()
    while True:
        time.sleep(sleep_period)
        t = time.time()
        logdl(8, "Scanning for idle targets")
        try:
            _systemd_keepalive(t)	# keepalive
            if t - cleanup_targets_last > ttbl.config.target_max_idle:
                _do_cleanup()
                cleanup_targets_last = t
            if t - cleanup_files_last > ttbl.config.cleanup_files_period:
                cleanup_files()
                cleanup_files_last = t
        except Exception as e:
            loge("Exception in clean thread: %s\n" % e)

main_pid = os.getpid()

# Define the login manager
login_manager = flask_login.LoginManager()

#! List of addresses to be considered local when they show in the
# remote_addr field of a request.
_local_addresses = set()
# Support for local users without having to go through local
# login--this is intended for running on single user workstations
# Start the daemon with --local-auth and just work as anonymous with
# full privilege.
#
# See https://flask-login.readthedocs.org/en/latest/#anonymous-users for
# the Flask details, but basically when flask needs to identify an
# Anonymous user, if the connection comes from one of the local ones,
# we just create an special anonymous user that has full privilege.
def _create_anonymous_user():
    local_auth_disabled_runtime = os.path.exists(
        os.path.join(ttbl.config.lib_path, "local_auth_disabled"))
    if not local_auth_disabled_runtime \
       and flask.request.remote_addr in _local_addresses:
        return ttbl.user_control.User('local', roles = [ "admin" ])
    return flask_login.AnonymousUserMixin()

login_manager.anonymous_user = _create_anonymous_user

#    Connect *login_manager*'s operations to the implementations of the
#    user database defined here in :class:`User`.
@login_manager.user_loader
def load_user(userid):
    """
    Called by the login manager when looking for information about
    a given user ID
    """
    return ttbl.user_control.User.search_user(userid)

@login_manager.unauthorized_handler
def unauthorized():
    """
    Called by the login manager when a user is no authorized to
    do an operation
    """
    flask_logi_abort(400, "User unauthorized; please login")

def auth_init(app):
    principals = flask_principal.Principal()
    principals.init_app(app)
    # The login manager provides session management for the Flask web
    # framework, handling login, logout and remembering user sessions
    # over extended periods of time (from
    # https://flask-login.readthedocs.org/en/latest/).
    global login_manager
    login_manager.init_app(app)
    # None means that the cookie will not contain information about
    # the IP address where the login came from. We want to be able to
    # carry our login information while the machine changes IP
    # address.
    # https://flask-login.readthedocs.io/en/latest/#session-protection
    login_manager.session_protection = None

    # Define this function here to connect FIXME how? This is kinda
    # confusing
    @flask_principal.identity_loaded.connect_via(app)
    def on_identity_loaded(sender, identity):
        """
        An identity has been loaded, add the information about the
        current user to the *identity* object
        """
        identity.user = flask_login.current_user

        # FIXME: we can do without the hasattr()? -- let's force
        # current_user to be at least a descentant of a certain base
        # class which enforces having those arguments
        if hasattr(flask_login.current_user, 'get_id'):
            identity.provides.add(flask_principal.UserNeed(
                flask_login.current_user.get_id()))

        if hasattr(flask_login.current_user, 'roles'):
            for role in flask_login.current_user.roles:
                identity.provides.add(flask_principal.RoleNeed(role))

local_auth = []

#
# Main
#
# Protect like this so this file can be easilly imported by the
# documentation generator with no side effects.
#
if __name__ == "__main__":

    # Drop the capabilities we don't really need but that we enabled
    # in the Ambient set so our helpers can inherit them. I might be
    # doing something wrong here, but I can't figure out how to
    # specify in systemd an inheritable set different than the ambient
    # set

    # Actually can't drop this -- I am definitely doing something
    # wrong and not sure what. If we drop this, then we can't read
    # /proc/PID/exe, which we need in commonl.process_alive() to
    # determine if the daemons have started properly -- instead it
    # will fail where 'Usually this means it has died while we
    # checked', which it shouldn't -- because we are running with the
    # same user/gid. Puzzled.
    #prctl.cap_effective.dac_read_search = False

    # Defaults
    # We want them here because we want these defaults to be overriden
    # by configuration to be overriden by command line options.
    host = "127.0.0.1"
    port = 5000

    arg_parser = argparse.ArgumentParser()
    commonl.cmdline_log_options(arg_parser)
    # Do it like this insead of adding a version to the main parser
    # because it will by default add adds -v as shortcut (when everyone and their grandma
    # knows -V is vor --version, -v for --verbose)
    arg_parser.add_argument(
        '-V', '--version',
        action = 'version', default = argparse.SUPPRESS,
        version = commonl.version_get(ttbl, "ttbd"),
        help= "show program's version number and exit")
    arg_parser.add_argument(
        "--config-path",
        action = "store", dest = "config_path",
        default = "~/.ttbd",
        help = "Path from where to load conf_*.py "
        "configuration files (in alphabetic order)")
    arg_parser.add_argument(
        "--config-file", "-c",
        action = "append", dest = "config_files", metavar = "CONFIG-FILE.py",
        # FIXME: s|/etc|system prefix from installation
        default = [ ],
        help = "Files to parse as configuration (this is used for testing, "
        "along with --config-path \"\"")
    arg_parser.add_argument(
        "--files-path", action = "store", default = "~/.ttbd/files",
        help = "directory where to store uploaded files [%(default)s]")
    arg_parser.add_argument(
        "--state-path", action = "store", default = "~/.ttbd/state",
        help = "Directory where to save state")
    arg_parser.add_argument(
        "--var-lib-path", "--state-persistent-path", dest = "var_lib_path",
        action = "store", default = "~/.ttbd/lib",
        help = "Directory where to save persistent state")
    arg_parser.add_argument(
        "--upload-max-size", action = "store", type = int,
        default = ttbl.config.upload_max_size,
        help = "Maximum (upload) file size (in bytes) [%(default)d]")
    arg_parser.add_argument(
        "--host", action = "store", default = None,
        help = "Hostname on which to listen [%(default)s]")
    arg_parser.add_argument(
        "--port", action = "store", type = int, default = None,
        help = "Port on which to listen [%d]" % port)
    arg_parser.add_argument(
        "--local-auth", metavar = "IP-ADDR", nargs = "?",
        action = "append", default = [],
        help = "Allow users as admins from the given IP "
        "addresses without caring; if no IP address is given, "
        "127.0.0.1 is assumed. Can be given multiple times with "
        "different addresses--no regexp or masks supported")
    arg_parser.add_argument(
        "--ssl-crt", default = None,
        help = "SSL certificate for HTTPS connections; if either "
        "(--ssl-key) not specified, it will use ad-hoc mode where "
        "it will be automatically generated")
    arg_parser.add_argument(
        "--ssl-key", default = None,
        help = "SSL key for HTTPS connections; see --ssl-crt")
    arg_parser.add_argument(
        "--no-ssl", dest = "ssl", action = "store_false", default = True,
        help = "Disable SSL support")
    arg_parser.add_argument(
        "--ssl-enabled-check-disregard", action = "store_true",
        default = False,
        help = "Do not check if SSL is enabled for certain things"
        "that shall be used only with it enabled, like LDAP")
    arg_parser.add_argument(
        "--target-max-idle", type = float, metavar = "SECONDS",
        action = "store", default = ttbl.config.target_max_idle,
        help = "What is the maximum amount of seconds a target can "
        "be idle before is auto-powered off (%(default).fs)")
    arg_parser.add_argument(
        "--system-wide", "-w", metavar = "NAME", action = "store",
        nargs = "?", type = str,
        const = "",	# If no arg, set to ""
        default = None,
        help = "Run in system wide mode, with an instance name (none by "
        "default, sets the config, state and file storage to "
        "{/etc/,/var/run,/var/cache}ttbd[-NAME]/")

    args = arg_parser.parse_args()

    if args.host != None:
        host = args.host
    if args.port != None:
        port = args.port

    log_format = "%(levelname)s %(module)s.%(funcName)s():%(lineno)d: %(message)s"
    log_format = commonl.log_format_compose(log_format, True)
    logging.basicConfig(format = log_format, level = args.level)

    # Cut down on verbosity from different packages
    logging.getLogger("requests").setLevel(logging.ERROR)
    logging.getLogger("flask").setLevel(logging.WARNING)
    logging.getLogger("werkzeug").setLevel(logging.WARNING)
    logging.getLogger("urllib3").setLevel(logging.ERROR)
    logging.getLogger("tornado.access").setLevel(logging.ERROR)

    ttbl.config.target_max_idle = args.target_max_idle

    if args.system_wide == None:
        ttbl.config.instance_suffix = ""
        ttbl.config.instance = ""
        # args.VAR stuff comes from settings on the argparser
        pass
    elif args.system_wide == "":
        ttbl.config.instance_suffix = ""
        ttbl.config.instance = ""
        args.config_path = "/etc/ttbd"
        args.files_path = "/var/cache/ttbd"
        args.state_path = "/var/run/ttbd"
        # State that stays through reboots
        args.var_lib_path = "/var/lib/ttbd"
    else:
        ttbl.config.instance = args.system_wide
        ttbl.config.instance_suffix = "-" + args.system_wide
        args.config_path = "/etc/ttbd" + ttbl.config.instance_suffix
        args.files_path = "/var/cache/ttbd" + ttbl.config.instance_suffix
        args.state_path = "/var/run/ttbd" + ttbl.config.instance_suffix
        # State that stays through reboots
        args.var_lib_path = os.path.join("/var/lib/ttbd", args.system_wide)
    args.var_lib_path = os.path.expanduser(args.var_lib_path)

    os.umask(0o007)

    if not os.path.exists(args.var_lib_path):
        try:
            os.makedirs(args.var_lib_path, 0o2770)
        except OSError as e:
            if e.errno != errno.EEXIST:
                loge("%s: cannot create path: %s" % (args.var_lib_path, e))
                raise
    else:
        commonl.check_dir_writeable(args.var_lib_path, "storing state")

    # Define things ttbl.config needs for configuring targets
    ttbl.test_target.state_path = os.path.expanduser(args.state_path)
    ttbl.config.state_path = os.path.expanduser(args.state_path)
    ttbl.config.lib_path = os.path.expanduser(args.var_lib_path)
    if not os.path.isdir(ttbl.test_target.state_path):
        os.makedirs(ttbl.test_target.state_path, 0o2770)
    else:
        commonl.check_dir_writeable(ttbl.test_target.state_path,
                                    "storing state")
    args.state_path = ttbl.test_target.state_path
    args.config_path = os.path.expanduser(args.config_path)

    # get the key for this instance; we need to do this before we read
    # the configuration, as we'll use ttbl._who_daemon
    key_filename = os.path.join(args.var_lib_path, "session.key")
    try:
        with open(key_filename) as f:
            sk = cPickle.load(f)
            if not isinstance(sk, str):
                raise Exception("Bad type in saved key")
            if len(sk) != 24:
                raise Exception("Bad length in saved key")
            logw("Reloaded key")
    except Exception as e:
        sk = os.urandom(24)
        logi("New key (because %s)" % e)
        umask_original = os.umask(0)
        commonl.rm_f(key_filename)
        try:
            fd = os.open(key_filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL,
                         0o600)
        finally:
            os.umask(umask_original)
        with os.fdopen(fd, 'w') as f:
            cPickle.dump(sk, f, protocol = 2)

    # this is the username we use for internal tasks who always gets a
    # pass on access checks if the target is unlocked
    # wee ttbl.with_target_locked_and_acquired()
    ttbl._who_daemon = "internal-" + commonl.mkid(sk, 6)

    # read the configuration
    commonl.check_dir(args.config_path, "storing configuration")
    # FIXME: this has to be done as the options are specified?
    try:
        if args.config_path != [ "" ]:
            commonl.config_import([ args.config_path ], re.compile("^conf[-_].*.py$"))
        for config_file in args.config_files:
            commonl.config_import_file(config_file, "__main__")
        logd("configuration: loaded")
    except Exception:
        # config_import* failures mean we cannot read the config files
        # properly; they have already printed their messages, so just
        # exit here.
        sys.exit(1)

    # Allow local users access as admins? [--local-auth]
    # Command line goes after configuration file
    local_auth += args.local_auth
    if local_auth:
        import ttbl.auth_party
        # _local_addresses is used by _create_anonymous_user()
        for i, remote in enumerate(local_auth):
            if remote == None:
                _local_addresses.add("127.0.0.1")
            elif remote == "":	# Override support
                del _local_address[:]
            else:
                _local_addresses.add(remote)
        # Allow any local login/password to work, so we won't worry
        # about passwords when calling 'login'
        ##ttbl.config._authenticators.insert(0, ttbl.auth_party.authenticator_party_c(
        ##    [ 'user', 'admin' ], local_addresses = _local_addresses))

    files_path = os.path.expanduser(args.files_path)
    ttbl.test_target.files_path = files_path
    commonl.check_dir_writeable(files_path, "storing uploaded files")

    # FIXME: needs a thread to clean up app.config['files_path']
    # enforcing LRU date and maximum size
    if ttbl.config.upload_max_size != args.upload_max_size:
        args.upload_max_size = ttbl.config.upload_max_size
    app.config['MAX_CONTENT_LENGTH'] = args.upload_max_size
    app.config['files_path'] = os.path.expanduser(args.files_path)
    app.config['state_path'] = ttbl.test_target.state_path
    ttbl.user_control.User.state_dir = args.var_lib_path
    app.config['REMEMBER_COOKIE_DURATION'] = datetime.timedelta(4)
    app.secret_key = base64.b64encode(sk)

    # SSL config: default to at least TLS v1.2
    ssl_options = ssl.OP_NO_SSLv3 | ssl.OP_NO_SSLv2 | ssl.OP_NO_TLSv1
    if args.ssl == False:
        logi("SSL: disabled")
        ssl_context = None
    elif args.ssl_crt and args.ssl_key:
        if os.path.exists(args.ssl_crt) and os.path.exists(args.ssl_key):
            logw("SSL: using disabling CRT %s, key %s"
                 % (args.ssl_crt, args.ssl_key))
        else:
            raise Exception("(%s,%s): cannot open SSL required files"
                            % (args.ssl_crt, args.ssl_key))
        ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
        ssl_context.options |= ssl_options
        ssl_context.load_cert_chain(args.ssl_crt, args.ssl_key)
    elif args.ssl == True:
        ttbl.config.ssl_enabled = args.ssl
        ttbl.config.ssl_enabled_check_disregard = \
                args.ssl_enabled_check_disregard
        logi("SSL: using adhoc context")
        ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
        ssl_context.options |= ssl_options
        old_umask = os.umask(0o077)	# Create files go=
        try:
            cert_fn, pkey_fn = werkzeug.serving.make_ssl_devcert(
                os.path.join(args.state_path, "cert"))
        finally:
            os.umask(old_umask)
        ssl_context.load_cert_chain(cert_fn, pkey_fn)
        os.chmod(cert_fn, 0o640)	# Make the cert g+w
    else:
        raise RuntimeError("can't give --no-ssl and --ssl* options")

    #
    # Make this process a reaper
    #
    # Any child, grandchild or other descendant will be held by this
    # process if its parent dies (rather than init), so it can waitpid
    # on it and stuff. This is very Linux specific [for now].
    #
    # int prctl(int option, unsigned long arg2, unsigned long arg3,
    #           unsigned long arg4, unsigned long arg5);
    #
    # option PR_SET_CHILD_SUBREAPER (36, from linux/prctl.h)
    # arg2   1
    # arg*   unused
    #
    # prctl(PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0);
    #
    import ctypes
    libc = ctypes.CDLL("libc.so.6")
    r = libc.prctl(ctypes.c_int(36), ctypes.c_ulong(1), ctypes.c_ulong(0),
                   ctypes.c_ulong(0), ctypes.c_ulong(0))


    # Handle death of children, so they don't become zombies
    #
    # Only children that are registered as daemons are reaped (so they
    # don't become zombies). If we just ignore SIGCHLD, then
    # subprocess.check_call() and friends fail to catch commands that
    # fail to execute and return a non-zero exit code.
    #
    # Note that this list of daemons is PER-PROCESS; so still haven't
    # really solved what happens if we reap a daemon from a Python web
    # framework process that exited.
    def sigchld_handler(signum, frame):
        pid = ttblc.waitid_poll()
        if pid == -1 or pid == 0:
            return
        if ttbl.daemon_pid_check(pid):
            ttbl.daemon_pid_rm(pid)
            # Reap a daemon so it doesn't zombie
            r = os.waitpid(pid, os.WNOHANG)
            return
        # Not a daemon, don't reap it, let other do it
        return

    signal.signal(signal.SIGCHLD, sigchld_handler)

    # Ensure we are session leaders, so we can kill the whole group
    try:
        os.setsid()
    except OSError as e:
        if e.errno != errno.EPERM:
            raise
        # It is already a session leader...can ignore

    # Cleanup process...now this is quite very ugly...see _cleanup()
    cleanup_process = multiprocessing.Process(target = cleanup_process_fn)
    cleanup_process.daemon = True
    cleanup_process.start()

    # See authentication in the file's doc header
    auth_init(app)

    # Have flask send HTTP keep alives by implementing 1.1
    werkzeug.serving.WSGIRequestHandler.protocol_version = "HTTP/1.1"

    if True:
        # Tornado is single threaded, so not good for what we need
        from tornado.wsgi import WSGIContainer
        from tornado.httpserver import HTTPServer
        from tornado.ioloop import IOLoop

        http_server = HTTPServer(WSGIContainer(app), ssl_options = ssl_context)
        http_server.bind(port, address = host)
        # This might not be the real good way to do this, but our APIs
        # might block for a long time, so we start a crapload of
        # servers to be able to service more requests in paralell.
        http_server.start(ttbl.config.processes)
        sd_notify.notify("READY=1")
        IOLoop.instance().start()
    else:
        app.run(host = host, port = port, debug = True, use_reloader = False,
                processes = 10, ssl_context = ssl_context)
