# -*- python -*-
# Qserv core/modules SConscript
#

"""
This SConscript file defines all targets produced from code in core/modules.

This file is loaded from top-level SConstruct and its main purpose is to
define all target being built and installed by code in modules directory.
It assigns those targets to three aliases:
  - "build" - target (e.g. shared libraries) being built in "build" directory
  - "install-notest" - targets installed in various places
  - "test" - all unit tests to run as a part of the build

The code in core/modules is split into a bunch of modules, modules do most
of the job definig targets for their own code, this includes:
  - making shared object (*.os) from C++ code
  - installing Python code and scripts
  - building test applications
  - defining unit tests

This file provides `standardModule()` method which does most of the above,
if a module has no special code then the module does not even need to create
its own `SConscript` file, we will automatically use `standardModule()` for
modules which do not have `SConscript` file. Some modules need small
customization which can be done via passing non-standard parameters to
`standardModule()`, in that case small `SConscript` file needs to be defined
in a module, e.g.:

    # Special SConscript which defines additional library
    # for linking with test applications
    Import('env')
    Import('standardModule')

    standardModule(env, test_libs='log4cxx')

See `standardModule()` for description of all optional parameters.

This file's job is to collect shared objects defined by individual modules
into shared libraries and make install targets for those libraries. Special
mapping below (`shlibs`) defines which modules contribute to which shared
libraries and also some additional options for linking and installing them.

Interface between module-level `SConscript` and this file is defined via the
special structure which contains all relevant info both at module-level and
library level (`build_data` dictionary below). This structure is stored in
the default environment so that any module's SConscript can access it without
importing.

Some modules can contribute their shared objects to more than one library
(e.g. `css` module has regular C++ code which goes to a library libqserv_css,
but it also builds Python wrapper which needs to be in a separate
shared library used as Python extension module). To support this feature
modules can define special "sub-modules" which are just special convention
for module names appearing in `build_data` dictionary. The same `css` module
defines targets for "css" module and "css:python" module and `shlibs`
mapping will put those into different shared libraries.
"""

from collections import defaultdict
import os

from SCons.Script import *
from SCons.Platform import platform_default

import detect
import genversion
import state

Import('env')


def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=None, bin_cc_files=None):
    """
    Implementation of SConscript logic for "standard" module.

    This method defines standard targets for:
      - shared objects produced from *.cc files (except test*.cc)
      - test applications built from test*.cc
      - unit test targets for tests applications
      - install targets for python/*.py and bin/*.py

    Behavior can be customized by providing non-standard values for parameters.

    @param exclude: pattern or list of patterns for *.cc files to exclude
                from shared object targets (see Glob's exclude). Typically
                this may be used for *.cc files used for Python wrappers which
                usually go into separate shared object.
    @param module: module name, used by this SConscript only, should not be
                used in module's SConscript
    @param test_libs: list (or space-separated string) of additional library
                names used for linking test applications
    @param unit_tests: if None then all test applications are used as unit
                tests, if only some applications are to be used for unit test
                then use this parameter to pass list (or space-separated
                string) of application names. Can be empty string or list to
                not run any unit tests.
    @param bin_cc_files: a dictionary defining which binary products (not
                the unit tests) should be built out of C/C++ files. Each key
                of the dictionary is the name of a C/C++ file (with function
                'main'). This file will be excluded from the unit tests and
                from a list of *.cc files which are complied into a shared
                library of the module. And the corresponding value of
                the dictionary is a list of libraries which are required to
                build the binary.
    """

    build_data = DefaultEnvironment()['build_data']

    path = '.' if module is None else module
    if module is None:
        module = os.path.basename(os.getcwd())
    state.log.debug('standardModule: module = %s path = %s' % (module, path))

    # using two optional lists provided by the nmodule to build names of
    # *.cc files to be excluded from being used to build module's
    # library or unit tests
    exclude_cc_files = []
    if exclude is not None:
        if isinstance(exclude, str): exclude_cc_files += [exclude,]
        else:                        exclude_cc_files += exclude
    if bin_cc_files is not None: exclude_cc_files += bin_cc_files.keys()

    # find all *.cc files and split then in two groups: files for libraries
    # and files for test apps
    cc_files = set(env.Glob(os.path.join(path, "*.cc"), source=True, strings=True, exclude=exclude_cc_files))
    cc_tests = set(fname for fname in cc_files if os.path.basename(fname).startswith("test"))
    # In Python3 order of set iteration is undefined, do manual sorting
    cc_objects = sorted(cc_files - cc_tests)
    cc_tests = sorted(cc_tests)
    state.log.debug('standardModule: exclude_cc_files = ' + str(exclude_cc_files))
    state.log.debug('standardModule: cc_objects = ' + str(cc_objects))
    state.log.debug('standardModule: cc_tests = ' + str(cc_tests))

    # make shared objects and add them to module's 'module_objects' list
    objects = Flatten([env.SharedObject(cc) for cc in cc_objects])
    build_data['module_objects'][module] = objects

    # Targets for test apps. Complication here is that we do not install those
    # apps and we run them from build/ and even before we install libraries
    # (e.g. as unit tests).
    # To support this we need to set RPATH to build/ dir where are libraries
    # are built also we include build/ in LIBBPATH to find libraries that are
    # being built.
    # Libraries included in the link are:
    #  - library that this module belongs to
    #  - all dependent libraries
    #  - any additional libs defined in test_libs
    mylib = build_data['module_lib'][module]
    libs = [mylib]
    libs += Split(build_data['shlibs'].get(mylib, {}).get('libs', []))
    libs += Split(test_libs or [])

    def Prog(cc_files):
        return env.Program(cc_files, LIBS=libs,
                           LIBPATH=['$build_dir'] + env["LIBPATH"],
                           RPATH='$build_dir')
    tests = Flatten([Prog(cc) for cc in cc_tests])
    build_data['tests'] += tests

    # by default all test apps are unit tests, if you want to filter out some
    # apps define unit_tests to be not None
    for test in tests:
        appName = os.path.basename(test.path)
        if unit_tests is None or appName in Split(unit_tests):
            utest = env.UnitTest(test)
            state.log.debug('standardModule: unit_test = ' + str(utest))
            build_data['unit_tests'] += utest

    # all python/*.py files are installed in lib/python/lsst/qserv/module
    py_modules = env.Glob(os.path.join(path, 'python', '*.py'), source=True, strings=True, exclude=exclude)
    state.log.debug('standardModule: py_modules = ' + str(py_modules))
    dst = os.path.join("$python_prefix/lsst/qserv", module)
    py_installed = env.Install(dst, py_modules)
    pyc_installed = env.PyCompile(py_installed)
    build_data['install'] += py_installed + pyc_installed

    # all bin/*.py files are installed in bin/
    py_scripts = env.Glob(os.path.join(path, 'bin', '*.py'), source=True, strings=True, exclude=exclude)
    state.log.debug('standardModule: py_scripts = ' + str(py_scripts))
    py_installed = env.Install("$prefix/bin", py_scripts)
    build_data['install'] += py_installed

    # inject binary targets built of the optionally provided C/C++ files
    if bin_cc_files is not None:
        for f,dependencies in bin_cc_files.items():
            p = env.Program(f, LIBS=dependencies, LIBPATH=['$build_dir'] + env['LIBPATH'])
            build_data['build_targets'] += p
            build_data['install'] += env.Install("$prefix/bin", p)

# ================== SConscript logic start here ====================

detect.importCustom(env, dict())
if not env.GetOption('clean'):

    detect.addExtern(env, ARGUMENTS.get('extern', None))

    canBuild = detect.checkMySql(env) and detect.setXrootd(env) and detect.checkXrootdLink(env)
    if not canBuild:
        raise StandardError("Can't build")

    env.Append(CPPPATH=[os.getcwd()])

    state.log.debug("Scons env:\n" + env.Dump())

# Generate version
versionFile = env.Command(['global/version.h'], None, genversion.buildVersionHeader)
env.AlwaysBuild(versionFile)  # Always rebuild this.


# Dictionary with definitions of shared libraries, shared library name
# (without 'lib' prefix) is a key in this dictionary, value is a map with
# the string keys:
#   mods: list (or space-separated string) of module names contributing to
#         this shared library, module name is a key in
#         build_data['module_objects'], and it can include sub-module name
#         separated by colon (e.g. 'css:python')
#   libs: list (or string) of names of libraries to link against, including
#         other qserv libraries
#   SHLIBPREFIX: optional prefix for library name, default is 'lib', can be
#         set to empty string, e.g. for Python extension modules
#   instDir: installation directory (absolute or relative to $prefix) for
#         the library, default is "lib"
shlibs = {}

cryptoLib = "crypto"
sslLib = "ssl"
if platform_default() == "darwin":
    cryptoLib = ""
    sslLib = ""

# library used by other shared libs
shlibs["qserv_common"] = dict(mods="""global memman proto mysql sql util""",
                              libs="""log protobuf mysqlclient_r """ +
                              cryptoLib)

# library implementing xrootd logging intercept (worker side)
shlibs["xrdlog"] = dict(mods="""xrdlog""",
                        libs="""log XrdSsiLib""")

# library implementing xrootd services (worker side)
shlibs["xrdsvc"] = dict(mods="""wbase wcontrol wconfig wdb wpublish wsched xrdsvc""",
                        libs="""qserv_common boost_regex
                             mysqlclient_r protobuf log """ + sslLib + " " +
                             cryptoLib + """ XrdSsiLib""")

# library with CSS code (regular C++ bindings)
shlibs["qserv_css"] = dict(mods="""css""",
                           libs="""qserv_common log mysqlclient_r""")

# library with QMeta code (regular C++ bindings)
shlibs["qserv_qmeta"] = dict(mods="""qmeta""",
                             libs="""qserv_common log mysqlclient_r""")

# library with all czar C++ code
shlibs["qserv_czar"] = dict(mods="""ccontrol czar parser qana query qdisp qproc rproc tests""",
                            libs="""qserv_css qserv_qmeta qserv_common antlr4-runtime sphgeom
                                 log XrdSsiLib boost_regex""")

# library implementing core functionality of the replication subsystem, tests and
# binary applications of the replication subsystem
shlibs["replica"] = dict(mods="""replica""",
                         libs="""qserv_common xrdsvc XrdCl XrdSsiLib qhttp util sphgeom protobuf
                              boost_filesystem boost_system log log4cxx""")

# library with Lua bindings for czar C++ code
shlibs["czarProxy"] = dict(mods="""proxy""",
                           libs="""qserv_czar""",
                           SHLIBPREFIX='',
                           instDir='lib/lua/qserv')

# library with qhttp C++ code
shlibs["qhttp"] = dict(mods="""qhttp""",
                       libs="""boost_filesystem boost_regex boost_system""")

# Python extension module wrapping qserv_css
shlibs["cssLib"] = dict(mods="css:python",
                        libs="qserv_css log boost_thread",
                        SHLIBPREFIX='',
                        instDir="$python_prefix/lsst/qserv/css")

# Python extension module wrapping qserv_qmeta
shlibs["qmetaLib"] = dict(mods="qmeta:python",
                          libs="qserv_qmeta log",
                          SHLIBPREFIX='',
                          instDir="$python_prefix/lsst/qserv/qmeta")

# get list of all modules
all_modules = sorted(str(d) for d in Glob('*', source=True) if os.path.isdir(d.srcnode().abspath))
state.log.debug('all_modules: ' + str(all_modules))

# setup build data structure, which is a map with these keys and values:
#    install: list of all install targets
#    module_objects: maps module name to list of shared object targets
#    shlibs: maps library name to dict as defined above
#            (copy of the above shlibs)
#    tests: list of all test app targets
#    unit_tests: list of unit test targets
#    module_lib: maps module name to library name
build_data = {}
build_data['install'] = []
build_data['module_objects'] = defaultdict(lambda: [])
build_data['shlibs'] = shlibs.copy()
build_data['tests'] = []
build_data['unit_tests'] = []
build_data['module_lib'] = defaultdict(lambda: None)
build_data['build_targets'] = []

# store in env so that modules can use it
defEnv = DefaultEnvironment()
defEnv['build_data'] = build_data

# fill module to library mapping
for shlib, shopt in shlibs.items():
    for mod in Split(shopt['mods']):
        if mod in build_data['module_lib']:
            state.log.warn("module %s appears in more than one library" % mod)
        build_data['module_lib'][mod] = shlib

state.log.debug('initialized build_data: ' + defEnv.Dump('build_data'))

# call all modules' SConscripts or use standard method
for mod in all_modules:
    scriptPath = os.path.join(mod, "SConscript")
    if env.Glob(scriptPath):  # os.access is faulty with VariantDir
        SConscript(scriptPath, exports={'env': env, 'standardModule': standardModule})
    else:
        standardModule(env, module=mod)

# define targets for shared libs
for shlib, shopt in shlibs.items():

    state.log.debug('defining target for ' + shlib)

    # collect all objects from all modules
    objects = []
    for mod in Split(shopt['mods']):
        mod_objects = build_data['module_objects'][mod]
        objects += mod_objects
        state.log.debug("module %s objects: %s" % (mod, map(str, mod_objects)))
    state.log.debug("library %s objects: %s" % (shlib, map(str, objects)))

    # add build/ to LIBPATH so that dependent qserv libs can be found
    # while linking
    libPath = ['$build_dir'] + env['LIBPATH']
    state.log.debug('libPath: %s' % libPath)

    # shared library target
    pfx = shopt.get('SHLIBPREFIX', env['SHLIBPREFIX'])
    out = env.SharedLibrary(shlib, objects, LIBS=Split(shopt['libs']),
                            SHLIBPREFIX=pfx, LIBPATH=libPath)
    build_data['build_targets'] += out

    # install target
    instDir = shopt.get('instDir', 'lib')
    # instDir can start with $ which confuses join() if substitution
    # is supposed to be absolute path, so we substitute in advance
    instDir = env.subst(instDir)
    target = env.Install(os.path.join("$prefix", instDir), out)
    build_data['install'] += target

state.log.debug('build_data: ' + defEnv.Dump('build_data'))
state.log.debug('build_data[install]: ' + str(map(str, build_data['install'])))

# unit tests to run
utests = build_data['unit_tests']
if utests:
    # just for convenience, all unit tests run after build target
    env.Requires(utests, "build")
    state.log.debug("Unit tests to run: %s" % map(str, utests))
    # special builder that checks that unit tests were successful
    utests = env.UnitTestCheck('unit-test-check-tag', utests)

build_data['build_targets'] += build_data['tests']

# define/extend all aliases
state.log.debug("build targets: %s" % map(str, build_data['build_targets']))
env.Alias("build", build_data['build_targets'])
state.log.debug("install targets: %s" % map(str, build_data['install']))
env.Alias("install-notest", build_data['install'])
if utests:
    env.Alias("test", utests)
    state.log.debug("Test tgts to run: %s" % map(str, utests))
