# -------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
# -------------------------------------------------------------------------
# !/usr/bin/env python
"""
entrypoint compiler
"""

# run this script with -h or --help to get all the available options.

import argparse
import json
import os
import sys
import tempfile
from textwrap import dedent, indent, fill

from code_fixer import fix_code, fix_code_core, fix_code_entrypoint, \
    run_autopep
from codegen_checker import check_codegen
from compiler_utils import convert_name, module_to_path, \
    COPYRIGHT_NOTICE, CODEGEN_WARNING, get_presteps, tabsize
from doc_builder import DocBuilder, DocParameter
from loss_processor import write_loss, get_loss_name
from manifest_diff_parser import parse_manifest_diff_entrypoints, \
    parse_manifest_diff_components

tab = ' ' * 8


def install_and_import(package):
    import importlib
    import pip
    try:
        importlib.import_module(package)
    except ImportError:
        pip.main(
            ['install', package])
    finally:
        globals()[package] = importlib.import_module(package)


install_and_import('autoflake')
install_and_import('isort')

my_path = os.path.realpath(__file__)
my_dir = os.path.dirname(my_path)
sys.path.append(os.path.join(my_dir, '..'))
script_args = None
verbose = None


class Role:
    """
    See same class in *nimbusml*.
    """

    Feature = 'Feature'
    Label = 'Label'
    Weight = 'Weight'
    GroupId = 'GroupId'
    User = 'User'
    Item = 'Item'
    Name = 'Name'
    RowId = 'RowId'


_allowed_roles = set(k for k in Role.__dict__ if k[0].upper() == k[0])

_predict_proba_template = \
    """
@trace
def predict_proba(self, X, **params):
    '''
    Returns probabilities
    '''
    return self._predict_proba(X, **params)
"""

_predict_proba_template_ova = \
    """
@trace
def predict_proba(self, X, **params):
    '''
    Returns probabilities
    '''
    self.classifier._check_implements_method('predict_proba')
    if not self.use_probabilities :
        raise ValueError('{}: use_probabilities needs to be set to True' \
        ' for predict_proba()'.format(self.__class__))
    return self._predict_proba(X, **params)
"""
_decision_function_template = \
    """
@trace
def decision_function(self, X, **params):
    '''
    Returns score values
    '''
    return self._decision_function(X, **params)
"""
_decision_function_template_ova = \
    """
@trace
def decision_function(self, X, **params):
    '''
    Returns score values
    '''
    self.classifier._check_implements_method('decision_function')
    if self.use_probabilities :
        raise ValueError('{}: use_probabilities needs to be set to False' \
         ' for decision_function()'.format(self.__class__))
    return self._decision_function(X, **params)
"""


def parse_manifest(manifest_json, pkg_path=None, overwrite=False):
    """
    """
    with open(manifest_json) as f:
        manifest = json.load(f)

    entrypoints = manifest["EntryPoints"]  # list
    nodes = []
    for e in entrypoints:
        try:
            nodes.append(
                parse_entrypoint(
                    e,
                    pkg_path=pkg_path,
                    overwrite=overwrite))
        except TypeError as exp:
            msg = '{} failed to generate. Reason: {}'.format(
                e['Name'], str(exp))
            print(msg)

    Components = manifest["Components"]  # list
    for Component in Components:
        kind = Component['Kind']
        for c in Component['Components']:
            try:
                parse_entrypoint(
                    c,
                    kind=kind,
                    pkg_path=pkg_path,
                    overwrite=overwrite)
            except TypeError as exp:
                msg = '{} failed to generate. Reason: {}'.format(
                    c['Name'], str(exp))
                print(msg)

    return [nodes]


def parse_manifest_diff(
        manifest_json,
        manifest_diff_json,
        pkg_path=None,
        overwrite=False):
    """
    """
    entrypoints = parse_manifest_diff_entrypoints(
        manifest_json, manifest_diff_json, verbose)
    entrypoint_nodes = [
        write_api(
            entrypoint,
            pkg_path=pkg_path,
            overwrite=overwrite) for entrypoint in entrypoints]
    components = parse_manifest_diff_components(
        manifest_json, manifest_diff_json, verbose)
    component_nodes = [
        write_api(
            component,
            kind=component['Kind'],
            pkg_path=pkg_path,
            overwrite=overwrite) for component in components]
    return [entrypoint_nodes] + [component_nodes]


def write_api(entrypoint, kind="node", pkg_path=None, overwrite=False):
    """
    """
    entrypoint_name = entrypoint['Name'].replace(".", "_").lower()
    class_name = entrypoint['NewName']
    class_dir = entrypoint['Module']
    class_type = entrypoint['Type']
    class_file = class_name.lower()

    doc_builder = DocBuilder()
    doc_builder.class_name = class_name
    doc_builder.class_module = class_dir
    doc_builder.desc = entrypoint['Desc']

    doc_builder_core = DocBuilder()
    doc_builder_core.class_name = class_name
    doc_builder_core.class_module = class_dir
    doc_builder_core.desc = entrypoint['Desc']

    banner = COPYRIGHT_NOTICE + CODEGEN_WARNING

    if verbose:
        print(class_name)

    ###################################
    # create function
    funobjs = create_py(entrypoint, kind=kind)
    visible_args = [
        arg for arg in funobjs['inputs'] if isinstance(
            arg.hidden, Missing)]
    doc_args = [
        DocParameter(
            name=arg.new_name_converted,
            desc=arg.desc) for arg in visible_args if
        arg.name_converted != 'column']
    doc_builder.add_manifest_args(doc_args)
    doc_builder_core.add_manifest_args(doc_args)

    # see what column param type is
    column_arg = [
        arg for arg in visible_args if arg.name_converted == 'column']

    # columns for entrypoint
    hidden_args = [arg for arg in funobjs['inputs']
                   if not isinstance(arg.hidden, Missing)]
    columns_entrypoint = [arg for arg in hidden_args]

    # In a function header, arguments must appear in this order:
    #   * any normal arguments(name);
    #   * any default arguments (name=value);
    #   * the *name (or* in 3.X) form;
    #   * any name or name=value keyword-only arguments (in 3.X);
    #   * the **name form.
    class_args = [arg.get_arg() for arg in visible_args if isinstance(
        arg.default, Missing) and arg.name_converted != 'column']
    class_args += [arg.get_arg() for arg in visible_args if not isinstance(
        arg.default, Missing) and arg.name_converted != 'column']
    class_args = ',\n        '.join(class_args)

    entrypoint_args_map = [arg for arg in visible_args if isinstance(
        arg.default, Missing) and arg.name_converted != 'column']
    entrypoint_args_map += [arg for arg in visible_args if not isinstance(
        arg.default, Missing) and arg.name_converted != 'column']
    entrypoint_args_map = [
        "%s=%s" %
        (arg.name_converted,
         arg.name_assignment) for arg in entrypoint_args_map]
    entrypoint_args_map = '\n'.join(entrypoint_args_map)

    args_map = [arg for arg in visible_args if isinstance(
        arg.default, Missing) and arg.name_converted != 'column']
    args_map += [arg for arg in visible_args if not isinstance(
        arg.default, Missing) and arg.name_converted != 'column']

    api_args_map = [
        "%s=%s" %
        (arg.new_name_converted,
         arg.new_name_converted) for arg in args_map]
    api_args_map = '\n'.join(api_args_map)

    core_args_map = [
        "%s=%s" %
        (arg.new_name_converted,
         arg.name_core_assignment) for arg in args_map]
    core_args_map = '\n'.join(core_args_map)

    fun_settings_body = None
    if class_type == 'Component':
        fun_settings_body = "\n    ".join(
            [arg.get_body() for arg in funobjs['settings']])

    dots = "..."
    if "." in class_dir:
        dots = "...."
    class_imports = [
        arg.get_import(
            prefix=(
                    "%sentrypoints." %
                    dots)) for arg in visible_args if
        arg.get_import() is not None]
    class_imports = '\n'.join(class_imports)

    dots = "..."
    if "." in class_dir:
        dots = "...."
    core_class_imports = [
        arg.get_import(
            prefix=(
                    "%sentrypoints." %
                    dots)) for arg in visible_args if
        arg.get_import() is not None]
    core_class_imports = '\n'.join(core_class_imports)

    # write the class to a file
    py_path = module_to_path(class_dir, pkg_path)
    if not os.path.exists(py_path):
        os.makedirs(py_path)
    file = os.path.join(py_path, ".".join([class_file, "py"]))
    if os.path.exists(file) and not overwrite:
        raise FileExistsError(
            "file {} exists, set 'overwrite = TRUE' to overwrite.".format(
                file))

    write_class(
        entrypoint,
        class_name,
        class_type,
        file,
        class_file,
        class_dir,
        banner,
        class_args,
        api_args_map,
        doc_builder,
        column_arg,
        hidden_args)

    # Generating test classes is broken. Commented out for now.
    # write the class test to a file
    # py_path = os.path.join(pkg_path, "tests", *class_dir.split("."))
    # if not os.path.exists(py_path): os.makedirs(py_path)
    #
    # file = os.path.join(py_path, "test_" + ".".join([class_file, "py"]))
    # if os.path.exists(file) and not overwrite:
    #     raise FileExistsError("file {} exists, set 'overwrite = TRUE'
    # to overwrite.".format(file))
    #
    # write_class_test(class_name, file)

    # write the core class to a file
    py_path = os.path.join(pkg_path, "internal", "core",
                           *class_dir.split("."))
    if not os.path.exists(py_path):
        os.makedirs(py_path)
    file = os.path.join(py_path, ".".join([class_file, "py"]))
    if os.path.exists(file) and not overwrite:
        raise FileExistsError(
            "file {} exists, set 'overwrite = TRUE' to overwrite.".format(
                file))

    write_core_class(
        entrypoint,
        entrypoint_name,
        class_name,
        class_type,
        file,
        class_file,
        class_dir,
        banner,
        core_class_imports,
        class_args,
        core_args_map,
        entrypoint_args_map,
        doc_builder_core,
        column_arg,
        columns_entrypoint,
        fun_settings_body,
        hidden_args)

    return funobjs


def parse_entrypoint(
        entrypoint,
        kind="node",
        prefix=None,
        pkg_path=None,
        overwrite=False):
    """
    """
    name = entrypoint['Name']
    # NOTE: '.' is not allowed in function name.
    fun_name = name.replace(".", "_")
    fun_file = fun_name
    # NOTE: components have to be imported before entrypoints,
    #   so use '_' prefix to differentiate components from entrypoints.
    if kind != "node":
        fun_file = "_{}_{}".format(kind, fun_file)
        fun_name = convert_name(fun_name)
    else:
        fun_name = fun_name.lower()

    if prefix is not None:
        fun_name = prefix + fun_name
    banner = CODEGEN_WARNING
    if verbose:
        print(fun_name)

    ###################################
    # create function
    funobjs = create_py(entrypoint, kind=kind)
    if entrypoint['Desc'] is not None:
        entrypoint['Desc'] = \
            fill(entrypoint['Desc'], 69, subsequent_indent=tab)
    doc_desc = "    **Description**\n        {}\n".format(
        entrypoint['Desc'])
    doc_args = [
        "    :param {name_converted}: {desc} ({inout}).".format(
            name_converted=setting.name_converted,
            desc=setting.desc,
            inout=setting.inout) for setting in funobjs['settings']]
    doc_args = [fill(item, 69, subsequent_indent=tab) for item in doc_args]
    fun_doc = "\n".join([doc_desc, *doc_args])
    fun_doc = '    """\n{}\n    """\n\n'.format(fun_doc)

    # In a function header, arguments must appear in this order:
    #   * any normal arguments(name);
    #   * any default arguments (name=value);
    #   * the *name (or* in 3.X) form;
    #   * any name or name=value keyword-only arguments (in 3.X);
    #   * the **name form.
    fun_args = [arg.get_arg() for arg in funobjs['settings']
                if isinstance(arg.default, Missing)]
    fun_args += [arg.get_arg() for arg in funobjs['settings']
                 if not isinstance(arg.default, Missing)]
    fun_args.append('**params')
    fun_args = ',\n        '.join(fun_args)
    # fun_args = ', '.join([arg.get_arg() for arg in funobjs['settings']])

    fun_imports = [arg.get_import() for arg in funobjs['settings']
                   if arg.get_import() is not None]
    fun_imports = '\n'.join(fun_imports)

    # fun_header = 'def {}(\n        {}) -> "{}":\n\n'.format(fun_name,
    # fun_args, kind)
    fun_header = 'def {}(\n        {}):\n\n'.format(fun_name, fun_args)
    fun_body = "\n    ".join(
        [arg.get_body() for arg in funobjs['settings']])

    # write the function to a file
    py_path = os.path.join(pkg_path, "internal", "entrypoints")
    if not os.path.exists(py_path):
        os.makedirs(py_path)
    file = os.path.join(py_path, ".".join([fun_file.lower(), "py"]))
    if os.path.exists(file) and not overwrite:
        raise FileExistsError(
            "file {} exists, set 'overwrite = TRUE' to overwrite.".format(
                file))

    if kind == "node":
        write_entrypoint(
            name,
            kind,
            file,
            banner,
            fun_imports,
            fun_header,
            fun_body,
            fun_doc)
    else:
        write_component(
            name,
            kind,
            file,
            banner,
            fun_imports,
            fun_header,
            fun_body,
            fun_doc)
        # write_component_import(name, fun_file)

    return funobjs


def check_read_file(name, to_write, do_check=True):
    if os.path.exists(name):
        with open(name, 'r') as f:
            content = f.read()
    else:
        content = ""
    if do_check:
        if "source=input" in to_write:
            raise Exception("name: {0}\n{1}".format(name, to_write))
        if "input=input" in to_write:
            raise Exception("name: {0}\n{1}".format(name, to_write))
        if "source: str" in to_write:
            raise Exception("name: {0}\n{1}".format(name, to_write))
    if "entrypoint" not in name:
        if ":param input:" in to_write:
            raise Exception("name: {0}\n{1}".format(name, to_write))
    return to_write == content


def write_entrypoint(
        name,
        kind,
        file,
        banner,
        fun_imports,
        fun_header,
        fun_body,
        fun_doc):
    module_doc = '"""\n{}\n"""\n'.format(name)

    imports = dedent("""
        import numbers
        {}
        from ..utils.utils import try_set, unlist
        from ..utils.entrypoints import EntryPoint
        \n\n""").format(fun_imports)

    fun_tail = indent(dedent("""\n
        input_variables = {
            x for x in unlist(inputs.values())
            if isinstance(x, str) and x.startswith("$")}
        output_variables = {
            x for x in unlist(outputs.values())
            if isinstance(x, str) and x.startswith("$")}

        entrypoint = EntryPoint(
            name=entrypoint_name, inputs=inputs, outputs=outputs,
            input_variables=input_variables,
            output_variables=output_variables)
        return entrypoint
        """), prefix=" " * tabsize)

    to_write = "".join([banner, module_doc, imports, fun_header, fun_doc,
                        "    entrypoint_name = '{}'\n".format(name),
                        "    inputs = {}\n    outputs = {}\n\n    ",
                        fun_body, fun_tail])

    if not check_read_file(file, to_write, do_check=False):
        with open(file, "w") as myfile:
            print("    [write_entrypoint]", os.path.abspath(file))
            myfile.write(banner)
            myfile.write(module_doc)
            myfile.write(imports)
            myfile.write(fun_header)
            myfile.write(fun_doc)
            myfile.write("    entrypoint_name = '{}'\n".format(name))
            myfile.write("    inputs = {}\n    outputs = {}\n\n    ")
            myfile.write(fun_body)
            myfile.write(fun_tail)

    fix_code_entrypoint(name, file)


def write_component(
        name,
        kind,
        file,
        banner,
        fun_imports,
        fun_header,
        fun_body,
        fun_doc):
    module_doc = '"""\n{}\n"""\n'.format(name)

    imports = dedent("""
        import numbers
        {}
        from ..utils.utils import try_set
        from ..utils.entrypoints import Component
        \n\n""").format(fun_imports)

    fun_tail = indent(dedent("""\n
        component = Component(
            name=entrypoint_name, settings=settings, kind='{}')
        return component
        """), prefix=" " * tabsize).format(kind)

    to_write = "".join([banner, module_doc, imports, fun_header, fun_doc,
                        "    entrypoint_name = '{}'\n".format(name),
                        "    settings = {}\n\n    ",
                        fun_body, fun_tail])

    if not check_read_file(file, to_write):
        with open(file, "w") as myfile:
            print("    [write_component]", os.path.abspath(file))
            myfile.write(banner)
            myfile.write(module_doc)
            myfile.write(imports)
            myfile.write(fun_header)
            myfile.write(fun_doc)
            myfile.write("    entrypoint_name = '{}'\n".format(name))
            myfile.write("    settings = {}\n\n    ")
            myfile.write(fun_body)
            myfile.write(fun_tail)

    run_autopep(file)


def write_class(
        entrypoint,
        class_name,
        class_type,
        file,
        class_file,
        class_dir,
        banner,
        class_args,
        args_map,
        doc_builder,
        column_arg,
        hidden_args):
    """
    Write classes not in folder ``internal``.
    """
    module_doc = '"""\n{}\n"""\n'.format(class_name)

    # Add parameter columns (part of the syntax) and roles.
    hidden = set(a.name for a in hidden_args)
    allowed_roles = sorted([k.lower()
                            for k in _allowed_roles if
                            k + 'Column' in hidden])
    sig_columns_roles = list(allowed_roles)

    base_file = "base_predictor"
    base_class = "BasePredictor"
    mixin_class = "ClassifierMixin"
    base_classes = "core, BasePredictor, ClassifierMixin"
    if class_type == "Transform":
        base_file = "base_transform"
        base_class = "BaseTransform"
        mixin_class = "TransformerMixin"
        base_classes = "core, BaseTransform, TransformerMixin"
        sig_columns_roles.append("columns")
    elif class_type == "Regressor":
        base_file = "base_predictor"
        base_class = "BasePredictor"
        mixin_class = "RegressorMixin"
        base_classes = "core, BasePredictor, RegressorMixin"
    elif class_type == "Clusterer":
        base_file = "base_predictor"
        base_class = "BasePredictor"
        mixin_class = "ClusterMixin"
        base_classes = "core, BasePredictor, ClusterMixin"
    elif class_type == "Component":
        base_classes = "core"

    dots = '.' * (2 + class_dir.count('.'))

    if class_type == "Component":
        import_lines = dedent("""
            __all__ = ["{}"]

            import numbers
            from {}internal.utils.utils import trace
            from {}internal.core.{}.{} import {} as core
            \n\n""").format(class_name, dots, dots, class_dir, class_file,
                            class_name)
    else:
        import_lines = dedent("""
            __all__ = ["{}"]

            import numbers
            from sklearn.base import {}
            from {}internal.utils.utils import trace
            from {}{} import {}
            from {}internal.core.{}.{} import {} as core
            \n\n""").format(class_name, mixin_class, dots, dots, base_file,
                            base_class, dots, class_dir, class_file,
                            class_name)

    # fun_imports, fun_body
    body = ''
    header = 'class {}({}):\n##CLASS_DOCSTRINGS##\n    @trace\n    ' \
             'def __init__(\n        self,'.format(class_name, base_classes)

    if column_arg:
        if column_arg[0].name_annotated == 'column: str':
            pass
        elif column_arg[0].name_annotated == 'column: list':
            arg_type = column_arg[0].type['ItemType']
            if arg_type == 'String':
                pass
            elif arg_type['Kind'] == 'Struct':
                # see if Source is among the fields
                source_field = [
                    f for f in arg_type['Fields'] if f['Name'] == 'Source']
                # see if Name is among the fields
                name_field = [
                    f for f in arg_type['Fields'] if f['Name'] == 'Source']
                if len(source_field) > 0:
                    pass
                if len(name_field) > 0:
                    pass
        elif column_arg[0].name_annotated == 'column: dict':
            pass
        else:
            print('unknown column type %s' % column_arg[0].name_annotated)

    if class_args:
        header += '\n        {},'.format(class_args)
    for h in sig_columns_roles:
        if h == 'groupid':
            h = 'group_id'
        header += '\n        {}=None,'.format(h)
    header += '\n        **params):\n'
    assert 'groupid' not in header
    assert 'colid' not in header
    assert 'rowid' not in header

    for h in sig_columns_roles:
        if h == "groupid":
            h = 'group_id'
        elif h == "rowid":
            h = 'row_id'
        elif h == "colid":
            h = 'col_id'
        if h == 'columns' and base_class == "BasePredictor":
            continue
        doc_builder._add_io_arg_desc(
            h,
            "see `Columns </nimbusml/concepts/columns>`_")

    class_docstrings = doc_builder.get_documentation()
    if 'supervised' in class_name.lower() and ':param label:' not in \
            class_docstrings:
        raise Exception(
            "Issue with '{0}'\n{1}".format(
                class_name, class_docstrings))
    if ':param input:' in class_docstrings:
        raise Exception(
            "Issue with '{0}'\n{1}".format(
                class_name, class_docstrings))
    header = header.replace('##CLASS_DOCSTRINGS##', class_docstrings)

    if args_map != '':
        for l in args_map.split('\n'):
            (var_name, var_value) = l.split('=')
            var_name = var_name.strip()
            if var_name == 'groupid':
                var_name = 'group_id'
            var_value = var_value.strip()
            body = body + '\n            {}={},'.format(var_name,
                                                        var_value)

    # input / output arguments
    body_header = ""
    body_sig_params = []
    for h in sig_columns_roles:
        # add roles as allowed parameters
        if h == 'groupid':
            h = 'group_id'
        elif h == 'colid':
            h = 'col_id'
        elif h == 'rowid':
            h = 'row_id'
        if h == "columns":
            body_header += "\n        if {0}: params['{0}'] = {0}".format(
                h)
        else:
            body_header += "\n        if '{0}_column' in params: raise " \
                           "NameError(\"'{0}_column' must be renamed to " \
                           "'{0}'\")".format(h)
            body_header += "\n        if {0}: params['{0}_column'] = {" \
                           "0}".format(h)
        body_sig_params.append(h)
    if 'input_columns' in header and 'columns=' in header:
        body_header += "\n        if columns: input_columns = " \
                       "sum(list(columns.values()),[]) if " \
                       "isinstance(list(columns.values())[0], list) " \
                       "else list(columns.values())"
    if 'output_columns' in header and 'columns=' in header:
        body_header += "\n        if columns: output_columns = " \
                       "list(columns.keys())"

    assert 'groupid' not in body_header
    assert 'colid' not in body_header
    assert 'rowid' not in body_header

    if class_type == "Component":
        body = "        core.__init__(\n            self,{}\n           " \
               " **params)\n".format(body)
    elif class_type == "Transform":
        clname = "BaseTransform"
        body = "{}\n        {}.__init__(self, **params)\n        " \
               "core.__init__(\n            self,{}\n            " \
               "**params)\n".format(body_header, clname, body)
    else:
        clname = "BasePredictor"
        body = "{}\n        {}.__init__(self, type='{}', **params)\n     " \
               "" \
               "   core.__init__(\n            self,{}\n            " \
               "**params)\n".format(body_header, clname,
                                    class_type.lower(), body)

    for h in body_sig_params:
        body += '        self.{0}{1}={1}\n'.format(
            '_' if h == 'columns' else '', h)

    if 'Predict_Proba' in entrypoint:
        if entrypoint['Predict_Proba'] is True:
            if class_name == 'OneVsRestClassifier':
                body += indent(_predict_proba_template_ova,
                               prefix=' ' * tabsize)
            else:
                body += indent(_predict_proba_template,
                               prefix=' ' * tabsize)

    if 'Decision_Function' in entrypoint:
        if entrypoint['Decision_Function'] is True:
            if class_name == 'OneVsRestClassifier':
                body += indent(_decision_function_template_ova,
                               prefix=' ' * tabsize)
            else:
                body += indent(_decision_function_template,
                               prefix=' ' * tabsize)

    # additional methods
    additional = """\n    def get_params(self, deep=False):\n
        \"\"\"\n        Get the parameters for this operator.
        \"\"\"\n        return core.get_params(self)\n"""

    # presteps
    additional += get_presteps(class_name)

    to_write = "".join(
        [banner, module_doc, import_lines, header, body, additional])

    assert 'groupid' not in to_write
    assert 'colid' not in to_write
    assert 'rowid' not in to_write

    if not check_read_file(file, to_write):
        with open(file, "w") as myfile:
            print("    [write_class]", os.path.abspath(file))
            myfile.write(banner)
            myfile.write(module_doc)
            myfile.write(import_lines)
            myfile.write(header)
            myfile.write(body)
            myfile.write(additional)

    fix_code(class_name, file)


def write_class_test(class_name, file):
    body = dedent("""
        import unittest
        from nimbusml.tests.estimator_checks import check_estimator
        from nimbusml import {}

        class Test{}(unittest.TestCase):

            def test_check_estimator_{}(self):
                check_estimator({})

        if __name__ == '__main__':
            unittest.main()
        \n\n""").format(class_name, class_name, class_name.lower(),
                        class_name)

    if not check_read_file(file, body):
        with open(file, "w") as myfile:
            print("    [write_class_test] write", os.path.abspath(file))
            myfile.write(body)


def write_core_class(
        entrypoint,
        entrypoint_name,
        class_name,
        class_type,
        file,
        class_file,
        class_dir,
        banner,
        imports,
        class_args,
        args_map,
        entrypoint_args_map,
        doc_builder,
        column_arg,
        columns_entrypoint,
        fun_settings_body,
        hidden_args):
    module_doc = '"""\n{}\n"""\n'.format(class_name)

    hidden = set(a.name for a in hidden_args)
    allowed_roles = [k.lower()
                     for k in _allowed_roles if k + 'Column' in hidden]

    dots = '.' * (1 + class_dir.count('.'))

    # Input/output extra information.
    if class_name in {'ColumnConcatenator'}:
        base_class = "BasePipelineItem"
        baseclass_sig = ", MultiOutputsSignature"
    elif class_name in {'RangeFilter'}:
        base_class = "BasePipelineItem"
        baseclass_sig = ", SingleInputAsStringSignature"
    elif class_name == 'OneHotVectorizer':
        base_class = "BasePipelineItem"
        baseclass_sig = ", EqualInputOutputSignature"
    elif class_name in {'NGramFeaturizer', 'OneHotHashVectorizer'}:
        base_class = "BasePipelineItem"
        baseclass_sig = ", SingleOutputSignature"
    elif class_name in {'Filter', 'ColumnDropper', 'TakeFilter',
                        'SkipFilter', 'RangerFilter', 'ColumnSelector'}:
        base_class = "BasePipelineItem"
        baseclass_sig = ", NoOutputSignature"
    elif class_name in {'MutualInformationSelector'}:
        base_class = "BasePipelineItem"
        baseclass_sig = ", SingleOutputSignatureWithRoles"
    elif class_name in {'MinMaxScaler'}:
        base_class = "BasePipelineItem"
        baseclass_sig = ", DefaultSignature"
    elif len(allowed_roles) > 0:
        base_class = "BasePipelineItem"
        baseclass_sig = ", DefaultSignatureWithRoles"
    else:
        base_class = "BasePipelineItem"
        baseclass_sig = ", DefaultSignature"

    if class_type == 'Component':
        base_class = "Component"

        import_lines = dedent("""
             __all__ = ["{0}"]

             import numbers
             {1}
             from {2}..utils.entrypoints import Component
             from {2}..utils.utils import trace, try_set
             \n\n""").format(class_name, imports, dots)
    else:
        import_base_class = "from {0}.base_pipeline_item import {1}{" \
                            "2}".format(dots, base_class, baseclass_sig)

        import_lines = dedent("""
            __all__ = ["{0}"]

            import numbers
            {1}
            from {2}..entrypoints.{3} import {4}
            from {2}..utils.utils import trace
            from {2}..utils.data_roles import Role
            {5}
            \n\n""").format(class_name, imports, dots, entrypoint_name,
                            entrypoint_name, import_base_class)

    # fun_imports, fun_body
    body_snip = ''
    tail_snip = ''
    tail_snip0 = ''

    if class_type == 'Component':
        header = 'class {}({}):\n##CLASS_DOCSTRINGS##\n    @trace\n    ' \
                 'def __init__(\n        self,'.format(class_name, base_class)
    else:
        header = 'class {}({}{}):\n##CLASS_DOCSTRINGS##\n    @trace\n    ' \
                 '' \
                 'def __init__(\n        self,'.format(class_name,
                                                       base_class,
                                                       baseclass_sig)

    if column_arg:
        if column_arg[0].name_annotated == 'column: str':
            # header += '\n        input: str = None,'
            # body_snip += '\n        self.input=input'
            # doc_builder.add_input_arg(is_list=False)
            tail_snip0 = indent(dedent("""
                            input_column = self.input
                            if input_column is None and 'input' in\
 all_args:
                                input_column = all_args['input']
                            if 'input' in all_args:
                                all_args.pop('input')

                            # validate input
                            if input_column is None:
                                raise ValueError("'None' input passed\
 when it cannot be none.")

                            if not isinstance(input_column, str):
                                raise ValueError("input has to be a\
 string, instead got %s" % type(
                                input_column))

                            """), prefix=" " * tabsize)
            tail_snip = '\n        column=input_column,'
        elif column_arg[0].name_annotated == 'column: list':
            arg_type = column_arg[0].type['ItemType']
            if arg_type == 'String':
                # header += '\n        input: list = None,'
                # body_snip += '\n        self.input=input'
                # doc_builder.add_input_arg(is_list=True)
                tail_snip0 = indent(dedent("""
                                input_columns = self.input
                                if input_columns is None and 'input' in\
 all_args:
                                    input_columns = all_args['input']
                                if 'input' in all_args:
                                    all_args.pop('input')

                                # validate input
                                if input_columns is None:
                                    raise ValueError("'None' input\
 passed when it cannot be none.")

                                if not isinstance(input_columns, list):
                                    raise ValueError("input has to be\
 a list of strings, instead got %s" % type(input_columns))

                                """), prefix=" " * tabsize)
                tail_snip = '\n        column=input_columns,'
            elif arg_type['Kind'] == 'Struct':
                # see if Source is among the fields
                source_field = [
                    f for f in arg_type['Fields'] if f['Name'] == 'Source']
                # see if Name is among the fields
                name_field = [
                    f for f in arg_type['Fields'] if f['Name'] == 'Name']
                if len(source_field) > 0:
                    # header += '\n        input: list = None,'
                    # body_snip += '\n        self.input=input'
                    # doc_builder.add_input_arg(is_list=True)
                    tail_snip0 = indent(dedent("""
                                    input_columns = self.input
                                    if input_columns is None and 'input'\
 in all_args:
                                        input_columns = all_args['input']
                                    if 'input' in all_args:
                                        all_args.pop('input')

                                    # validate input
                                    if input_columns is None:
                                        raise ValueError("'None' input\
 passed when it cannot be none.")

                                    if not isinstance(input_columns, list):
                                        raise ValueError("input has to\
 be a list of strings, instead got %s" % type(input_columns))

                                 """), prefix=" " * tabsize)
                    tail_snip = '\n            column=[dict(Source=col) ' \
                                'for col in input_columns] if ' \
                                'input_columns else None,'
                if len(name_field) > 0:
                    # header += '\n        output: list = None,'
                    # body_snip += '\n        self.output=output'
                    # doc_builder.add_output_arg(is_list=True)
                    tail_snip0 = indent(dedent("""
                                    input_columns = self.input
                                    if input_columns is None and 'input'\
 in all_args:
                                        input_columns = all_args['input']
                                    if 'input' in all_args:
                                        all_args.pop('input')

                                    output_columns = self.output
                                    if output_columns is None and\
 'output' in all_args:
                                        output_columns = all_args['output']
                                    if 'output' in all_args:
                                        all_args.pop('output')
                                    __CONC__
                                    # validate input
                                    if input_columns is None:
                                        raise ValueError("'None' input\
 passed when it cannot be none.")

                                    if not isinstance(input_columns, list):
                                        raise ValueError("input has to\
 be a list of strings, instead got %s" % type(input_columns))

                                    # validate output
                                    if output_columns is None:
                                        output_columns = input_columns

                                    if not isinstance(output_columns,\
 list):
                                        raise ValueError("output has to\
 be a list of strings, instead got %s" % type(output_columns))

                                 """), prefix=" " * tabsize)
                    if entrypoint_name == "transforms_pcacalculator":
                        rep = indent(dedent("""
                                            if any(isinstance(el, list)\
 for el in input_columns):
                                                input_columns = [y for x\
 in input_columns for y in x]
                                            conc = None
                                            if isinstance(input_columns,\
 list) and len(input_columns) > 1:
                                                # We concatenate the\
 columns.
                                                data = all_args.pop("data")
                                                outcol = "temp_%s" %\
 str(id(self))
                                                if type(output_columns)\
 == list:
                                                    if len(output_columns)\
 == 1:
                                                        outcol =\
 output_columns[0]
                                                output_data = "%s_c%s" %\
 (all_args['output_data'], str(id(self)))
                                                model = "%s_c%s" %\
 (all_args['model'], str(id(self)))
                                                conc =\
 self._add_concatenator_node(
                                             data, input_columns,\
 output_data, outcol, model)
                                                input_columns = [outcol]
                                                all_args["data"] =\
 output_data
                                    """), prefix=" " * tabsize)
                        tail_snip0 = tail_snip0.replace("__CONC__", rep)
                    else:
                        tail_snip0 = tail_snip0.replace(
                            " " * tabsize + "__CONC__", "")
                    tail_snip = '\n        column=[dict(Source=i, ' \
                                'Name=o) for i, o in zip(input_columns, ' \
                                'output_columns)] if input_columns else ' \
                                'None,'
            else:
                print('unknown arg type %s' % column_arg[0].name_annotated)
        elif column_arg[0].name_annotated == 'column: dict':
            arg_type = column_arg[0].type
            if arg_type['Kind'] == 'Struct':
                # see if Source is among the fields
                source_field = [
                    f for f in arg_type['Fields'] if f['Name'] == 'Source']
                # see if Name is among the fields
                name_field = [
                    f for f in arg_type['Fields'] if f['Name'] == 'Name']
                if source_field[0]['Type']['Kind'] != 'Array' or \
                        name_field[0]['Type'] != 'String':
                    print('unknown column type')
                else:
                    tail_snip0 = indent(dedent("""
                        input_columns = self.input
                        if input_columns is None and 'input' in all_args:
                            input_columns = all_args['input']
                        if 'input' in all_args:
                            all_args.pop('input')

                        output_column = self.output
                        if output_column is None and 'output' in all_args:
                            output_column = all_args['output']
                        if 'output' in all_args:
                            all_args.pop('output')

                        # validate input
                        if input_columns is None:
                            raise ValueError("'None' input passed when\
 it cannot be none.")

                        if not isinstance(input_columns, list):
                            raise ValueError("input has to be a list of\
 strings, instead got %s"
                                             % type(input_columns))

                        # validate output
                        if output_column is None:
                            raise ValueError("'None' output passed when\
 it cannot be none.")

                        if not isinstance(output_column, str):
                            raise ValueError("output has to be a string,\
 instead got %s" % type(
                                             output_column))

                        """), prefix=" " * tabsize)
                    tail_snip = '\n        column=[dict(Source=i, ' \
                                'Name=o) for i, o in zip(input_columns, ' \
                                'output_columns)] if input_columns else ' \
                                'None,'
        else:
            print('unknown column type %s' % column_arg[0].name_annotated)

    if entrypoint_name == "schemamanipulation_concatcolumns":
        print('    [write_core_class] !=', entrypoint_name)
        tail_snip0 = indent(dedent("""
                            input_columns = self.input
                            if input_columns is None and 'input' in\
 all_args:
                                input_columns = all_args['input']
                            if 'input' in all_args:
                                all_args.pop('input')

                            output_columns = self.output
                            if output_columns is None and 'output' in\
 all_args:
                                output_columns = all_args['output']
                            if 'output' in all_args:
                                all_args.pop('output')

                            # validate input
                            if input_columns is None:
                                raise ValueError("'None' input passed\
 when it cannot be none.")

                            if not isinstance(input_columns, list):
                                raise ValueError("input has to be a list\
 of strings, instead got %s" % type(input_columns))

                            for i in input_columns:
                                if not isinstance(i, list):
                                    raise ValueError("input has to be a\
 list of list strings, instead got input element of type %s" % type(i))

                            # validate output
                            if output_columns is None:
                                raise ValueError("'None' output passed\
 when it cannot be none.")

                            if not isinstance(output_columns, list):
                                raise ValueError("output has to be a\
 list of strings, instead got %s" % type(output_columns))

                            if (len(input_columns) != len(output_columns)):
                                raise ValueError("input and output have\
 to be of same length, instead input %s and output %s" %\
 (len(input_columns), len(output_columns)))

                            column = []
                            for i in range(len(input_columns)):
                                source = []
                                for ii in input_columns[i]:
                                    source.append(ii)
                                column.append(dict([('Source', source),\
 ('Name', output_columns[i])]))
                            """), prefix=" " * tabsize)
        entrypoint_args_map = ''
        tail_snip = "column=column"

    if len(columns_entrypoint) > 0:
        for c in columns_entrypoint:
            name = c.new_name_converted
            if name.endswith('_column'):
                tail_snip += "\n        {0}=self._getattr_role('{0}', " \
                             "all_args),".format(name)
            elif name == "source" or c.name == "Source":
                tail_snip += "\n        source=self.source,"
            elif name == "name":
                tail_snip += "\n        name=self._name_or_source,"

    if class_args:
        header += '\n        {},'.format(class_args)
    header += '\n        **params):\n'

    assert 'groupid' not in header
    assert 'colid' not in header
    assert 'rowid' not in header
    assert 'groupid' not in tail_snip
    assert 'colid' not in tail_snip
    assert 'rowid' not in tail_snip

    class_docstrings = doc_builder.get_documentation(write_report=False)
    header = header.replace('##CLASS_DOCSTRINGS##', class_docstrings)

    body = ''
    if class_type != 'Component':
        body = "        {}.__init__(self, type='{}', **params)\n".format(
            base_class, class_type.lower())
    if body_snip != '':
        body += body_snip

    if args_map != '':
        for l in args_map.split('\n'):
            (var_name, var_value) = l.split('=')
            var_name = var_name.strip()
            var_value = var_value.strip()
            body = body + '\n        self.{}={}'.format(var_name,
                                                        var_value)

    if class_type == 'Component':
        body = body + \
               indent("\nself.kind = '{}'".format(entrypoint['Kind']),
                      prefix=" " * tabsize * 2) + \
               indent("\nself.name = '{}'".format(entrypoint['Name']),
                      prefix=" " * tabsize * 2) + \
               indent('\nself.settings={}', prefix=" " * tabsize * 2)

        fun_settings_body = fun_settings_body.replace(
            'settings[', 'self.settings[')
        body = body + '\n\n' + indent(
            '    ' + fun_settings_body, prefix=" " * tabsize)
        tail = indent(
            '\n\nsuper(' + class_name +
            ', self).__init__(name=self.name, settings=self.settings,'
            ' kind=self.kind)', prefix=" " * 2 * tabsize)
    else:
        tail = dedent("""\n
            @property
            def _entrypoint(self):
                return {0}

            @trace
            def _get_node(self, **all_args):
            """.format(entrypoint_name))

        if class_type == 'Transform':
            tail += tail_snip0

        tail += "    algo_args = dict("

        if tail_snip != '':
            tail += tail_snip

        tail = indent(tail, prefix=" " * tabsize)

        if entrypoint_args_map != '':
            for l in entrypoint_args_map.split('\n'):
                (var_name, var_value) = l.split('=')
                var_name = var_name.strip()
                var_value = var_value.strip()
                if var_name != 'loss_function':
                    tail = tail + \
                           '\n            {}=self.{},'.format(var_name,
                                                              var_value)
                else:
                    tail = tail + \
                           '\n            {}={},'.format(var_name,
                                                         var_value)

        tail = tail.rstrip(',') + ")\n\n"
        tail = tail + "        all_args.update(algo_args)\n"
        if entrypoint_name == "transforms_pcacalculator":
            tail = tail + """
        return [conc, self._entrypoint(**all_args)] if conc\
 else self._entrypoint(**all_args)
            """
        else:
            tail = tail + "        return self._entrypoint(**all_args)"

    assert 'groupid' not in tail_snip
    assert 'groupid' not in tail
    assert 'groupid' not in body
    assert 'groupid' not in header
    assert 'groupid' not in module_doc

    custom_methods = core_custom_methods(class_name)
    to_write = "".join([banner, module_doc, import_lines,
                        header, body, tail, custom_methods])

    if len(to_write.split('expression=self.expression')) > 2:
        raise Exception(
            "-----entrypoint_args_map=\n{0}\n-----to_write=\n{1}".format(
                entrypoint_args_map, to_write))
    assert 'groupid' not in to_write
    assert 'colid' not in to_write
    assert 'rowid' not in to_write

    if not check_read_file(file, to_write):
        with open(file, "w") as myfile:
            print("    [write_core_class]", os.path.abspath(file))
            myfile.write(banner)
            myfile.write(module_doc)
            myfile.write(import_lines)
            myfile.write(header)
            myfile.write(body)
            myfile.write(tail)
            myfile.write(custom_methods)
    fix_code_core(class_name, file)


def core_custom_methods(class_name):
    #
    if class_name == "KMeansPlusPlus":
        return '''

    @trace
    def fit_predict(self, X, y=None, **params):
        """
        Fits and predicts.
        """
        self.fit(X, y=y, **params)
        return self.predict(X)
        '''
    else:
        return ""


def write_component_import(name, fun_file):
    # from .modules.linear_model.fastlinearregressor import
    # FastLinearRegressor
    file = os.path.join(pkg_path, "__init__.py")
    import_line = "from .entrypoints.{} import {}\n".format(
        fun_file.lower(), name.lower())
    with open(file, "a") as myfile:
        print("    [write_component_import]", os.path.abspath(file))
        myfile.write(import_line)


def create_py(entrypoint, kind="node"):
    # print('processing {} {}'.format(kind, entrypoint['name']))
    # name = entrypoint['Name']  # str
    # desc = entrypoint['Desc']  # str
    #
    funobjs = {}
    if kind == "node":
        inputs = entrypoint['Inputs']  # list
        outputs = entrypoint['Outputs']  # list
        funobjs['inputs'] = [parse_arg(input, 'inputs') for input in
                             inputs]
        funobjs['outputs'] = [parse_arg(output, 'outputs')
                              for output in outputs]
        funobjs['settings'] = funobjs['inputs'] + funobjs['outputs']
    else:
        settings = entrypoint['Settings']  # list
        # aliases = entrypoint['aliases']     #list
        funobjs['settings'] = [
            parse_arg(
                setting,
                'settings') for setting in settings]
        funobjs['inputs'] = funobjs['settings']

    return funobjs


def parse_arg(argument, inout):
    """
    """
    if ("column" in argument['Desc'].lower() and 'column' in argument[
        "Name"].lower(
    )) or argument["Name"] in ('Name', 'Source', 'Column', "Features"):
        is_column = True
    else:
        is_column = False

    arg_type = argument['Type']
    if arg_type in ["Int", "UInt", "Float"]:
        assert not is_column
        arg_obj = NumericScalarArg(argument, inout)
    elif arg_type in ["Bool"]:
        assert not is_column
        arg_obj = BooleanScalarArg(argument, inout)
    elif arg_type in ["String", "DataView", "PredictorModel",
                      "TransformModel", "FileHandle", "Char", "Bindings"]:
        arg_obj = StringScalarArg(argument, inout, is_column=is_column)
    elif isinstance(arg_type, dict):
        kind = arg_type['Kind']
        if kind == "Enum":
            assert not is_column
            arg_obj = EnumArg(argument, inout)
        elif kind == "Struct":
            if argument['Name'] in ['Inputs', 'Outputs']:
                arg_obj = StructSubgraphArg(
                    argument, inout, is_column=is_column)
            else:
                arg_obj = StructScalarArg(argument, inout,
                                          is_column=is_column)
        elif kind == "Array":
            itemType = arg_type['ItemType']
            if itemType in ["Int"]:
                assert not is_column
                arg_obj = NumericArrayArg(argument, inout)
            elif itemType in ["String", "DataView", "PredictorModel",
                              "TransformModel", "Node"]:
                arg_obj = StringArrayArg(argument, inout,
                                         is_column=is_column)
            elif isinstance(itemType, dict):
                arg_obj = StructArrayArg(argument, inout,
                                         is_column=is_column)
            else:
                raise TypeError(
                    "unknow itemType ({}) of argument kind ({}).".format(
                        itemType, kind))
        elif kind == "Dictionary":
            itemType = arg_type['itemType']
            if itemType == "String":
                arg_obj = StringDictionaryArg(argument, inout)
            else:
                raise TypeError(
                    "unknow itemType ({}) of argument kind ({}).".format(
                        itemType, kind))
        elif kind == "Component":
            componentKind = arg_type['ComponentKind']
            if componentKind in ["NetDefinition",
                                 "Optimizer",
                                 "EarlyStoppingCriterion",
                                 "MathPlatformKind",
                                 "MultiClassClassifierNetDefinition",
                                 "CalibratorTrainer",
                                 "ParallelTraining",
                                 "AnomalyKernel",
                                 "StopWordsRemover",
                                 "NgramExtractor",
                                 "CountTableBuilder",
                                 "DistributedTrainer",
                                 "BoosterParameterFunction",
                                 "ParallelLightGBM",
                                 "AutoMlEngine",
                                 "SearchTerminator"]:
                arg_obj = ComponentArg(argument, inout)
            elif componentKind in ["ClassificationLossFunction",
                                   "RegressionLossFunction",
                                   "SDCAClassificationLossFunction",
                                   "SDCARegressionLossFunction",
                                   "LossFunction"]:
                arg_obj = LossComponentArg(componentKind, argument, inout)
            else:
                raise TypeError(
                    "unknown componentKind ({}) of argument kind ({"
                    "}).".format(componentKind, kind))
        else:
            raise TypeError(
                "unknown kind ({}) of argument type ({}).".format(
                    kind, arg_type))
    else:
        raise TypeError("unknown argument type ({}).".format(arg_type))
    #
    return arg_obj


class Missing():
    """
    represent a missing object
    """

    def __str__(self):
        return ""


def quote(obj):
    """
    convert to a string
    """
    if isinstance(obj, str):
        res = "'{}'".format(obj)
    else:
        res = str(obj)
    return res


class Argument:
    """
    argument
    """

    def __init__(self, argument, inout):  # dict
        if not isinstance(argument, dict):
            raise ValueError("argument should be of type 'dict'.")
        self.inout = inout
        self.name = argument.get('Name', Missing())
        self.new_name = argument.get('NewName', Missing())
        self.hidden = argument.get('Hidden', Missing())
        self.type = argument.get('Type', Missing())
        self.desc = argument.get('Desc', Missing())
        self.range = argument.get('Range', Missing())
        self.default = argument.get('Default', Missing())
        self.required = argument.get('Required', Missing())
        self.aliases = argument.get('Aliases', Missing())

        self.name_converted = convert_name(self.name)
        self.new_name_converted = convert_name(
            self.name) if isinstance(
            self.new_name,
            Missing) else convert_name(
            self.new_name)
        self.name_assignment = self.new_name_converted
        self.name_core_assignment = self.new_name_converted
        # self.name_annotated = '{}: """{}"""'.format(self.name, self.type)
        self.name_annotated = '{}: {}'.format(
            self.new_name_converted, self.type_python)

        # NOTE: the default values specified in the
        # manifest.json for some inputs do not work.
        if self.name in ('WeightColumn', 'GroupIdColumn', 'GroupColumn'):
            self.default = None

    def __str__(self):
        return self.name

    def get_import(self):
        """
        get the import statement portion for the argument
        """
        pass

    @property
    def type_python(self):
        return


class NumericScalarArg(Argument):
    """
    argument of numeric scalar type
    """

    def get_arg(self):
        """
        get the function header portion for the argument
        """
        if isinstance(self.default, Missing):
            if self.required is True:
                arg = ' = '.join([self.new_name_converted, '0'])
            else:
                arg = ' = '.join([self.new_name_converted, str(None)])
        else:
            arg = ' = '.join([self.new_name_converted, str(self.default)])
        return arg

    def get_body(self):
        """
        get the function body portion for the argument
        """
        template = "if {name_converted} is not None:\n        {inout}['{" \
                   "name}'] = try_set(\nobj={name_converted},\n" \
                   "none_acceptable={none_acceptable},\n" \
                   "is_of_type=numbers.Real"
        body = template.format(
            inout=self.inout,
            name=self.name,
            name_converted=self.name_converted,
            none_acceptable=not self.required)
        if not isinstance(self.range, Missing):
            sorted_range = json.dumps(
                self.range, sort_keys=True).replace(
                '"', "'")
            range_check = ", valid_range={0}".format(sorted_range)
        else:
            range_check = ""
        return body + range_check + ")"

    @property
    def type_python(self):
        return "numbers.Real"


class BooleanScalarArg(NumericScalarArg):
    """
    argument of boolean scalar type
    """

    def get_body(self):
        """
        get the function body portion for the argument
        """
        template = "if {name_converted} is not None:\n        {inout}['{" \
                   "name}'] = try_set(obj={name_converted}, " \
                   "none_acceptable={none_acceptable}, is_of_type=bool"
        body = template.format(
            inout=self.inout,
            name=self.name,
            name_converted=self.name_converted,
            none_acceptable=not self.required)
        return body + ")"

    @property
    def type_python(self):
        return "bool"


class StringScalarArg(Argument):
    """
    argument of string scalar type
    """

    def __init__(self, argument, inout, is_column=False):
        Argument.__init__(self, argument, inout)
        self.is_column = is_column

    def get_arg(self):
        """
        get the function header portion for the argument
        """
        if isinstance(self.default, Missing):
            if self.required is True:
                arg = self.new_name_converted
            else:
                arg = ' = '.join([self.new_name_converted, str(None)])
        else:
            if self.default is None:
                arg = ' = '.join([self.new_name_converted, str(None)])
            else:
                arg = ' = '.join(
                    [self.new_name_converted, repr(str(self.default))])
        return arg

    def get_body(self):
        """
        get the function body portion for the argument
        """
        template = "if {name_converted} is not None:\n        {inout}['{" \
                   "name}'] = try_set(obj={name_converted}, " \
                   "none_acceptable={none_acceptable}, is_of_type=str"
        if self.is_column:
            template += ", is_column=True"
        body = template.format(
            inout=self.inout,
            name=self.name,
            name_converted=self.name_converted,
            none_acceptable=not self.required)
        return body + ")"

    @property
    def type_python(self):
        return "str"


class EnumArg(StringScalarArg):  # kind = 'Enum', values = []
    """
    argument of enum type
    """

    def get_body(self):
        """
        get the function body portion for the argument
        """
        template = "if {name_converted} is not None:\n        {inout}['{" \
                   "name}'] = try_set(obj={name_converted}, " \
                   "none_acceptable={none_acceptable}, is_of_type=str"
        body = template.format(
            inout=self.inout,
            name=self.name,
            name_converted=self.name_converted,
            none_acceptable=not self.required)
        value_check = ", values={0}".format(str(self.type['Values']))
        return body + value_check + ")"


class ArrayArg(Argument):
    """
    argument of array type
    """

    def get_arg(self):
        """
        get the function header portion for the argument
        """
        if isinstance(self.default, Missing):
            arg = self.new_name_converted
        else:
            arg = ' = '.join([self.new_name_converted, str(self.default)])
        return arg

    def get_body(self):
        """
        get the function body portion for the argument
        """
        template = "if {name_converted} is not None:\n        {inout}['{" \
                   "name}'] = try_set(obj={name_converted}, " \
                   "none_acceptable={none_acceptable}, is_of_type=list"
        body = template.format(
            inout=self.inout,
            name=self.name,
            name_converted=self.name_converted,
            none_acceptable=not self.required)
        return body + ")"

    @property
    def type_python(self):
        return "list"


class NumericArrayArg(ArrayArg):  # kind = Array, itemType = Int
    """
    argument of numeric array type
    """
    # TODO: check subtypes?
    pass


class StringArrayArg(ArrayArg):
    # kind = Array, itemType = String, DataView,
    # PredictorModel, TransformModel
    """
    argument of string array type
    """

    # TODO 2017-05-06: check subtypes?

    def __init__(self, argument, inout, is_column=False):
        ArrayArg.__init__(self, argument, inout)
        self.is_column = is_column

    def get_body(self):
        """
        get the function body portion for the argument
        """
        template = "if {name_converted} is not None:\n        {inout}['{" \
                   "name}'] = try_set(obj={name_converted}, " \
                   "none_acceptable={none_acceptable}, is_of_type=list"
        if self.is_column:
            template += ', is_column=True'
        body = template.format(
            inout=self.inout,
            name=self.name,
            name_converted=self.name_converted,
            none_acceptable=not self.required)
        return body + ")"


class StructArrayArg(ArrayArg):  # kind = Array, itemType = dict
    """
    argument of struct array type
    """

    # TODO 2017-05-06: check subtypes?

    def __init__(self, argument, inout, is_column=False):
        ArrayArg.__init__(self, argument, inout)
        self.is_column = is_column

    def get_body(self):
        """
        get the function body portion for the argument
        """
        template = "if {name_converted} is not None:\n        {inout}['{" \
                   "name}'] = try_set(obj={name_converted}, " \
                   "none_acceptable={none_acceptable}, is_of_type=list"
        if self.is_column:
            template += ', is_column=True'
        body = template.format(
            inout=self.inout,
            name=self.name,
            name_converted=self.name_converted,
            none_acceptable=not self.required)
        return body + ")"


class DictionaryArg(NumericScalarArg):
    """
    argument of dictionary type
    """

    def get_body(self):
        # NOTE 2017-06-20: a feature extractor has to be set to
        # None/null in order to disable it.
        # If not set, the default setting for those feature extractors
        # will get
        # used, which is not null.
        if self.name in ('CharFeatureExtractor', 'WordFeatureExtractor'):
            template = "{inout}['{name}'] = try_set(obj={" \
                       "name_converted}, none_acceptable={" \
                       "none_acceptable}, is_of_type=dict"
        else:
            template = "if {name_converted} is not None:\n        {" \
                       "inout}['{name}'] = try_set(obj={name_converted}, " \
                       "" \
                       "none_acceptable={none_acceptable}, is_of_type=dict"
        body = template.format(
            inout=self.inout,
            name=self.name,
            name_converted=self.name_converted,
            none_acceptable=not self.required)
        return body + ")"

    @property
    def type_python(self):
        return "dict"


class StringDictionaryArg(DictionaryArg):
    # kind = Dictionary, itemType = String
    """
    argument of string dictionary type
    """
    # TODO: check subtypes?
    pass


class StructScalarArg(DictionaryArg):
    """
    argument of scalar struct type
    """

    def __init__(self, argument, inout, is_column=False):
        DictionaryArg.__init__(self, argument, inout)
        self.is_column = is_column

    def get_body(self):
        template = "if {name_converted} is not None:\n        {inout}['{" \
                   "name}'] = try_set(obj={name_converted}, " \
                   "none_acceptable={none_acceptable}, is_of_type=dict"
        if self.is_column:
            template += ", is_column=True"
        body = template.format(
            inout=self.inout,
            name=self.name,
            name_converted=self.name_converted,
            none_acceptable=not self.required)
        field_check = ", field_names={0}".format(
            str([field['Name'] for field in self.type['Fields']]))
        return body + field_check + ")"


class StructSubgraphArg(StructScalarArg):
    """
    argument of struct type for subgraph input/output
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.name_converted = self.name_converted + '_subgraph'
        self.new_name_converted = self.name_converted


class ComponentArg(DictionaryArg):
    # kind = Component, componentKind =
    # NetDefinitionComponent, Optimizer, LossFunction,
    # EarlyStoppingCriterion, MathPlatformKind,
    # MultiClassClassifierNetDefinition
    """
    argument of component type
    """

    def get_arg(self):
        if isinstance(self.default, Missing):
            arg = self.new_name_converted
        else:
            if self.default is None:
                arg = ' = '.join(
                    [self.new_name_converted, str(self.default)])
            else:
                component_name = convert_name(self.default['Name'])
                if self.default.get("Settings") is None:
                    arg = '{} = None'.format(
                        self.new_name_converted)  # , component_name)
                else:
                    settings = self.default['Settings']
                    cargs = [' = '.join([convert_name(key), quote(
                        settings[key])]) for key in settings]
                    arg = '{} = {}({})'.format(
                        self.new_name_converted, component_name,
                        ', '.join(cargs))
        return arg

    def get_import(self, prefix="."):
        if not isinstance(self.default,
                          Missing) and self.default is not None:
            component_name = convert_name(self.default['Name'])
            fun_file = "_{}_{}".format(
                self.type['ComponentKind'].lower(),
                self.default['Name'].lower())
            return "from {}{} import {}".format(
                prefix, fun_file, component_name)


class LossComponentArg(ComponentArg):
    """
    argument of loss component type
    """

    def get_factory_statement(self, statement_type):
        func = statement_type + '_loss'
        return "{}('{}', self.__class__.__name__, self.loss)".format(
            func, self.componentKind)

    def __init__(self, componentKind, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.componentKind = componentKind
        self.name_assignment = self.get_factory_statement('create')
        self.name_core_assignment = "loss; {}".format(
            self.get_factory_statement('check'))
        if (self.new_name_converted == 'loss'):
            self.is_api = True
        else:
            # This is an argument for creating entrypoint class, not api or
            # core class
            self.is_api = False

    def get_arg(self):
        if self.is_api:
            default_name = get_loss_name(self.default['Name'])
            arg = "{} = '{}'".format(self.new_name_converted, default_name)
        else:
            arg = "{} = None".format(self.name_converted)
        return arg

    def get_import(self, prefix="."):
        if self.is_api:
            prefix = prefix.replace('entrypoints', 'core')
            imp = "from {}loss.loss_factory import create_loss, " \
                  "check_loss".format(prefix)
        else:
            imp = super().get_import(prefix)
        return imp


def generate_code(pkg_path, generate_entrypoints, generate_api):
    manifest_json = os.path.join(my_dir, r'manifest.json')
    manifest_diff_json = os.path.join(my_dir, r'manifest_diff.json')
    doc_path = os.path.join(my_dir, '..', 'docs', 'docstrings')
    DocBuilder.set_doc_path(doc_path)

    if verbose:
        print("manifest_json: {}".format(os.path.realpath(manifest_json)))
        print("manifest_diff_json: {}".format(
            os.path.realpath(manifest_diff_json)))
        print("pkg_path: {}".format(os.path.realpath(pkg_path)))
        print("doc_path: {}".format(os.path.realpath(doc_path)))

    if generate_entrypoints:
        if verbose:
            print("Generating entrypoint classes...")
        parse_manifest(manifest_json, pkg_path=pkg_path, overwrite=True)

    if generate_api:
        if verbose:
            print("Generating public and core API classes...")
        write_loss(manifest_json, manifest_diff_json, pkg_path)
        parse_manifest_diff(
            manifest_json,
            manifest_diff_json,
            pkg_path=pkg_path,
            overwrite=True)


arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
    '--generate_api',
    action='store_true',
    help='Generates both public API and core API classes.')
arg_parser.add_argument(
    '--generate_entrypoints',
    action='store_true',
    help='Generates entrypoint classes.')
arg_parser.add_argument(
    '--check_manual_changes',
    action='store_true',
    help='Generates all classes in a temp ' +
         'location and checks that codegen and local changes match.')
arg_parser.add_argument(
    '--folder',
    default="temp",
    help='Where to generate files for manual checking ' +
         '(temp means use a temporary folder created by the OS is '
         'temporary)')

if __name__ == '__main__':

    # NOTE: manifest.json has to be checked in as core
    # build does not generate it.

    script_args = arg_parser.parse_args()
    pkg_path = os.path.join(my_dir, r'..\nimbusml')

    if script_args.check_manual_changes:
        verbose = False
        if script_args.folder == 'temp':
            codegen_dir = tempfile.mkdtemp(prefix='ep_compiler_')
        else:
            codegen_dir = os.path.abspath(script_args.folder)
            if not os.path.exists(codegen_dir):
                os.makedirs(codegen_dir)
        generate_code(codegen_dir, True, True)
        passed = check_codegen(
            pkg_path,
            codegen_dir,
            remove=script_args.folder == 'temp')
        if not passed:
            # Notify cake about failure with non-zero value.
            exit(1)
    elif script_args.generate_api or script_args.generate_entrypoints:
        verbose = True
        generate_code(
            pkg_path,
            script_args.generate_entrypoints,
            script_args.generate_api)
    else:
        print(
            'No valid option was provided. Use --help to view the '
            'options. Exiting without any operation.')
