"""Helps build config packages for installer-specific templates.

Takes in a bunch of configuration files, as well as functions to calculate the values/strings which
need to be put into the configuration.

Operates strictly:
  - All parameters are strings. All things calculated / derived are strings.
  - Every given parameter must map to some real config option.
  - Every config option must be given only once.
  - Defaults can be overridden. If no default is given, the parameter must be specified
  - empty string is not the same as "not specified"
"""

import importlib.machinery
import json
import logging as log
import os
import os.path
import pprint
import textwrap
from copy import copy, deepcopy
from typing import List

import yaml

import gen.calc
import gen.exhibitor_tls_bootstrap
import gen.internals
import gen.template
import gen.util
import pkgpanda.exceptions
from gen.exceptions import ValidationError
from pkgpanda import PackageId
from pkgpanda.constants import (
    cloud_config_yaml,
    config_dir,
    dcos_config_yaml,
    dcos_services_yaml,
)
from pkgpanda.util import (
    hash_checkout,
    is_absolute_path,
    json_prettyprint,
    load_string,
    split_by_token,
    write_json,
    write_string,
    write_yaml,
)

# List of all roles all templates should have.
role_names = {"master", "slave", "slave_public"}

role_template = config_dir + '/roles/{}'

CLOUDCONFIG_KEYS = {'coreos', 'runcmd', 'apt_sources', 'root', 'mounts', 'disk_setup', 'fs_setup', 'bootcmd'}
PACKAGE_KEYS = {'package', 'root'}


# Allow overriding calculators with a `gen_extra/calc.py` if it exists
gen_extra_calc = None
if os.path.exists('gen_extra/calc.py'):
    gen_extra_calc = importlib.machinery.SourceFileLoader('gen_extra.calc', 'gen_extra/calc.py').load_module()


def validate_downstream_entry(entry: dict) -> None:
    """Raise an exception if entry is an invalid downstream gen.internals.Source entry."""
    version_key = 'dcos_version'
    entry_keys = set(entry.get('must', {}).keys()) | set(entry.get('default', {}).keys())
    if version_key in entry_keys:
        raise Exception(
            'The downstream entry redefines config param {}, which must be inherited from upstream'.format(version_key)
        )


def stringify_configuration(configuration: dict):
    """Create a stringified version of the complete installer configuration
    to send to gen.generate()"""
    gen_config = {}
    for key, value in configuration.items():
        if isinstance(value, list) or isinstance(value, dict):
            log.debug("Caught %s for genconf configuration, transforming to JSON string: %s", type(value), value)
            value = json.dumps(value)

        elif isinstance(value, bool):
            if value:
                value = 'true'
            else:
                value = 'false'

        elif isinstance(value, int):
            log.debug("Caught int for genconf configuration, transforming to string: %s", value)
            value = str(value)

        elif isinstance(value, str):
            pass

        else:
            log.error("Invalid type for value of %s in config. Got %s, only can handle list, dict, "
                      "int, bool, and str", key, type(value))
            raise Exception()

        gen_config[key] = value

    log.debug('Stringified configuration: \n{}'.format(gen_config))
    return gen_config


def add_roles(cloudconfig, roles):
    for role in roles:
        cloudconfig['write_files'].append({
            "path": role_template.format(role),
            "content": ""})

    return cloudconfig


def add_units(cloudconfig, services, cloud_init_implementation='coreos'):
    '''
    Takes a services dict in the format of CoreOS cloud-init 'units' and
    injects into cloudconfig a transformed version appropriate for the
    cloud_init_implementation.  See:
    https://coreos.com/os/docs/latest/cloud-config.html for the CoreOS 'units'
    specification. See: https://cloudinit.readthedocs.io/en/latest/index.html
    for the Canonical implementation.

    Parameters:
    * cloudconfig is a dict
    * services is a list of dict's
    * cloud_init_implementation is a string: 'coreos' or 'canonical'
    '''
    if cloud_init_implementation == 'canonical':
        cloudconfig.setdefault('write_files', [])
        cloudconfig.setdefault('runcmd', [])
        for unit in services:
            unit_name = unit['name']
            if 'content' in unit:
                write_files_entry = {'path': '/etc/systemd/system/{}'.format(unit_name),
                                     'content': unit['content'],
                                     'permissions': '0644'}
                cloudconfig['write_files'].append(write_files_entry)
            if 'enable' in unit and unit['enable']:
                runcmd_entry = ['systemctl', 'enable', unit_name]
                cloudconfig['runcmd'].append(runcmd_entry)
            if 'command' in unit:
                opts = []
                if 'no_block' in unit and unit['no_block']:
                    opts.append('--no-block')
                if unit['command'] in ['start', 'stop', 'reload', 'restart', 'try-restart', 'reload-or-restart',
                                       'reload-or-try-restart']:
                    runcmd_entry = ['systemctl'] + opts + [unit['command'], unit_name]
                else:
                    raise Exception("Unsupported unit command: {}".format(unit['command']))
                cloudconfig['runcmd'].append(runcmd_entry)
    elif cloud_init_implementation == 'coreos':
        cloudconfig.setdefault('coreos', {}).setdefault('units', [])
        cloudconfig['coreos']['units'] += services
    else:
        raise Exception("Parameter value '{}' is invalid for cloud_init_implementation".format(
            cloud_init_implementation))

    return cloudconfig


# For converting util -> a namespace only.
class Bunch(object):

    def __init__(self, adict):
        self.__dict__.update(adict)


def render_cloudconfig(data):
    return "#cloud-config\n" + render_yaml(data)


utils = Bunch({
    "role_template": role_template,
    "add_roles": add_roles,
    "role_names": role_names,
    "add_services": None,
    "add_stable_artifact": None,
    "add_channel_artifact": None,
    "add_units": add_units,
    "render_cloudconfig": render_cloudconfig
})


def render_yaml(data):
    return yaml.dump(data, default_style='|', default_flow_style=False)


# Recursively merge to python dictionaries.
# If both base and addition contain the same key, that key's value will be
# merged if it is a dictionary.
# This is unlike the python dict.update() method which just overwrites matching
# keys.
def merge_dictionaries(base, additions):
    base_copy = base.copy()
    for k, v in additions.items():
        try:
            if k not in base:
                base_copy[k] = v
                continue
            if isinstance(v, dict) and isinstance(base_copy[k], dict):
                base_copy[k] = merge_dictionaries(base_copy.get(k, dict()), v)
                continue

            # Append arrays
            if isinstance(v, list) and isinstance(base_copy[k], list):
                base_copy[k].extend(v)
                continue

            # Merge sets
            if isinstance(v, set) and isinstance(base_copy[k], set):
                base_copy[k] |= v
                continue

            # Unknown types
            raise ValueError("Can't merge type {} into type {}".format(type(v), type(base_copy[k])))
        except ValueError as ex:
            raise ValueError("{} inside key {}".format(ex, k)) from ex
    return base_copy


def load_templates(template_dict):
    result = dict()
    for name, template_list in template_dict.items():
        result_list = list()
        for template_name in template_list:
            result_list.append(gen.template.parse_resources(template_name))

            extra_filename = "gen_extra/" + template_name
            if os.path.exists(extra_filename):
                result_list.append(gen.template.parse_str(
                    load_string(extra_filename)))
        result[name] = result_list
    return result


# Render the Jinja/YAML into YAML, then load the YAML and merge it to make the
# final configuration files.
def render_templates(template_dict, arguments):
    rendered_templates = dict()
    templates = load_templates(template_dict)
    for name, templates in templates.items():
        full_template = None
        for template in templates:
            rendered_template = template.render(arguments)

            # If not yaml, just treat opaquely.
            if not name.endswith('.yaml'):
                # No merging support currently.
                assert len(templates) == 1
                full_template = rendered_template
                continue
            template_data = yaml.safe_load(rendered_template)

            if full_template:
                full_template = merge_dictionaries(full_template, template_data)
            else:
                full_template = template_data

        rendered_templates[name] = full_template

    return rendered_templates


# Collect the un-bound / un-set variables from all the given templates to build
# the schema / configuration target. The templates and their structure serve
# as the schema for what configuration a user must provide.
def target_from_templates(template_dict):
    # NOTE: the individual yaml template targets are merged into one target
    # since we never want to target just one template at a time for now (they
    # all merge into one config package).
    target = gen.internals.Target()
    templates = load_templates(template_dict)
    for template_list in templates.values():
        for template in template_list:
            target += template.target_from_ast()

    return [target]


def write_to_non_taken(base_filename, json):
    number = 0

    filename = base_filename
    while (os.path.exists(filename)):
        number += 1
        filename = base_filename + '.{}'.format(number)

    write_json(filename, json)

    return filename


def do_gen_package(config, package_filename):
    # Generate the specific dcos-config package.
    # Version will be setup-{sha1 of contents}
    with gen.util.pkgpanda_package_tmpdir() as tmpdir:
        # Only contains package, root
        assert config.keys() == {"package"}

        # Write out the individual files
        for file_info in config["package"]:
            assert file_info.keys() <= {"path", "content", "permissions"}
            if is_absolute_path(file_info['path']):
                fileinfo_drive, fileinfo_path = os.path.splitdrive(file_info['path'])
                path = tmpdir + fileinfo_path
            else:
                path = tmpdir + '/' + file_info['path']
            try:
                if os.path.dirname(path):
                    os.makedirs(os.path.dirname(path), mode=0o755)
            except FileExistsError:
                pass

            with open(path, 'w') as f:
                f.write(file_info['content'] or '')

            # the file has special mode defined, handle that.
            if 'permissions' in file_info:
                assert isinstance(file_info['permissions'], str)
                os.chmod(path, int(file_info['permissions'], 8))
            else:
                os.chmod(path, 0o644)

        gen.util.make_pkgpanda_package(tmpdir, package_filename)


def render_late_content(content, late_values):

    def _dereference_placeholders(parts):
        for part, is_placeholder in parts:
            if is_placeholder:
                if part not in late_values:
                    log.debug('Found placeholder for unknown value "{}" in late config: {}'.format(part, repr(content)))
                    raise Exception('Bad late config file: Found placeholder for unknown value "{}"'.format(part))
                yield late_values[part]
            else:
                yield part

    return ''.join(_dereference_placeholders(split_by_token(
        gen.internals.LATE_BIND_PLACEHOLDER_START,
        gen.internals.LATE_BIND_PLACEHOLDER_END,
        content,
        strip_token_decoration=True,
    )))


def _late_bind_placeholder_in(string_):
    return gen.internals.LATE_BIND_PLACEHOLDER_START in string_ or gen.internals.LATE_BIND_PLACEHOLDER_END in string_


def resolve_late_package(config, late_values):
    resolved_config = {
        'package': [
            {k: render_late_content(v, late_values) if k == 'content' else v for k, v in file_info.items()}
            for file_info in config['package']
        ]
    }

    assert not any(
        _late_bind_placeholder_in(v) for file_info in resolved_config['package'] for v in file_info.values()
    ), 'Resolved late package must not contain late value placeholder: {}'.format(resolved_config)

    return resolved_config


def extract_files_containing_late_variables(start_files):
    found_files = []
    left_files = []

    for file_info in deepcopy(start_files):
        assert not any(_late_bind_placeholder_in(v) for k, v in file_info.items() if k != 'content'), (
            'File info must not contain late config placeholder in fields other than content: {}'.format(file_info)
        )

        if file_info['content'] and _late_bind_placeholder_in(file_info['content']):
            found_files.append(file_info)
        else:
            left_files.append(file_info)

    # All files still belong somewhere
    assert len(found_files) + len(left_files) == len(start_files)

    return found_files, left_files


# Validate all arguments passed in actually correspond to parameters to
# prevent human typo errors.
# This includes all possible sub scopes (Including config for things you don't use is fine).
def flatten_parameters(scoped_parameters):
    flat = copy(scoped_parameters.get('variables', set()))
    for name, possible_values in scoped_parameters.get('sub_scopes', dict()).items():
        flat.add(name)
        for sub_scope in possible_values.values():
            flat |= flatten_parameters(sub_scope)

    return flat


def validate_all_arguments_match_parameters(parameters, setters, arguments):
    errors = dict()

    # Gather all possible parameters from templates as well as setter parameters.
    all_parameters = flatten_parameters(parameters)
    for setter_list in setters.values():
        for setter in setter_list:
            all_parameters |= setter.parameters
            all_parameters.add(setter.name)
            all_parameters |= {name for name, value in setter.conditions}

    # Check every argument is in the set of parameters.
    for argument in arguments:
        if argument not in all_parameters:
            errors[argument] = 'Argument {} given but not in possible parameters {}'.format(argument, all_parameters)

    if len(errors):
        raise ValidationError(errors, set())


def validate(
        arguments,
        extra_templates=list(),
        extra_sources=list()):
    sources, targets, _ = get_dcosconfig_source_target_and_templates(arguments, extra_templates, extra_sources)
    return gen.internals.resolve_configuration(sources, targets).status_dict


def user_arguments_to_source(user_arguments) -> gen.internals.Source:
    """Convert all user arguments to be a gen.internals.Source"""

    # Make sure all user provided arguments are strings.
    # TODO(cmaloney): Loosen this restriction  / allow arbitrary types as long
    # as they all have a gen specific string form.
    gen.internals.validate_arguments_strings(user_arguments)

    user_source = gen.internals.Source(is_user=True)
    for name, value in user_arguments.items():
        user_source.add_must(name, value)
    return user_source


# TODO(cmaloney): This function should disolve away like the ssh one is and just become a big
# static dictonary or pass in / construct on the fly at the various template callsites.
def get_dcosconfig_source_target_and_templates(
        user_arguments: dict,
        extra_templates: List[str],
        extra_sources: List[gen.internals.Source]):
    log.info("Generating configuration files...")

    # TODO(cmaloney): Make these all just defined by the base calc.py
    config_package_names = ['dcos-config', 'dcos-metadata']

    template_filenames = [dcos_config_yaml, cloud_config_yaml, 'dcos-metadata.yaml', dcos_services_yaml]

    # TODO(cmaloney): Check there are no duplicates between templates and extra_template_files
    template_filenames += extra_templates

    # Re-arrange templates to be indexed by common name. Only allow multiple for one key if the key
    # is yaml (ends in .yaml).
    templates = dict()
    for filename in template_filenames:
        key = os.path.basename(filename)
        templates.setdefault(key, list())
        templates[key].append(filename)

        if len(templates[key]) > 1 and not key.endswith('.yaml'):
            raise Exception(
                "Internal Error: Only know how to merge YAML templates at this point in time. "
                "Can't merge template {} in template_list {}".format(filename, templates[key]))

    # Include a base target that references variables we need to calculate cluster_packages.
    base_target = gen.internals.Target({
        'config_package_names',
        'dcos_image_commit',
        'package_ids',
        'template_filenames',
    })
    targets = [base_target] + target_from_templates(templates)
    base_source = gen.internals.Source(is_user=False)
    base_source.add_entry(gen.calc.entry, replace_existing=False)

    if gen_extra_calc:
        validate_downstream_entry(gen_extra_calc.entry)
        base_source.add_entry(gen_extra_calc.entry, replace_existing=True)

    def add_builtin(name, value):
        base_source.add_must(name, json_prettyprint(value))

    sources = [base_source, user_arguments_to_source(user_arguments)] + extra_sources

    # Add builtin variables.
    # TODO(cmaloney): Hash the contents of all the templates rather than using the list of filenames
    # since the filenames might not live in this git repo, or may be locally modified.
    add_builtin('template_filenames', template_filenames)
    add_builtin('config_package_names', list(config_package_names))

    # Add placeholders for builtin variables whose values will be calculated after all others, so that we won't get
    # unset argument errors. The placeholder value with be replaced with the actual value after all other variables are
    # calculated.
    temporary_str = 'DO NOT USE THIS AS AN ARGUMENT TO OTHER ARGUMENTS. IT IS TEMPORARY'
    add_builtin('cluster_packages', temporary_str)
    add_builtin('cluster_package_list_id', temporary_str)
    add_builtin('user_arguments_full', temporary_str)
    add_builtin('user_arguments', temporary_str)
    add_builtin('config_yaml_full', temporary_str)
    add_builtin('config_yaml', temporary_str)
    add_builtin('expanded_config', temporary_str)
    add_builtin('expanded_config_full', temporary_str)

    return sources, targets, templates


def build_late_package(late_files, config_id, provider):
    if not late_files:
        return None

    # Add a empty pkginfo.json to the late package after validating there
    # isn't already one.
    for file_info in late_files:
        assert file_info['path'] != '/pkginfo.json'
        assert is_absolute_path(file_info['path'])

    late_files.append({
        "path": "/pkginfo.json",
        "content": "{}"})

    return {
        'package': late_files,
        'name': 'dcos-provider-{}-{}--setup'.format(config_id, provider)
    }


def validate_and_raise(sources, targets):
    # TODO(cmaloney): Make it so we only get out the dcosconfig target arguments not all the config target arguments.
    resolver = gen.internals.resolve_configuration(sources, targets)
    status = resolver.status_dict

    if status['status'] == 'errors':
        raise ValidationError(errors=status['errors'], unset=status['unset'])

    return resolver


def get_late_variables(resolver, sources):
    # Gather out the late variables. The presence of late variables changes
    # whether or not a late package is created
    late_variables = dict()
    # TODO(branden): Get the late vars and expressions from resolver.late
    for source in sources:
        for setter_list in source.setters.values():
            for setter in setter_list:
                if not setter.is_late:
                    continue

                if setter.name not in resolver.late:
                    continue

                # Skip late vars that aren't referenced by config.
                if not resolver.arguments[setter.name].is_finalized:
                    continue

                # Validate a late variable should only have one source.
                assert setter.name not in late_variables

                late_variables[setter.name] = setter.late_expression
    log.debug('Late variables:\n{}'.format(pprint.pformat(late_variables)))

    return late_variables


def get_secret_variables(sources):
    return list(set(var_name for source in sources for var_name in source.secret))


def get_final_arguments(resolver):
    return {k: v.value for k, v in resolver.arguments.items() if v.is_finalized}


def format_expanded_config(config):
    return textwrap.indent(json_prettyprint(config), prefix=('  ' * 3))


def user_arguments_to_yaml(user_arguments: dict):
    return textwrap.indent(
        yaml.dump(user_arguments, default_style='|', default_flow_style=False, indent=2),
        prefix=('  ' * 3),
    )


def validate_cluster_packages(cluster_packages):
    for pkg_id in cluster_packages:
        try:
            PackageId(pkg_id)
        except pkgpanda.exceptions.ValidationError as ex:
            raise Exception('Invalid cluster package ID: {}'.format(str(ex))) from ex


def get_config_id(argument_dict: dict):
    """Return a unique ID for the configuration represented by argument_dict."""
    # dcos_image_commit and template_filenames should be included in argument_dict, but we reference them explicitly to
    # ensure that the config ID reflects the current commit and config templates.
    return hash_checkout({
        'commit': argument_dict['dcos_image_commit'],
        'template_filenames': argument_dict['template_filenames'],
        'argument_dict': argument_dict,
    })


def generate(
        arguments,
        extra_templates=list(),
        extra_sources=list(),
        extra_targets=list()):
    # To maintain the old API where we passed arguments rather than the new name.
    user_arguments = arguments
    arguments = None

    sources, targets, templates = get_dcosconfig_source_target_and_templates(
        user_arguments, extra_templates, extra_sources)

    resolver = validate_and_raise(sources, targets + extra_targets)
    argument_dict = get_final_arguments(resolver)
    late_variables = get_late_variables(resolver, sources)
    secret_builtins = ['expanded_config_full', 'user_arguments_full', 'config_yaml_full']
    secret_variables = set(get_secret_variables(sources) + secret_builtins)
    masked_value = '**HIDDEN**'

    # Calculate config ID after all variables are resolved, to make sure any change in config yields a new config ID.
    config_id = get_config_id(argument_dict)

    # Calculate values that depend on the config ID.
    config_package_names = json.loads(argument_dict['config_package_names'])
    package_ids = json.loads(argument_dict['package_ids'])
    config_package_ids = ['{}--setup_{}'.format(name, config_id) for name in config_package_names]
    cluster_packages = sorted(config_package_ids + package_ids)
    validate_cluster_packages(cluster_packages)
    cluster_package_list_id = hash_checkout(cluster_packages)

    # Calculate values for builtin variables.
    argument_dict['cluster_packages'] = json.dumps(cluster_packages)
    argument_dict['cluster_package_list_id'] = cluster_package_list_id
    user_arguments_masked = {k: (masked_value if k in secret_variables else v) for k, v in user_arguments.items()}
    argument_dict['user_arguments_full'] = json_prettyprint(user_arguments)
    argument_dict['user_arguments'] = json_prettyprint(user_arguments_masked)
    argument_dict['config_yaml_full'] = user_arguments_to_yaml(user_arguments)
    argument_dict['config_yaml'] = user_arguments_to_yaml(user_arguments_masked)

    # The expanded_config and expanded_config_full variables contain all other variables and their values.
    # expanded_config is a copy of expanded_config_full with secret values removed. Calculating these variables' values
    # must come after the calculation of all other variables to prevent infinite recursion.
    # TODO(cmaloney): Make this late-bound by gen.internals
    expanded_config_full = {
        k: v for k, v in argument_dict.items()
        # Omit late-bound variables whose values have not yet been calculated.
        if not v.startswith(gen.internals.LATE_BIND_PLACEHOLDER_START)
    }
    expanded_config_scrubbed = {k: v for k, v in expanded_config_full.items() if k not in secret_variables}
    argument_dict['expanded_config_full'] = format_expanded_config(expanded_config_full)
    argument_dict['expanded_config'] = format_expanded_config(expanded_config_scrubbed)

    # Initialize CA and add arguments (exhibitor_ca_certificate and exhibitor_ca_certificate_path)
    gen.exhibitor_tls_bootstrap.initialize_exhibitor_ca(argument_dict)

    log.debug(
        "Final arguments:" + json_prettyprint({
            # Mask secret config values.
            k: (masked_value if k in secret_variables else v) for k, v in argument_dict.items()
        })
    )

    # Fill in the template parameters
    # TODO(cmaloney): render_templates should ideally take the template targets.
    rendered_templates = render_templates(templates, argument_dict)

    # Validate there aren't any unexpected top level directives in any of the files
    # (likely indicates a misspelling)
    for name, template in rendered_templates.items():
        if name == dcos_services_yaml:  # yaml list of the service files
            assert isinstance(template, list)
        elif name == cloud_config_yaml:
            assert template.keys() <= CLOUDCONFIG_KEYS, template.keys()
        elif isinstance(template, str):  # Not a yaml template
            pass
        else:  # yaml template file
            log.debug("validating template file %s", name)
            assert template.keys() <= PACKAGE_KEYS, template.keys()

    stable_artifacts = []
    channel_artifacts = []

    # Find all files which contain late bind variables and turn them into a "late bind package"
    # TODO(cmaloney): check there are no late bound variables in cloud-config.yaml
    late_files, regular_files = extract_files_containing_late_variables(
        rendered_templates[dcos_config_yaml]['package'])
    # put the regular files right back
    rendered_templates[dcos_config_yaml] = {'package': regular_files}

    # Render cluster package list artifact.
    cluster_package_list_filename = 'package_lists/{}.package_list.json'.format(cluster_package_list_id)
    os.makedirs(os.path.dirname(cluster_package_list_filename), mode=0o755, exist_ok=True)
    write_string(cluster_package_list_filename, argument_dict['cluster_packages'])
    log.info('Cluster package list: {}'.format(cluster_package_list_filename))
    stable_artifacts.append(cluster_package_list_filename)

    def make_package_filename(package_id, extension):
        return 'packages/{0}/{1}{2}'.format(
            package_id.name,
            repr(package_id),
            extension)

    # Render all the cluster packages
    cluster_package_info = {}

    # Prepare late binding config, if any.
    late_package = build_late_package(late_files, config_id, argument_dict['provider'])
    if late_variables and late_package:
        # Render the late binding package. This package will be downloaded onto
        # each cluster node during bootstrap and rendered into the final config
        # using the values from the late config file.
        late_package_id = PackageId(late_package['name'])
        late_package_filename = make_package_filename(late_package_id, '.dcos_config')
        os.makedirs(os.path.dirname(late_package_filename), mode=0o755, exist_ok=True)
        write_yaml(late_package_filename, {'package': late_package['package']}, default_flow_style=False)
        log.info('Package filename: {}'.format(late_package_filename))
        stable_artifacts.append(late_package_filename)

        # Add the late config file to cloud config. The expressions in
        # late_variables will be resolved by the service handling the cloud
        # config (e.g. Amazon CloudFormation). The rendered late config file
        # on a cluster node's filesystem will contain the final values.
        rendered_templates[cloud_config_yaml]['root'].append({
            'path': config_dir + '/setup-flags/late-config.yaml',
            'permissions': '0644',
            'owner': 'root',
            # TODO(cmaloney): don't prettyprint to save bytes.
            # NOTE: Use yaml here simply to make avoiding painful escaping and
            # unescaping easier.
            'content': render_yaml({
                'late_bound_package_id': late_package['name'],
                'bound_values': late_variables
            })})

    # Collect metadata for cluster packages.
    for package_id_str in cluster_packages:
        package_id = PackageId(package_id_str)
        package_filename = make_package_filename(package_id, '.tar.xz')

        cluster_package_info[package_id.name] = {
            'id': package_id_str,
            'filename': package_filename
        }

    # Render config packages.
    for package_id_str in config_package_ids:
        package_id = PackageId(package_id_str)
        package_filename = cluster_package_info[package_id.name]['filename']
        do_gen_package(rendered_templates[package_id.name + '.yaml'], cluster_package_info[package_id.name]['filename'])
        stable_artifacts.append(package_filename)

    # Convert cloud-config to just contain write_files rather than root
    cc = rendered_templates[cloud_config_yaml]

    # Shouldn't contain any packages. Providers should pull what they need to
    # late bind out of other packages via cc_package_file.
    assert 'package' not in cc
    cc_root = cc.pop('root', [])
    # Make sure write_files exists.
    assert 'write_files' not in cc
    cc['write_files'] = []
    # Do the transform
    for item in cc_root:
        assert is_absolute_path(item['path'])
        cc['write_files'].append(item)
    rendered_templates[cloud_config_yaml] = cc

    # Add utils that need to be defined here so they can be bound to locals.
    def add_services(cloudconfig, cloud_init_implementation):
        return add_units(cloudconfig, rendered_templates[dcos_services_yaml], cloud_init_implementation)

    utils.add_services = add_services

    def add_stable_artifact(filename):
        assert filename not in stable_artifacts + channel_artifacts
        stable_artifacts.append(filename)

    utils.add_stable_artifact = add_stable_artifact

    def add_channel_artifact(filename):
        assert filename not in stable_artifacts + channel_artifacts
        channel_artifacts.append(filename)

    utils.add_channel_artifact = add_channel_artifact

    return Bunch({
        'arguments': argument_dict,
        'cluster_packages': cluster_package_info,
        'stable_artifacts': stable_artifacts,
        'channel_artifacts': channel_artifacts,
        'templates': rendered_templates,
        'utils': utils
    })
