#============================================================================ 
#Name        : mappers.py 
#Part of     : Helium 

#Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
#All rights reserved.
#This component and the accompanying materials are made available
#under the terms of the License "Eclipse Public License v1.0"
#which accompanies this distribution, and is available
#at the URL "http://www.eclipse.org/legal/epl-v10.html".
#
#Initial Contributors:
#Nokia Corporation - initial contribution.
#
#Contributors:
#
#Description:
#===============================================================================

""" Archive mappers that map how the input files are divided into archives.


"""

import buildtools
import os
import codecs
import fileutils
import logging
import symrec
import re
import csv
import shutil

_logger = logging.getLogger('logger.mappers')
_logger.setLevel(logging.INFO)

# Default value for missing/invalid policy files.
MISSING_POLICY = "9999"


class Mapper(object):
    """ Mapper Abstract class. Any custom implementation must derive it!.
    
    It handles metadata creation.
    """
    
    def __init__(self, config, tool):
        self._tool = tool
        self._config = config
        self._metadata = None
        if not os.path.exists(self._config['archives.dir']):
            os.makedirs(self._config['archives.dir'])
        
        if self._config.has_key("grace.metadata"):
            raise Exception('grace.metadata not supported, see documentation for correct configuration')
        
        if self._config.has_key("release.metadata") and self._config.get_boolean("release.metadata", False):
            if self._config.has_key("release.template") and os.path.exists(self._config["release.template"]) and \
             not os.path.exists(os.path.join(self._config['archives.dir'], self._config['name'] + ".metadata.xml")):
                shutil.copy(config["release.template"], os.path.join(self._config['archives.dir'], self._config['name'] + ".metadata.xml"))
            self._metadata = symrec.ReleaseMetadata(os.path.join(self._config['archives.dir'], self._config['name']+ ".metadata.xml"),
                                       service=self._config['release.service'],
                                       product=self._config['release.product'],
                                       release=self._config['release.name'])
            self._metadata.save()            
        
    def declare_package(self, filename, extract="single"):
        """ Add a package to the metadata file. """
        if self._metadata is None:
            return
        self._metadata.add_package(os.path.basename(filename), extract=extract, filters=self._config.get_list('release.filters', None), default=self._config.get_boolean('release.default', True))
        self._metadata.save()
    
    def create_commands(self, manifest):
        """ Return a list of command list. """
        return [[self._tool.create_command(self._config['name'], manifests=[manifest])]]
        
    
class DefaultMapper(Mapper):
    """ The default mapper. It splits the content based on size characteristics.
    
    'the max.files.per.archive' and 'max.uncompressed.size' properties define how the input files
    are split between a number of part zips.
    """
    def __init__(self, config, archiver):
        """ Initialization. """
        Mapper.__init__(self, config, archiver)

    def create_commands(self, manifest):
        """ Return a list of command lists. """
        result = []

        _logger.info("  * Input manifest: " + manifest)
        manifests = self._split_manifest_file(self._config['name'], manifest)
        if not os.path.exists(self._config['archives.dir']):
            _logger.info("  * Mkdir " + self._config['archives.dir'])
            os.makedirs(self._config['archives.dir'])

        for manifest in manifests:
            _logger.info("  * Creating command for manifest: " + manifest)
            filename = os.path.join(self._config['archives.dir'], os.path.splitext(os.path.basename(manifest))[0])
            if len(manifests) == 1:
                filename = os.path.join(self._config['archives.dir'], self._config['name'])
            _logger.info("  * " + filename + self._tool.extension())
            self.declare_package(filename + self._tool.extension(), self._config.get('release.extract', 'single'))
            result.extend(self._tool.create_command(self._config.get('zip.root.dir', self._config['root.dir']), filename, manifests=[manifest]))
        
        return [result]

    def _split_manifest_file(self, name, manifest_file_path, key=None):
        """ This method return a list of files that contain the content of the zip parts to create. """
        filenames = []
        
        if (self._config.has_key('max.files.per.archive') or self._config.has_key('max.uncompressed.size')):
            size = 0
            files = 0
            part = 0
            filename = ""
            output = None
                        
            if os.path.exists(self._config['root.dir']) and os.path.isdir(self._config['root.dir']):
                curdir = os.path.abspath(os.curdir)
                os.chdir(self._config.get('zip.root.dir', self._config['root.dir']))            
                maxfiles = self._config.get('max.files.per.archive', 100000000)
                _logger.info("Max number of files per archive: " + str(maxfiles))
                max_uncompressed_size = self._config.get('max.uncompressed.size', 100000000)
                _logger.info("Max uncompressed size per archive: " + str(max_uncompressed_size))
                
                file_handle = codecs.open(manifest_file_path, "r", "utf-8" )
        
                for line in file_handle.readlines():
                    line = line.rstrip()
        
                    if(os.path.isfile(line)):
                        if part == 0 or files == int(maxfiles) or size + os.path.getsize(line) >= int(max_uncompressed_size):
                            if output != None:
                                output.close()
        
                            size = 0
                            files = 0
                            part += 1
        
                            if key is not None:
                                filename = "%s_part%02d_%s" % (name, part, key)
                            else:
                                filename = "%s_part%02d" % (name, part)
                            filenames.append(os.path.join(self._config['temp.build.dir'], filename + ".txt"))
        
                            output = codecs.open(os.path.join(self._config['temp.build.dir'], filename + ".txt"), 'w', "utf-8" )
        
                        files += 1
                        size += os.path.getsize(line)
        
                        output.write(u"".join([line, u'\n']))
                    elif(os.path.isdir(line)):
                        if (len(os.listdir(line)) == 0):
                            if part == 0 or files == int(maxfiles):
                                if output != None:
                                    output.close()
        
                                size = 0
                                files = 0
                                part += 1
        
                                if key is not None:
                                    filename = "%s_part%02d_%s" % (name, part, key)
                                else:
                                    filename = "%s_part%02d" % (name, part)
                                filenames.append(os.path.join(self._config['temp.build.dir'], filename + ".txt"))
                                
                                output = open(os.path.abspath(os.path.join(self._config['temp.build.dir'], filename + ".txt")), 'w')
        
                            files += 1
        
                            output.write(u"".join([line, u'\n']))
                    else:
                        _logger.warning('Not recognized as file or directory: %s' % line)
        
                if output != None:
                    output.close()
        
                file_handle.close()
                os.chdir(curdir)
        else:
            filenames.append(manifest_file_path)
        
        return filenames


class PolicyMapper(DefaultMapper):
    """ Implements a policy content mapper.
    
    It transforms a list of files into a list of commands with their inputs.
    All files with policy 0 will be under the main archive.
    All other files will get backed up by policy and then store into an second archive. 
    """
    
    def __init__(self, config, archiver):
        """ Initialization. """
        DefaultMapper.__init__(self, config, archiver)
        self._policies = {}
        self._policy_cache = {}
        self._binary = {}
        # Load csv
        if self._config.has_key('policy.csv'):
            if os.path.exists(self._config['policy.csv']):
                self.load_policy_binary(self._config['policy.csv'])
            else:
                _logger.error("POLICY_ERROR: File not found '%s'." % self._config['policy.csv'])

    def load_policy_binary(self, csvfile, column=1):
        """ Loads the binary IDs from the CSV file. """
        _logger.info("POLICY_INFO: Loading policy definition '%s'." % csvfile)
        reader = csv.reader(open(csvfile, "rU"))
        for row in reader:
            if re.match(r"^((?:\d+)|(?:0842[0-9a-zA-Z]{3}))$", row[0].strip()):                
                _logger.info("POLICY_INFO: Adding policy: '%s' => '%s'" % (row[0].strip(), row[column].strip().lower()))
                self._binary[row[0].strip()] = row[column].strip().lower()
            else:
                _logger.warning("POLICY_WARNING: Discarding policy: '%s'." % row[0].strip())

    def zip2zip(self):
        """ Should the non public zip be zipped up under a specific zip. """
        return self._config.get_boolean('policy.zip2zip', False)
    
    def create_commands(self, manifest):
        """ Generates a list of build commands. """
        result = []
        stages = []

        # Create the archive output directory
        if not os.path.exists(self._config['archives.dir']):
            _logger.info("  * Mkdir " + self._config['archives.dir'])
            os.makedirs(self._config['archives.dir'])
        
        # Sort the manifest content, splitting it by policy
        file_handle = codecs.open(manifest, "r", "utf-8")
        for line in file_handle.readlines():
            line = line.rstrip()
            self._sort_by_policy(line)
        file_handle.close()
        
        # Generating sublists.
        for key in self._policies.keys():
            manifests = []
            self._policies[key].close()
            manifest = os.path.join(self._config['temp.build.dir'], self._config['name'] + "_%s" % key + ".txt")
            _logger.info("  * Input manifest: " + manifest)
            if self._config.has_key("split.on.uncompressed.size.enabled") and self._config.get_boolean("split.on.uncompressed.size.enabled", "false"):
                manifests = self._split_manifest_file(self._config['name'], manifest, key)
            else:
                manifests.append(manifest)
            for manifest in manifests:
                _logger.info("  * Creating command for manifest: " + manifest)
                filename = os.path.join(self._config['archives.dir'], os.path.splitext(os.path.basename(manifest))[0])
                if len(manifests) == 1:
                    filename = os.path.join(self._config['archives.dir'], self._config['name'] + "_%s" % key)
                _logger.info("  * " + filename + self._tool.extension())
                self.declare_package(filename + self._tool.extension(), self._config.get('release.extract', 'single'))
                result.extend(self._tool.create_command(self._config.get('zip.root.dir', self._config['root.dir']), filename, manifests=[manifest]))
        stages.append(result)
        
        # See if any internal archives need to be created
        content = []
        for key in self._policies.keys():
            if not self.zip2zip():
                self.declare_package(self._config['name'] + "_%s" % key + self._tool.extension())
            else:
                if key != "0":
                    content.append(os.path.join(self._config['archives.dir'], self._config['name'] + "_%s" % key + self._tool.extension()))
                else:
                    self.declare_package(self._config['name'] + "_%s" % key + self._tool.extension())

        # Creating zip that contains each policy zips.
        if self.zip2zip() and len(content) > 0:
            manifest = os.path.join(self._config['temp.build.dir'], self._config['name'] +  ".internal.txt")
            file_handle = codecs.open( manifest, "w+", "utf-8" )
            file_handle.write(u"\n".join(content))
            file_handle.close()
            internal = "internal"
            if self._config.has_key('policy.internal.name'):
                internal = self._config['policy.internal.name']
            filename = os.path.join(self._config['archives.dir'], self._config['name'] +  "_" + internal)
            _logger.info("  * " + filename + self._tool.extension())
            self.declare_package(filename + self._tool.extension(), "double")
            stages.append(self._tool.create_command(self._config['archives.dir'], filename, manifests=[manifest]))

            cmds = []
            for filename in content:
                cmds.append(buildtools.Delete(filename=filename))
            stages.append(cmds)
        return stages
    
    def get_dir_policy(self, dirname):
        """ Get policy value for a specific directory. """
        dirname = os.path.normpath(dirname)
        if not self._policy_cache.has_key(dirname):
            policyfile = None
            for name in self.get_policy_filenames():
                if os.sep != '\\':
                    for filename in os.listdir(dirname):
                        if filename.lower() == name.lower():
                            policyfile = os.path.join(dirname, filename)
                            break
                elif os.path.exists(os.path.join(dirname, name)): 
                    policyfile = os.path.join(dirname, name)
                    break
            
            value = self._config.get('policy.default.value', MISSING_POLICY)
            if policyfile != None:      #policy file present
                try:
                    value = fileutils.read_policy_content(policyfile)
                    if value not in self._binary.keys():    #check policy file is valid
                        _logger.error("POLICY_ERROR: policy file found %s but policy %s value not exists in csv" % (policyfile, value))
                except IOError, exc:
                    _logger.error("POLICY_ERROR: %s" % exc)         
                    value = self._config.get('policy.default.value', MISSING_POLICY)
            else:       #no policy file present
                filePresent = False
                dirPresent = False
                for ftype in os.listdir(dirname):   #see if files or directories are present
                    if os.path.isdir(os.path.join(dirname, ftype)):
                        dirPresent = True
                    if os.path.isfile(os.path.join(dirname, ftype)):
                        filePresent = True
                        
                if filePresent:    #files present : error     
                    _logger.error("POLICY_ERROR: could not find a policy file under: '%s'" % dirname)
                elif dirPresent and not filePresent:  #directories only : warning
                    _logger.error("POLICY_WARNING: no policy file, no files present, but sub-folder present in : '%s'" % dirname)
                else:       #no files no dirs : warning
                    _logger.error("POLICY_WARNING: empty directory at : '%s'" % dirname)
                
            # saving the policy value for that directory.
            self._policy_cache[dirname] = value
        return self._policy_cache[dirname]
        
    def get_policy_filenames(self):
        """ Returns the list of potential policy filenames. """
        return self._config.get_list('policy.filenames', ['Distribution.policy.s60'])
        
    def _sort_by_policy(self, filename):
        """ Store the input file sorted by its policy number. """
        path = os.path.join(self._config['root.dir'], filename)
        parentdir = os.path.dirname(path)
        if os.path.isdir(path):
            parentdir = path
        value = self.get_dir_policy(parentdir)
        if not value in self._policies:
            self._policies[value] = codecs.open( os.path.join(self._config['temp.build.dir'], self._config['name'] + "_%s" % value + ".txt"), "w+", "utf-8" )
        self._policies[value].write(u"%s\n" % filename)


class PolicyRemoverMapper(PolicyMapper):
    """ This class implements a variant of the policy mapper.
        
    It removes the internal source. Only binary flagged content is kept.
    """
    
    def __init__(self, config, archiver):
        """ Initialization. """
        PolicyMapper.__init__(self, config, archiver)
        self._rm_policy_cache = {}

    def get_policy_root_dir(self):
        """ Return the policy.root.dir or root.dir if not set or not under root.dir."""
        if not self._config.has_key("policy.root.dir"):
            return os.path.normpath(self._config['root.dir'])
        else:
            if fileutils.destinsrc(self._config['root.dir'], self._config['policy.root.dir']):
                return os.path.normpath(self._config['policy.root.dir'])
            else:
                return os.path.normpath(self._config['root.dir'])

    def get_rmdir_policy(self, dirname):
        """ check if the directory should be dropped or not"""
        dirname = os.path.normpath(dirname)
        # check if parent is banned...
        prootdir = os.path.normpath(self.get_policy_root_dir())
        rootdir = os.path.normpath(self._config['root.dir'])
        if os.sep == '\\':
            dirname = dirname.lower()
            prootdir = prootdir.lower()
            rootdir = rootdir.lower()        
        
        # else get real value...
        if not self._rm_policy_cache.has_key(dirname):
            self._rm_policy_cache[dirname] = self.get_dir_policy(dirname)
        
        return self._rm_policy_cache[dirname]
        
    def create_commands(self, manifest):
        """ Generates a list of build commands. """
        stages = PolicyMapper.create_commands(self, manifest)
        
        if not self._config.has_key('policy.csv'):
            _logger.error("POLICY_ERROR: Property 'policy.csv' not defined everything will get removed.")
        cmds = []
        file_handle = codecs.open( manifest, "r", "utf-8" )
        for line in file_handle.readlines():
            line = line.rstrip()
            filepath = os.path.normpath(os.path.join(self._config.get('zip.root.dir', self._config['root.dir']), line))
            value = self.get_rmdir_policy(os.path.dirname(filepath))            
            delete = True
            if value in self._binary.keys():
                if self._binary[value] == "yes":
                    _logger.info("POLICY_INFO: Keeping %s (%s=>yes)!" % (filepath, value))
                    delete = False
                elif self._binary[value] == "bin":
                    _logger.info("POLICY_INFO: Keeping %s (%s=>bin)!" % (filepath, value))
                    delete = False
            else:
                _logger.error("POLICY_ERROR: %s value for %s not in csv file. Will be removed!!" % (value, filepath))
               
            if delete:
                _logger.info("POLICY_INFO: File %s will be removed!" % filepath)
                cmds.append(buildtools.Delete(filename=filepath))
        file_handle.close()
        if len(cmds) > 0:
            stages.append(cmds)
        return stages


class SFPolicyRemoverMapper(PolicyRemoverMapper):
    """ Implement an SFL column based policy remover. """

    def __init__(self, config, archiver):
        """ Initialization. """
        PolicyRemoverMapper.__init__(self, config, archiver)

    def load_policy_binary(self, csvfile, column=1):
        """ Loading the policy using the 3rd column. """
        _logger.info("POLICY_INFO: Loading actions from the 3rd column")
        PolicyRemoverMapper.load_policy_binary(self, csvfile, column=3)

class EPLPolicyRemoverMapper(PolicyRemoverMapper):
    """ Implement an EPL column based policy remover. """
    def __init__(self, config, archiver):
        """ Initialization. """
        PolicyRemoverMapper.__init__(self, config, archiver)

    def load_policy_binary(self, csvfile, column=1):
        """ Loading the policy using the 4th column. """
        _logger.info("POLICY_INFO: Loading actions from the 4th column")
        PolicyRemoverMapper.load_policy_binary(self, csvfile, column=4)
        
        
MAPPERS = {'default': DefaultMapper,
             'policy': PolicyMapper,
             'policy.remover': PolicyRemoverMapper,
             'sfl.policy.remover': SFPolicyRemoverMapper,
             'epl.policy.remover': EPLPolicyRemoverMapper,}

def get_mapper(name, config, archiver):
    """ Get mapper instance from its string id. """
    if name in MAPPERS:
        return MAPPERS[name](config, archiver)
    raise Exception("ERROR: Could not find mapper '%s'." % name)
