import os
import re
import time
import json
import yaml
from sys import exit
import logging
from datetime import datetime
from tempfile import NamedTemporaryFile
try:
    # python 2
    from urllib import quote as urlencode
    from urlparse import urlparse, urlunparse
except ImportError:
    # python 3
    from urllib.parse import urlparse, urlunparse, quote as urlencode

from insights.client.connection import InsightsConnection
from insights.client.constants import InsightsConstants as constants
from insights.client.utilities import (
    generate_machine_id, write_data_to_file, get_time
)
from insights.util.subproc import call, CalledProcessError

logger = logging.getLogger(__name__)
MIN_YARA_VERSION = "4.1.0"
MALWARE_APP_URL = 'https://console.redhat.com/insights/malware'
MALWARE_CONFIG_FILE = os.path.join(constants.default_conf_dir, "malware-detection-config.yml")
LAST_SCAN_FILE = os.path.join(constants.default_conf_dir, '.last_malware-detection_scan')
DEFAULT_MALWARE_CONFIG = """
# Configuration file for the Red Hat Insights Malware Detection Client app
# File format is YAML
---
# Scan the filesystem?
scan_filesystem: true

# Scan the running processes?
# Disabled by default due to its potential performance impact on the host when scanning numerous or large processes
scan_processes: false

# Perform a simple test scan of the insights-client config directory and process to verify installation and scanning
# are working correctly.  The results from this scan do not show up in the webUI.
# Once verified, disable this option to perform actual malware scans.
test_scan: true

# scan_only: a single or list of files/directories/process IDs to scan, for example:
# scan_only:
# - /var/www
# - /home
# ... means scan only files in /var/www and /home.  May also be written as scan_only: [/var/www, /home]
# scan_only: 12345
#... means scan only process ID 12345
# No value means scan all files/directories/PIDs
scan_only:

# scan_exclude: a single or list of files/directories to be excluded from filesystem scanning
# If an item appears in both scan_only and scan_exclude, scan_exclude takes precedence and the item will be excluded
# The scan_exclude list is pre-populated with a number of top level directories that are recommended to be excluded,
# as well as the insights-client log directory which could cause extra matches itself
scan_exclude:
- /proc
- /sys
- /cgroup
- /selinux
- /net
- /mnt
- /media
- /dev
- /var/log/insights-client

# scan_since: scan files created or modified since X days ago or since the 'last' scan.
# Valid values are integers >= 1 or the string 'last'.  For example:
# scan_since: 1
# ... means scan files created/modified since 1 day ago
# scan_since: last
# ... means scan files created/modified since the last successful scan
# No value means scan all files regardless of created/modified date
scan_since:

# Exclude mounted network/external filesystems mountpoints?
# Scanning files within mounted network filesystems may be slow and cause extra network traffic.
# They are excluded by default, meaning that files in network/externally mounted filesystems are not scanned.
# Their mountpoints will be added to the scan_exclude list of directories to be excluded from scanning
exclude_network_filesystem_mountpoints: true

# List of network/external filesystem types to search for mountpoints on the system.
# If any mountpoints are found for these filesystem types, the value of the exclude_network_filesystem_mountpoints
# option will determine if files within the mountpoints are scanned or not.
network_filesystem_types: [nfs, nfs4, cifs, smbfs, fuse.sshfs, ceph, glusterfs, gfs, gfs2]

# Add extra metadata about each scan match (if possible), eg file type & md5sum, matching line numbers, process name
# The extra metadata will display in the webUI along with the scan matches
add_metadata: true

# Specific location of the yara binary file.  Autodetected if not specified.  For example:
# yara_binary: /usr/local/bin/yara
yara_binary:

# Abort a particular scan if it takes longer than scan_timeout seconds.  Default is 3600 seconds (1 hour)
scan_timeout: # 3600

# Run the yara process with this nice priority value.  Default is 19 (lowest priority)
nice_value: # 19

# The max number of CPUs threads used by yara when scanning.  Autodetected, but default is 2
cpu_thread_limit: # 2
""".lstrip()

# All the config options have corresponding environment variables
# Env vars are initially strings and need to be parsed to their appropriate type to match the yaml types
ENV_VAR_TYPES = {
    'boolean': ['SCAN_FILESYSTEM', 'SCAN_PROCESSES', 'TEST_SCAN', 'ADD_METADATA',
                'EXCLUDE_NETWORK_FILESYSTEM_MOUNTPOINTS'],
    'list': ['SCAN_ONLY', 'SCAN_EXCLUDE', 'NETWORK_FILESYSTEM_TYPES'],
    'integer': ['SCAN_TIMEOUT', 'NICE_VALUE', 'CPU_THREAD_LIMIT', 'STRING_MATCH_LIMIT'],
    'int_or_str': ['SCAN_SINCE'],
    'string': ['YARA_BINARY']
}


class MalwareDetectionClient:
    def __init__(self, insights_config):
        # insights_config is used to access insights-client auth and network config when downloading rules
        self.insights_config = insights_config

        # Load the malware-detection config file
        self.config = self._load_config()

        # Early check if the yara binary exists.  No point continuing if it doesn't
        self.yara_binary = self._find_yara()

        # Get/set the values of assorted integer config values - mainly options used with the yara command
        for option, value in [('nice_value', 19),
                              ('scan_timeout', 3600),
                              ('cpu_thread_limit', 2),
                              ('string_match_limit', 10)]:
            try:
                setattr(self, option, int(self._get_config_option(option, value)))
            except Exception as e:
                logger.error("Problem setting configuration option %s: %s", option, str(e))
                exit(constants.sig_kill_bad)

        # If doing a test scan, then ignore the other scan_* options because test scan sets its own values for them
        if not self._process_test_scan_option():
            self._process_scan_options()

        # Obtain the rules to be used by yara
        self.rules_file = self._get_rules()

        # Build the yara command, with its various command line options, that will be run
        self.yara_cmd = self._build_yara_command()

        # host_scan is a dictionary into which all the scan matches are stored.  Its structure is like:
        # host_scan = {rule_name: [{source: ..., stringData: ..., stringIdentifier: ..., stringOffset: ...},
        #                          {source: ...}],
        #              rule_name: [{...}, {...}, {...}],
        #              ... }
        # host_scan_mutation is the host_scan dict converted to a GraphQL mutation query string
        self.host_scan = {}
        self.host_scan_mutation = ''

        # Check if we are adding extra metadata to each scan match
        self.add_metadata = self._get_config_option('add_metadata', False)

        self.matches = 0

    def run(self):
        # Start the scans and record the time they were started
        scan_start = get_time()
        self.scan_filesystem()
        self.scan_processes()

        if self.do_filesystem_scan or self.do_process_scan:
            # If any scans were performed then get the results as a GraphQL mutation query
            # This mutation query is what is uploaded to the malware backend
            host_scan_mutation = self._create_host_scan_mutation()

            # Write a message to user informing them if there were matches or not and what to do next
            if self.matches == 0:
                logger.info("No rule matches found.\n")
            else:
                logger.info("Found %d rule match%s.", self.matches, 'es' if self.matches > 1 else '')
                if not self.test_scan:
                    logger.info("Please visit %s for more information\n", MALWARE_APP_URL)

            # Write the scan start time to disk (its used by the 'scan_since: last' option)
            # Only write the scan time after scans have completed without error or interruption, and its not a test scan
            if not self.test_scan:
                write_data_to_file(scan_start, LAST_SCAN_FILE)
                os.chmod(LAST_SCAN_FILE, 0o644)
            else:
                logger.info("\nRed Hat Insights malware-detection app test scan complete.\n"
                            "Test scan results are not recorded in the Insights UI (%s)\n"
                            "To perform proper scans, please set test_scan: false in %s\n",
                            MALWARE_APP_URL, MALWARE_CONFIG_FILE)

            # This is what is uploaded to the malware backend
            return host_scan_mutation
        else:
            logger.error("No scans performed, no results to upload.")
            exit(constants.sig_kill_bad)

    @staticmethod
    def _load_config():
        # Load the malware-detection config file.  Write out a default one first if it doesn't already exist
        if not os.path.isfile(MALWARE_CONFIG_FILE):
            logger.info("Writing the malware-detection app default configuration to %s", MALWARE_CONFIG_FILE)
            write_data_to_file(DEFAULT_MALWARE_CONFIG, MALWARE_CONFIG_FILE)
            os.chmod(MALWARE_CONFIG_FILE, 0o644)

        try:
            with open(MALWARE_CONFIG_FILE) as m:
                return yaml.safe_load(m)
        except Exception as e:
            logger.error("Error encountered loading the malware-detection app config file %s:\n%s",
                         MALWARE_CONFIG_FILE, str(e))
            exit(constants.sig_kill_bad)

    def _find_yara(self):
        """
        Find the yara binary on the local system and check it's version >= MIN_YARA_VERSION
        """
        def yara_version_ok(yara):
            # Check the installed yara version >= MIN_YARA_VERSION
            installed_yara_version = call([[yara, '--version']]).strip()
            try:
                if float(installed_yara_version[:3]) < float(MIN_YARA_VERSION[:3]):
                    raise RuntimeError("Found yara version %s, but malware-detection requires version >= %s\n"
                                       "Please install a later version of yara."
                                       % (installed_yara_version, MIN_YARA_VERSION))
            except RuntimeError as e:
                logger.error(str(e))
                exit(constants.sig_kill_bad)
            except Exception as e:
                logger.error("Error getting the version of the specified yara binary %s: %s" % (yara, str(e)))
                exit(constants.sig_kill_bad)
            # If we are here then the version of yara was ok
            return True

        yara = self._get_config_option('yara_binary')
        if yara and not os.path.isfile(yara):
            logger.error("Couldn't find the specified yara binary %s.  Please check it exists", yara)
            exit(constants.sig_kill_bad)
        elif yara and yara_version_ok(yara):
            logger.debug("Using specified yara binary: %s", yara)
            return yara

        try:
            yara = str(call([['which', 'yara']])).strip()
        except CalledProcessError:
            logger.error("Couldn't find yara.  Please ensure the yara package is installed")
            exit(constants.sig_kill_bad)
        yara_version_ok(yara)   # Generates an error if not ok
        logger.debug("Using yara binary: %s", yara)
        return yara

    def _process_scan_options(self):
        """
        Initialize the various scan flags and lists and run methods that may change/populate them
        """
        self.do_filesystem_scan = self._get_config_option('scan_filesystem', True)
        self.do_process_scan = self._get_config_option('scan_processes', False)
        if not (self.do_filesystem_scan or self.do_process_scan):
            logger.error("Both filesystem and process scans are disabled.  Nothing to do.")
            exit(constants.sig_kill_bad)

        self.scan_fsobjects = []
        self.scan_pids = []

        self._process_scan_only_option()
        self._process_scan_exclude_option()
        self._process_scan_since_option()
        self._process_exclude_network_filesystem_mountpoints_option()

    def _process_test_scan_option(self):
        self.test_scan = self._get_config_option('test_scan', False)
        if not self.test_scan:
            return False

        self.scan_since_dict = {'timestamp': None}
        self.scan_exclude_list = []
        self.network_filesystem_mountpoints = []

        # For matching the test rule, scan the insights config file and the currently running process
        # Make sure the config file exists first though!
        if os.path.isfile(MALWARE_CONFIG_FILE):
            self.do_filesystem_scan = True
            self.scan_fsobjects = [MALWARE_CONFIG_FILE]
        else:
            self.do_filesystem_scan = False
            self.scan_fsobjects = []

        self.do_process_scan = True
        self.scan_pids = [str(os.getpid())]
        logger.info("\nPerforming a test scan of %sthe current process (PID %s) "
                    "to verify the malware-detection app is installed and scanning correctly ...\n",
                    "%s and " % self.scan_fsobjects[0] if self.do_filesystem_scan else "", self.scan_pids[0])
        return True

    def _process_scan_only_option(self):
        """
        Parse the scan_only option, if specified, to get a list of things to scan
        """
        scan_only = self._get_config_option('scan_only')
        if scan_only:
            # Process the scan_only option as a list of items to scan
            # There may be both files/directories and PIDs all together
            # Strings are assumed to be files/directories and ints are assumed to be process IDs
            if not isinstance(scan_only, list):
                scan_only = [scan_only]
            for item in scan_only:
                if isinstance(item, str) and self.do_filesystem_scan:
                    # Remove extras slashes (/) in the file name and leading double slashes too (normpath doesn't)
                    item = os.path.normpath(item).replace('//', '/')
                    # Assume the item represents a filesystem item
                    if os.path.exists(item):
                        self.scan_fsobjects.append(item)
                    else:
                        logger.info("Skipping missing scan_only filesystem item: '%s'", item)
                elif isinstance(item, int) and self.do_process_scan:
                    # Assume the item represents a process ID
                    if os.path.exists('/proc/%s' % item):
                        self.scan_pids.append(str(item))
                    else:
                        logger.info("Skipping missing scan_only PID: %s", item)
                else:
                    logger.info("Skipping scan_only item: %s", item)

            if self.scan_fsobjects:
                logger.info("Scan only the specified filesystem item%s: %s", "s" if len(self.scan_fsobjects) > 1 else "",
                            self.scan_fsobjects)
            if self.scan_pids:
                logger.info("Scan only the specified process ID%s: %s", "s" if len(self.scan_pids) > 1 else "",
                            self.scan_pids)
            if not (self.scan_fsobjects or self.scan_pids):
                logger.error("Unable to scan the items specified for the scan_only option")
                exit(constants.sig_kill_bad)

    def _process_scan_since_option(self):
        """
        scan_since is specified as an integer representing the number of days ago to scan for modified files
        If the option was specified and valid, then get the corresponding unix timestamp for the specified
        number of days ago from now, which is used for comparing file modification times
        """
        self.scan_since_dict = {'timestamp': None, 'datetime': None}
        scan_since = self._get_config_option('scan_since')
        if scan_since is not None:
            timestamp = get_scan_since_timestamp(scan_since)
            if timestamp:
                self.scan_since_dict['timestamp'] = timestamp
                self.scan_since_dict['datetime'] = datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
                message = "Scan for files created/modified since %s%s"
                if isinstance(scan_since, str):
                    submessage = 'last successful scan on '
                else:
                    submessage = '%s day%s ago on ' % (scan_since, "s" if scan_since > 1 else "")
                logger.info(message, submessage, self.scan_since_dict['datetime'])

    def _process_scan_exclude_option(self):
        """
        Simple parse of the scan_exclude option (if specified) to get a list of valid items to exclude
        """
        if not self.do_filesystem_scan:
            return

        self.scan_exclude_list = []
        scan_exclude = self._get_config_option('scan_exclude')
        if scan_exclude:
            if not isinstance(scan_exclude, list):
                # Convert scan_exclude to a list if only a single non-list item was specified
                scan_exclude = [str(scan_exclude)]
            for item in scan_exclude:
                item = os.path.normpath(item).replace('//', '/')
                if os.path.exists(item):
                    self.scan_exclude_list.append(item)
                else:
                    logger.debug("Skipping missing scan_exclude item: '%s'", item)
            logger.info("Excluding specified filesystem item%s: %s", "s" if len(self.scan_exclude_list) > 1 else "",
                        self.scan_exclude_list)

    def _process_exclude_network_filesystem_mountpoints_option(self):
        """
        If exclude_network_filesystem_mountpoints is true, get a list of mountpoints of mounted network filesystems.
        The network_filesystem_types option has the list of network filesystems types to look for mountpoints for,
        eg NFS, CIFS, SMBFS, SSHFS, Ceph, GlusterFS, GFS.
        The list of network filesystem mountpoints will be added to the list of directories to exclude from scanning
        """
        if not self.do_filesystem_scan:
            return

        self.network_filesystem_mountpoints = []
        if not self._get_config_option('exclude_network_filesystem_mountpoints'):
            # We aren't excluding network filesystems, leave it as a blank list (ie nothing to exclude)
            return

        network_filesystem_types = self._get_config_option('network_filesystem_types')
        if not network_filesystem_types:
            logger.error("No value specified for 'network_filesystem_types' option")
            exit(constants.sig_kill_bad)

        if isinstance(network_filesystem_types, list):
            network_filesystem_types = ','.join(network_filesystem_types)
        cmd = ['findmnt', '-t', network_filesystem_types, '-n', '-o', 'TARGET']
        logger.debug("Command to find mounted network filesystems: %s", ' '.join(cmd))
        try:
            output = call([cmd])
        except CalledProcessError as err:
            logger.error("Unable to get network filesystem mountpoints: %s", err.output.strip())
            exit(constants.sig_kill_bad)

        self.network_filesystem_mountpoints = str(output).strip().split('\n') if output else []
        if self.network_filesystem_mountpoints:
            logger.info("Excluding network filesystem mountpoints: %s", self.network_filesystem_mountpoints)
        else:
            logger.debug("No mounted network filesystems found")

    def _get_rules(self):
        """
        Obtain the rules used by yara for scanning from the rules_location option.
        They can either be downloaded from the malware backend or obtained from a local file.
        """
        self.rules_location = self._get_config_option(
            'rules_location', "https://console.redhat.com/api/malware-detection/v1/signatures.yar"
        )

        # If rules_location starts with a /, assume its a file rather than a URL
        if self.rules_location.startswith('/'):
            # Remove any extra slashes from the file name and from the start too (normpath doesn't remove those)
            rules_file = os.path.normpath(self.rules_location).replace('//', '/')
            if not os.path.isfile(rules_file):
                logger.error("Couldn't find specified rules file: %s", rules_file)
                exit(constants.sig_kill_bad)
            logger.debug("Using specified rules file: %s", rules_file)
            return rules_file

        # If we are here, then we are downloading the rules from the malware backend
        # Check if insights-config is defined first because we need to access its auth and network config
        if not self.insights_config:
            logger.error("Couldn't access the insights-client configuration")
            exit(constants.sig_kill_bad)

        # Make sure the rules_location starts with https://
        if not re.match('^https?://', self.rules_location):
            self.rules_location = 'https://' + self.rules_location

        # if insights-client is using cert auth or basic auth without a username and password, append 'cert.' to the url
        authmethod = self.insights_config.authmethod if hasattr(self.insights_config, 'authmethod') else 'CERT'
        username = self.insights_config.username if hasattr(self.insights_config, 'username') else ''
        password = self.insights_config.password if hasattr(self.insights_config, 'password') else ''
        if authmethod == 'CERT' or (authmethod == 'BASIC' and not (username or password)):
            self.insights_config.authmethod = 'CERT'
            parsed_url = urlparse(self.rules_location)
            if not parsed_url.netloc.startswith('cert.'):
                self.rules_location = urlunparse(parsed_url._replace(netloc='cert.' + parsed_url.netloc))

        # If doing a test scan, replace signatures.yar (or any other file suffix) with test-rule.yar
        if self.test_scan:
            self.rules_location = self._get_test_rule_location(self.rules_location)

        logger.debug("Downloading rules from: %s", self.rules_location)
        try:
            self.insights_config.cert_verify = True
            conn = InsightsConnection(self.insights_config)
            response = conn.get(self.rules_location)
            if response.status_code != 200:
                logger.error("%s %s: %s", response.status_code, response.reason, response.text)
                exit(constants.sig_kill_bad)
        except Exception as e:
            logger.error("Unable to download rules from %s: %s", self.rules_location, str(e))
            exit(constants.sig_kill_bad)

        self.temp_rules_file = NamedTemporaryFile(prefix='tmp_malware-detection-client_rules.', mode='wb', delete=True)
        self.temp_rules_file.write(response.content)
        self.temp_rules_file.flush()
        return self.temp_rules_file.name

    def _build_yara_command(self):
        """
        Get all the switches for the yara command to be run, for example:
        - whether the rules file is compiled or not (-C)
        - the number of CPU threads to use (-p)
        - the nice command and its value to use (nice -n 'value')
        - scan timeouts (-a)
        """
        # Detect if the rules file is a text or binary (compiled) file (or otherwise)
        output = call([['file', '-b', self.rules_file]])
        rule_type = output.strip().lower()
        if os.path.getsize(self.rules_file) == 0 or rule_type == 'empty':
            logger.error("Rules file %s is empty", self.rules_file)
            exit(constants.sig_kill_bad)

        compiled_rules_flag = '-C' if rule_type.startswith('yara') or rule_type == 'data' else ''
        logger.debug("Rules file type: '%s', Compiled rules: %s", rule_type, compiled_rules_flag == '-C')

        # Quickly test the rules file to make sure it contains usable rules!
        # Note, if the compiled_rules_flag is '' it must be removed from the list or it causes problems
        cmd = list(filter(None, [self.yara_binary, '--fail-on-warnings', '-p', '1', '-f', compiled_rules_flag,
                                 self.rules_file, '/dev/null']))
        try:
            call([cmd])
        except CalledProcessError as err:
            logger.error("Unable to use rules file %s: %s", self.rules_file, err.output.strip())
            exit(constants.sig_kill_bad)

        # Limit the number of threads used by yara to limit the CPU load of the scans
        # If system has 2 or fewer CPUs, then use just one thread
        nproc = call('nproc').strip()
        if not nproc or int(nproc) <= 2:
            self.cpu_thread_limit = 1
        logger.debug("Using %s CPU thread(s) for scanning", self.cpu_thread_limit)

        # Construct the (partial) yara command that will be used later for scanning files and processes
        # The argument for the files and processes that will be scanned will be added later
        yara_cmd = list(filter(None, ['nice', '-n', str(self.nice_value), self.yara_binary, '-s', '-N',
                                      '-a', str(self.scan_timeout), '-p', str(self.cpu_thread_limit), '-r', '-f',
                                      compiled_rules_flag, self.rules_file]))
        logger.debug("Yara command: %s", yara_cmd)
        return yara_cmd

    def scan_filesystem(self):
        if not self.do_filesystem_scan:
            return False

        # Process the filesystem items to scan
        # If self.scan_fsobjects is set, then just scan its items, less any items in the exclude list
        # And exclude the rules file, unless that's the thing we specifically want to scan
        # scan_dict will contain all the toplevel directories to scan, and any particular files/subdirectories to scan
        if self.rules_file not in self.scan_fsobjects:
            self.scan_exclude_list.append(self.rules_file)
        scan_dict = process_include_exclude_items(include_items=self.scan_fsobjects,
                                                  exclude_items=self.scan_exclude_list,
                                                  exclude_mountpoints=self.network_filesystem_mountpoints)
        logger.debug("Filesystem objects to be scanned in: %s", sorted(scan_dict.keys()))

        logger.info("Starting filesystem scan ...")
        fs_scan_start = time.time()

        for toplevel_dir in sorted(scan_dict):
            # Make a copy of the self.yara_cmd list and add to it the thing to scan
            cmd = self.yara_cmd[:]
            dir_scan_start = time.time()

            specified_log_txt = "specified " if 'include' in scan_dict[toplevel_dir] else ""
            if self.scan_since_dict['timestamp']:
                logger.info("Scanning %sfiles in %s modified since %s ...", specified_log_txt, toplevel_dir,
                            self.scan_since_dict['datetime'])
                # Find the recently modified files in the given top level directory
                scan_list_file = NamedTemporaryFile(prefix='%s_scan_list.' % os.path.basename(toplevel_dir),
                                                    mode='w', delete=True)
                if 'include' in scan_dict[toplevel_dir]:
                    find_modified_include_items(scan_dict[toplevel_dir]['include'], self.scan_since_dict['timestamp'], scan_list_file)
                else:
                    find_modified_in_directory(toplevel_dir, self.scan_since_dict['timestamp'], scan_list_file)

                scan_list_file.flush()
                cmd.extend(['--scan-list', scan_list_file.name])
            else:
                logger.info("Scanning %sfiles in %s ...", specified_log_txt, toplevel_dir)
                if 'include' in scan_dict[toplevel_dir]:
                    scan_list_file = NamedTemporaryFile(prefix='%s_scan_list.' % os.path.basename(toplevel_dir),
                                                        mode='w', delete=True)
                    scan_list_file.write('\n'.join(scan_dict[toplevel_dir]['include']))
                    scan_list_file.flush()
                    cmd.extend(['--scan-list', scan_list_file.name])
                else:
                    cmd.append(toplevel_dir)

            logger.debug("Yara command: %s", cmd)
            try:
                output = call([cmd]).strip()
            except CalledProcessError as cpe:
                logger.error("Unable to scan %s: %s", toplevel_dir, cpe.output.strip())
                continue

            self.parse_scan_output(output.strip())

            dir_scan_end = time.time()
            logger.info("Scan time for %s: %d seconds", toplevel_dir, (dir_scan_end - dir_scan_start))

        fs_scan_end = time.time()
        logger.info("Filesystem scan time: %s", time.strftime("%H:%M:%S", time.gmtime(fs_scan_end - fs_scan_start)))
        return True

    def scan_processes(self):
        if not self.do_process_scan:
            return False

        logger.info("Starting processes scan ...")
        pids_scan_start = time.time()

        if not self.scan_pids:
            # Get list of process ids to scan
            all_pids = [entry for entry in os.listdir('/proc') if entry.isdigit()]
            exclude_pids = [str(os.getpid())]  # exclude our script's pid at least
            self.scan_pids = sorted(list(set(all_pids) - set(exclude_pids)), key=lambda pid: int(pid))

        for scan_pid in self.scan_pids:
            pid_scan_start = time.time()
            logger.info("Scanning process %s ...", scan_pid)
            cmd = self.yara_cmd + [str(scan_pid)]
            logger.debug("Yara command: %s", cmd)
            try:
                output = call([cmd]).strip()
            except CalledProcessError as cpe:
                logger.error("Unable to scan process %s: %s", scan_pid, cpe.output.strip())
                continue

            self.parse_scan_output(output)

            pid_scan_end = time.time()
            logger.info("Scan time for process %s: %d seconds", scan_pid, (pid_scan_end - pid_scan_start))

        pids_scan_end = time.time()
        logger.info("Processes scan time: %s", time.strftime("%H:%M:%S", time.gmtime(pids_scan_end - pids_scan_start)))
        return True

    def parse_scan_output(self, output, exclude_items=[]):
        if not output:
            return

        # Each 'set' of output lines consists of 1 line containing the rule and file/pid (aka source) it matches
        # Followed by one or more related lines of matching string data from that source, eg
        # ...
        # rule_name source                            + Set of 3 related lines
        # 0x_offset:string_identifier:string_data     |
        # 0x_offset:string_identifier:string_data     +
        # rule_name source                            + Set of 2 related lines
        # 0x_offset:string_identifier:string_data     +
        # ...

        def skip_string_data_lines(string_data_lines):
            # Skip the 0x... lines containing string match data
            while string_data_lines and string_data_lines[0].startswith('0x'):
                logger.debug("Skipping string data line '%s'", string_data_lines[0])
                string_data_lines.pop(0)

        output_lines = output.split("\n")
        while output_lines:
            if 'error scanning ' in output_lines[0]:
                if output_lines[0].endswith('error: 4'):
                    # Yara 'could not map file' error - only display this error if debugging (spammy)
                    logger.debug(output_lines[0])
                else:
                    logger.info(output_lines[0])
                output_lines.pop(0)  # Skip the error scanning line
                # Skip any string match lines after the error scanning line
                skip_string_data_lines(output_lines)
                continue
            # Get the rule_name and source from the first line in the set
            try:
                rule_name, source = output_lines[0].rstrip().split(" ", 1)
            except ValueError as err:
                # Hopefully shouldn't happen but log it and continue processing
                logger.debug("Error parsing rule match '%s': %s", output_lines[0], err)
                output_lines.pop(0)  # Skip the erroneous line
                # Skip any string match lines afterwards until we get to the next rule match line
                skip_string_data_lines(output_lines)
                continue

            # All good so far, skip over the line containing the rule name and matching source file/pid
            output_lines.pop(0)

            # If the rule or the source file/pid is to be excluded, then skip over its scan matches
            # and move onto the next match
            # if any([exclude_rule.lower() in rule_name.lower() for exclude_rule in self.exclude_rules]) \
            #         or any([ei in source for ei in exclude_items]):
            #     skip_string_data_lines(output_lines)
            #     continue

            # Check if the rule name contains a ':' or doesn't start with a char/string
            # It shouldn't and its likely to be due to a malformed string_offset line
            # Skip any further scan matches until the next rule match
            if ':' in rule_name or not re.match('^[a-zA-Z]+', rule_name):
                skip_string_data_lines(output_lines)
                continue

            rule_match = {'rule_name': rule_name, 'matches': []}
            source_type = "process" if source.isdigit() else "file"

            # Parse the string match data for the remaining lines in the set
            string_matches = 0
            while output_lines and output_lines[0].startswith('0x'):
                if string_matches < self.string_match_limit:
                    try:
                        string_offset, string_identifier, string_data = output_lines[0].split(':', 2)
                        string_offset = int(string_offset, 0)
                    except ValueError as err:
                        logger.debug("Error parsing string match '%s': %s", output_lines[0], err)
                        output_lines.pop(0)
                        continue
                    rule_match_dict = {'source': source,
                                       'string_data': string_data.strip(),
                                       'string_identifier': string_identifier,
                                       'string_offset': string_offset,
                                       'metadata': {'source_type': source_type}}
                    rule_match['matches'].extend([rule_match_dict])
                output_lines.pop(0)
                string_matches += 1

            # If string_match_limit is 0 or there was no string data, there will be no rule_matches,
            # but still record the file/pid source that was matched
            if not rule_match['matches']:
                rule_match_dict = {'source': source,
                                   'string_data': '',
                                   'string_identifier': '',
                                   'string_offset': -1,
                                   'metadata': {'source_type': source_type}}
                rule_match['matches'] = [rule_match_dict]

            if self.add_metadata:
                # Add extra data to each rule match, beyond what yara provides
                # Eg, for files: line numbers & context, checksums; for processes: process name
                # TODO: find more pythonic ways of doing this stuff instead of using system commands
                metadata_func = self._add_file_metadata if source_type == 'file' else self._add_process_metadata
                metadata_func(rule_match['matches'])

            self.matches += 1
            logger.info("Matched rule %s in %s %s", rule_name, source_type, source)
            logger.debug(rule_match)
            if self.host_scan.get(rule_match['rule_name']):
                self.host_scan[rule_match['rule_name']].extend(rule_match['matches'])
            else:
                self.host_scan[rule_match['rule_name']] = rule_match['matches']

    def _add_process_metadata(self, rule_matches):
        """
        Add extra data to the process scan matches beyond what is provided by yara, eg process name
        """
        # All passed in rule_matches will have the same source PID
        # Check the process still exists before obtaining the metadata about it
        source = rule_matches[0]['source']
        if not os.path.exists('/proc/%s' % source):
            return

        # Get name of process from ps command
        # -h: no output header, -q: only the specified process, -o args: just the process name and args
        try:
            process_name = call([['ps', '-hq', source, '-o', 'args']])
        except CalledProcessError:
            process_name = 'unknown'

        for rule_match in rule_matches:
            rule_match['metadata'].update({'process_name': process_name})

    def _add_file_metadata(self, rule_matches):
        """
        Add extra data to the file scan matches beyond what is provided by yara
        - eg matching line numbers, line context, file checksum
        - Use grep to get the line numbers & sed to get the line
        """
        def get_line_from_file(file_name, line_number):
            # Extract the line at line_number from file_name
            line_length_limit = 120
            try:
                line = call([['sed', '%dq;d' % line_number, file_name]]).strip()
            except CalledProcessError:
                line = ""
            # Limit line length if necessary and urlencode it to minimize problems with GraphQL when uploading
            return urlencode(line if len(line) < line_length_limit else line[:line_length_limit] + "...")

        # All passed in rule_matches will have the same source file
        # Check the file still exists before obtaining the metadata about it
        source = rule_matches[0]['source']
        if not os.path.exists(source):
            return

        # Get the file type, mime type and md5sum hash of the source file
        try:
            file_type = call([['file', '-b', source]]).strip()
            mime_type = call([['file', '-bi', source]]).strip()
            md5sum = call([['md5sum', source]]).strip().split()[0]
        except Exception:
            file_type = mime_type = md5sum = ""

        grep_string_data_match_list = []
        if mime_type and 'charset=binary' not in mime_type:
            # Get the line numbers for each of yara's string_data matches in the source file, but not for binary files
            # Build a grep command that searches for each of the string_data patterns in the source file
            # For each string_data pattern that grep finds, the grep output will have the form...
            # line_number:offset_from_0:string_data_pattern

            # Get the set of patterns to grep for, eg ['pattern1', 'pattern2', etc], ie remove duplicate patterns
            grep_string_data_pattern_set = set([match['string_data'] for match in rule_matches])
            if grep_string_data_pattern_set:
                # Build an option list for grep, eg ['-e', 'pattern1', '-e', 'pattern2', ... etc]
                # zip creates a list of tuples, eg [('-e', 'pattern'), ('-e', 'pattern2'), ...], then flatten the list
                grep_string_data_patterns = [item for tup in list(zip(['-e'] * len(grep_string_data_pattern_set),
                                                                      grep_string_data_pattern_set))
                                             for item in tup]
                # Create the grep command to execute.  -F means don't interpret regex special chars in the patterns
                grep_command = ['/bin/grep', '-Fbon'] + grep_string_data_patterns + [source]
                logger.debug("grep command: %s", grep_command)
                try:
                    grep_output = call([grep_command])
                except CalledProcessError:
                    grep_output = ""

                # Now turn the grep output into a list of tuples for easier searching a little later, ie
                # [(line_number, offset_from_0, string_data_pattern), (...), ]
                if grep_output and not grep_output.lower().startswith('binary'):
                    grep_string_data_match_list = list(map(lambda grep_output_line: tuple(grep_output_line.split(':', 3)),
                                                           grep_output.splitlines()))

        for rule_match in rule_matches:
            metadata = rule_match['metadata']
            metadata.update({'file_type': file_type,
                             'mime_type': mime_type,
                             'md5sum': md5sum})
            if grep_string_data_match_list:
                # Now, for each offset_from_0 in the grep output, we want to match it with the corresponding
                # string_offset value from the yara output so we can get the line number for that string_data match
                # And while we are here, get the line from the source file at that line number
                line_number = None
                for grep_list_item in grep_string_data_match_list:
                    if int(grep_list_item[1]) == rule_match['string_offset']:
                        line_number = int(grep_list_item[0])
                        break
                if line_number:
                    metadata.update({'line_number': line_number,
                                     'line': get_line_from_file(source, line_number)})

    def _create_host_scan_mutation(self):
        # Build the mutation text
        mutation_header = """
        mutation HostScan {
          recordHostScan(
            input: {
              scannedhost: {
                insightsId: "%s"
                rulesScanned: [""" % generate_machine_id()

        mutation_footer = """
                ]
              }
            }
          ) {
            success
          }
        }"""

        mutation = mutation_header
        for rule_name in self.host_scan.keys():
            rule_scan = """{
                ruleName: "%s"
                stringsMatched: [""" % rule_name
            for match in self.host_scan[rule_name]:
                rule_scan += """{
                    source: "%s"
                    stringData: %s
                    stringIdentifier: %s
                    stringOffset: "%s"
                    metadata: "%s"
                }, """ % (match['source'],
                          json.dumps(match['string_data']),
                          json.dumps(match['string_identifier']),
                          match['string_offset'],
                          json.dumps(match['metadata']).replace('"', '\\"'))
            rule_scan += "]}, "
            mutation += rule_scan

        mutation += mutation_footer
        return mutation

    def _get_config_option(self, option, default_value=None):
        """
        Get the value of a config option or if it doesn't exist or is None, the default_value
        """
        value = os.getenv(option.upper())
        if value is not None:
            return self._parse_env_var(option.upper(), value)
        value = self.config.get(option)
        return value if value is not None else default_value

    @staticmethod
    def _parse_env_var(env_var, value):
        """
        Parse specific environment variables to make sure they have appropriate values
        """
        logger.debug("Found environment variable: %s, value: %s", env_var, value)
        # Parse these env vars as booleans
        if env_var in ENV_VAR_TYPES['boolean']:
            return value.lower() in ('true', 'yes', 't', 'y')

        # Parse these as lists and convert any numeric values into ints
        if env_var in ENV_VAR_TYPES['list']:
            if value:
                value_list = value.split(',') if ',' in value else [value]
                return list(map(lambda x: int(x) if str.isdigit(x) else x, value_list))
            else:
                return []

        # Parse scan_since, can be either an int or a string (ie 'last')
        if env_var in ENV_VAR_TYPES['int_or_str']:
            return int(value) if str.isdigit(value) else value

        # Parse these as ints
        if env_var in ENV_VAR_TYPES['integer']:
            try:
                return int(value)
            except ValueError as e:
                logger.error("Problem parsing environment variable %s: %s", env_var, str(e))
                exit(constants.sig_kill_bad)

        # env_var value doesn't require parsing, just return it as is (ie. as a string)
        return value

    @staticmethod
    def _get_test_rule_location(rules_location):
        test_rule = 'test-rule.yar'
        # Nothing to do if the location already ends with test_rule
        if rules_location.endswith(test_rule):
            return rules_location
        # Replace the last entry from the path with the test-rule
        # A bit crude but it seems to work ok with both URL and file paths
        return os.path.join(os.path.dirname(rules_location), test_rule)


#
# Utility functions
# Mainly for including / excluding certain directories for scanning
# And also for finding files that have been modified recently
#
def get_toplevel_dirs():
    """
    Returns a list of the top level directories directly under root (/),
    """
    toplevel_dirs = sorted(filter(lambda x: not os.path.islink(x), map(lambda x: "/" + x, os.listdir('/'))))
    return toplevel_dirs


def get_parent_dirs(item, parent_dir_list, base_case='/'):
    """
    Get a list of parent directories of a particular filesystem item, stopping at base_case (root by default)
    Eg for get_parent_dirs('/path/to/some/item', parent_dir_list) ->
        parent_dir_list = ['/path', '/path/to', '/path/to/some', '/path/to/some/item']
    """
    if os.path.samefile(item, base_case) or os.path.samefile(item, '/'):
        return
    get_parent_dirs(os.path.dirname(item), parent_dir_list, base_case)
    parent_dir_list.append(item)


def process_include_items(include_items=[]):
    """
    Process the include items to a get list of directories to be scanned
    If there are no entries then get the list of top level directories under root (/),
    :return: a list of directories to be scanned.  It never returns an empty list.
    """
    default_values = get_toplevel_dirs()

    logger.debug("Parsing include items ...")
    parsed_list = []
    for item in include_items:
        item = item.strip()
        if not item or item.startswith('#'):
            continue
        include_item = os.path.normpath(item).replace('//', '/')
        if os.path.exists(include_item):
            # ignore the include_item if its not a full directory path
            if not include_item.startswith('/'):
                logger.debug("Skipping partial directory path '%s' ...", include_item)
                continue
            elif os.path.islink(include_item):
                logger.debug("Skipping link '%s' ...", include_item)
                continue
            elif os.path.samefile(include_item, '/'):
                # Found / in include item list.  No need to get the other items because / trumps all
                logger.debug("Found root directory in list of items to scan.  Ignoring the other items ...")
                parsed_list = default_values
                break
            else:
                parsed_list.append(include_item)
        else:
            logger.debug("Skipping missing item '%s' ...", include_item)

    if not parsed_list:
        logger.debug("No items specified to be scanned.  Using default values %s ...", default_values)
        parsed_list = default_values
    else:
        # Remove any duplicates and any children of parent directories before returning
        parsed_list = remove_child_items(sorted(list(set(parsed_list))))

    logger.debug("Include items: %s", parsed_list)
    return parsed_list


def process_exclude_items(exclude_items=[]):
    """
    Process the exclude items to get list of directories to NOT be scanned
    :return: a list of directories to not be scanned if any, otherwise an empty list
    """
    logger.debug("Parsing exclude items ...")
    parsed_list = []
    for item in exclude_items:
        item = item.strip()
        if not item or item.startswith('#'):
            continue
        exclude_item = os.path.normpath(item).replace('//', '/')
        if os.path.exists(exclude_item):
            # ignore the exclude_item if its not a full directory path
            if os.path.samefile(exclude_item, '/'):
                # Found / in exclude list.  No need to get the other items because / trumps all
                logger.debug("Found root directory in the exclude list.  Expanding it to all toplevel directories ...")
                parsed_list = get_toplevel_dirs()
                break
            elif not exclude_item.startswith('/'):
                logger.debug("Skipping partial directory path '%s' ...", exclude_item)
                continue
            else:
                parsed_list.append(exclude_item)
        else:
            logger.debug("Skipping missing item '%s' ...", exclude_item)

    if not parsed_list:
        logger.debug("No items specified to be excluded")
    else:
        # Remove any duplicates and any children of parent directories before returning
        parsed_list = remove_child_items(sorted(list(set(parsed_list))))

    logger.debug("Exclude items: %s", parsed_list)
    return parsed_list


def remove_child_items(item_list):
    """
    For a list of filesystem items, remove those items that are duplicates or children of other items
    Eg, for remove_child_items['/path/to/some/item/child', '/path/to/another/item', '/path/to/some/item']
        returns ['/path/to/another/item', '/path/to/some/item']
    If one if the items is root, then it wins
    Also, all items should be the full path starting at root (/).  Any that aren't are removed
    """
    if '/' in item_list:
        return ['/']

    # Remove duplicates and any non-full path items
    item_list = sorted(list(set(filter(lambda x: x.startswith('/'), item_list))))
    remove_items = set([])
    for i, item1 in enumerate(item_list[:-1]):
        for item2 in item_list[i + 1:]:
            if item1 != item2 and item2.startswith(item1 + '/'):
                remove_items.add(item2)
    for remove_item in remove_items:
        item_list.remove(remove_item)
    return sorted(list(set(item_list)))


def remove_included_excluded_items(included_items, excluded_items):
    """
    Go through the list of included items and remove any that are in the exclude list,
        or are children of excluded items (no need to scan an included item if its parent is to be excluded)
    """
    # Clean up the lists, just in case this hasn't been done already
    included_items = remove_child_items(included_items)
    excluded_items = remove_child_items(excluded_items)

    remove_items = set([])
    for included_item in included_items:
        for excluded_item in excluded_items:
            if excluded_item == included_item or included_item.startswith(excluded_item + '/'):
                remove_items.add(included_item)
    for remove_item in remove_items:
        included_items.remove(remove_item)
    return included_items


def process_include_exclude_items(include_items=[], exclude_items=[], exclude_mountpoints=[]):
    """
    Process the include and exclude items, where the exclude items are effectively subtracted from the include_items.
    It builds a scan_dict dictionary of items to scan keyed by the filesystem top level directories.
    Only the toplevel directories from items in the include_items list will be present in scan_dict.
    scan_dict = {'/boot': {'include': ['/boot/include/me', ...], 'exclude: ['/boot/exclude/me', ...]},
                 '/etc': {'include': ['/etc/include/me', ...], 'exclude: ['/etc/exclude/me', ...]},
                 ...
    :return: scan_dict
    """
    # Get a list of excluded items from the exclude file and network filesystem mountpoints
    initial_exclude_list = process_exclude_items(exclude_items)
    final_exclude_list = remove_child_items(list(set(exclude_mountpoints) | set(initial_exclude_list)))
    logger.debug("Final exclude items: %s", final_exclude_list)

    # Get a list of included items from the include file, minus the excluded items
    initial_include_list = process_include_items(include_items)
    if not initial_include_list:
        logger.error("No items to scan because the include items doesn't contain any valid items")
        exit(constants.sig_kill_bad)
    final_include_list = remove_included_excluded_items(initial_include_list, final_exclude_list)
    logger.debug("Final include items after removing exclude items: %s", final_include_list)
    if not final_include_list:
        logger.error("No items to scan because the specified exclude items cancel them out")
        exit(constants.sig_kill_bad)

    # This is the dictionary that will hold all the items to scan (after processing the include and exclude items)
    # It will be keyed by each of the toplevel directories containing items to scan
    # yara will scan each of the toplevel dir's 'include' keys (if present), or just the toplevel dir itself
    scan_dict = {}

    # Populate the scan_dict by creating keys for each toplevel directory of the items to include/scan
    # Create an 'include' key for each toplevel directory containing items to include in that toplevel directory
    logger.debug("Populating scan_dict's include items ...")
    for include_item in final_include_list:
        item_subpaths = []
        get_parent_dirs(include_item, item_subpaths)
        include_item_toplevel_dir = item_subpaths[0]
        if include_item_toplevel_dir not in scan_dict:
            # Create an 'include' key if the item to scan isn't just the toplevel directory itself
            scan_dict[include_item_toplevel_dir] = {'include': set([include_item])}\
                if include_item != include_item_toplevel_dir else {}
        else:
            scan_dict[include_item_toplevel_dir]['include'].add(include_item)

    logger.debug("Scan dict after adding include items: %s", scan_dict)

    # Populate an 'exclude' key for the toplevel dirs in the scan_dict that also have items to exclude
    # Or remove the toplevel dirs from the scan dict where the toplevel dir itself is to be excluded
    logger.debug("Populating scan_dict's exclude items ...")
    for exclude_item in final_exclude_list:
        item_subpaths = []
        get_parent_dirs(exclude_item, item_subpaths)
        exclude_item_toplevel_dir = item_subpaths[0]
        if exclude_item_toplevel_dir not in scan_dict:
            # This exclude_item's toplevel dir isn't in the scan dict, so skip it (since its not being included)
            continue
        if 'exclude' not in scan_dict[exclude_item_toplevel_dir]:
            # Create the 'exclude' key if it doesn't already exist
            scan_dict[exclude_item_toplevel_dir]['exclude'] = {'items': [], 'subpaths': set([])}

        scan_dict[exclude_item_toplevel_dir]['exclude']['items'].append(exclude_item)

        # Add the list of subpaths leading to this exclude item.
        # The subpaths are needed later for listing the contents each subpath
        scan_dict[exclude_item_toplevel_dir]['exclude']['subpaths'].update(item_subpaths)

    logger.debug("Scan dict after adding exclude items: %s", scan_dict)

    # For each toplevel dir with items to exclude, re-populate the include key with directory content listings
    # of the subpaths, minus the items to exclude and only including items to include.  Yep, its complicated.
    # These directory listings will be used with yara's --scan-list option
    logger.debug("Re-populating scan_dict's include items with directory content listings to pass to yara ...")
    for toplevel_dir in scan_dict:
        if 'exclude' not in scan_dict[toplevel_dir]:
            continue

        # Get directory listings of each of the subpaths
        if 'include' in scan_dict[toplevel_dir]:
            scan_items = set(scan_dict[toplevel_dir]['include'])
        else:
            scan_items = set([])
        toplevel_dir_exclude = scan_dict[toplevel_dir]['exclude']
        for exclude_item in toplevel_dir_exclude['items']:
            subpaths = []
            get_parent_dirs(exclude_item, subpaths)
            for i, subpath in enumerate(subpaths[:-1]):
                dir_list = os.listdir(subpath)
                dir_list = sorted(map(lambda x: subpath + '/' + x, dir_list))
                dir_list.remove(subpaths[i + 1])
                scan_items.update(dir_list)

        # Go through the list of scan items and remove any exclude items or exclude item subpaths
        for scan_item in list(scan_items):
            for exclude_item in toplevel_dir_exclude['items']:
                if scan_item == exclude_item or scan_item.startswith(exclude_item + '/'):
                    scan_items.remove(scan_item)
                    break
            else:
                for exclude_subpath in toplevel_dir_exclude['subpaths']:
                    if scan_item == exclude_subpath:
                        scan_items.remove(scan_item)

        # If there is an include list, make sure the scan_items only include items in the include list
        if 'include' in scan_dict[toplevel_dir]:
            for maybe_include in list(scan_items):
                if os.path.islink(maybe_include) or (not os.path.isfile(maybe_include) and not os.path.isdir(maybe_include)):
                    scan_items.remove(maybe_include)
                    continue
                if any([maybe_include == definitely_include or maybe_include.startswith(definitely_include + '/')
                        for definitely_include in scan_dict[toplevel_dir]['include']]):
                    continue
                else:
                    scan_items.remove(maybe_include)

        # Overwrite the existing include key list with the new list of scan_items
        scan_dict[toplevel_dir]['include'] = sorted(list(scan_items))

    logger.debug("Final scan_dict: %s", scan_dict)
    return scan_dict


def get_scan_since_timestamp(since):
    """
    Return a unix timestamp corresponding to how long ago to scan for files created/modified since this timestamp.
    Valid values of 'since' are integers > 0 meaning the number of days back in time from now,
                             or 'last' meaning get the timestamp of the last scan
    If 'since' is not one of these valid values, then terminate
    """
    now = time.time()

    def get_lastscan_timestamp(lastscan):
        try:
            # Convert the datetime string into a unix timestamp
            lastscan_seconds = float(datetime.strptime(lastscan, '%Y-%m-%dT%H:%M:%S.%f').strftime('%s'))
            if lastscan_seconds > now:
                raise RuntimeError("Last scan time is in the future.")
        except Exception as err:
            logger.error("Error getting time of last malware scan: %s.  Ignoring 'scan_since: last' option ...", str(err))
            return None
        return lastscan_seconds

    if isinstance(since, str) and since.lower().startswith('l'):
        # Get the timestamp of the last scan
        if os.path.isfile(LAST_SCAN_FILE):
            with open(LAST_SCAN_FILE) as f:
                lastscan = f.readline().strip()
            return get_lastscan_timestamp(lastscan)
        else:
            logger.info("File %s doesn't exist for 'scan_since: last' option.  Continuing ...",
                        LAST_SCAN_FILE)
            return None
    elif isinstance(since, str):
        logger.error("Unknown value '%s' for scan_since option.  Valid values are integers >= 1 and 'last'", since)
        exit(constants.sig_kill_bad)

    try:
        since_int = int(since)
        if since_int >= 1:
            return now - (since_int * 86400)  # 86400 seconds in a day
        else:
            raise ValueError("Invalid scan_since value %s.  Valid values are integers >= 1 and 'last'" % since)
    except ValueError as e:
        logger.error(str(e))
        exit(constants.sig_kill_bad)


def is_recent_mtime(item, timestamp):
    """
    Return True if the given 'item' has a modification time that is newer than 'timestamp'
    Return False otherwise, or if the the 'item' is a link or another non-file type (eg pipes)
    """
    if os.path.exists(item) and not os.path.islink(item) and os.path.isfile(item):
        return os.path.getmtime(item) > timestamp
    return False


def find_modified_in_directory(directory, timestamp, output_file):
    """
    Find files in 'directory' that have been created/modified since 'timestamp'
    and write their names to 'output_file'
    """
    for root, dirs, files in os.walk(directory):
        for afile in files:
            path = os.path.join(root, afile)
            if is_recent_mtime(path, timestamp):
                output_file.write(path + "\n")


def find_modified_include_items(item_list, timestamp, output_file):
    """
    Find files in the given list of items (files/directories) that have been created/modified since 'timestamp'
    and write their names to 'output_file'
    """
    for item in item_list:
        if os.path.isdir(item):
            find_modified_in_directory(item, timestamp, output_file)
        else:
            if is_recent_mtime(item, timestamp):
                output_file.write(item + '\n')
