#!/usr/bin/env python2
# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security
# See the file license.txt for copying permission

import os
import sys
import re
import random
import string
from HTMLParser import HTMLParser
try:
    import xml.etree.ElementTree as et
except ImportError:
    print "drone-burp requires the lxml module"
    sys.exit(1)

sys.path.append(os.path.abspath(
    os.path.join(os.path.dirname(__file__), '..'))
)

from optparse import OptionParser
from urlparse import urlparse
from lairdrone import api, drone_models as models
from lairdrone import helper

OS_WEIGHT = 75
TOOL = "burp"

burp_issue_model = {
    'name': '',
    'issue_background': '',
    'issue_detail': '',
    'remediation_background': '',
    'vulnerability_classifications': [],
    'references': [],
    'ip': '',
    'port': 0,
    'url': '', # https://api.example.com
    'scheme': '', # https
    'hostname': '', # api.example.com
    'path': '',
    'severity': '',
}

class MLStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return ''.join(self.fed)

def strip_tags(html):
    s = MLStripper()
    s.feed(html)
    txt = s.get_data().strip()
    txt = re.sub('\n\n\n(\s*)', '\n\n', txt)
    return txt

# add_issue_hosts adds the host for each issue in the list of issues to the vuln.
def add_issue_hosts(vuln, issues):
    for issue in issues:
        host_key_dict = dict(models.host_key_model)
        host_key_dict['string_addr'] = issue['ip']
        host_key_dict['port'] = issue['port']

        # Check if host/port is already associated with vuln, add if not
        if host_key_dict not in vuln['hosts']:
            vuln['hosts'].append(host_key_dict)

# merge_hosts merges hosts from the src list of hosts into the dst list.
def merge_hosts(dst, src):
    for shost in src:
        has_host = False
        for dhost in dst:
            if shost['string_addr'] == dhost['string_addr']:
                has_host = True

                if not dhost.get('os'):
                    dhost['os'] = shost['os']

                for p in shost['ports']:
                    has_port = False
                    for dp in dhost['ports']:
                        if p['port'] == dp['port']:
                            has_port = True

                    if not has_port:
                        dhost['ports'].append(p)

                has_hostname = False
                for h in shost['hostnames']:
                    for dh in dhost['hostnames']:
                        if h == dh:
                            has_hostname = True

                    if not has_hostname:
                        dhost['hostnames'].append(h)

                # TODO: handle notes

        if not has_host:
            dst.append(shost)


# get_issue_hosts gets a list of models.host_model's from the list of issues, one per issue.  These
# must be de-duped later.
def get_issue_hosts(issues):
    hosts = list()
    for issue in issues:
        host = dict(models.host_model)
        host['os'] = list() # no OS
        host['ports'] = list()
        host['hostnames'] = list()

        if issue['ip'] == '':
            raise Exception('issue has blank ip')

        host['string_addr'] = issue['ip']
        host['long_addr'] = helper.ip2long(issue['ip'])

        port = dict(models.port_model)
        port['port'] = issue['port']
        port['protocol'] = models.PROTOCOL_TCP
        port['service'] = issue['scheme']
        host['ports'].append(port)

        # Don't set an OS
        os_dict = dict(models.os_model)
        os_dict['tool'] = TOOL
        host['os'].append(os_dict)

        merge_hosts(hosts, [host])

    return hosts


def get_severity(severity):
    if severity == 'High':
        return 10.0
    if severity == 'Medium':
        return 5.0
    if severity == 'Low':
        return 3.0
    return 0.0

def process_retirejs(plugin_id, issues):
    v = dict(models.vulnerability_model)

    v['title'] = 'Use of Components with Known Vulnerabilities'
    v['cvss'] = 6.5
    v['description'] = '''We identified third-party software components that have known vulnerabilities in the versions in use.

Third party software components are frequently used in the creation of software. They provide functionality that the application creators would otherwise need to write themselves. Third party software components are frequently, but not exclusively, open source and can have a wide range of quality, bug reporting, and security practices.

Security vulnerabilities in third-party software components can expose the parent application to security issues. The full range of weaknesses is possible, including injection, broken access control, XSS, etc. The impact could range from minimal to complete host takeover and data compromise.

Tracking weaknesses in software applications can be a difficult task. Vulnerability reports for commercial or open source software do not always specify exactly which versions of a component are vulnerable in a standard, searchable way. Further, not all libraries use an understandable version numbering system and many do not report to a central clearinghouse such as CVE and NVD.'''
    v['solution'] = '''Update the affected components to the latest secure version.

The projects for most software components or libraries do not create vulnerability patches for old versions. Instead, most simply fix the problem in the next release. Therefore, upgrading to these new versions is critical.

One option for mitigation is to not use components that you didn't write. Because this is usually not a realistic option, software projects should have a process in place to:

1. Identify all components and the versions you are using, including all dependencies. (e.g., the versions plugin).
2. Monitor the security of these components in public databases, project mailing lists, and security mailing lists, and keep them up to date.
3. Establish security policies governing component use, such as requiring certain software development practices, passing security tests, and acceptable licenses.
4. Where appropriate, consider adding security wrappers around components to disable unused functionality and/ or secure weak or vulnerable aspects of the component.

Services such as [snyk.io](https://snyk.io/) can help manage many aspects of this process.

For more background on the dangers posed by software dependencies in general, see this blog post by Russ Cox: [research.swtch.com/deps](https://research.swtch.com/deps).'''

    v['tags'] = ['cat:application']

    plugin_dict = dict(models.plugin_id_model)
    plugin_dict['tool'] = TOOL
    plugin_dict['id'] = plugin_id
    v['plugin_ids'].append(plugin_dict)

    retirejs_detail = re.compile(r'The library <b>(?P<library>[^<]+)</b> version <b>(?P<version>[^<]+)</b>.*\n.*\n.*\n.*\n<ul>\n(?P<urls>(.*\n)+)</ul>\n.*\n.*\n.*\nThe vulnerability is affecting all versions prior <b>(?P<before>[^<]+)</b> \(between <b>(?P<from>[^<]+)</b> and <b>(?P<to>[^<]+)</b>\)')
    href = re.compile('href="([^"]+)"')

    libraries = dict() # '<library>-<version>': {'groupdict': {}, 'issues': []}

    for issue in issues:
        tag = 'dhostname:%s->%s:%s/tcp' % (issue['hostname'], issue['ip'], issue['port'])
        has_tag = False
        for t in v['tags']:
            if t == tag:
                has_tag = True
        if not has_tag:
            v['tags'].append(tag)

        m = retirejs_detail.search(issue['issue_detail'])
        if m is None:
            raise Exception("retirejs issue detail doesn't match regex")

        gd = m.groupdict()

        libkey = '%s-%s' % (gd['library'], gd['version'])
        if libkey not in libraries:
            libraries[libkey] = {'groupdict': gd, 'issues': []}
        libraries[libkey]['issues'].append(issue)

    # NOTE: I am omitting the component url for now, since we don't know what it is.
    detail_tpl = '''The library **%s** version **%s** is in use and has known security issues. The vulnerabilities affect all versions %s.

We identified the following affected pages, though there are likely more:

%s

More details can be found in the following issue disclosures:

%s'''

    # each unique library gets its own output

    for library in libraries.values():
        libdict = library['groupdict']
        version_range = 'between **%s** and **%s**' % (libdict['from'], libdict['to'])
        if libdict['from'] == '*':
            version_range = 'prior to **%s**' % libdict['to']

        pages = []
        for issue in library['issues']:
            pages.append(issue['url'] + issue['path'])

        disclosures = []
        for url in libdict['urls'].split():
            m = href.search(url)
            if m is not None and len(m.groups()) > 0:
                disclosures.append(m.group(1))

        evidence = detail_tpl % (
            libdict['library'],
            libdict['version'],
            version_range,
            '\n'.join(['- ' + page for page in pages]),
            '\n'.join(['- <%s>' % disc for disc in disclosures]),
        )

        if v['evidence']:
            v['evidence'] += '\n\n---\n\n'

        v['evidence']+= evidence

    # we would pass a filtered list of issues if we were weeding some out
    add_issue_hosts(v, issues)
    return v, get_issue_hosts(issues)


def process_default(plugin_id, issues):
    # add in hostnames
    # for details, add in the issue path, followed by the unique content, but group by matching content.
    v = dict(models.vulnerability_model)

    v['title'] = issues[0]['name'].title()
    v['cvss'] = get_severity(issues[0]['severity'])
    v['description'] = strip_tags(issues[0]['issue_background']).replace('\n', '\n\n')
    v['solution'] = strip_tags(issues[0]['remediation_background']).replace('\n', '\n\n')
    if issues[0]['references']:
        v['solution'] += '\n\nAdditional Resources:\n\n'
        v['solution'] +=  '\n'.join(['- <%s>' % ref for ref in issues[0]['references']])


    v['tags'] = ['cat:application']

    plugin_dict = dict(models.plugin_id_model)
    plugin_dict['tool'] = TOOL
    plugin_dict['id'] = plugin_id
    v['plugin_ids'].append(plugin_dict)

    evidences = {} # issue_detail: [issue]
    for issue in issues:
        tag = 'dhostname:%s->%s:%s/tcp' % (issue['hostname'], issue['ip'], issue['port'])
        has_tag = False
        for t in v['tags']:
            if t == tag:
                has_tag = True
        if not has_tag:
            v['tags'].append(tag)

        if issue['issue_detail'] not in evidences:
            evidences[issue['issue_detail']] = []

        has_issue = False
        for ei in evidences[issue['issue_detail']]:
            if ei['url'] == issue['url']:
                has_issue = True

        if not has_issue:
            evidences[issue['issue_detail']].append(issue)

    for evidence, evid_issues in evidences.items():
        if v['evidence']:
            v['evidence'] += '\n\n---\n\n'

        v['evidence'] += '%s:\n\n' % ', '.join([issue['url'] + issue['path'] for issue in evid_issues])
        v['evidence'] += strip_tags(evidence.strip()).replace('\n', '\n\n')

    add_issue_hosts(v, issues)
    return v, get_issue_hosts(issues)


def parse(project, burp_file, db, options):
    """Parses a Burp file and updates the Lair database

    :param project: The project id
    :param burp_file: The Burp xml file to be parsed
    :param db: A database connection
    """
    try:
        import xml.etree.ElementTree as et
    except ImportError as e:
        print "[!] Error: {0}. Install/upgrade module".format(e)
        exit(1)

    tree = et.parse(burp_file)
    root = tree.getroot()

    # Create the project dictionary which acts as foundation of document
    project_dict = dict(models.project_model)
    project_dict['commands'] = list()
    project_dict['vulnerabilities'] = list()
    project_dict['project_id'] = project

    # Temp dicts used to ensure no duplicate hosts or ports are added
    temp_vulns = dict()
    temp_hosts = list()

    command_dict = dict(models.command_model)
    command_dict['tool'] = TOOL
    command_dict['command'] = 'Active scan'
    project_dict['commands'].append(command_dict)

    # Group vuln instances by their plugin_id
    temp_issues = dict()

    for issue_elem in root.iter('issue'):
        issue = dict(burp_issue_model)
        issue['references'] = list()
        issue['vulnerability_classifications'] = list()

        name = issue_elem.find('name')
        if name is not None:
            issue['name'] = name.text

        issue_background = issue_elem.find('issueBackground')
        if issue_background is not None:
            # don't strip here, we need original for regex matches in post processing
            issue['issue_background'] = issue_background.text

        issue_detail = issue_elem.find('issueDetail')
        if issue_detail is not None:
            issue['issue_detail'] = issue_detail.text

        remediation_background = issue_elem.find('remediationBackground')
        if remediation_background is not None:
            issue['remediation_background'] = remediation_background.text

        classifications = issue_elem.find('vulnerabilityClassifications')
        if classifications is not None:
            issue['vulnerability_classifications'] = classifications.text

        references = issue_elem.find('references')
        if references is not None:
            for ref in references.text.split('href="'):
                if ref.find('"') != -1:
                    issue['references'].append(ref[0:ref.index('"')])

        host_elem = issue_elem.find('host')
        if host_elem is not None:
            issue['ip'] = host_elem.attrib['ip']
            issue['url'] = host_elem.text
            issue['port'] = 80

            url = urlparse(host_elem.text)
            issue['scheme'] = url.scheme
            issue['hostname'] = url.hostname
            if url.port:
                issue['port'] = url.port
            if url.scheme == 'https':
                issue['port'] = 443

        path = issue_elem.find('path')
        if path is not None:
            issue['path'] = path.text

        severity = issue_elem.find('severity')
        if severity is not None:
            issue['severity'] = severity.text

        type_elem = issue_elem.find('type')
        if type_elem is not None:
            plugin_id = type_elem.text
            if type_elem.text == "134217728":
                # For these we need persistent IDs for each type we want to group otherwise things like retire.js won't group propertly.
                if issue['name'].startswith('Vulnerable version of the library'):
                    plugin_id = 'retirejs01'
                if issue['name'].startswith('CSP: The domain is hosting user content'):
                    plugin_id = 'csp-domain-uc01'
                if plugin_id == "134217728":
                    print "\n\nWARNING: Issue is using a Burp extension generated issue type (plugin_id) which may conflict with other project vulns and not be added as a result --",  issue['name'] + '\n\n'
                else:
                    print "CHANGED:", issue['name'], plugin_id

        if plugin_id not in temp_issues:
            temp_issues[plugin_id] = []

        temp_issues[plugin_id].append(issue)

    temp_vulns = list() # [models.vulnerability_model]
    temp_hosts = list() # [models.host_model]

    for plugin_id, issues in temp_issues.items():
        # issues shouldn't be handling hosts. We're going to map from

        if issue['severity'] == 'Information' and not options.include_informational:
            continue

        # handle special cases:
        if issues[0]['name'].startswith('Vulnerable version of the library'):
            vuln, hosts = process_retirejs(plugin_id, issues)
            # hosts must be already added to vuln as models.host_key_model, v['hosts']

        else:
            vuln, hosts = process_default(plugin_id, issues)

        # vuln will be unique
        temp_vulns.append(vuln)

        # only add hosts if they are unique:
        merge_hosts(temp_hosts, hosts)

    project_dict['vulnerabilities'] = temp_vulns
    project_dict['hosts'] = temp_hosts

    return project_dict

if __name__ == '__main__':

    usage = "usage: %prog <project_id> <file>"
    description = "%prog imports Burp files into Lair"

    parser = OptionParser(usage=usage, description=description,
                          version="%prog 0.0.1")
    parser.add_option(
        "--include-informational",
        dest="include_informational",
        default=False,
        action="store_true",
        help="Forces informational plugins to be loaded"
    )

    (options, args) = parser.parse_args()

    if len(args) != 2:
        print parser.get_usage()
        exit(1)

    # Connect to the database
    db = api.db_connect()

    project = parse(args[0], args[1], db, options)

    api.save(project, db, TOOL)

    exit(0)
