#!/usr/bin/env python2
# Author: Kees Cook <kees@ubuntu.com>
# Author: Marc Deslauriers <marc.deslauriers@ubuntu.com>
# Copyright (C) 2011-2013 Canonical Ltd.
#
# Reports CVEs that are not reported in USNs correctly.
#
# If you're going to use --update, you'll need to fetch the -all database and
# use --database too so you can copy it back after updating it.
#  wget -N http://people.canonical.com/~ubuntu-security/usn/database-all.pickle.bz2
#  bunzip2 -c database-all.pickle.bz2 > database-all.pickle
#
# This script is distributed under the terms and conditions of the GNU General
# Public License, Version 3 or later. See http://www.gnu.org/copyleft/gpl.html
# for details.
#
import cPickle, sys, os, os.path, optparse, usn_lib, cve_lib, glob
from source_map import version_compare
from lp_lib import UCTLaunchpad

parser = optparse.OptionParser()
parser.add_option("-v", "--verbose", dest="verbose", help="Report specific CVEs", action='store_true')
parser.add_option("-D", "--database", help="Specify location of USN data (default '%default')", default='database-all.pickle')
parser.add_option("-R", "--reverted", help="Specify location of reverted CVE list (default '%default')", default="meta_lists/reverted-CVEs.txt")
parser.add_option("-p", "--packages", help="Comma separated list of packages to examine (default is all kernel source packages)", default=None)
parser.add_option("-d", "--debug", help="Report additional debugging while processing USNs", action='store_true')
parser.add_option("--show-multipkg-usns", help="Include mismatches for USNs that contain multiple packages per USN", action='store_true')
parser.add_option("--drop-cve", help="Explicitly drop a CVE from all USNs", action='store', metavar='CVE[,CVE...]', default=None)
parser.add_option("--ignore-mismatched", help="Ignore CVEs reported in the wrong USN", action='store_true')
parser.add_option("--ignore-unreported", help="Ignore CVEs not yet reported in a USN", action='store_true')
parser.add_option("--include-retired", help="Examine retired CVEs when reporting missing CVEs in a USN", action='store_true')
parser.add_option("-u", "--update", help="Add missing CVEs to local USN database", action='store_true')
(opt, args) = parser.parse_args()

uctlp = UCTLaunchpad(opt)

if opt.packages == None:
    opt.packages = cve_lib.kernel_srcs
else:
    opt.packages = set(opt.packages.split(','))

reverted = usn_lib.get_reverted(opt.reverted)
if not os.path.exists(opt.database):
    raise ValueError("Missing database %s" % (opt.database))
db       = usn_lib.load_database(opt.database)

# Find all CVE paths
cvelist = []
cves = dict()
active = set()
check_dirs = [cve_lib.active_dir, cve_lib.retired_dir, cve_lib.ignored_dir]
for dirname in check_dirs:
    if len(args)==0:
        items = glob.glob('%s/CVE-*' % (dirname))
    else:
        items = [y for y in ['%s/%s' % (dirname, x) for x in args] if os.path.exists(y)]
    items = sorted(items)
    for cvefile in items:
        cvelist += [cvefile]

# Load and keep only kernel CVEs
if opt.debug:
    print >>sys.stderr, "Loading CVEs ..."
for cvefile in cvelist:
    state, cve = cvefile.split('/')[-2:]
    info = cve_lib.load_cve(cvefile)
    keep = False
    for pkg in opt.packages:
        if pkg in info['pkgs']:
            keep = True
            break
    if keep:
        cves[cve] = info
        if state == "active":
            active.add(cve)

pkgs = usn_lib.packages_dict(db, opt.packages, opt=opt)
touched = set()

# Build map of pkgs-updated-per-usn
updated_packages = dict()
for usn in sorted(db, cmp=version_compare):
    updated_packages[usn] = set()
    for rel in db[usn]['releases']:
        if not db[usn]['releases'][rel].has_key('sources'):
            continue
        for pkg in db[usn]['releases'][rel]['sources']:
            updated_packages[usn].add(pkg)

# Looks at all USNs and see if CVEs are reported in the wrong USNs
if not opt.ignore_mismatched:
  for usn in sorted(db, cmp=version_compare):
    # This USN is ancient and lacks any CVE information
    if not db[usn].has_key('cves'):
        if (opt.debug):
            print "%s lacks CVEs" % (usn)
        continue

    # cves[cve]
    # 'pkgs' -> dict(  pkg -> dict(  release ->  (state, notes)   ) )
    # db[usn]
    # 'releases', rel, 'sources', pkg, 'version'
    for rel in db[usn]['releases']:
        # Ignore EOL releases
        if rel in cve_lib.eol_releases:
            continue
        # Ignore old USNs
        if not db[usn]['releases'][rel].has_key('sources'):
            continue
        # Look at fixed packages
        for pkg in opt.packages:
            if not db[usn]['releases'][rel]['sources'].has_key(pkg):
                continue

            # Find versions fixed
            for cve in db[usn]['cves']:
                if not cve.startswith('CVE-'):
                    continue
                if not cve in cves:
                    # Must depend on CVEs being accurate, so if it's missing, it's missing
                    #print "USN %s: mentions unloaded %s" % (usn, cve)
                    continue

                # If not mentioned in the CVE, it wasn't vulnerable
                if not cves[cve]['pkgs'].has_key(pkg):
                    continue

                # If release not mentioned in the CVE, it wasn't vulnerable
                if not cves[cve]['pkgs'][pkg].has_key(rel):
                    continue

                if cves[cve]['pkgs'][pkg][rel][0] == 'pending':
                    # sync-from-usns.py needs to be run...
                    print "USN %s: %s in %s is still pending for %s (fixed in %s)" % (usn, pkg, rel, cve, db[usn]['releases'][rel]['sources'][pkg]['version'])
                    continue

                if cves[cve]['pkgs'][pkg][rel][0] in ['released', 'released-esm']:
                    fixed = cves[cve]['pkgs'][pkg][rel][1]
                    if fixed == db[usn]['releases'][rel]['sources'][pkg]['version']:
                        # Released and correct
                        continue

                    # If it was fixed in the devel release, there is no USN
                    release_version = uctlp.get_earliest_version(rel, pkg)
                    if version_compare(release_version, fixed) >= 0:
                        if opt.debug:
                            print >>sys.stderr, "Skipping %s in %s (fixed in %s, release was %s)" % (pkg, rel, fixed, release_version)
                        continue

                    latest = uctlp.get_latest_version(rel, pkg, key='updsec', pockets=['Updates','Security'])
                    if version_compare(fixed, latest) > 0:
                        if opt.debug:
                            print >>sys.stderr, "Skipping %s in %s (fixed in %s, latest was %s)" % (pkg, rel, fixed, latest)
                        continue

                    usn_to = None
                    single_pkg = len(updated_packages[usn]) == 1

                    # If "fixed" is in a published USN, we have a definitive
                    # mismatch.
                    if fixed in pkgs[pkg][rel]:
                        if cve == 'CVE-2008-3831' and pkg == 'linux' and usn == '679-1':
                            # Hardcoded exceptions
                            continue
                        if opt.show_multipkg_usns or single_pkg:
                            print "USN %s: %s in %s version mismatched, exact (USN is %s, %s shows %s, expected in USN %s)" % (usn, pkg, rel, db[usn]['releases'][rel]['sources'][pkg]['version'], cve, cves[cve]['pkgs'][pkg][rel][1], pkgs[pkg][rel][fixed])
                            usn_to = pkgs[pkg][rel][fixed]
                    else:
                        # If "fixed" is between USN versions, find the first
                        # USN fix is >fixed
                        for ver in (sorted(pkgs[pkg][rel], cmp=version_compare)):
                            if version_compare(ver, fixed) > 0:
                                if pkgs[pkg][rel][ver] != usn:
                                    if opt.show_multipkg_usns or single_pkg:
                                        print "USN %s: %s in %s version mismatched, offset (USN is %s, %s shows %s, expected in USN %s)" % (usn, pkg, rel, db[usn]['releases'][rel]['sources'][pkg]['version'], cve, cves[cve]['pkgs'][pkg][rel][1], pkgs[pkg][rel][ver])
                                        usn_to = pkgs[pkg][rel][ver]
                                break

                    # Did we find a CVE we need to move?
                    if usn_to != None:
                        usn_lib.del_cves(db, usn, [cve])
                        touched.add(usn)
                        usn_lib.add_cves(db, usn_to, [cve])
                        touched.add(usn_to)

# Create a per-CVE list of USNs that mention the CVE
usns = dict()
for usn in sorted(db, cmp=version_compare):
    if not db[usn].has_key('cves'):
        continue
    for cve in db[usn]['cves']:
        if not cve.startswith('CVE-'):
            continue
        usns.setdefault(cve, set())
        usns[cve].add(usn)

# Looks at all active CVEs and sees which USN they should have been reported in
if not opt.ignore_unreported:
    for cve in sorted(cves):
        if not cve in active and not opt.include_retired:
            continue
        usns.setdefault(cve, set())
        for pkg in opt.packages:

            if not pkgs.has_key(pkg):
                # silently skip packages that don't have USNs published
                if pkg == "linux-armadaxp":
                    continue
                print "Unknown package %s not in database-all.pickle: ignoring" % (pkg)
                continue
            if not cves[cve]['pkgs'].has_key(pkg):
                continue

            for rel in sorted(cves[cve]['pkgs'][pkg]):
                if rel in cve_lib.eol_releases + ['upstream','devel','product','snap']:
                    # Ignore non-stable or EOL releases
                    continue
                status = cves[cve]['pkgs'][pkg][rel][0]
                fixed = cves[cve]['pkgs'][pkg][rel][1]
                if not status in ['pending', 'released', 'released-esm']:
                    # Skip everything that isn't pending or released
                    continue
                if fixed == "":
                    # Skip pending with no destination version
                    continue

                # If it was fixed in the devel release, there is no USN
                release_version = uctlp.get_earliest_version(rel, pkg)
                if version_compare(release_version, fixed) >= 0:
                    if opt.debug:
                        print >>sys.stderr, "Skipping %s in %s (fixed in %s, release was %s)" % (pkg, rel, fixed, release_version)
                    continue

                latest = uctlp.get_latest_version(rel, pkg, key='updsec', pockets=['Updates','Security'])
                if version_compare(fixed, latest) > 0:
                    if opt.debug:
                        print >>sys.stderr, "Skipping %s in %s (fixed in %s, latest was %s)" % (pkg, rel, fixed, latest)
                    continue

                reported = False
                for usn in sorted(usns[cve], cmp=version_compare):
                    if not db[usn]['releases'].has_key(rel):
                        continue
                    if not db[usn]['releases'][rel].has_key('sources'):
                        continue
                    if not db[usn]['releases'][rel]['sources'].has_key(pkg):
                        continue
                    reported = True
                    break
                if not reported:
                    usn = None
                    # find the first USN reported version is >=fixed
                    for ver in (sorted(pkgs[pkg][rel], cmp=version_compare)):
                        if version_compare(ver, fixed) >= 0:
                            usn = pkgs[pkg][rel][ver]
                            break
                    if usn == None:
                        print "%s for %s in %s (%s %s) may need a USN if it's in -security" % (cve, pkg, rel, status, fixed)
                    else:
                        print "%s for %s in %s (%s %s) should be reported in USN %s" % (cve, pkg, rel, status, fixed, usn)
                        desc = cves[cve].get('Ubuntu-Description','').strip()
                        if len(desc) == 0:
                            raise ValueError("USN %s needs %s, but it lacks a Ubuntu-Description" % (usn, cve))
                        try:
                            usn_lib.add_cves(db, usn, [cve])
                        except Exception as e:
                            print >>sys.stderr, "ERROR: could not add %s to %s:\n%s" % (cve, usn, e)
                        usns[cve].add(usn)
                        touched.add(usn)

if opt.drop_cve:
    for cve in opt.drop_cve.split(','):
        for usn in usns[cve]:
            usn_lib.del_cves(db, usn, [cve])
            touched.add(usn)

if opt.update:
    usn_lib.save_database(db, opt.database)
    print "Updated: %s" % (" ".join(sorted(touched)))
