#!/usr/bin/python2.7
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved.
#
#
# userland-fetch - a file download utility
#
#  A simple program similar to wget(1), but handles local file copy, ignores
#  directories, and verifies file hashes.
#

import errno
import os
import re
import sys
import shutil
import json
import subprocess
from urllib import splittype, splithost
from urllib2 import urlopen, HTTPError
import hashlib
import getopt

def printIOError(e, txt):
	""" Function to decode and print IOError type exception """
	print "I/O Error: " + txt + ": "
	try:
		(code, message) = e
		print str(message) + " (" + str(code) + ")"
	except:
		print str(e)

def validate_signature(path, signature):
	"""Given paths to a file and a detached PGP signature, verify that
	the signature is valid for the file.  Current configuration allows for
	unrecognized keys to be downloaded as necessary."""

	# Find the root of the repo so that we can point
	# GnuPG at the right configuration and keyring.
	for get_root_cmd in [["hg", "root"], ["git", "rev-parse", "--show-toplevel"]]:
		try:
			proc = subprocess.Popen(get_root_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
			proc.wait()
			if proc.returncode == 0:
				out, _ = proc.communicate()
				gpgdir = os.path.join(out.strip(), "tools", ".gnupg")
				break
		except OSError as e:
			if e.errno != os.errno.ENOENT:
				raise
	else:
		message = "Warning: No Mercurial or Git repository was detected!\n"
		message += "Warning: Signature validation is disabled."
		return (True, message)

	try:
		# Suppress the permissions warning: none of the information here is private,
		# so not having to worry about getting mercurial keeping the directory
		# unreadable is just simplest.
		proc = subprocess.Popen(["/usr/bin/gpg2", "--verify", "--no-permission-warning",
		                         "--homedir", gpgdir, signature, path], stdin=open("/dev/null"),
		                        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

		proc.wait()

		if proc.returncode == 0:
			return (True, "")
		else:
			return (False, proc.stdout.read())

	except OSError as e:
		# If the executable simply couldn't be found, just skip the validation.
		if e.errno == errno.ENOENT:
			message = "Warning: Cannot validate the signature, command 'gpg2' not found!"
			return (True, message)
		else:
			raise    # Raise the unhandled exceptions to the caller function.

	assert False, "This statement should be unreachable!"

def validate(file, hash):
	"""Given a file-like object and a hash string, verify that the hash
	matches the file contents."""

	try:
		algorithm, hashvalue = hash.split(':')
	except:
		algorithm = "sha256"

	# force migration away from sha1
	if algorithm == "sha1":
		algorithm = "sha256"
	try:
		m = hashlib.new(algorithm)
	except ValueError:
		return False

	while True:
		try:
			block = file.read()
		except IOError, err:
			print str(err),
			break

		m.update(block)
		if block == '':
			break

	return "%s:%s" % (algorithm, m.hexdigest())

def validate_container(filename, hash):
	"""Given a file path and a hash string, verify that the hash matches the
	file contents."""

	try:
		file = open(filename, 'r')
	except IOError as e:
		printIOError(e, "Can't open file " + filename)
		return False
	return validate(file, hash)


def validate_payload(filename, hash):
	"""Given a file path and a hash string, verify that the hash matches the
	payload (uncompressed content) of the file."""

	import gzip
	import bz2

	expr_bz = re.compile('.+\.bz2$', re.IGNORECASE)
	expr_gz = re.compile('.+\.gz$', re.IGNORECASE)
	expr_tgz = re.compile('.+\.tgz$', re.IGNORECASE)

	try:
		if expr_bz.match(filename):
			file = bz2.BZ2File(filename, 'r')
		elif expr_gz.match(filename):
			file = gzip.GzipFile(filename, 'r')
		elif expr_tgz.match(filename):
			file = gzip.GzipFile(filename, 'r')
		else:
			return False
	except IOError as e:
		printIOError(e, "Can't open archive " + filename)
		return False
	return validate(file, hash)

def download(url, timeout, filename=None, quiet=False):
	"""Download the content at the given URL to the given filename
	(defaulting to the basename of the URL if not given.  If 'quiet' is
	True, throw away any error messages.  Returns the name of the file to
	which the content was donloaded."""

	src = None

	try:
		src = urlopen(url=url, timeout=timeout)
	except IOError as e:
		if not quiet:
			printIOError(e, "Can't open url " + url)
		return None
	except URLError as e:
		if not quiet:
			print "Can't open url " + url + ": " + str(e)
		return None

	# 3xx, 4xx and 5xx (f|ht)tp codes designate unsuccessfull action
	if 3 <= int(src.getcode()/100) <= 5:
		if not quiet:
			print "Error code: " + str(src.getcode())
		return None

	if not filename:
		filename = src.geturl().split('/')[-1]

	try:
		dst = open(filename, 'wb')
	except IOError as e:
		if not quiet:
			printIOError(e, "Can't open file " + filename + " for writing")
		src.close()
		return None

	while True:
		block = src.read()
		if not block:
			break
		dst.write(block)

	src.close()
	dst.close()

	# return the name of the file that we downloaded the data to.
	return filename

def pypi_url(url, filename):
	"""Given a pypi: URL, return the real URL for that component/version.

	The pypi scheme has a host (with an empty host defaulting to
	pypi.python.org), and a path that should be of the form
	"component==version".  Other specs could be supported, but == is the
	only thing that makes sense in this context.

	The filename argument is the name of the expected file to download, so
	that when pypi gives us multiple archives to choose from, we can pick
	the right one.
	"""

	host, path = splithost(splittype(url)[1])

	# We have to use ==; anything fancier would require pkg_resources, but
	# really that's the only thing that makes sense in this context.
	try:
		name, version = re.match("/(.*)==(.*)$", path).groups()
	except AttributeError:
		print "PyPI URLs must be of the form 'pypi:///component==version'"
		return None

	if not host:
		host = "pypi.python.org"

	for protocol in ["https", "http"]:
		success = True
		jsurl = "%s://%s/pypi/%s/json" % (protocol, host, name)
		try:
			# Don't wait very long for the connection
			f = urlopen(jsurl, None, 2)
		except HTTPError as e:
			success = False
			if e.getcode() == 404:
				printIOError(e, "Unknown component '%s'" % name)
			else:
				printIOError(e, "Can't open PyPI JSON url %s - http error code = %d" % (jsurl, e.getcode()))
		except IOError as e:
			success = False
			printIOError(e, "Can't open PyPI JSON url %s - IOError encountered" % jsurl)

		if success:
			break

	if not success:
		printIOError(e, "No protocol worked - bailing out.")
		return None

	js = json.load(f)
	try:
		verblock = js["releases"][version]
	except KeyError:
		print "Unknown version '%s'" % version
		return None

	urls = [d["url"] for d in verblock]
	for archiveurl in urls:
		if archiveurl.endswith("/%s" % os.path.basename(filename)):
			return archiveurl

	if urls:
		print "None of the following URLs delivers '%s':" % filename
		print "  " + "\n  ".join(urls)
	else:
		print "Couldn't find any suitable URLs"
	return None

def get_trusted_paths():
	""" Extracts trusted search paths from which archives
	can be downloaded. Archives are uploaded to trusted paths
	when the hash sum and signature are verified after an integration.
	"""
	trusted_paths = os.getenv('DOWNLOAD_SEARCH_PATH')
	if trusted_paths:
		return trusted_paths.split(' ')
	else:
		return []

def download_paths(search, filename, url):
	"""Returns a list of URLs where the file 'filename' might be found,
	using 'url', 'search', and $DOWNLOAD_SEARCH_PATH as places to look.

	If 'filename' is None, then the list will simply contain 'url'.
	"""

	urls = list()

	if filename:

		search += get_trusted_paths()

		file = os.path.basename(filename)

		urls = [base + '/' + file for base in search]

		# filename should always be first
		if filename in urls:
			urls.remove(filename)
		urls.insert(0, filename)

	# command line url is a fallback, so it's last
	if url != None and url not in urls:
		urls.append(url)

	return urls

def download_from_paths(search_list, file_arg, url, timeout_arg, link_arg, quiet=False):
	"""Attempts to download a file from a number of possible locations.
	Generates a list of paths where the file ends up on the local
	filesystem.  This is a generator because while a download might be
	successful, the signature or hash may not validate, and the caller may
	want to try again from the next location.  The 'link_arg' argument is a
	boolean which, when True, specifies that if the source is not a remote
	URL and not already found where it should be, to make a symlink to the
	source rather than copying it.
	"""
	for url in download_paths(search_list, file_arg, url):
		if not quiet:
			print "Source %s..." % url,

		scheme, path = splittype(url)
		name = file_arg

		if scheme in [None, 'file']:
			if os.path.exists(path) == False:
				if not quiet:
					print "not found, skipping file copy"
				continue
			elif name and name != path:
				if link_arg == False:
					if not quiet:
						print "\n    copying..."
					shutil.copy2(path, name)
				else:
					if not quiet:
						print "\n    linking..."
					os.symlink(path, name)
		elif scheme in ['http', 'https', 'ftp']:
			if not quiet:
				print "\n    downloading...",
			name = download(url, timeout_arg, file_arg, quiet)
			if name == None:
				if not quiet:
					print "failed"
				continue
		# Hook back to component Makefile to allow special download cases:
		# make://[component make target]/[archive name]
		elif scheme == "make":
			os.system("gmake " + path.split('/')[2])
		elif scheme == "pypi":
			nurl = pypi_url(url, file_arg)
			if nurl:
				if not quiet:
					print "\n    translated %s to %s..." % (url, nurl),
					print "\n    downloading...",
			else:
				if not quiet:
					print "\n    unable to contact PyPI",
				continue
			name = download(nurl, timeout_arg, file_arg, quiet)
			if name is None:
				if not quiet:
					print "failed"
				continue
		else:
			print " unknown scheme '%s'" % scheme
			return

		# Indicate if the file is local or downloaded from a trusted source.
		filename = os.path.basename(file_arg)
		if filename == url or any(p in url for p in get_trusted_paths()):
			trusted = True
		else:
			trusted = False

		yield (name, trusted)

def usage():
	print "Usage: %s [-f|--file (file)] [-l|--link] [-h|--hash (hash)] " \
          "[-s|--search (search-dir)] [-S|--sigurl (signature-url)] " \
	  "[-t|--timeout (timeout)] --url (url)" % \
          (sys.argv[0].split('/')[-1])
	sys.exit(1)

def main():
	# FLUSH STDOUT
	sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

	file_arg = None
	link_arg = False
	hash_arg = None
	url_arg = None
	sig_arg = None
	skip_sig_arg = False
	timeout_arg = 300
	search_list = list()

	try:
		opts, _ = getopt.getopt(sys.argv[1:], "f:h:ls:S:nt:u:",
			["file=", "link", "hash=", "search=", "sigurl=", "nosig",
			"timeout=", "url="])
	except getopt.GetoptError, err:
		print str(err)
		usage()

	for opt, arg in opts:
		if opt in ["-f", "--file"]:
			file_arg = arg
		elif opt in ["-l", "--link"]:
			link_arg = True
		elif opt in ["-h", "--hash"]:
			hash_arg = arg
		elif opt in ["-s", "--search"]:
			search_list.append(arg)
		elif opt in ["-S", "--sigurl"]:
			sig_arg = arg
		elif opt in ["-n", "--nosig"]:
			skip_sig_arg = True
		elif opt in ["-t", "--timeout"]:
			try:
				timeout_arg = int(arg)
			except ValueError:
				print "Invalid argument for %s, should be a " \
				    "number, but is %s" % (opt, arg)
				sys.exit(1)
			if timeout_arg < 0:
				print "Invalid argument for %s, should be a " \
				    "positive number, but is %s" % (opt, arg)
				sys.exit(1)
		elif opt in ["-u", "--url"]:
			url_arg = arg
		else:
			assert False, "unknown option"

	for (name, trusted) in download_from_paths(search_list, file_arg, url_arg,
	                                           timeout_arg, link_arg):


		# Variable sig_valid is set to True provided the signature is verified.
		# If the verification is skipped, value in warning_message is emitted.
		sig_valid = False
		warning_message = ""

		if not skip_sig_arg:

			print "\n    validating signature...",
			if sig_arg:
				# Put the signature file in the same directory as the
				# file we're downloading.
				sig_file = os.path.join(os.path.dirname(file_arg),
				                        os.path.basename(sig_arg))

				# Validate with the first signature we find.
				for (sig_file, _) in download_from_paths(search_list, sig_file, sig_arg,
				                                         timeout_arg, link_arg, True):
					if sig_file:
						(ok, msg) = validate_signature(name, sig_file)
						if ok == True and len(msg) == 0:
							print "ok"
							sig_valid = True
						elif ok == True or trusted == True:
							print "skipped"
							warning_message = '\n'+msg
						else:
							print "failed"
							print '\n'+msg    # Error output from 'gpg2'.

							print "Archive saved as %s may be tampered." % (name)
							try:
								os.remove(name)
								print "File '%s' was removed." % (name)
							except OSError:
								print "Failed to remove the corrupted file!"

							# Bad signature - exit with an error code.
							sys.exit(1)

						assert (ok == True or trusted == True), "Invalid signature handling!"
						break

				else:
					if trusted:
						print "skipped"
						warning_message = "\nWarning: Cannot fetch the signature!"
					else:
						print "failed (couldn't fetch signature)"
						sys.exit(1)

			else:
				print "skipped (no signature URL)"

		else:
			print ""
			warning_message = "Warning: Signature validation was disabled!"


		print "    validating hash...",
		realhash = validate_container(name, hash_arg)

		hash_failed = False
		if not hash_arg:
			print "skipping (no hash)"
			print "hash is: %s" % realhash
		elif realhash == hash_arg:
			print "ok"
		else:
			payloadhash = validate_payload(name, hash_arg)
			if payloadhash == hash_arg:
				print "ok"
			else:
				# If the signature validated, we assume
				# that the expected hash is just a typo,
				# but we warn just in case.
				if sig_valid:
					print "invalid hash! Did you forget to update it?"
				else:
					print "corruption detected"

				print "    expected: %s" % hash_arg
				print "    actual:   %s" % realhash
				print "    payload:  %s" % payloadhash

				if sig_valid:
					# If the hash is invalid and the signature validation
					# succeeded, rename the downloaded archive (so the
					# user doesn't have to re-download it) and fail.
					newname = name + ".invalid-hash"
					try:
						os.rename(name, newname)
						print "Archive saved as %s; " \
						    "if it isn't corrupted, " \
						    "rename it to %s." % (newname, name)
					except OSError:
						print "Failed to rename %s file!" % name

				hash_failed = True

		# Print the accumulated warning messages before the exit.
		if len(warning_message) > 0:
			print warning_message

		if not hash_failed:
			# No corruption detected - exit with success.
			sys.exit(0)
		else:
			# Invalid archive hash - exit with an error code.
			sys.exit(1)

	else:
		# Failed to download the package.
		print "Cannot download the archives! Terminating..."
		sys.exit(1)

	assert False, "This statement should be unreachable!"

if __name__ == "__main__":
	main()
