#!/usr/bin/python
import cStringIO
import glob
import json
import os
import pycurl
import re
import shutil
import tempfile
import types
import sys
import unicodedata
from optparse import OptionParser
from zipfile import ZipFile


usage = "usage: %prog [options]\nConfigure a disconnected environment for katello using a subscription manifest and pulp-admin."
parser = OptionParser(usage=usage, description="")
parser.add_option("-m", "--manifest", dest="manifest",
                  help="read manifest from FILE", metavar="FILE")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="verbose mode")
parser.add_option("-d", "--debug", action="store_true", dest="debug", help="debug mode")
parser.add_option("-e", "--everything", action="store_true", dest="everything",
                  help="create repos for everything (beta, iso, srpms, debug rpms)")
parser.add_option("-s", "--script-output-dir", dest="script_output_dir",
                  help="write generation scripts to DIRECTORY", metavar="DIRECTORY")
parser.add_option("-o", "--output-dir", dest="output_dir",
                  help="write repo tree to DIRECTORY", metavar="DIRECTORY")
parser.add_option("-u", "--uncommon", dest="enable_uncommon", default=False, action="store_true", 
                  help="sync repos that are not commonly used (betas, srpms, isos, debug rpms)")
parser.add_option("-C", "--ca-cert", dest="cacert", default="/etc/rhsm/ca/redhat-uep.pem",
                  help="specify CA cert to use", metavar="FILE")
parser.add_option("-f", "--force", dest="force", default=False, action="store_true", 
                  help="overwrite any existing directories and scripts (output,script output)")
parser.add_option("-a", "--arches", dest="arches", default=None, help="only create repos for a specific architecture (e.g. x86_64,i386)")
parser.add_option("-r", "--releases", dest="releases",
                  help="Only create repos for specific versions (e.g. 6Server,5.4)")
parser.add_option("-c", "--channels", dest="channels",
                  help="Only create repos for specific channels (e.g. cf-tools_1.0,subscription-asset-manager_1)")
parser.add_option("-n", "--dry-run", dest="dryrun", default=False, action="store_true", 
                  help="Do not create repos, only write scripts to show what would be created")
parser.add_option("-t", "--cdnhost", dest="cdn_hostname", default="https://cdn.redhat.com/",
        help="CDN hostname (https://cdn.redhat.com by default)")

(options, args) = parser.parse_args()

VALID_ARCHES = ['i386','x86_64','ia64']
VALID_RELEASES = ['5.6','5.7','5.8','5Server','6.1','6.2','6.3','6Server']
VALID_CHANNELS = ['supplementary','vt','optional','cf-tools','productivity','devtoolset','subscription-asset-manager']
DEFAULT_CLIENT_CA_CERT = '/etc/rhsm/ca/redhat-uep.pem'
REPO_CREATE_STR = "pulp-admin repo create --id %s --feed %s --feed_ca %s --feed_cert %s"
REPO_SYNC_STR = "pulp-admin repo sync --id %s"
REPO_EXPORT_STR = "pulp-admin repo export --id %s -t %s"

opt_err = False
if not options.manifest:
    print "error: --manifest is required."
    opt_err = True
if not options.output_dir:
    print "error: --output-dir is required."
    opt_err = True
if not options.script_output_dir:
    print "error: --script-output-dir is required."
    opt_err = True
if options.arches:
    arches = options.arches.split(',')
    for arch in arches:
        if not arch in VALID_ARCHES:
            print "error: %s is not in valid list of arches: %s" % (arch, VALID_ARCHES)
            opt_err = True
            break
if options.releases:
    releases = options.releases.split(',')
    for release in releases:
        if not release in VALID_RELEASES:
            print "error: %s is not in valid list of releases: %s" % (release, VALID_RELEASES)
            opt_err = True
            break
if options.channels:
    channels = options.channels.split(',')
    for channel in channels:
        if not channel in VALID_CHANNELS:
            print "error: %s is not in valid list of channels: %s" % (channel, VALID_CHANNELS)
            opt_err = True
            break
if opt_err:
    print usage
    sys.exit(1)

url_cache = {}

# print verbose
def p_v(str):
    if options.verbose:
        print str

# print debug
def p_d(str):
    if options.debug:
        print str

def fetch_listing(url, cert_location):
    if type(url) == types.UnicodeType:
        #pycurl does not accept unicode strings for a URL, so we need to convert
        url = unicodedata.normalize('NFKD', url).encode('ascii','ignore')

    if url_cache.get(url, False):
        return url_cache[url]
    p_v("fetching %s" % url)

    buf = cStringIO.StringIO()
    curl.setopt(curl.URL, url)
    curl.setopt(curl.WRITEFUNCTION, buf.write)
    curl.setopt(curl.SSLCERT, cert_location)
    curl.setopt(curl.CAINFO, cacert)
    curl.perform()
    status = curl.getinfo(curl.HTTP_CODE)
    if status != 200:
      raise Exception("ERROR: status code returned from cdn was %s" % status)

    listing = buf.getvalue()
    buf.close()
    url_cache[url] = listing
    return listing

def expand_variable(url, var, cert_location):
    url_prefix = url[0:url.find('$')]
    if url_prefix.endswith('/'):
        url_prefix = url_prefix[:-1]
    fetch_url = options.cdn_hostname + url_prefix + '/listing'
    releases = fetch_listing(fetch_url, cert_location).split('\n')
    expanded_urls = []
    for release in releases:
        if release.strip() == '':
            continue # skip empty lines
        expanded_urls.append(url.replace(var, release))
    return expanded_urls

def create_dir(directory, cert_location, with_listing=False):
    try:
        p_v("creating dir %s" % options.output_dir + directory)
        os.makedirs(options.output_dir + directory)
    except KeyboardInterrupt:
        raise
    except OSError, e:
        # if the dir is already there, pass
        if e.errno == 17:
            pass
    if with_listing:
        if directory.endswith('/'):
            directory = directory[:-1]
        listing_file = open(options.output_dir + directory + '/listing', 'w')
        listing_file.writelines(fetch_listing(options.cdn_hostname + directory + '/listing', cert_location))
        listing_file.close()

def chown_dirs(path):
    p_v("setting permissions to apache(48) on %s" % path)
    try:
        os.chown(path,48,48)
        for root, dirs, files in os.walk(path):
            for dir in dirs:
                os.chown(os.path.join(root, dir),48,48)
    except KeyboardInterrupt:
        raise
    except:
        print("unable to set permissions for apache on %s" % path)
        pass

curl = pycurl.Curl()

if options.cacert:
    cacert = options.cacert
else:
    cacert = DEFAULT_CLIENT_CA_CERT

try:
    workdir = tempfile.mkdtemp() # create dir
    shutil.copy(options.manifest, workdir)
    print("extracting manifest") 
    zip_file = ZipFile(options.manifest, "r")
    zip_file.extractall(workdir)
    zip_file = ZipFile(os.path.join(workdir, "consumer_export.zip"), "r")
    zip_file.extractall(workdir)
except KeyboardInterrupt:
    raise
except:
    print "unable to exttract manifest"
    raise

try:
    if os.path.exists(options.output_dir):
        if options.force:
            shutil.rmtree(options.output_dir)
    if os.path.exists(options.script_output_dir):
        if options.force:
            shutil.rmtree(options.script_output_dir)
    os.makedirs(options.script_output_dir)
    sync_writer = open(options.script_output_dir + "/sync_all.sh", "wb")
    export_writer = open(options.script_output_dir + "/export_all.sh", "wb")
    repos_writer = open(options.script_output_dir + "/repos.list", "wb")
        
except:
    print "unable to create output scripts"
    raise

entitlement_files = glob.glob(os.path.join(workdir, "export", "entitlements", "*.json"))
product_cert_mapping = {}

for entitlement_file in entitlement_files:
    entitlement_fh = open(entitlement_file)
    entitlement_json = json.load(entitlement_fh)
    # we only ever have one cert in here for now, but this might change in the future.
    cert = entitlement_json['certificates'].pop()

    cert_path = os.path.join(workdir, "export", "entitlement_certificates", "%s.pem" % cert['serial']['serial'])
    product_cert_mapping[entitlement_json['pool']['productId']] = cert_path
    if 'providedProducts' in entitlement_json['pool']:
        for pp in entitlement_json['pool']['providedProducts']:
            p_v("provided products found: %s" % pp['productId'])
            product_cert_mapping[pp['productId']] = cert_path

p_v("cert to product mapping: %s" %  product_cert_mapping)

print("reading products")
products = glob.glob(os.path.join(workdir, "export", "products", "*.json"))
p_v("products found: %s" % products)



full_content_urls = []
for product in products:
    product_file = open(product)
    product_json = json.load(product_file)
    try:
        cert_location = product_cert_mapping[product_json['id']]
    except KeyboardInterrupt:
        raise
    except KeyError:
        # if there's no mapping for this product, just skip
        p_v("skipping product json %s, not found in mapping" % product)
        continue
    
    p_v("examining product json %s" % product)
    for productContent in product_json['productContent']:
        contentURL = productContent['content']['contentUrl']
        contentName = productContent['content']['name']
        sys.stdout.write(contentName)
        sys.stdout.flush()
            
        # we have the contentURL, now write the dirs out locally and expand/walk the content tree
        # start at the first var that needs expansion
        content_prefix = contentURL[0:contentURL.find('$')]
        create_dir(content_prefix, cert_location, with_listing=True)
        # top-level listings are created, step through the releases now
        release_dirs = expand_variable(contentURL, '$releasever', cert_location)
        for rd in release_dirs:
            sys.stdout.write('.')
            sys.stdout.flush()
            create_dir(rd[0:rd.find('$')], cert_location, with_listing=True)
            arches_dirs = expand_variable(rd, '$basearch', cert_location)
            for ad in arches_dirs:
                # if two products have the same repo listed, do not add twice to url list
                if ad not in full_content_urls:
                    p_v("adding url: %s" % ad)
                    full_content_urls.append(ad)
                create_dir(ad, cert_location)
        print "\n"

chown_dirs(options.output_dir)
print "\ncreating repos in pulp"
beta_parser = re.compile(r'beta')
if options.arches:
    arches_parser = re.compile(r'\b(?:%s)\b' % '|'.join(arches))
if options.releases:
    releases_parser = re.compile(r'\b(?:%s)\b' % '|'.join(releases))
if options.channels:
    channels_parser = re.compile(r'\b(?:%s)\b' % '|'.join(channels))
if not options.releases:		# Added for channel matching, so releases are not excluded
    releases_parser = re.compile(r'\b(?:%s)\b' % '|'.join(VALID_RELEASES))
uncommon_parser = re.compile(r'beta|source|debug|iso')
for u in full_content_urls:
    p_d("processing %s" % u)
    s = u.split('/')
    if not options.enable_uncommon and uncommon_parser.search(u):
        p_v("uncommon, skipping %s" % u)
        continue
    if options.arches and not arches_parser.search(u):
        p_v("arch not specified, skipping %s" % u)
        continue
    if options.releases and not releases_parser.search(u):
        p_v("release not specified, skipping %s" % u)
        continue
    if options.channels and not channels_parser.search(u):
        p_v("checking if %s is a release OS" % u)
        if u.endswith("os") and releases_parser.search(s[-3]): 	# e.g. it is default or specified release 
            p_v("valid release, ignoring channel specification for %s" %u)
        else:
            p_v("channel not specified, skipping %s" % u)
            continue
    try:
        if u.endswith("os"):
            repo_id = s[6] + '_' + s[4] + '_' + '_'.join(s[7:-1])
        else:
            repo_id = s[6] + '_' + s[4] + '_' + '_'.join(s[7:])
    except:
        # if anything goes wrong create at least working repo id
        repo_id = '_'.join(filter(None, s[-5:]))

    # we need to append this to the repoid, otherwise it will conflict with the dist version
    if beta_parser.search(u):
        repo_id += "_beta"

    sys.stdout.write('.')
    sys.stdout.flush()
    try:
        # TODO: this should call directly to pulp's rest api, instead of using pulp-admin
        p_v("executing " +  REPO_CREATE_STR % (repo_id, options.cdn_hostname+u, cacert, cert_location) + "> /dev/null")
        if not options.dryrun:
            os.system(REPO_CREATE_STR % (repo_id, options.cdn_hostname+u, cacert, cert_location) + "> /dev/null")
        p_v("writing commands to files")
        sync_writer.write(REPO_SYNC_STR % repo_id + '\n')
        export_writer.write(REPO_EXPORT_STR % (repo_id, options.output_dir + u) + '\n')
	# vvaldez adding single list file with accompanying script
        repos_writer.write(repo_id + '\t' + u + '\n')
    except KeyboardInterrupt:
        raise
    except:
        print "unable to write file"
        raise

sync_writer.close()
export_writer.close()
repos_writer.close()
p_v("exiting")
print "\n"
