#!/usr/bin/env python
# Authors Dominig ar Foll (Intel OTC) (first versions in Bash)
#         Florent Vennetier (Intel OTC) (third version in Python)
# Date 16 February 2012
# Version 3.4
# License GPLv2
#
# Credit Thanks to Yan Yin for providing the initial version used as a base
#

import sys
import os
import urllib
import urllib2
import urlparse
import tempfile
from xml.etree import ElementTree

from osc import conf, core

DEBUG_MODE = False

def help_(progName="obs2obscopy"):
    print """HELP
Function : Copy a project from an OBS to an OBS, using as input a package list
           with the desired MD5 to represent the package revision.
           If target package is already at the correct revision, its copy
           is ignored. Running the script several time is possible until zero
           errors is acheived.
           A log file named <MD5_FILE>.log is created for everyrun.

Usage:     %s obs-alias source_prj target_prj md5_file
       or  %s obs-alias source_prj other-obs-alias target_prj md5_file
           md5_file is created by the script obstag

Example1: Local copy
       %s http://myObs.mynetwork:81 MyMeeGo:1.2:oss myTest:MeeGo:1.2:oss my_revision_tag.md5
Example2: Remote copy (by link)
       %s http://myObs.mynetwork:81 meego.com:MeeGo:1.2:oss myobs:MeeGo:1.2:oss my_revision_tag.md5
           where meego.com is a local projet which is a link to a remote OBS public api
Example3: Remote copy
       %s http://api.pub.meego.com Project:MINT:Testing http://myObs.mynetwork:81 myTest:MINT:Testing my_revision_tag.md5

Return code:
          0 success
          1 some packages not copied
          2 wrong (number of) arguments
          3 problem reading MD5 file or writing log file
          4 network error
          5 interrupted by user

Environment:
       If the DEBUG environment variable is non-empty, all URLs will
       be printed to stderr before being called.

Version 3.4  License GPLv2""" % (progName, progName, progName, progName, progName)

logFilePath = "/tmp/obs2obscopy.log"
logFile = None

old_stderr = sys.stderr
old_stdout = sys.stdout

class PrintHandler(object):
    """
    Wrapper for sys.stderr and sys.stdout that writes to a log file.
    """

    def __init__(self, stream, log_file):
        self.stream = stream
        self.log_file = log_file
        self.skipline = False

    def write(self, buf):
        # osc has the bad habit to print a warning when we do not use HTTPS.
        # And worse, it also prints a newline...
        if buf.startswith("WARNING: SSL certificate checks disabled"):
            self.skipline = True
        elif self.skipline:
            self.skipline = False
        else:
            self.stream.write(buf)
            self.log_file.write(buf)

    def flush(self):
        self.stream.flush()
        self.log_file.flush()

    def close(self):
        self.stream.close()

def printerror(message):
    print >> sys.stderr, message

def printmessage(message, newline=True):
    if newline:
        print >> sys.stdout, message
    else:
        print >> sys.stdout, message,

def printdebug(message):
    if DEBUG_MODE:
        print >> sys.stderr, "DEBUG:", message


def makeurl(baseurl, l, query=[]):
    """
    Replacement for `osc.core.makeurl` which preserves the "path"
    part of the url.
    """

#    if conf.config['verbose'] > 1:
#        print 'makeurl:', baseurl, l, query,

    if type(query) == type(list()):
        query = '&'.join(query)
    elif type(query) == type(dict()):
        query = urllib.urlencode(query)

    scheme, netloc, path = urlparse.urlsplit(baseurl)[0:3]
    if len(path) > 0:
        l.insert(0, path)
    finalurl = urlparse.urlunsplit((scheme, netloc, '/'.join(l), query, ''))
    if conf.config['verbose'] > 1 or DEBUG_MODE:
        printdebug("-> %s" % finalurl)
    return finalurl

def getPackageFiles(apiurl, project, package, src_revision=None):
    query = {}
    if src_revision:
        query["rev"] = src_revision
    query["expand"] = "1"
    url = makeurl(apiurl, ['source', project, package], query=query)
    res = core.http_request("GET", url)
    xmlResult = res.read()
    directoryList = ElementTree.fromstring(xmlResult)
    result = {}
    for directory in directoryList:
        for packageEntry in directory.iter("entry"):
            name = packageEntry.get("name")
            md5 = packageEntry.get("md5")
            if name.startswith('_service:') or name.startswith('_service_'):
                continue
            result[name] = md5
    return result

def testPackageFiles(src_apiurl, src_project, src_package,
                     dst_apiurl, dst_project, dst_package,
                     src_revision):

    listToUpdate = set()
    dicoSrcFiles = getPackageFiles(src_apiurl, src_project, src_package, src_revision)
    dicoDstFiles = getPackageFiles(dst_apiurl, dst_project, dst_package)

    listSrcPackageFiles = set(dicoSrcFiles.keys())
    if '_link' in listSrcPackageFiles and len(listSrcPackageFiles)>1:
        listSrcPackageFiles.remove('_link')

    listDstPackageFiles = set(dicoDstFiles.keys())

    listToTest = listSrcPackageFiles.intersection(listDstPackageFiles)

    for aFile in listToTest:
        if dicoSrcFiles[aFile] != dicoDstFiles[aFile]:
            listToUpdate.add(aFile)

    for aFile in listSrcPackageFiles.difference(listDstPackageFiles):
        listToUpdate.add(aFile)
    return listToUpdate

def copy_pac(src_apiurl, src_project, src_package,
             dst_apiurl, dst_project, dst_package,
             client_side_copy=False,
             keep_maintainers=False,
             keep_develproject=False,
             expand=False,
             revision=None,
             comment=None):
    """
    Patched copy of osc 0.132.5 copy_pac() function.


    Create a copy of a package.

    Copying can be done by downloading the files from one package and commit
    them into the other by uploading them (client-side copy) --
    or by the server, in a single api call.
    """
    if not (src_apiurl == dst_apiurl and src_project == dst_project \
        and src_package == dst_package):
        src_meta = core.show_package_meta(src_apiurl, src_project, src_package)
        dst_userid = conf.get_apiurl_usr(dst_apiurl)
        src_meta = core.replace_pkg_meta(src_meta, dst_package, dst_project, keep_maintainers,
                                         dst_userid, keep_develproject)
        print 'Sending meta data...'
        u = makeurl(dst_apiurl, ['source', dst_project, dst_package, '_meta'])

        core.http_PUT(u, data=src_meta)


    query = {'rev': 'upload'}
    revision = core.show_upstream_srcmd5(src_apiurl, src_project, src_package, expand=expand, revision=revision)

    listSrcPackageFiles = set(core.meta_get_filelist(src_apiurl,
                                                     src_project,
                                                     src_package,
                                                     expand=expand,
                                                     revision=revision))
    if '_link' in listSrcPackageFiles:
        listSrcPackageFiles.remove('_link')

    listDstPackageFiles = set(core.meta_get_filelist(dst_apiurl,
                                                     dst_project,
                                                     dst_package,
                                                     expand=expand,
                                                     revision="upload"))

    listToDel = listDstPackageFiles.difference(listSrcPackageFiles)
    listToUpDate = testPackageFiles(src_apiurl, src_project, src_package,
                                    dst_apiurl, dst_project, dst_package,
                                    src_revision=revision)

    if len(listToDel) > 0:
        print 'Deleting files...'
    for n in listToDel:
        print '  ', n
        delQuery = 'rev=upload'
        u = makeurl(dst_apiurl, ['source', dst_project, dst_package, core.pathname2url(n)], query=delQuery)
        core.http_DELETE(u)

    if len(listToUpDate) > 0:
        print 'Copying files...'
    else:
        print 'Nothing to Copy files...'

    for n in listToUpDate:
        if n.startswith('_service:') or n.startswith('_service_'):
            continue
        print '  ', n

        tmpfile = None
        try:
            (fd, tmpfile) = tempfile.mkstemp(prefix='osc-copypac')
            os.close(fd)
            core.get_source_file(src_apiurl, src_project, src_package, n, targetfilename=tmpfile, revision=revision)
            u = makeurl(dst_apiurl, ['source', dst_project, dst_package, core.pathname2url(n)], query=query)
            core.http_PUT(u, file=tmpfile)
        finally:
            if not tmpfile is None:
                os.unlink(tmpfile)
    if comment:
        query['comment'] = comment
    query['cmd'] = 'commit'
    u = makeurl(dst_apiurl, ['source', dst_project, dst_package], query=query)
    core.http_POST(u)
    return 'Done.'

def getApiUrlFromAlias(alias):
    """
    Return the full API URL corresponding to `alias`.
    Return `alias` if it is an URL.
    """
    if not alias.startswith("http"):
        apiUrl = conf.config["apiurl_aliases"].get(alias, None)
        if apiUrl is None:
            message = "Wrong API URL or alias: '%s'" % alias
            printerror(message)
            sys.exit(2)
        printdebug("%s is an alias for %s" % (alias, apiUrl))
        return apiUrl
    else:
        return alias
def creatPackage(apiurl, projectObsName, package):

    url = makeurl(apiurl, ['source', projectObsName, package, "_meta"])
    tmp = '<package project="%s" name="%s"><title>%s</title>'
    tmp += '<description>%s</description></package>'
    tmp = tmp % (projectObsName, package, "", "")
    aElement = ElementTree.fromstring(tmp)
    core.http_PUT(url, data=ElementTree.tostring(aElement))


def copyproject(apiUrl, srcProjectName, dstApiUrl, dstProjectName, tagFilePath):
    printdebug("Source API URL or alias:      %s" % apiUrl)
    printdebug("Destination API URL or alias: %s" % dstApiUrl)
    printdebug("Source project:               %s" % srcProjectName)
    printdebug("Destination project:          %s" % dstProjectName)
    printdebug("Tag file path:                %s" % tagFilePath)

    totalPkg = 0
    existPkg = 0
    copiedPkg = 0
    goodPkg = 0
    failPkg = 0
    goodLink = 0
    failLink = 0

    conf.get_config()
    core.makeurl = makeurl

    global logFilePath
    logFilePath = tagFilePath + ".log"
    logFile = open(logFilePath, "wau")
    sys.stdout = PrintHandler(sys.stdout, logFile)
    sys.stderr = PrintHandler(sys.stderr, logFile)

    apiUrl = getApiUrlFromAlias(apiUrl)
    dstApiUrl = getApiUrlFromAlias(dstApiUrl)

    try:
        tagFile = open(tagFilePath, "ru")
    except IOError as e:
        message = "Cannot open %s: %s" % (tagFilePath, e.strerror)
        printerror(message)
        sys.exit(3)
    message = "Copying revision version of source packages as defined in "
    message += tagFilePath
    printmessage(message)

    print "info: Checking connectivity with target project...",
    sys.stdout.flush()
    try:
        core.meta_get_packagelist(dstApiUrl, dstProjectName)
    except KeyboardInterrupt:
        raise
    except (urllib2.HTTPError, urllib2.URLError) as e:
        print
        if type(e) == urllib2.HTTPError:
            strerror = str(e) + " (%s)" % e.url
        else:
            strerror = str(e.reason)
        message = "FAILED\nError: Target project '%s' cannot be reached: %s"\
            % (dstProjectName, strerror)
        printerror(message)
        sys.exit(4)
    print "OK"

    print "info: Checking connectivity with source project...",
    sys.stdout.flush()
    try:
        dstPackageList = core.meta_get_packagelist(dstApiUrl, srcProjectName)
    except KeyboardInterrupt:
        raise
    except (urllib2.HTTPError, urllib2.URLError) as e:
        print

        if type(e) == urllib2.HTTPError:
            strerror = str(e) + " (%s)" % e.url
        else:
            strerror = str(e.reason)
        message = "FAILED\nError: Source project '%s' cannot be reached: %s"\
            % (srcProjectName, strerror)
        printerror(message)
        sys.exit(4)
    print "OK"

    # Parse the tag file. Make a list of tuples containing
    # package name and md5.
    packageList = []
    for line in tagFile:
        parts = line.split("|", 4) # only first 4 fields are relevant

        md5 = parts[0].strip()
        if len(parts) < 4 or parts[0].lstrip().startswith("#"):
            continue
        elif len(md5) < 1:
            md5 = "latest"

        totalPkg += 1
        pkgName = parts[3].strip()
        packageList.append((pkgName, md5))
    tagFile.close()

    packageCounter = 0
    failedPackageList = []

    for (pkgName, md5) in packageList:
        packageCounter += 1
        packagePresent = False
        if  not pkgName in dstPackageList:
            creatPackage(dstApiUrl, dstProjectName, pkgName)

        try:
            fileList = core.meta_get_filelist(dstApiUrl,
                                              dstProjectName,
                                              pkgName,
                                              revision=md5)
            if len(fileList) > 0 :
                packagePresent = True

        except KeyboardInterrupt:
            raise
        except urllib2.HTTPError as e:
            if e.code == 404 or e.code == 400:
                printdebug("Got status %d: package does not exist on destination" % e.code)
                packagePresent = False
            else:
                message = "Error: Cannot list files of package '%s': %s"\
                    % (pkgName, str(e))
                printerror(message)
                failPkg += 1
                failedPackageList.append(pkgName)
                continue
        except Exception as e:
            message = "Error: Cannot list files of package '%s': %s: %s\n"\
                % (pkgName, str(type(e)), str(e))
            printerror(message)
            failPkg += 1
            failedPackageList.append(pkgName)
            continue

        packageIsLink = False
        doCopy = True
        if not packagePresent:

            try:
                fileList = core.meta_get_filelist(apiUrl, srcProjectName, pkgName)
                if "_link" in fileList:
                    printmessage("(following the link) ", False)
                    packageIsLink = True
                    if packageIsLink:
                        listFileToUpdate = testPackageFiles(apiUrl,
                                                            srcProjectName,
                                                            pkgName,
                                                            dstApiUrl,
                                                            dstProjectName,
                                                            pkgName,
                                                            md5)
                        if len(listFileToUpdate) == 0:
                            packagePresent = True

            except Exception as e:
                message = "\nError checking if %s is a link: '%s', trying standard copy"\
                    % (pkgName, str(e))
                printerror(message)
                doCopy = False

        if not packagePresent and doCopy:
            copiedPkg += 1
            message = "info (%d/%d): Copying %s from %s" % (packageCounter, totalPkg,
                                                            pkgName, srcProjectName)
            printmessage(message, False)

            try:
                
                copy_pac(apiUrl,
                         srcProjectName,
                         pkgName,
                         dstApiUrl,
                         dstProjectName,
                         pkgName,
                         client_side_copy=(apiUrl != dstApiUrl),
                         revision=(None if packageIsLink else md5),
                         expand=packageIsLink)
                # I don't know if it can fail without returning an exception.
                # And we can't analyse the return value because it is sometimes
                # "Done." and sometimes an XML string (depending on the
                # value of parameter client_side_copy).
                if packageIsLink:
                    goodLink += 1
                else:
                    goodPkg += 1
                printmessage("DONE")

            except Exception as e:
                if packageIsLink:
                    failLink += 1
                else:
                    failPkg += 1
                failedPackageList.append(pkgName)
                message = "FAILED: %s: %s" % (str(type(e)), str(e))
                if type(e) == urllib2.HTTPError:
                    if e.code == 400 and md5 != "latest":
                        message += "\nMaybe revision '%s' is not available" % md5
                        message += "\n try 'curl -u user:password \"%s\"'" % e.url
                printmessage(message)

        elif not  doCopy:
            failPkg += 1
            failedPackageList.append(pkgName)
            message = "FAILED: %s: %s" % (str(type(e)), str(e))
        else:
            existPkg += 1
            message = "info (%d/%d): %s is already present on %s" % (packageCounter,
                                                                     totalPkg,
                                                                     pkgName,
                                                                     dstProjectName)
            printmessage(message)

    printmessage("\nPackages in error: %s\n" % (", ".join(failedPackageList)))

    printmessage("Final reports")
    printmessage("   Total packages requested     = %s" % totalPkg)
    printmessage("   Packages existing on target  = %s" % existPkg)
    printmessage("   Packages needed copying      = %s" % copiedPkg)
    printmessage("   Packages copied              = %s" % goodPkg)
    printmessage("   Packages in error            = %s" % failPkg)
    printmessage("   Linked packages copied       = %s" % goodLink)
    printmessage("   Linked packages in error     = %s" % failLink)

    print "Log file available in %s" % logFilePath
    if failPkg != 0:
        sys.exit(1)

if __name__ == '__main__':
    if os.environ.has_key("DEBUG") and os.environ["DEBUG"] is not None:
        DEBUG_MODE = True
    try:
        if len(sys.argv) == 5:
            # Use same OBS URL as source and destination
            copyproject(sys.argv[1], sys.argv[2], sys.argv[1],sys.argv[3], sys.argv[4])
        elif len(sys.argv) == 6:
            copyproject(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5])
        else:
            print >> sys.stderr, "Wrong number of arguments !\n"
            help_(sys.argv[0])
            sys.exit(2)
    except KeyboardInterrupt:
        print >> sys.stderr, "Interrupted by user..."
        sys.exit(5)
