#!/usr/bin/env python
'''
USAGE:   gpseginstall -f|--file HOSTFILE [-u|--user USER] [-g|--group GROUP] [-p|--password PASSWORD] [-c|--commands COMMAND_OPTIONS] [--verbose]
         where HOSTFILE lists all the hosts to install the software on
         where USER will be the user to install the software as. The default is gpadmin. (root only option)
         where GROUP will be the group to install the software as. The default is gpadmin.
         where PASSWORD will be set for the USER default is changeme (root only option)

         where COMMAND_OPTIONS list a subset of commands to be run from the list below.  The default is all commands.
            u -- add user  (root only)
            c -- chown software on master host (root only)
            s -- tar, zip and copy over software to machines in cluster
            p -- change passwords (root only)
            E -- exchange keys for root (root only)
            e -- exchange keys for user
            l -- check and fix user limits for new user -- requires software to be installed on cluster with command 's' or equivalent (root only)
            v -- verify software on remote machines
'''

import os, sys, re
import subprocess, warnings, logging, tarfile
warnings.simplefilter('ignore', DeprecationWarning)
sys.path.append(sys.path[0] + '/lib')

try:
    import paramiko, getpass, pexpect
    import gppylib.userinput
    from optparse import Option, OptionParser 
    from gppylib.gpparseopts import OptParser, OptChecker
    from gppylib.gplog import get_default_logger, setup_tool_logging
    from gppylib.commands.unix import getLocalHostname, getUserName, SYSTEM
    from gppylib.commands.base import WorkerPool, Command, NakedExecutionPasswordMap, NakedExecutionInfo, NAKED
except ImportError, e:    
    sys.exit('Cannot import modules.  Please check that you have sourced greenplum_path.sh.  Detail: ' + str(e))

EXECNAME = os.path.split(__file__)[-1]

logger = get_default_logger()
setup_tool_logging(EXECNAME,getLocalHostname(),getUserName()) 
hosts = dict() # address to HostMapping object
uniqueseghosts = dict() # hostname to UniqueHost object
options = None
passwordMap = None
nullFile = logging.FileHandler('/dev/null')
logging.getLogger('paramiko.transport').addHandler(nullFile)
pool = None
gphome = None
installation_info = None
commands = set() # selected commands
rootCommands = set(['u', 'p', 'c', 's', 'e', 'E', 'l', 'v']) # valid command types for root
nonRootCommands = set(['s', 'e', 'v'])                       # valid command types for non-root user
defaultNonRootCommands = set(['s', 'v'])                     # commands to run by default for non-root user
currentUser = None
isLinux = False
GPSSH_EXKEYS_TIMEOUT = 600

class InstallationInfo:
    def __init__(self, link_name, binary_path, binary_dir_location, binary_dir_name):
        self.install_link_name = link_name                      # greenplum-db  
        self.install_binary_path = binary_path                  # /data/release-4.0
        self.install_binary_dir_location = binary_dir_location  # /data
        self.install_binary_dir_name = binary_dir_name          # release-4.0
        self.install_md5 = None
        self.install_version_string = None

    def debugStr(self):
        return "link_name %s\nbinary_path %s\nbinary_dir_location %s\nbinary_dir_name %s" % (self.install_link_name, self.install_binary_path, self.install_binary_dir_location, self.install_binary_dir_name)

class HostMapping:
    def __init__(self, address):
        self.address = address
        self.hostname = None

    def debugStr(self):
        return "address %s hostname %s" % (self.address, self.hostname)

class UniqueHost:
    def __init__(self, address, hostname):
        self.address = address
        self.hostname = hostname
        self.userExists = False
    
    def debugStr(self):
        return "address %s hostname %s userExists %s" % (self.address, self.hostname, self.userExists)

def cli_help():
    help_path = os.path.join(sys.path[0], '..', 'docs', 'cli_help', EXECNAME + '_help')
    f = None
    try:
        try:
            f = open(help_path);
            return f.read(-1)
        except:
            return ''
    finally:
        if f: f.close()


def usage():
    print cli_help() or __doc__

# True is an error
def parseargs():

    global options, isLinux

    parser = OptParser(option_class=OptChecker)
    parser.remove_option('-h')
    parser.add_option('-h', '-?', '--help', action='store_true')
    parser.add_option('-f', '--file', type='string')
    parser.add_option('-u', '--user', type='string', default='gpadmin')
    parser.add_option('-g', '--group', type='string')
    parser.add_option('-p', '--password', type='string')
    parser.add_option('-c', '--commands', type='string')
    parser.add_option('--verbose', action='store_true')
    (options, args) = parser.parse_args()

    global gphome
    gphome = os.environ.get('GPHOME')
    if not gphome:
        logger.error("GPHOME not set")
        return True

    if SYSTEM.getName() != "linux" and SYSTEM.getName() != "sunos":
        logger.error("This utility is only supported on the linux and solaris operating system")
        return True

    if SYSTEM.getName() == "linux":
        isLinux = True

    if options.help:
        usage()
        return True

    if not options.file:
        logger.error("--file must be specified")
        return True

    if not os.path.exists(options.file):
        logger.error("file %s does not exist" % options.file)
        return True

    if options.user == "root":
        logger.error("can not install the software into the account of user root")
        return True

    global currentUser
    currentUser = getpass.getuser()

    if currentUser == "root":
        validCommands = rootCommands
    else:
        validCommands = nonRootCommands

        if options.user != currentUser:
            logger.error("--user option '%s' does not equal non-root user running this utility '%s'" % (options.user, currentUser))
            return True

    if not options.group:
        options.group = options.user

    global commands
    if options.commands and len(options.commands):
        for i in range(len(options.commands)):
            if options.commands[i] in validCommands:
                if options.commands[i] not in commands:
                    commands.add(options.commands[i])
            else:
                logger.error("'%s' is not a valid command for user(%s)" % (options.commands[i], currentUser))
                return True
    else:
        if currentUser == "root":
            commands = rootCommands
        else:
            commands = defaultNonRootCommands

    location = os.path.abspath(gphome)
    if os.path.islink(location):
        link_name = os.path.split(location)[1]
        binary_path = os.path.realpath(location)
    else:
        link_name = None
        binary_path = location

    (binary_dir_location, binary_dir_name) = os.path.split(binary_path)
 
    global installation_info
    installation_info = InstallationInfo(link_name, binary_path, binary_dir_location, binary_dir_name)
    logger.info("Installation Info:\n" + installation_info.debugStr())
 
    return False

# True is an error
def readHostList():

    try:
        for line in open(options.file, "r"):
            hostname = line.strip()
            if not len(hostname):
                continue
            hosts[hostname] = HostMapping(hostname)
    except Exception, e:
        logger.error("Error while reading file: %s" % options.file)
        logger.error(e.__str__())
        return True

    if not len(hosts.keys()):
        logger.error("No hosts read from hostfile: %s" % options.file)
        return True

    return False

# True is an error
def discoverPasswordMap():

    logger.info("check cluster password access")

    global passwordMap

    try:    
        passwordMap = NakedExecutionPasswordMap(hosts.keys())
        passwordMap.discover()
    except Exception, e:
        logger.error("could not successfully access all machines")
        msg = e.__str__()
        if msg:
            logger.error("trace: %s" % msg)
        return True

    if passwordMap.complete:
        return False
    else:
        return True

# True is an error
def dedupeHosts():

    logger.info("de-duplicate hostnames")

    masterHostName = None

    try:
        cmd = Command("master", "hostname")
        pool.addCommand(cmd)
        pool.join()
        items = pool.getCompletedItems()
        for i in items:
            if i.results.rc or i.results.halt or not i.results.completed:
                logger.error("error trying to obtain hostname on master %s" % i.results.stderr)
                return True
            masterHostName = i.results.stdout.strip()
            break
    except Exception, e:
        logger.error("exception trying to obtain hostname on master: %s" % e.__str__())
        return True

    if not masterHostName:
        logger.error("Could not find master hostname")
        return True

    logger.info("master hostname: %s" % masterHostName)

    try:
        for host in hosts.keys():
            cmd = Command(host, "hostname", NAKED, host, nakedExecutionInfo=NakedExecutionInfo(passwordMap))
            pool.addCommand(cmd)

        pool.join()
        items = pool.getCompletedItems()
        for i in items:
            address = i.remoteHost
            if i.results.rc or i.results.halt or not i.results.completed:
                logger.error("error obtaining information from host %s" % address)
                return True
            hostname = i.results.stdout[0].strip()
            hosts[address].hostname = hostname
    except Exception, e:
        logger.error("%s" % e.__str__())
        return True

    global uniqueseghosts
    for k in hosts.keys():
        hostname = hosts[k].hostname
        address = hosts[k].address

        if hostname in uniqueseghosts:
            # Here we do a heuristic:
            # there are several interfaces to each host
            # and we don't want to pick a slow interface
            # it is very likely that a hostname with a '-' will represent a fast link
            # so try to pick an address that contains a '-'
            # for example choose sdw1-1 over vert1
            if (not re.search("-", uniqueseghosts[hostname].address)) and (re.search("-", address)):
                uniqueseghosts[hostname].address = address
            continue

        if masterHostName == hostname:
            continue

        uniqueseghosts[hostname] = UniqueHost(address, hostname)

    if options.verbose:
        for k in uniqueseghosts.keys():
            logger.info("segment hostname: %s (%s)" % (uniqueseghosts[k].address, k))

    return False

# True is an error
def checkUsernames():

    logger.info("check for user %s on cluster" % options.user)

    try:
        for k in uniqueseghosts.keys():
            uh = uniqueseghosts[k]
            cmd = Command(uh.address, "id %s" % options.user, NAKED, uh.address, nakedExecutionInfo=NakedExecutionInfo(passwordMap))
            pool.addCommand(cmd)
        pool.join()
        items = pool.getCompletedItems()
        for i in items:
            if not i.results.rc:
                address = i.remoteHost
                hostname = hosts[address].hostname
                uniqueseghosts[hostname].userExists = True
    except Exception, e:
        logger.error("%s" % e.__str__())
        return True

    return False

def getAddUserCommand():
    if isLinux:
        return "useradd -r -m %s" % options.user
    else:
        return "groupadd %s; useradd -d /export/home/%s -m -g %s -s /bin/bash %s" % (options.user, options.user, options.user, options.user)

# True is an error
def addUserIfRequired():

    ###################################################################################
    logger.info("add user %s on master" % options.user)
    try:
        cmd = Command("useradd", getAddUserCommand())
        pool.addCommand(cmd)
        pool.join()
        items = pool.getCompletedItems() # get the completed items but ignore them
    except Exception, e:
        logger.error("%s" % e.__str__())
        return True

    try:
        cmd = Command("checkId", "id %s" % options.user)
        pool.addCommand(cmd)
        pool.join()
        items = pool.getCompletedItems()
        for i in items:
            if i.results.rc:
                logger.error("failed to add user %s to master host: %s" % (options.user, i.results.stderr))
                return True
    except Exception, e:
        logger.error("%s" % e.__str__())
        return True
     
    ###################################################################################
    logger.info("add user %s on cluster" % options.user)

    failedToAddUser = set() # set of address's where useradd failed
    try:
        for k in uniqueseghosts.keys():
            uh = uniqueseghosts[k]
            if uh.userExists:
                continue

            cmd = Command(uh.address, getAddUserCommand(), NAKED, uh.address, nakedExecutionInfo=NakedExecutionInfo(passwordMap))
            pool.addCommand(cmd)
        pool.join()
        items = pool.getCompletedItems()
        for i in items:
            address = i.remoteHost
            if not i.results.rc:
                hostname = hosts[address].hostname
                uniqueseghosts[hostname].userExists = True
            else:
                logger.error("%s: %s" % (address, i.results.stderr))
                failedToAddUser.add(address)
                
    except Exception, e:
        logger.error("%s" % e.__str__())
        return True

    if len(failedToAddUser):
        for h in failedToAddUser:
            logger.error("could not create user %s on host %s" % (options.user, h))
        return True

    return False

# True is an error
def  getLocalSoftwareVersion():

    global installation_info
    cmdStr = "%s/bin/gpssh --version" % gphome
    try:
        cmd = Command("version", cmdStr)
        pool.addCommand(cmd)
        pool.join()
        items = pool.getCompletedItems()
        for i in items:
            if i.results.rc:
                logger.error("Failed to run command: %s" % cmd.cmdStr)
                logger.error(i.results.stderr)
                return True

            installation_info.install_version_string = i.results.stdout.strip()
    except Exception, e:
        logger.error("%s" % e.__str__())
        return True

    return False

# True is an error
def  simpleLocalCommand(cmdStr, checkError=True, verbose=True):
    if verbose:
        logger.info(cmdStr)
    try:
        cmd = Command("simpleLocalCommand", cmdStr)
        pool.addCommand(cmd)
        pool.join()
        items = pool.getCompletedItems()
        if checkError:
            for i in items:
                if i.results.rc:
                    logger.error("Failed to run command: %s" % cmd.cmdStr)
                    logger.error(i.results.stderr)
                    return True 
    except Exception, e:
        logger.error("%s" % e.__str__())
        return True

    return False

# True is an error
def  simpleRemoteCommand(cmdStr, checkError=True, verbose=True):

    failures = set()
    if verbose:
        logger.info("remote command: %s" % cmdStr)

    try:
        for k in uniqueseghosts.keys():
            uh = uniqueseghosts[k]
            cmd = Command(uh.address, cmdStr, NAKED, uh.address, nakedExecutionInfo=NakedExecutionInfo(passwordMap))
            pool.addCommand(cmd)
        pool.join()
        items = pool.getCompletedItems()
        if checkError:
            for i in items:
                if i.results.rc:
                    logger.error("%s: %s" % (i.remoteHost, i.results.stderr))
                    failures.add(i.remoteHost)
                    
    except Exception, e:
        logger.error("%s" % e.__str__())
        return True

    if len(failures):
        for h in failures:
            logger.error("error running command %s on host %s" % (cmdStr, h))
        return True

    return False


# True is an error
def chownMasterSoftware():

    ###################################################################################
    if installation_info.install_link_name:

        cmdStr = "chown -R %s:%s %s/%s" % (options.user, options.group, installation_info.install_binary_dir_location, installation_info.install_link_name)
        if (simpleLocalCommand(cmdStr, True)):
            return True

    ###################################################################################
    cmdStr = "chown -R %s:%s %s" % (options.user, options.group, installation_info.install_binary_path)
    if (simpleLocalCommand(cmdStr, True)):
        return True


# True is an error
def changeUserPassword():

    if not options.password:
        password = gppylib.userinput.ask_create_password()
        if not password:
            logger.error("error obtaining password from user")
            return True
    else:
        password = options.password
    cmd = "%s/sbin/gpchangeuserpassword --user %s --password %s" % (gphome, options.user, password)

    logger.info("Changing system passwords ...")
    if (simpleLocalCommand(cmd, False, verbose=False)):
        return True

    cmdStr = ". %s/greenplum_path.sh; %s" % (gphome, cmd)

    if (simpleRemoteCommand(cmdStr, True, verbose=False)):
        return True

def md5Command():
    if isLinux:
        return "md5sum"
    else:
        return "digest -a md5"

def md5OutputWords():
    if isLinux:
        return 2
    else:
        return 1


# True is an error
def setMd5Locally():

    try:
        cmd = Command("md5", "%s %s.tar" % (md5Command(), installation_info.install_binary_path))
        pool.addCommand(cmd)
        pool.join()
        items = pool.getCompletedItems()
        for i in items:
            if i.results.rc:
                logger.error("Failed to run command: %s" % cmd.cmdStr)
                logger.error(i.results.stderr)
                return True 
            else:
                fields = i.results.stdout.split()
                if len(fields) != md5OutputWords():
                    raise Exception("Unexpected output from md5sum: %s" % i.results.stdout)
                installation_info.install_md5 = fields[0].strip()
                break
    except Exception, e:
        logger.error("%s" % e.__str__())
        return True

    return False

# True is an error
def copyOverSoftware():

    ###################################################################################
    cmdStr = "rm -f %s.tar; rm -f %s.tar.gz" % (installation_info.install_binary_path, installation_info.install_binary_path)
    if (simpleLocalCommand(cmdStr, False)):
        return True

    cmdStr = "cd %s; tar cf %s.tar %s" % (installation_info.install_binary_dir_location, installation_info.install_binary_dir_name, installation_info.install_binary_dir_name)
    if (simpleLocalCommand(cmdStr, False)):
        return True

    ###################################################################################
    if setMd5Locally():
        return True

    ###################################################################################

    cmdStr = "gzip %s.tar" % installation_info.install_binary_path
    if (simpleLocalCommand(cmdStr, False)):
        return True

    ###################################################################################
    cmdStr = "mkdir -p %s" % installation_info.install_binary_dir_location
    if (simpleRemoteCommand(cmdStr, True)):
        return True

    ###################################################################################
    # a couple paranoid checks
    if installation_info.install_binary_path == "/" or  \
       installation_info.install_binary_path == "/usr" or \
       installation_info.install_binary_path == "/usr/bin" or \
       re.search('boot', installation_info.install_binary_path):
        raise Exception("illegal path for installaiton %s" % installation_info.install_binary_path)
       
    cmdStr = "rm -rf %s" % (installation_info.install_binary_path)
    if (simpleRemoteCommand(cmdStr, True)):
        return True

    ###################################################################################
    logger.info("scp software to remote location")

    failures = set()

    filename = installation_info.install_binary_path + ".tar.gz"
    try:
        for k in uniqueseghosts.keys():
            uh = uniqueseghosts[k]
            cmdStr = "scp %s %s:%s" % (filename, uh.address, installation_info.install_binary_dir_location)
            cmd = Command(uh.address, cmdStr)
            pool.addCommand(cmd)
        pool.join()
        items = pool.getCompletedItems()
        for i in items:
            if i.results.rc:
                logger.error("command failed: '%s': %s" % (i.cmdStr, i.results.stderr.strip()))
                failures.add(i.name)
                
    except Exception, e:
        logger.error("%s" % e.__str__())
        return True

    if len(failures):
        return True


    ###################################################################################
    cmdStr = "gzip -f -d %s.tar.gz" % installation_info.install_binary_path
    if (simpleRemoteCommand(cmdStr, True)):
        return True

    ###################################################################################
    logger.info("md5 check on remote location")
    failures.clear()

    try:
        for k in uniqueseghosts.keys():
            uh = uniqueseghosts[k]
            cmdStr = "%s %s.tar" % (md5Command(), installation_info.install_binary_path)
            cmd = Command(uh.address, cmdStr, NAKED, uh.address, nakedExecutionInfo=NakedExecutionInfo(passwordMap))
            pool.addCommand(cmd)
        pool.join()
        items = pool.getCompletedItems()
        for i in items:
            address = i.remoteHost
            if i.results.rc:
                logger.error("%s: %s" % (address, i.results.stderr))
                failures.add(address)
            else:
                fields = i.results.stdout[0].split()
                if len(fields) == md5OutputWords():
                    md5 = fields[0].strip()
                    if md5 != installation_info.install_md5:
                        logger.error("on host %s md5sum %s != expected %s" % (address, md5, installation_info.install_md5))
                        failures.add(address)
                else:
                    logger.error("Unexpected output on host %s from md5sum: %s" % (address, i.results.stdout))
                    failures.add(address)
                
    except Exception, e:
        logger.error("%s" % e.__str__())
        return True

    if len(failures):
        for h in failures:
            logger.error("md5sum check of %s.tar on host %s failed" % (installation_info.install_binary_path, h))
        return True

    ###################################################################################
    cmdStr = "cd %s; tar xf %s.tar" % (installation_info.install_binary_dir_location, installation_info.install_binary_dir_name)
    if (simpleRemoteCommand(cmdStr, True)):
        return True

    ###################################################################################
    cmdStr = "rm -f %s.tar" % (installation_info.install_binary_path)
    if (simpleRemoteCommand(cmdStr, True)):
        return True

    ###################################################################################
    if installation_info.install_link_name:
        cmdStr = "cd %s; rm -f %s; ln -fs %s %s" % (installation_info.install_binary_dir_location, installation_info.install_link_name, installation_info.install_binary_dir_name, installation_info.install_link_name)
        if (simpleRemoteCommand(cmdStr, True)):
            return True

        if currentUser == "root":
            cmdStr = "chown -R %s:%s %s/%s" % (options.user, options.group, installation_info.install_binary_dir_location, installation_info.install_link_name)
            if (simpleRemoteCommand(cmdStr, True)):
                return True

    ###################################################################################
    if currentUser == "root":
        cmdStr = "chown -R %s:%s %s" % (options.user, options.group, installation_info.install_binary_path)
        if (simpleRemoteCommand(cmdStr, True)):
            return True

    ###################################################################################
    cmdStr = "rm -f %s.tar.gz" % (installation_info.install_binary_path)
    if (simpleLocalCommand(cmdStr, True)):
        return True

    ###################################################################################
    return False


# True is an error
def verifyVersionAtPath(usepath):

    cmdStr = ". %s/greenplum_path.sh; %s/bin/gpssh --version" % (usepath, usepath)
    logger.info("remote command: %s" % cmdStr)

    try:
        for k in uniqueseghosts.keys():
            uh = uniqueseghosts[k]
            cmd = Command(uh.address, cmdStr, NAKED, uh.address, nakedExecutionInfo=NakedExecutionInfo(passwordMap))
            pool.addCommand(cmd)
        pool.join()
        items = pool.getCompletedItems()
        for i in items:
            if i.results.rc:
                logger.error("error on host %s with command: %s" % (i.remoteHost, cmdStr))
                return True
            if not i.results.stdout:
                logger.error("could not find version string from host %s with command: %s" % (i.remoteHost, cmdStr))
                return True
            version_string = i.results.stdout[0].strip()
            if version_string != installation_info.install_version_string:
                logger.error("version string on host %s: '%s' does not match expected: '%s'" % (i.remoteHost, version_string, installation_info.install_version_string))
                return True

    except Exception, e:
        logger.error("%s" % e.__str__())
        return True

    return False


# True is an error
def verifySoftware():

    if (getLocalSoftwareVersion()):
        return True

    logger.info("version string on master: %s" % installation_info.install_version_string)

    if verifyVersionAtPath(gphome):
        return True

    if gphome != installation_info.install_binary_path:
        if verifyVersionAtPath(installation_info.install_binary_path):
            return True

    return False

# True is an error
def checkAndFixUserLimits():

    if not isLinux:
        return False

    cmd = "%s/sbin/gpfixuserlimts -f /etc/security/limits.conf -u %s" % (gphome, options.user)
    if (simpleLocalCommand(cmd, True)):
        return True

    cmdStr = ". %s/greenplum_path.sh; %s" % (gphome, cmd)
    if (simpleRemoteCommand(cmdStr, True)):
        return True

    return False


# True is an error
def interactiveCommand(cmdStr):

    try:
        p = subprocess.Popen(cmdStr, shell=True, executable="/bin/bash")
        sts = os.waitpid(p.pid, 0)[1]
        if sts:
            logger.error("error on cmd: %s" % cmdStr)
    except Exception, e:
        logger.error("Exception running cmd: %s" % cmdStr)
        logger.error(e.__str__())
        return True
        
    return False
 
# True is an error
def exchangeKeysUser():

    if currentUser == "root":
        return exchangeKeysAsRootForUser()
    else:
        return exchangeKeysAsSelf()

# True is an error
def exchangeKeysAsSelf():

    cmdStr = 'gpssh-exkeys -f %s' % options.file
    return interactiveCommand(cmdStr)

# True is an error
def exchangeKeysAsRootForUser():

    ###################################################################################
    logger.info("exchange ssh keys for user %s" % options.user)

    testCmd = 'su %s -c "cat %s &> /dev/null"' % (options.user, options.file)
    try:
        p = subprocess.Popen(testCmd, shell=True, executable="/bin/bash")
        sts = os.waitpid(p.pid, 0)[1]
        if sts:
            logger.error("failed doing a test read of file: %s" % testCmd)
            logger.error("%s is not accessible as user %s" % (options.file, options.user))
            return True
    except Exception, e:
        logger.error("failed doing a test read of file: %s" % testCmd)
        logger.error("%s is not accessible as user %s" % (options.file, options.user))
        logger.error(e.__str__())
        return True
 
    done = False
    badPassword = False
    child = None
    cmdStr = None

    try:
        cmdStr = 'su %s -c "gpssh-exkeys -f %s"' % (options.user, options.file)
        child = pexpect.spawn(cmdStr)

        index = 0
        while 1:
            index = child.expect(["password", "bad", pexpect.EOF, pexpect.TIMEOUT], timeout=GPSSH_EXKEYS_TIMEOUT)
            if index == 0:
                child.sendline(options.password)
                continue
            if index == 1:
                badPassword = True
                break
            if index == 2:
                done = True
                break
            if index == 3:
                logger.info("Timeout running command: %s" % cmdStr)
                break

    except Exception, e:
        logger.info("Exception running cmd: %s" % cmdStr)
        logger.info(e.__str__())

    if done:
        child.close()
        if child.exitstatus:
            logger.info("Cmd '%s' failed with error code %s" % (cmdStr, child.exitstatus))
        else:
            return False

    # using the supplied password did not work... lets try again in interactive mode
    logger.info("gppsh-exkeys failed running from within pexpect ... now try outside of pexpect")
    return interactiveCommand(cmdStr)
   
# True is an error
def exchangeKeysRoot():
    ###################################################################################
    logger.info("exchange ssh keys for user root")

    rootPasswords = list(passwordMap.unique_passwords)

    done = False
    child = None
    cmdStr = None
    passwordIndex = 0
    

    try:
        cmdStr = 'gpssh-exkeys -f %s' % options.file
        child = pexpect.spawn(cmdStr)

        index = 0
        while 1:
            index = child.expect(["password", "bad", pexpect.EOF, pexpect.TIMEOUT], timeout=GPSSH_EXKEYS_TIMEOUT)
            if index == 0:
                passwordIndex = 0
                child.sendline(rootPasswords[passwordIndex])
                continue
            if index == 1:
                passwordIndex += 1
                if passwordIndex >= len(rootPasswords):
                    raise Exception("could not determine root password on all machines")
                child.sendline(rootPasswords[passwordIndex])
                continue
            if index == 2:
                done = True
                break
            if index == 3:
                logger.error("Timeout running command: %s" % cmdStr)
                break

    except Exception, e:
        logger.info("Error running cmd: %s" % cmdStr)
        logger.info(e.__str__())

    if done:
        child.close()
        if child.exitstatus:
            logger.info("Cmd '%s' failed with error code %s" % (cmdStr, child.exitstatus))
        else:
            return False

    # using the supplied password did not work... lets try again in interactive mode
    logger.info("gppsh-exkeys failed running from within pexpect ... now try outside of pexpect")
    return interactiveCommand(cmdStr)


def earlyExit(error = False):
    if error:
        logger.fatal("early exit from gpseginstall")
    if pool:
        try:
            pool.join()
            pool.haltWork()
            pool.joinWorkers()
        except KeyboardInterrupt:
            logger.info("Exiting, please wait ...")

    if error:
        sys.exit(1)
    else:
        logger.info("SUCCESS -- Requested commands completed")
        sys.exit(0)

if (parseargs()):
    sys.exit(0)

pool = WorkerPool()

try:

    if (readHostList()):
        earlyExit(True)

    if (discoverPasswordMap()):
        earlyExit(True)

    if (dedupeHosts()):
        earlyExit(True)

    if 'u' in commands:
        if (checkUsernames()):
            earlyExit(True)
    
        if (addUserIfRequired()):
            earlyExit(True)
    
    if 'c' in commands:
        if (chownMasterSoftware()):
            earlyExit(True)
    
    if 's' in commands:
        if (copyOverSoftware()):
            earlyExit(True)
    
    if 'p' in commands:
        if (changeUserPassword()):
            earlyExit(True)
    
    if 'E' in commands:
        if (exchangeKeysRoot()):
            earlyExit(True)
    
    if 'e' in commands:
        if (exchangeKeysUser()):
            earlyExit(True)
    
    if 'l' in commands:
        if (checkAndFixUserLimits()):
            earlyExit(True)
    
    if 'v' in commands:
        if (verifySoftware()):
            earlyExit(True)

except KeyboardInterrupt:
    logger.info("Job stopped by user")
    earlyExit(True)
    
earlyExit(False)
