#!/usr/bin/python
#
#	fc_gluster - service control routines
#
#	Mad Pengiun <madpenguin@linux.co.uk> (c) 2012 
#
#	TODO:: modprobe for flashcache and nbd @ start
#	TODO:: fix auto-boot on node1 and node2 re; mount gluster
# 	TODO:: submit patches for FC re; more stats info
#
###############################################################################
#
#	External Routines and Constants
#
###############################################################################
#
import sys
import socket
import subprocess as sub
import os
import stat
import time
import shelve
import libvirt
import datetime
from datetime import datetime
#
#	Useful constants from libvirt
#
from libvirt import VIR_MIGRATE_LIVE		# make migrations "live"
from libvirt import VIR_MIGRATE_PEER2PEER	# migrate directly, !via libirt
from libvirt import VIR_MIGRATE_PERSIST_DEST	# make the result persistent
from libvirt import VIR_MIGRATE_UNDEFINE_SOURCE	# remove froms src on completion
from libvirt import VIR_DOMAIN_RUNNING		# is domain running on this node
#
#	Our Code Libraries
#
from fc_mongo import Database
from fc_logging import logBegin,logEnd,logMiddle,logYN,logError
#
#	Handy shortcuts
#
from sys 	import argv,exit
from termcolor 	import colored
#
#	Local Constants
#
CONFIG_DIR 	= '/etc/fc_gluster/'
DATABASE 	= '/etc/flashcache.dbm'
HOSTNAME 	= os.uname()[1]
FALLOW_DELAY 	= 300
#
#	Initialise !
#
mdb 		= Database()
#
###############################################################################
#
def isKVMRunning(io,inst):

	active = []
	for id in io.listDomainsID():
		domain = io.lookupByID(id)
		info = domain.info()
		if info[0] == VIR_DOMAIN_RUNNING: active.append(domain.name())
	return inst in active
#
###############################################################################
#
def getDomain(io,inst):
	active = []
	for id in io.listDomainsID():
		domain = io.lookupByID(id)
		if domain.name() == inst: return domain
	return None
#
###############################################################################
#
def delay(secs):
	while secs > 0:
		logMiddle(secs,blue)
		time.sleep(10)
		secs -= 10
#
###############################################################################
#
def getInstances(host=HOSTNAME):
	""" return the names of the instances to process """
	if len(argv)>2:	return argv[2:]
	else:
		instances = []
		results = mdb.getInstances(host)
		for result in results:
			instances.append(result['name'])
		return instances
#
###############################################################################
#
def isRunning(inst):
	return os.path.exists("/dev/mapper/"+inst)
#
###############################################################################
#	
def setParam(inst,dev,param,v):
	""" set a flashcache parameter """
	path = "/proc/sys/dev/flashcache/"+inst['name']+"+"+dev+"/"+param
	while True:
		if os.path.exists(path):
			print "["+colored(param,"yellow")+"="+colored(v,"green")+"]",
			io = open(path,"w")
			io.write(str(v)+"\n")
			io.close()
			return

		print colored(".","red"),
		if not wait: return
		time.sleep(1)
#
###############################################################################
#	
def run(cmd):
	""" run a shell command in a sub-shell """
	pipe = sub.Popen(cmd,bufsize=8192,stdout=sub.PIPE,stderr=sub.PIPE,shell=True)
	result = pipe.communicate()
	return pipe.returncode
#
def background(cmd):
	return sub.Popen(cmd,shell=False)
#
def remote(cmd):
	""" run a shell command in a sub-shell """
	pipe = sub.Popen(cmd,bufsize=8192,stdout=sub.PIPE,stderr=sub.PIPE,shell=True)
	result = pipe.communicate()
	return [pipe.returncode,result]
#
###############################################################################
#
#	do_stats - print out our current status in tabular format
#
def do_stats():
	""" print out our status in tabular format """
	print "+"+25*"-"+"+"+7*"-"+"+"+17*"-"+"+"+17*"-"+"+"+17*"-"+"+"+17*'-'+'+'
	print "| %s %s" % (colored("%-14s" % "Instance"	,"blue"),colored("%8s" % "Size ","green")),
	print "| %s" 	%  colored("%-5s"  % "Raw"	,"blue"),
	print "| %s" 	%  colored("%15s"  % "Cached"	,"blue"),
	print "| %s" 	%  colored("%15s"  % "Dirty"	,"blue"),
	print "| %s"	%  colored("%15s"  % "Read"	,"blue"),
	print "| %s"	%  colored("%15s"  % "write"	,"blue")+" |"
	print "+"+25*"-"+"+"+7*"-"+"+"+17*"-"+"+"+17*"-"+"+"+17*"-"+"+"+17*'-'+'+'
	for inst in mdb.getInstances(HOSTNAME):
		name = inst['name']
		ssd = "/dev/%s/%s" % (inst['cache'],name)
		dev = mdb.getDevice(HOSTNAME,name)

		if not isRunning(name):
			print "| %s (%s) |" % (colored("%-14s" % name,"blue"),colored("%6s" % "down","red")),
			print 6*" "+"|"+17*" "+"|"+17*" "+"|"+17*' '+'|'+17*' '+'|'
			continue

		stats = ""
		path = "/proc/flashcache/"+name+"+"+name+"/flashcache_stats"
		if os.path.exists(path): 
			io = open(path,"r")
			stats = io.read()
			io.close()

		cmd = "dmsetup table %s" % name
		pipe = sub.Popen(cmd,bufsize=8192,stdout=sub.PIPE,stderr=sub.PIPE,shell=True)
		result = pipe.communicate()
		if pipe.returncode:
			print "* FAILURE RUNNING DMSETUP for %s" % name
			sys.exit(0)

		result = result[0]
		result = result.expandtabs(0).split("\n")
		record = result[1].split(",")
		ssd = record[0].split("(")[1].replace(")","")
		aoe = record[1].split("(")[1].replace(")","").split(" ")[0]
		record = result[2].split(",")
		capacity = record[0].split("(")[1].replace(")","")
		record  = result[3].split(",")
		try:
			cached  = int(record[1].split("(")[1].replace(")",""))
			percent = int(record[2].split("(")[1].replace(")",""))
			record  = result[4].split(",")
			dirtyb = int(record[0].split("(")[1].replace(")",""))
			dirtyp = int(record[1].split("(")[1].replace(")",""))
		except:
			record  = result[4].split(",")
			cached  = int(record[1].split("(")[1].replace(")",""))
			percent = int(record[2].split("(")[1].replace(")",""))
			record  = result[5].split(",")
			dirtyb = int(record[0].split("(")[1].replace(")",""))
			dirtyp = int(record[1].split("(")[1].replace(")",""))

		stats = stats.replace("\n"," ")
		stats = stats.split(" ")
		
		ssd_reads = 0
		ssd_writes = 0
		disk_reads = 0
		disk_writes = 0
		
		for stat in stats:
			if len(stat)<2: continue
			key,val = stat.split("=")
			if key == 'ssd_reads'	: ssd_reads = int(val)
			if key == 'disk_reads'	: disk_reads = int(val)
			if key == 'ssd_writes'	: ssd_writes = int(val)
			if key == 'disk_writes'	: disk_writes = int(val)			

		total_reads = ssd_reads + disk_reads + 1
		rp = int(100 * float(ssd_reads) / float(total_reads))
		total_writes = ssd_writes + disk_writes + 1
		wp = int(100 * float(ssd_writes) / float(total_writes))

		print "| %s (%s)"   % (colored("%-14s" 	% name,"green"),colored("%6s" % capacity,"yellow")),
		print "| %s" 	    %  colored("%-5s" 	% dev,"green"),
		print "| %s (%s)"   % (colored("%8d" 	% cached,"green"),	colored("%3d%%" % percent,"yellow")),
		print "| %s (%s)"   % (colored("%8d" 	% dirtyb,"green"),	colored("%3d%%" % dirtyp,"yellow")),
		print "| %s (%s)"   % (colored("%8d" 	% total_reads,"green"),	colored("%3d%%" % rp,"yellow")),
		print "| %s (%s) |" % (colored("%8d" 	% total_writes,"green"),colored("%3d%%" % wp,"yellow")),
		if inst.has_key('type'):
			if inst['type'] == 'RAID':
				devices = mdb.getDevices(HOSTNAME,name)
				for device in devices: device,
		print

	print "+"+25*"-"+"+"+7*"-"+"+"+17*"-"+"+"+17*"-"+"+"+17*"-"+"+"+17*'-'+'+'
#
###############################################################################
#
#	do_stop - stop a given instance
#
def do_stop(remove=False):
	""" stop one or more instances """
	for name in getInstances():	
		inst = mdb.getInstance(name)
		if not inst:
			logError('Instance does not exist on this node',name)
			continue

		if not isRunning(name):
			logError('Instance is already stopped',name)
			continue
				
		logBegin('Stopping',name)			
		ssd = "/dev/%s/%s" % (inst['cache'],name)		
		
		if run("/sbin/dmsetup remove %s" % name):
			logEnd('dmsetup Failed',False)
			continue

		logMiddle('dm stopped',"green")
		if inst['type'] == 'RAID':
			if not do_down(inst): continue
		else:
			dev = mdb.getDevice(HOSTNAME,name)
			if not dev:
				logError('No device known for',name)
				continue
			
			if run("qemu-nbd -d /dev/"+dev):
				logMiddle(dev,"red")
				logEnd('Fail',False)
				continue
			else:	
				logMiddle(dev,"blue")

		if not remove:
			logEnd('Ok',True)
		else:
			logMiddle('remove','blue')
			if run("lvremove -f "+ssd):
				logEnd('Fail',False)	
			else:	
				mdb.delDevice(HOSTNAME,name)
				logEnd('Ok',True)
#
###############################################################################
#
def do_start():
	""" start one or more instances """
	for name in getInstances():	
		inst = mdb.getInstance(name)
		if not inst:
			logError('Instance does not exist on this node',name)
			continue

		if isRunning(name):
			logError('Instance is already running',name)
			continue
		
		if inst['type'] == 'RAID':
			do_up(inst)
			continue
		
		if inst['type'] != 'gluster':
			logError('Unknown instance type',name)
			continue
				
		img = "/"+inst['path']+"/"+name+".img"
		if not os.path.exists(img):
			logEnd('missing: '+img,False)
			continue

		ssd = "/dev/%s/%s" % (inst['vol_group'],name)
		
		logBegin('Starting',name)			
		
		dev = mdb.getDevice(HOSTNAME,name)
		if dev:
			logMiddle('old::'+dev,"blue")
		else:
			dev = mdb.getNextDevice()
			logMiddle('new::'+dev,"green")
			mdb.updateDevice(HOSTNAME,name,dev)
			
		if run("qemu-nbd -c /dev/"+dev+" "+img+" --nocache"):	
			logEnd(dev,False)
			continue

		logMiddle("connect","blue")

		if os.path.exists(ssd):
			logMiddle('load',"blue")
			if run("/sbin/flashcache_load "+ssd+" "+name):
				logEnd('fail',False)
				continue
		else:
			logMiddle('lvm',"blue")
			if run("lvcreate -L%dG -n%s %s" % (inst['cache_size'],name,inst['cache'])):
				logEnd('fail',False)
				continue
			
			logMiddle('ndb',"blue")
			if run("/sbin/flashcache_create -p back -s%dg %s %s /dev/%s" % (inst['cache_size'],name,ssd,dev)):
				logEnd('fail',False)
				continue

		logMiddle('RA::8192',"blue")
		sleep(3)
		if run("blockdev --setra 8192 /dev/mapper/"+name): logMiddle('fail',"red")
		setParam(inst,dev,"reclaim_policy",1,True)
		setParam(inst,dev,"fallow_delay",FALLOW_DELAY,True)
		logEnd('Ok',"green")
#
###############################################################################
#
def do_migrate(pingpong=False):
	""" migrate a running instance from one machine to another """
	#
	#	Params: instance dst
	#
	if len(argv)<4:
		logBegin('Migration needs <instance> and <destination>')
		logEnd('Fail',False)
		return

	inst = sys.argv[2]
	dest = sys.argv[3]

	logBegin('Validating source')
	src_conn = libvirt.open("qemu:///system")

	if not isKVMRunning(src_conn,inst):
		if not pingpong:
			logMiddle('Instance is not running',"blue")
			logEnd('Fail',False)
			return

		logEnd('Ok',True)

		while True:
			logBegin('Waiting to see instance')
			if isKVMRunning(src_conn,inst): 
				delay(10)
				break
			delay(40)
			logEnd('Not found',True)

	logEnd('Ok',True)

	logBegin('Validating destination')
	dst_conn = libvirt.open("qemu+ssh://"+dest+"/system")
	if isKVMRunning(dst_conn,inst):
		logMiddle('Instance is already running',"blue")
		logEnd('Fail',False)
		return

	logEnd('Ok',True)

	logBegin('Turning caching off on source')
	config = doLoad(inst)
	config.ssd,config.dev = db[inst]
	setParam(inst,config,'cache_all',0)
	setParam(inst,config,"do_sync",1)
	logEnd('Ok',True)

	logBegin('Checking mapper on %s' % dest)
	blockdev = "/dev/mapper/"+inst

	status,result = remote('ssh root@'+dest+' "if [ -e '+blockdev+' ]; then echo -n 1; else echo -n 0; fi"')
	if status:
		logEnd(result,False)
		return

	logEnd('Ok',True)

	if result[0]=='1':
		yn = logYN('Mapper exists on target, remove?')
		if not yn:
			logBegin('Aborting procedure')
			logEnd('Fail',False);
			return

		logBegin('Shutting down Mapper on %s' % dest)
		status,result = remote('ssh root@'+dest+' "service fc_gluster remove '+inst+'"')
		if status:
			logEnd('Failed',False)
			print result
			return
	
		logEnd('Ok',True)

	logBegin('Creating Mapper on %s' % dest)
	status,result = remote('ssh root@'+dest+' "service fc_gluster start '+inst+'"')
	if status:
		logEnd('Failed',False)
		print result
		return
	
	logEnd('Ok',True)

	logBegin('Waiting for Dirty Blocks => 0')
	while True:
		cmd = "dmsetup table %s" % inst
		pipe = sub.Popen(cmd,bufsize=8192,stdout=sub.PIPE,stderr=sub.PIPE,shell=True)
		result = pipe.communicate()
		if pipe.returncode:
			print " * - FAILURE RUNNING DMSETUP for %s" % inst
			exit(0)
	
		result = result[0]
		result = result.expandtabs(0).split("\n")
		record = result[1].split(",")
		ssd = record[0].split("(")[1].replace(")","")
		aoe = record[1].split("(")[1].replace(")","").split(" ")[0]
		record = result[2].split(",")
		capacity = record[0].split("(")[1].replace(")","")
		record  = result[3].split(",")
		try:
			cached  = int(record[1].split("(")[1].replace(")",""))
			percent = int(record[2].split("(")[1].replace(")",""))
			record  = result[4].split(",")
			dirtyb = int(record[0].split("(")[1].replace(")",""))
			dirtyp = int(record[1].split("(")[1].replace(")",""))
		except:
			record  = result[4].split(",")
			cached  = int(record[1].split("(")[1].replace(")",""))
			percent = int(record[2].split("(")[1].replace(")",""))
			record  = result[5].split(",")
			dirtyb = int(record[0].split("(")[1].replace(")",""))
			dirtyp = int(record[1].split("(")[1].replace(")",""))

		if dirtyb == 0: 
			logEnd('Ok',True)
			break

		logMiddle('wait',"blue")
		setParam(inst,config,"do_sync",1)
		time.sleep(1)

	logBegin('Waiting')
	delay(10)
	logEnd('Ok',True)

	logBegin('Migrating %s to %s' % inst,dest)

	flags = VIR_MIGRATE_LIVE|VIR_MIGRATE_PEER2PEER|VIR_MIGRATE_PERSIST_DEST|VIR_MIGRATE_UNDEFINE_SOURCE
	uri = "qemu+ssh://%s/system" % (dest)
	src = getDomain(src_conn,inst)

	dst = src.migrate(dst_conn,flags,inst,uri,0)

	if not dst:
		logEnd('Failed',False)
		print result
		return
	
	logEnd('Ok',True)

	if not pingpong:
		yn = logYN('Are we Ok to remove the local Mapper?')
		if not yn:
			logBegin('Aborting procedure')
			logEnd('Fail',False);
			return

	logBegin('Shutting down obsolete Mapper')
	if run('service fc_gluster remove '+inst):
		logEnd('Failed',False)
		print result
		return
	
	logEnd('Ok',True)

	logBegin('Turning cache back on')
	status,result = remote('ssh root@'+dest+' "service fc_gluster cacheon '+inst+'"')
	if status:
		logEnd('Failed',False)
		print result
		return
	
	logEnd('Ok',True)
#
###############################################################################
#
def do_pingpong():
	while True:
		logBegin("Sleeping")
		delay(20)
		logEnd("Ok",True)
		print
		do_migrate(pingpong=True)
#
###############################################################################
#
#	setParameters - cycle through the chosen instances and set given parameters
#
def setParameters(intro,param):
	""" set specific parameters based on command line list """
	for name in getInstances():	
		if not isRunning(name):
			logError('Instance is not running',name)
			continue
		inst = mdb.getInstance(name)
		if not inst:
			logError('Instance does not exist on this node',name)
			continue
				
		dev = mdb.getDevice(HOSTNAME,name)
		if not dev: dev = name
		#	logError('No NBD device registered for',name)
		#	continue
			
		logBegin(intro,name)
		logMiddle(dev,'blue')
		for key in param.keys():
			val = param[key]
			setParam(inst,dev,key,val)
		logEnd('Ok',True)		
#
###############################################################################
#
#	Generic routines to change system bahaviour
#
def do_sync():		setParameters('Synchronizing',{"fallow_delay":FALLOW_DELAY,'do_sync':1})
def do_nosync():	setParameters('Synchronizing',{"fallow_delay":0})
def do_cacheon():	setParameters('Enabling Cache',{"cache_all":1,"do_sync":1,"reclaim_policy":1})
def do_cacheoff():	setParameters('Enabling Cache',{"cache_all":0,"do_sync":1,"reclaim_policy":1})
def do_clearstats():	setParameters('Enabling Cache',{"zero_stats":1})
def do_remove():	do_stop(remove=True)
#
def do_settype():
	try:
		typ = argv[3]
		name = argv[2]
	except:
		do_usage()
		
	inst = mdb.getInstance(name)
	if not inst:
		logError('Instance does not exist on this node',name)
		return
		
	if not mdb.setType(name,typ):
		logError('Set type failed for',name)
		return
		
	print "Done."

def do_sethosts():
	try:
		name = argv[2]
		update = {}
		for host in argv[3:]:
			host,port = host.split(":")
			update[host] = str(port)
	except:
		do_usage()
	
	inst = mdb.getInstance(name)
	if not inst:
		logError('Instance does not exist on this node',name)
		return
		
	if not mdb.setHosts(name,update):
		logError('Set hosts failed for',name)
		return
		
	print "Done."

def do_setdevice():
	try:
		name = argv[2]
		dev = argv[3]
	except:
		do_usage()
	
	inst = mdb.getInstance(name)
	if not inst:
		logError('Instance does not exist on this node',name)
		return
		
	if not mdb.setDevice(HOSTNAME,name,dev):
		logError('Set device failed for',name)
		return
		
	print "Done."
###############################################################################
#
#	do_up - start a remote RAID array / fc combo
#
def do_up(inst):
	""" start one or more instances """
	name = inst['name']
	ssd = "/dev/%s/%s" % (inst['cache'],name)
		
	logBegin('Starting',name)			
		
	devices = mdb.getDevices(HOSTNAME,name)
		
	if not devices:
		if not mdb.setDevices(inst):	
			logEnd('Missing devices',False)
			return
		devices = mdb.getDevices(HOSTNAME,name)
		if not devices:
			logEnd('Device creation failed',False)
			return

	dlist = devices
	mdadm = "mdadm --assemble --run /dev/md/"+name
	up = 0
	for host in inst['hosts']:
		port = inst['port']
		dev = dlist.pop()
		cmd = "nbd-client -d /dev/%s" % dev
		run(cmd)
		cmd = "nbd-client "+host+" "+port+" "+"/dev/"+dev
		if not run(cmd):
			up += 1
			logMiddle(dev,"green")
			mdadm += " "+"/dev/"+dev			
		else:	logMiddle(dev,"red")
		mdb.regClient(HOSTNAME,name,dev,host)
		
			
	if not up:
		logEnd("Fail",False)
		logError("Insufficient devices to start RAID",name)
		return
	
	if run(mdadm):	
		logEnd(mdadm,False)
		return
		
	logMiddle("RAID-up","blue")

	if os.path.exists(ssd):
		logMiddle('load',"blue")
		if run("/sbin/flashcache_load "+ssd+" "+name):
			logEnd('fail',False)
			return
	else:
		logMiddle('create',"blue")
		cmd = "lvcreate -L%sG -n%s %s" % (inst['cache_size'],name,inst['cache'])
		if run(cmd):
			logEnd('fail',False)
			print cmd			
			return
			
		logMiddle('lvm-create',"blue")
		cmd = "/sbin/flashcache_create -p back -s%sg %s %s /dev/md/%s" % (inst['cache_size'],name,ssd,name)
		if run(cmd):
			logEnd('fail',False)
			print cmd
			return

	logMiddle('RA::8192',"blue")
	if run("blockdev --setra 8192 /dev/mapper/"+name): logMiddle('fail',"red")
	setParam(inst,dev,"reclaim_policy",1)
	setParam(inst,dev,"fallow_delay",FALLOW_DELAY)
	logEnd('Ok',"green")
#
###############################################################################
#
#	do_down - take down a RAID instance
#
def do_down(inst):
	
	name = inst['name']
	
	if run("mdadm --stop /dev/md/"+name):
		logMiddle('failed to stop RAID',"red")
		logEnd('Fail',False)
		return False

	logMiddle('RAID','blue')
	devices = mdb.getDevices(HOSTNAME,name)
		
	if not devices:
		logEnd('Missing devices',Fail)
		return False
		
	for dev in devices:
		cmd = "nbd-client -d /dev/%s" % dev
		if run(cmd):
			logMiddle(dev,'red')
		else:	logMiddle(dev,"green")
		mdb.unregClient(HOSTNAME,name,dev)		
		
	return True
#
###############################################################################
#
#	do_backup - bring a back-end up
#
def do_backup():

	HOST = HOSTNAME.split(".")[0]	
	host = mdb.getHost(HOST)
	if not host:
		logError("This host is not registered as a storage node",HOSTNAME)
		return
	
	ip = host['ip']
	started = []
	waitfor = {}
	logBegin("Starting backends for",HOST)
	for name in getInstances(False):
		inst = mdb.getInstance(name)
		if not inst:
			logEnd('Instance does not exist: '+name,False)
			continue
		if not inst.has_key('type') or inst['type'] != "RAID": continue		
		for host in inst['hosts']:
			if host == HOST:
				
				if mdb.procRunning(HOST,name):
					process = mdb.procQuery(HOST,name)
					logMiddle("%s=%d" % (name,process['pid']),"blue")					
					continue
				
				path = "/srv/%s/%s.img" % (inst["vol"],name)
				if not os.path.exists(path):
					cmd = 'qemu-img create -f qcow2 %s %sG' % (path,inst['vol_size'])
					if run(cmd):
						logEnd('Create failed',False)
						logError(cmd)
						continue					
				
				pid = os.fork()
				if not pid:
					io = open("/tmp/fc_gluster.log","a")
					sys.stdin.close()
					os.dup2(io.fileno(), sys.stdout.fileno())
					os.dup2(io.fileno(), sys.stderr.fileno())
					os.execl("/usr/bin/qemu-nbd","qemu-nbd","-p",inst['port'],"-b",ip,"-t","-e","2",path)

				logMiddle(name,"yellow")
				started.append([name,pid])
				waitfor[pid]=True
	logEnd("Done",True)
	count = 0
	while count<4:
		for pid in waitfor.keys():
			pid,status = os.waitpid(pid, os.WNOHANG)
			if pid: del waitfor[pid]
		if not len(waitfor): break
		time.sleep(0.5)
		count += 1
			
	logBegin("Checking status")
	for entry in started:
		name,pid = entry
		if os.path.exists("/proc/%d" % pid):
			logMiddle("%s=%d" % (name,pid),"yellow")
			mdb.procRegister(HOST,name,pid)
		else:	logMiddle("%s" % (name),"red")
	
	logEnd("Ok",True)	
#

#
###############################################################################
#
#	do_backdown - down a back-end
#
def do_backdown():

	HOST = HOSTNAME.split(".")[0]	
	host = mdb.getHost(HOST)
	if not host:
		logError("This host is not registered as a storage node",HOSTNAME)
		return
	
	ip = host['ip']
	started = []
	waitfor = {}
	logBegin("Stopping backends for",HOST)
	for name in getInstances(False):
		inst = mdb.getInstance(name)
		if not inst:
			logEnd('Instance does not exist: '+name,False)
			continue
		if not inst.has_key('type') or inst['type'] != "RAID": continue		
		for host in inst['hosts']:
			if host == HOST:
				if not mdb.procRunning(HOST,name): continue
				process = mdb.procQuery(HOST,name)
				pid = process['pid']
				os.kill(pid,6)
				logMiddle(name,"yellow")
				process = mdb.procKill(HOST,name)
				
	logEnd("Done",True)
#
###############################################################################
#
#	do_backstatus - status of back-end servers
#
def do_backstatus():
	
	HOST = HOSTNAME.split(".")[0]	
	print "+"+22*"-"+"+"+12*"-"+"+"+17*"-"+"+"+7*"-"+"+"+7*"-"+"+"
	print "| %-20s | %-10s | %-15s | %5s | %5s |" % ('Started @','on Host','Instance','Port','PID')	
	print "+"+22*"-"+"+"+12*"-"+"+"+17*"-"+"+"+7*"-"+"+"+7*"-"+"+"
	for name in getInstances(False):
		inst = mdb.getInstance(name)
		if not inst:
			logEnd('Instance does not exist: '+name,False)
			continue
		if not inst.has_key('type') or inst['type'] != "RAID": continue		
		for host in inst['hosts']:
			process = mdb.procQuery(host,name)
			if not process: continue
			if not process.has_key('start'): continue
			if process:
				start = str(process['start']).split(".")[0]
				start = colored("%-20s" % start,"green")
				pid = colored("%5d" % process['pid'],"blue")
			else:
				start = colored("%-20s" % '(not running)',"red")
				pid = colored("%5s" % ' ',"blue")
			
			port = colored("%5d" % int(inst['port']),"blue")
			host = colored("%-10s" % host,"yellow")
			n = colored("%-15s" % name,"green")
			img = colored("%s" % name,"yellow")
			vol = colored("%s" % inst["vol"],"blue")
			
			path = "/srv/%s/%s.img" % (vol,img)
			print "| %s | %s | %s | %s | %s | %s" % (start,host,n,port,pid,path)

	print "+"+22*"-"+"+"+12*"-"+"+"+17*"-"+"+"+7*"-"+"+"+7*"-"+"+"
#
###############################################################################
#
#	do_addraid - add a new RAID instance
#
def do_addraid():
	name		= argv[2]
	port		= argv[3]
	cache		= argv[4]
	cache_size	= argv[5]
	vol		= argv[6]
	vol_size	= argv[7]
	hosts		= argv[8:]
	if mdb.getInstance(name):
		logError("Instance already exists",name)
		return
	mdb.addRaid(name,port,cache,cache_size,vol,vol_size,hosts)
	print "Done."
#
###############################################################################
#
#	do_createR10 - create a RAID10 on the block devices
#
def do_createR10():
	
	name = argv[2]
	HOST = HOSTNAME.split(".")[0]
	
	inst = mdb.getInstance(name)
	if not inst:
		logError("Instance does not exist",inst)
		return
			
	if os.path.exists("/dev/md/"+name):
		logError("Array is already running, please stop it first!",name)
		return
			
	if not logYN('This will DELETE / RE-INITIALIZE instance :: '+name):
		logError("Aborted - delete on ",name)
		return

				
	logBegin('Creating new RAID array',name)				
	devices = mdb.getDevices(HOSTNAME,name)
		
	if not devices:
		if not mdb.setDevices(inst):	
			logEnd('Missing devices',False)
			return
		devices = mdb.getDevices(HOSTNAME,name)
		if not devices:
			logEnd('Device creation failed',False)
			return

	dlist = devices
	mdadm = "mdadm --create /dev/md/"+name+" --name="+name+" --level=10 -pf2 -c64 --bitmap-chunk=4M --bitmap=internal -n%d" % len(inst['hosts'])
	
	for host in inst['hosts']:
		port = inst['port']
		dev = dlist.pop()
		cmd = "nbd-client -d /dev/%s" % dev
		run(cmd)
		
		cmd = "nbd-client "+host+" "+port+" "+"/dev/"+dev
		if not run(cmd):
			mdadm += " "+"/dev/"+dev
			logMiddle("zero::%s" % dev,"blue")
			cmd = "mdadm --zero-superblock /dev/%s" % dev
			run(cmd)			
		else:
			mdadm += " missing"
			logMiddle(dev,"red")
				
	pid = os.fork()
	if not pid:
		io = open("/tmp/fc_gluster.log","a")
		sys.stdin.close()
		os.dup2(io.fileno(), sys.stdout.fileno())
		os.dup2(io.fileno(), sys.stderr.fileno())
		args = mdadm.split(" ")		
		os.execv('/sbin/mdadm',args)
		exit(1)
				
	pid,status = os.wait()
	if status == 0:
		time.sleep(2)
		if run("mdadm --stop /dev/md/"+name):
			logEnd('Error stopping array',False)
			return	
	else:
		logEnd('Error on MDADM',False)
		return
	
	logEnd("Ok",True)
	
#
###############################################################################
#
#	do_fixraid
#
def do_fixraid():
	
	for name in getInstances(False):	
		if not isRunning(name): continue
		inst = mdb.getInstance(name)
		path = "/dev/md/"+name
		device = os.readlink(path).split("/")[1]
			
		devices = []
		io = open("/proc/mdstat","r")
		while True:
			line = io.readline()
			if not line: break
			parts = line.split(" ")
			if parts[0] != device: continue
			for part in parts:
				if part[:3] != "nbd": continue
				if "(F)" in part: continue
				dev = part.split("[")[0]
				devices.append(dev)
				
		needed = mdb.getNBDs(HOSTNAME,name)
		for dev in needed:
			if not dev in devices:
				if logYN('Instance %s [RAID %s] is missing %s, rebuild' % (colored(name,"yellow"),colored(device,"yellow"),colored(dev,"yellow"))):
					rec = mdb.getClient(HOSTNAME,name,dev)			
					cmd = "nbd-client "+rec['server']+" "+inst['port']+" "+"/dev/"+dev
					logError(cmd,name)
					run(cmd)
					cmd = "mdadm --fail /dev/md/"+name+" /dev/"+dev
					logError(cmd,name)
					run(cmd)
					cmd = "mdadm --remove /dev/md/"+name+" /dev/"+dev
					logError(cmd,name)
					run(cmd)				
					cmd = "mdadm --re-add --run /dev/md/"+name+" /dev/"+dev
					logError(cmd,name)
					run(cmd)
#
###############################################################################
#
#	do_local
#
def do_local():
	""" make a given instance local to this machine """
	for name in getInstances():
		if not mdb.getInstance(name):
			logError("Unknown instance",name)
			continue
		logBegin('Setting host as local',name)
		mdb.setLocal(name)
		logEnd('Ok',True)
			

#		parts = line.split(" ")
#		device = parts[0]
#		line = io.readline()
#		parts = line.split
#			for part in parts[1:]:
#				if "(F)" in part:
#					dev = part.split("[")[0]
#					print device,dev
#	io.close()
#	
	
	
#		pids=os.listdir('/proc') 
#		for pid in sorted(pids): 
#			try: 
#				int(pid) 
#			except ValueError: 
#				continue 
#			fd_dir=os.path.join('/proc', pid, 'fd') 
#			for file in os.listdir(fd_dir): 
#				try: 
#					link=os.readlink(os.path.join(fd_dir, file)) 
#				except OSError: 
#					continue 
#				files.append(link)
#		
##		index = 0
#		while index < MAX_BLOCK:
#			path = "/dev/nbd%d" % index
#			if not path in files: return "nbd%d" % index
#			index += 1
#			
#	
	
#
###############################################################################
#
#	All possible entry points for the routine
#
routines = {
	'sync'		:do_sync,
	'nosync'	:do_nosync,
	'cacheon'	:do_cacheon,
	'cacheoff'	:do_cacheoff,
	'clearstats'	:do_clearstats,
	'remove'	:do_remove,
	#
	'stats'		:do_stats,
	'stop' 		:do_stop,
	'start'		:do_start,
	'migrate'	:do_migrate,
	'pingpong'	:do_pingpong,
	'settype'	:do_settype,
	'sethosts'	:do_sethosts,
	'setdevice'	:do_setdevice,
	'addraid'	:do_addraid,
	'backup'	:do_backup,
	'backdown'	:do_backdown,
	'backstatus'	:do_backstatus,
	'createR10'	:do_createR10,
	'fixRAID'	:do_fixraid,
	"local"		:do_local
}
#
#	Display a list of possible command (dynamic)
#
def do_usage():
	s = ''
	for routine in routines.keys():
		if s <> '': s+='|'
		s += routine
	print 'service fc_gluster [%s]' % s
	exit(1)
#
#	This is the main command parser
#
if len(argv)<2:
	cmd = ""
else:	cmd = argv[1]

if cmd in routines:
	routines[cmd]()
else:	do_usage()
