#!/usr/pythoncontroller/python

import telnetlib
import socket
import sys
import os
import diskutil
import time
import util
from subprocess import *
VM_ACCESS_RETRY = 30

def findMountDev(mntpoint):
    f = open('/proc/mounts','r')
    lines = f.readlines()
    dev = ''
    for line in lines:
        entry = line.split()
        if entry[1] == mntpoint:
            dev = entry[0]
            break
    f.close()
    return dev

def rpclog(logstr):
    logstr = time.strftime('%d-%m-%Y-%H:%M:%S ', time.gmtime()) + str(logstr)
    f = open('/var/log/onappstoresan','a')
    print >>f, logstr
    f.close()

def openSock(hostname, port):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect((hostname, port))
    return s

def netcat(sock):
    rpclog("Starting netcat")
    running = True
    while running:
        data = ''
        try:
            #Read 64K from stdin
            data = sys.stdin.read(65536)
            if not len(data):
                running = False
                break
        except:
            running = False
            break

        sock.sendall(data)
    sock.shutdown(socket.SHUT_WR)
    sock.close()

def cmd_exec(cmd, node):
    rpclog("Executing command: %s against node %s" % (cmd,node))
    tn = telnetlib.Telnet(node)
    time.sleep(1)
    tn.read_until("\r\n")
    tn.write("ulimit -s 256\n")
    time.sleep(1)
    tn.read_until("\r\n")
    tn.write("%s\n" % cmd)
    time.sleep(1)
    tn.read_until("\r\n")
    time.sleep(1)
    output = tn.read_until("# ")
    tn.close()
    return "%s" % output[:-4]

def scp_file_to_node_tmp(localpath, ip, remotepath):
    try:
        rpclog("Doing scp: %s to node %s:%s" % (localpath,ip,remotepath))
        filename = os.path.basename(localpath)
        check = cmd_exec("ls -la %s" % remotepath, ip)
        scpcmd = ['scp', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'LogLevel=quiet', localpath, "%s:%s" % (ip,remotepath)]
        output = util.runCommand(scpcmd)[0].split('\n')
        check = cmd_exec("ls -la %s/%s | wc -l" % (remotepath,filename), ip)
        if int(check) != 1:
            raise Exception("ERROR: failed to scp %s to node %s:%s (file not found after scp)!" % (localpath,ip,remotepath))
        return True
    except:
        raise Exception("ERROR: scp %s to node %s:%s failed!" % (localpath,ip,remotepath))


def start_node(dev, ram=1024, initrd=None, mode=diskutil.CMD_STATUS, address=None, netmask=None, gw=None, bridge=None,
               livecheck=True, name=None, vifname=None):
    '''Default RAM size for node 1024 Mb'''
    config = diskutil.parseCurrentConfig()
    while True:
        rpclog("Starting node for dev %s" % dev[0])
        print "Basename for dev %s" % dev[0]
        if name != None:
            diskID = name
        else:
            diskID = os.path.basename(dev[0])
        l = diskutil.getLock(diskutil.VMIPLOCK)
        c = diskutil.initConfigArgs()
        if address == None:
            ip = c['address']
        else:
            c['address'] = address
            c['netmask'] = netmask
            c['gw'] = gw
            ip = address
        c['name'] = diskID
        c['mem'] = ram
        c['devs'] = dev
        c['status'] = mode
        if bridge != None:
            c['bridge'] = bridge
        if initrd != None:
            c['ramdisk'] = initrd
        if vifname != None:
            c['vifname'] = vifname

        rpclog("Node IP: %s" % ip)
        conf = diskutil._genVMConfig(c)
        diskutil._nodeControl(conf, 0)
        diskutil.releaseLock(l, diskutil.VMIPLOCK)
        booted = False
        if not livecheck:
            retry = VM_ACCESS_RETRY
            booted = True
        else:
            retry = 0
        while retry < VM_ACCESS_RETRY:
            try:
                cmd_exec("true",ip)
                rpclog("Node %s booted with ip: %s" % (diskID, ip))
                booted = True
                break
            except:
                time.sleep(1)
                retry += 1
        #Set MTUs
        intfs = diskutil.bridgeList(listall=True)[c['bridge']]
        for intf in intfs:
            if not intf.startswith("eth") and not intf.startswith("onappstorebond"):
                diskutil.setMTU(intf, config['mtu'])
        diskutil.setMTU(c['bridge'], config['mtu'])

        if booted:
            break
        else:
            rpclog("Node %s did not come up in %d seconds, destroying and trying again" % (diskID, VM_ACCESS_RETRY))
            stop_node(diskID, conf)

    return ip, conf

def stop_node(diskID, conf, silent = False):
    if not silent:
        rpclog("Stopping node: %s" % diskID)

    diskutil._nodeControl(diskID, 1)
    l = diskutil.getLock(diskutil.VMIPLOCK)
    os.unlink(conf)
    diskutil.releaseLock(l, diskutil.VMIPLOCK)

def mkfs(args):
    ID = os.path.basename(args[1])
    (ip, conf) = start_node([args[1]], _getMemSizeFromParams(args))
    print cmd_exec("mkfs.ext2 /dev/xvda", ip)
    cmd_exec("sync", ip)
    stop_node(ID, conf)

def storageControllerRun(args):
    if len(args) < 2:
        print "Usage: scrun <storage node IP address> <cmd to run on storage node>"
        raise Exception("Invalid argument! scrun requires at least 2 arguments!")
    ip = args[0]
    cmdline = " ".join(args[1:])
    print cmd_exec(cmdline,ip)

def getLocalHostId():
    cmd = "onappstore getid"
    process=Popen(cmd, shell=True, stdout = PIPE, stderr=PIPE)
    res, err = process.communicate()
    if process.returncode != 0:
        print 'result=FAILURE error=%s output= \"%s\"' % ( err, res.strip() )
        return
    rval = util.parseNameValuePairs(res.strip().split())
    if not rval.has_key('uuid'):
        raise Exception("Error parsing node uuid!")
    cmd = "onappstore nodeinfo uuid=%s" % rval['uuid']
    process=Popen(cmd, shell=True, stdout = PIPE, stderr=PIPE)
    res, err = process.communicate()
    if process.returncode != 0:
        print 'result=FAILURE error=%s output= \"%s\"' % ( err, res.strip() )
        return
    rval = util.parseNameValuePairs(res.strip().split())
    if not rval.has_key('hostid'):
        raise Exception("Error parsing node hostid!")
    return rval['hostid']

def storageControllerRunAll(args):
    if len(args) < 1:
        print "Usage: scrunall <cmd to run in all local storage nodes>"
        raise Exception("Invalid argument! scrunall requires at least 1 argument!")
    cmd = "onappstore nodes minimal=true showips=true nodetype=backend"
    process=Popen(cmd, shell=True, stdout = PIPE, stderr=PIPE)
    res, err = process.communicate()
    if process.returncode != 0:
        print 'result=FAILURE error=%s output= \"%s\"' % ( err, res.strip() )
        return
    # Use the hostid to run only on the LOCAL nodes!!
    hostid = getLocalHostId()
    rval = util.parseNameValuePairs(res.strip().split())
    if not rval.has_key('nodes'):
        raise Exception("Error parsing nodes!")
    lnips = [ n for n in rval['nodes'].split(',') if n.startswith('10.200.%s' % hostid) ]
    cmdline = " ".join(args)
    print "Local Nodes=",lnips
    for ip in lnips:
        print "Node=%s" % ip
        print cmd_exec(cmdline,ip)

def replicationHelper(args):
    br = args[0]
    mem = int(args[1])
    addr = args[2]
    mask = args[3]
    g = args[4]
    n = args[5]
    rd = args[6]
    vif = args[7]
    VM_ACCESS_RETRY = 1

    (ip,conf) = start_node(args[8:],mem,initrd=rd,address=addr,netmask=mask,gw=g,bridge=br,livecheck=False,name=n,vifname=vif)
    print "Launched VM: %s,%s" % (ip,conf)

def launchHelper(args):
    ID = os.path.basename(args[-1])
    (ip,conf) = start_node([args[-1]], _getMemSizeFromParams(args))
    print "Launched VM: %s,%s" % (ip,conf)

def launchFakeSN(args):
    conf = diskutil.parseCurrentConfig()
    devs = args[-1].split(',')
    print devs
    ID = os.path.basename(devs[0])
    (ip,conf) = start_node(devs, _getMemSizeFromParams(args), mode=diskutil.BACKEND_STATUS)
    print "Launched VM: %s,%s" % (ip,conf)

def killHelper(args):
    ID = os.path.basename(args[-1]).split('-')[1]
    stop_node(ID, args[-1])

def get_blockdev_size(device):
    if not os.path.exists(device):
        raise Exception('ERROR: device %s does not exist!' % device)
    cmd = "blockdev --getsize64 %s" % device
    process=Popen(cmd, shell=True, stdout = PIPE, stderr=PIPE)
    res, err = process.communicate()
    if process.returncode != 0 or res.lower().find('error') != -1 or res.lower().find('fail') != -1:
        raise Exception('ERROR: failed to get size for device %s!' % device)
    return int(res)


# Example of usage and options (reads from /dev/zero, writes to /dev/null):
#
# ddwrapper device=/dev/mapper/uuid bs=8M count=300
# ddwrapper device=/dev/mapper/uuid rawdev=true buffered=true memsize=1024 debugout=1 bs=8M count=300
#
def dd(args):
    retVal = 'result=SUCCESS '
    try:
        strargs = ''
        ID = None
        device = None
        rawdev = True
        fsbase = "/mnt"
        fstype = "xfs"
        memsize = 1024
        debugout = False
        buffered = False
        for arg in args:
            if arg.find('device') != -1:
                device = arg.split('=')[1]
                continue
            if arg.startswith('fsbase='):
                fsbase = arg.split('=')[1]
                continue
            if arg.startswith('fstype='):
                fstype = arg.split('=')[1]
                continue
            if arg.startswith('memsize='):
                msize = int(arg.split('=')[1])
                if msize >= 128 and msize <= 16384:
                    memsize = msize
                continue
            if arg.startswith('debugout='):
                if int(arg.split('=')[1]) != 0:
                    debugout = True
                continue
            if arg.startswith('rawdev='):
                if arg.split('=')[1].lower() == 'true':
                    rawdev = True
                elif arg.split('=')[1].lower() == 'false':
                    rawdev = False
                else:
                    raise Exception("Invalid rawdev argument (need true or false).")
                continue
            if arg.startswith('buffered='):
                if arg.split('=')[1].lower() == 'true':
                    buffered = True
                elif arg.split('=')[1].lower() == 'false':
                    buffered = False
                else:
                    raise Exception("Invalid buffered argument (need true or false).")
                continue
            strargs += arg
            strargs += ' '
        strargs = strargs.strip()

        # Need at least a device or a mounted fs base name
        if device == None:
            print "Usage: ddwrapper device=DEV [fsbase=BASEDIR] [rawdev=true/false] [buffered=false/true] [memsize=256] [debugout=1] [fstype=FS] <misc dd options, e.g. bs=8M count=300>"
            raise Exception("Invalid device arguments (need device).")
        elif not os.path.exists(device):
            raise Exception('ERROR: device %s does not exist!' % device)
        elif fstype not in ["ext2", "ext3", "ext4", "xfs"]:
            raise Exception("Invalid fstype argument (needed for mkfs).")

        if fstype == "xfs":
            mkfsbin = "/onappstore/sbin/mkfs.xfs -f "
        elif fstype == "ext2":
            mkfsbin = "/sbin/mkfs.ext2"
        else:
            mkfsbin = "mkfs." + fstype

        if rawdev:
            retVal += "memsize=%s rawdev=True fstype=None buffered=%s " % (memsize,buffered)
        else:
            retVal += "memsize=%s rawdev=False fstype=%s buffered=%s " % (memsize,fstype,buffered)

        # then create a VM with the device attached
        (ip,conf) = start_node([device], memsize, '/onappstore/testinitrd.img')
        ID = os.path.basename(device)

        if diskutil._isXenHV():
            dev = "/dev/xvda"
        elif diskutil._isKVMHV():
            dev = "/dev/vda"
        else:
            raise Exception("HV not recognised\n")

        # are we running dd on a raw device or a filesystem?
        if rawdev:
            fname = dev
            if buffered:
                wflags = " "
                rflags = " "
            else:
                wflags = "oflag=direct"
                rflags = "iflag=direct"
        else:
            if buffered:
                wflags = " "
                rflags = " "
            else:
                wflags = "conv=fdatasync"
                rflags = "conv=fdatasync"

            # In the rawdev=false case, the fsbase is where we need to mount the new FS INSIDE the VM !
            if fsbase == None or fsbase.find('.') != -1 or not fsbase.startswith('/mnt'):
                raise Exception("Invalid fsbase directory! (need to start with /mnt without any \'.\'")

            # ok, we have a plain device and need mkfs and mount to fsbase...
            #print 'Doing mkfs => mkfsbin=%s dev=%s fsbase=%s' % (mkfsbin,dev,fsbase)
            res = str(cmd_exec("mkdir -p %s && %s %s && mount %s %s" % (fsbase,mkfsbin,dev,dev,fsbase), ip)).strip()
            # make sure mkfs and mount succeeded
            if res.lower().find('error') != -1 or res.lower().find('fail') != -1 or res.lower().find('not found') != -1:
                raise Exception("mkfs or mount failed:\n" + res)

            mounted = cmd_exec("mount | grep %s" % dev,ip)
            if len(mounted) < 8 or mounted.find(dev) < 0:
                raise Exception('ERROR: failed to mount %s at %s !' % (dev,fsbase))

            dir = "%s/test" % fsbase
            fname = "%s/test.dat" % dir
            res = str(cmd_exec("mkdir -p %s && chmod a+rwx %s" % (dir,dir), ip)).strip()
            if res.lower().find('error') != -1 or res.lower().find('fail') != -1 or res.lower().find('not found') != -1:
                raise Exception("mkdir failed:\n" + res)

        cmdstr = "onappdd of=%s if=/dev/zero %s %s" % (fname,strargs,wflags)
        res = str(cmd_exec( cmdstr, ip)).strip()
        if debugout:
            print "cmd = \"%s\"" % cmdstr
            print "res = %s" % res

        if res.lower().find('error') != -1:
            dmesg_output = str(cmd_exec("dmesg", ip)).strip()
            retVal += 'dmesg_write=%s' % dmesg_output
            raise Exception("Data write failed with I/O error. Output: %s, Dmesg output: %s" % (res, dmesg_output))

        write_raww = res.split('\r\n')[-1].strip()
        write_time = write_raww.split(',')[1].split()[0]
        write_mbps = write_raww.split(',')[2].split()[0]
        write_raw = write_raww.replace(' ', '_')
        retVal += 'write_results=%s write_time=%s write_mbps=%s' % ( write_raw, write_time, write_mbps )

        # now flush data and empty caches (before the read)
        res = str(cmd_exec("sync && echo 3 > /proc/sys/vm/drop_caches && sync", ip)).strip()

        # now exec a dd read test
        cmdstr = "onappdd if=%s of=/dev/null %s %s" % (fname,strargs,rflags)
        res = str(cmd_exec( cmdstr, ip)).strip()
        if debugout:
            print "cmd = \"%s\"" % cmdstr
            print "res = %s" % res

        if res.lower().find('error') != -1:
            dmesg_output = str(cmd_exec("dmesg", ip)).strip()
            retVal += 'dmesg_read=%s' % dmesg_output
            raise Exception("Data read failed with I/O error. Output: %s, Dmesg output: %s" % (res, dmesg_output))

        read_raww = res.split('\r\n')[-1].strip()
        read_time = read_raww.split(',')[1].split()[0]
        read_mbps = read_raww.split(',')[2].split()[0]
        read_raw = read_raww.replace(' ', '_')
        retVal += ' read_results=%s read_time=%s read_mbps=%s' % ( read_raw, read_time, read_mbps )

        if not rawdev:
            str(cmd_exec("umount %s" % fsbase, ip)).strip() # umount

    except Exception, e:
        retVal = 'result=FAILURE error=%s' % str(e)
    finally:
        # delete the created VM
        if ID != None:
            stop_node(ID, conf, silent=True)

    print retVal

# Example of usage and options (xdd options passed through after device=X):
#
# xddwrapper device=/dev/mapper/uuid -op read -blocksize 4096 -mbytes 1024 -passes 2 -queuedepth 8 -timelimit 120 -dio -verbose -reqsize 32
# xddwrapper device=/dev/mapper/:uuid -op write -blocksize 4096 -mbytes 1024 -passes 2 -queuedepth 8 -timelimit 120 -dio -verbose -reqsize 256
# xddwrapper device=/dev/mapper/uuid -rwratio 70 -blocksize 4096 -seek random -mbytes 1024 -passes 2 -queuedepth 8 -timelimit 120 -dio -verbose -reqsize 256
#
def xdd(args):
    retVal = 'result=SUCCESS '
    try:
        strargs = ''
        ID = None
        for arg in args:
            if arg.find('device') != -1:
                device = arg.split('=')[1]
                continue
            strargs += arg
            strargs += ' '
        strargs = strargs.strip()

        # then create a VM with the device attached
        (ip,conf) = start_node([device], _getMemSizeFromParams(args), '/onappstore/testinitrd.img')
        ID = os.path.basename(device)

        if diskutil._isXenHV():
            dev = "/dev/xvda"
        elif diskutil._isKVMHV():
            dev = "/dev/vda"
        else:
            print "HV not recognised"
            return

        # Do a single run of xdd using the args provided in the cmd line...
        res = str(cmd_exec("xdd.linux -targets 1 %s %s" % (dev,strargs), ip)).strip()

        # process xdd internal error messages
        if res.find('Error creating thread') != -1 or res.find('Segmentation fault') != -1:
            raise Exception("XDD ERROR (benchmark failure): Output: \n" + res)

        # I/O error...
        if res.lower().find('failure') != -1:
            dmesg_output = str(cmd_exec("dmesg", ip)).strip()
            retVal += 'dmesg_write=%s' % dmesg_output
            raise Exception("xdd failed with I/O error. Output: %s, Dmesg output: %s" % (res, dmesg_output))

        avgline = [ l.strip() for l in res.split('\r\n') if l.lower().find('average') > 0 ]
        if len(avgline) != 1:
            dmesg_output = str(cmd_exec("dmesg", ip)).strip()
            retVal += 'dmesg_write=%s' % dmesg_output
            raise Exception("Error parsing xdd output: %s, Dmesg output: %s" % (res, dmesg_output))

        cols = ['target','pass','T','io_queue_depth','total_bytes','total_ops','total_time_secs', \
                'throughput_mbps','iops','latency','cpu','op_type','req_size']
        kvres = dict( zip( cols, avgline[0].split() ) )

        cols_wanted = ['io_queue_depth','total_bytes','total_ops','total_time_secs','throughput_mbps', \
                        'iops','latency','op_type','req_size']
        for (k,v) in kvres.items():
            if k in cols_wanted:
                retVal += " %s=%s" % (k,v)

    except Exception, e:
        retVal = 'result=FAILURE error=%s' % str(e)
    finally:
        # delete the created VM
        if ID != None:
            stop_node(ID, conf, silent=True)

    print retVal


def kvstringToDict(s, pairsplitter = ':', separator = ','):
    dict = {}
    if len(s):
        for pair in s.split(separator):
            l = pair.split(pairsplitter)
            dict[l[0]] = str(l[1]).strip("'").strip('"')
    return dict


# Example of usage and options (dcc options passed through after device=X):
#
# dccwrapper device=/dev/mapper/uuid -r 1 -m 5119 -p 0x99332
# dccwrapper device=/dev/mapper/uuid -w 1 -r 1 -m 5119 -p 0x99332
#
def dcc(args):
    retVal = 'result=SUCCESS '
    try:
        strargs = ''
        ID = None
        for arg in args:
            if arg.find('device') != -1:
                device = arg.split('=')[1]
                continue
            strargs += arg
            strargs += ' '
        strargs = strargs.strip()

        # then create a VM with the device attached
        (ip,conf) = start_node([device], _getMemSizeFromParams(args), '/onappstore/testinitrd.img')
        ID = os.path.basename(device)

        if diskutil._isXenHV():
            dev = "/dev/xvda"
        elif diskutil._isKVMHV():
            dev = "/dev/vda"
        else:
            print "HV not recognised"
            return

        # Do a single run of dcc using the args provided in the cmd line...
        res = str(cmd_exec("dcc %s -n %s" % (strargs,dev), ip)).strip()

        # process dcc internal error messages
        lres = res.lower()
        if lres.find('terminated') != -1 or lres.find('segmentation') != -1 or lres.find('abort') != -1:
            raise Exception("DCC ERROR (failure): Output: \n" + res)

        dccout = [ l.strip() for l in lres.split('\r\n') if l.strip().startswith('//dcc:') ]

        if len(dccout) != 1:
            #dmesg_output = str(cmd_exec("dmesg", ip)).strip()
            raise Exception("Error parsing dcc output: %s, len: %d, dccout: %s" % (res, len(dccout), str(dccout)))

        dccRes= kvstringToDict(dccout[0].replace('//dcc:',''),'=',',')
        dccRes['errcount'] = int(dccRes['errcount'])
        dccRes['bsize'] = int(dccRes['bsize'])

        # I/O error...
        if int(dccRes['errcount']) != 0:
            #dmesg_output = str(cmd_exec("dmesg", ip)).strip()
            raise Exception("dcc failed with error: %s. Output: %s" % (dccRes['error'],res))

        for (k,v) in dccRes.items():
            retVal += " %s=%s" % (k,v)

    except Exception, e:
        retVal = 'result=FAILURE error=%s' % str(e)
    finally:
        # delete the created VM
        if ID != None:
            stop_node(ID, conf, silent=True)

    print retVal


# Example of usage and options (iozone options passed through after device=X):
#
#  iozwrapper device=/dev/mapper/uuid threads=1 -r 1m
#  iozwrapper device=/dev/mapper/uuid threads=2 -r 512k
#  iozwrapper device=/dev/mapper/uuid memsize=1024 threads=2 mintimesec=180 -r 128k
#  iozwrapper device=/dev/mapper/uuid dom0=true threads=1 -r 1m
#
# also working options: [memsize=1024] [mintimesec=60] [debugout=1]
#
# CAUTION: using more than 2 threads may result in iozone error (benchmark does
#          not seem to handle many threads well...
#
def iozone(args):
    retVal = 'result=SUCCESS '
    try:
        dom0 = False
        strargs = ''
        ID = None
        device = None
        fsbase = "/mnt"
        fstype = "xfs"
        threads = 1
        totsizemb = 1024
        memsize = 1024
        mintimesec = 0
        debugout = False

        for arg in args:
            if arg.startswith('device='):
                device = arg.split('=')[1]
                continue
            if arg.startswith('fsbase='):
                fsbase = arg.split('=')[1]
                continue
            if arg.startswith('fstype='):
                fstype = arg.split('=')[1]
                continue
            if arg.startswith('memsize='):
                msize = int(arg.split('=')[1])
                if msize >= 128 and msize <= 16384:
                    memsize = msize
                continue
            if arg.startswith('threads='):
                threads = int(arg.split('=')[1])
                continue
            if arg.startswith('dsize='):
                totsizemb = int(arg.split('=')[1])
                continue
            if arg.startswith('mintimesec='):
                timesec = int(arg.split('=')[1])
                if timesec >= 10 and timesec <= 1800:
                    mintimesec = timesec
                continue
            if arg.startswith('debugout='):
                if int(arg.split('=')[1]) != 0:
                    debugout = True
                continue
            if arg.startswith('dom0='):
                if arg.split('=')[1].lower() == 'true':
                    dom0 = True
                elif arg.split('=')[1].lower() == 'false':
                    dom0 = False
                else:
                    raise Exception("Invalid dom0 argument (need true or false).")
                continue
            if arg.strip() in ['-i', '-s', '-F', '-t']:
                print "Usage: iozwrapper device=DEV fsbase=BASEDIR [dom0=true/false] [memsize=1024] [mintimesec=60] [debugout=1] [fstype=FS] [threads=#threads] [dsize=#MBytes] [misc iozone options]"
                raise Exception("Invalid argument! Iozone arguments -i, -F and -t are provided by the wrapper!")
            strargs += arg
            strargs += ' '
        strargs = strargs.strip()

        # Need at least a device or a mounted fs base name
        if device == None or fsbase == None:
            print "Usage: iozwrapper device=DEV [fsbase=BASEDIR] [dom0=true/false] [memsize=1024] [mintimesec=60] [debugout=1] [fstype=FS] [threads=#threads] [dsize=#MBytes] [misc iozone options]"
            raise Exception("Invalid device or fsbase arguments (need device or fsbase).")
        elif not os.path.exists(device):
            raise Exception('ERROR: device %s does not exist!' % device)
        elif fstype not in ["ext2", "ext3", "ext4", "xfs"]:
            raise Exception("Invalid fstype argument (needed for mkfs).")

        if fstype == "xfs":
            mkfsbin = "/onappstore/sbin/mkfs.xfs -f "
        elif fstype == "ext2":
            mkfsbin = "/sbin/mkfs.ext2"
        else:
            mkfsbin = "mkfs." + fstype

        if threads < 1 or threads > 32:
            raise Exception("Invalid number of threads (max 32 threads supported).")
        elif totsizemb < threads or totsizemb > 32*1024:
            raise Exception("Invalid total data size in MBytes (max size: 32768 MB supported).")

        # if I/O data size is less than 2 * memsize, increase it...
        retVal += "memsize=%s fstype=%s" % (memsize,fstype)
        totsizemb = memsize * 2 if totsizemb < memsize * 2 else totsizemb

        # then check the device size and adjust the max size
        devsize = get_blockdev_size(device)
        totsizemb = (devsize/(1024*1024))-100 if  totsizemb > (devsize/(1024*1024))-100 else totsizemb

        # Example cmd line: iozone -i 0 -i 1 -i 5 -i 7 -i 8 -j 1024 -R -e -c -s 250m -t 4 \
        #                   -F /mnt/iozt/ioztest1.tmp /mnt/iozt/ioztest2.tmp /mnt/iozt/ioztest3.tmp /mnt/iozt/ioztest4.tmp -r 4k -t 4 -+p 70

        """ -i #  Test to run (0=write/rewrite, 1=read/re-read, 2=random-read/write
                 3=Read-backwards, 4=Re-write-record, 5=stride-read, 6=fwrite/re-fwrite
                 7=fread/Re-fread, 8=random_mix, 9=pwrite/Re-pwrite, 10=pread/Re-pread
                 11=pwritev/Re-pwritev, 12=preadv/Re-preadv) """
        #testargs = "-i 0 -i 1 -i 5 -i 6 -i 7 -i 8 -j 64 -R -e -c " # these are the default iozone tests to run and parse results...
        testargs = "-i 0 -i 1 -i 5 -i 8 -j 64 -R -e -c -T " # these are the default iozone tests to run and parse results...
        #print 'dom0 = %s ' % str(dom0)
        #print 'strargs = ', strargs
        testargs += "-s %dm -t %d -F " % ( int(totsizemb/threads), threads)
        for t in range(0, threads):
            testargs += "%s/t%d/test%d.dat " % (fsbase, t, t)
        testargs += "-t %d " % threads
        strargs = testargs + strargs + " -+p 70"
        #print 'strargs = ', strargs

        # do we use a mounted FS on dom0??
        if dom0:
            if fsbase != None:
                dirs = " "
                for t in range(0, threads):
                    dirs += "%s/t%d " % (fsbase, t)
                cmd = "mkdir -p %s" % dirs
                process=Popen(cmd, shell=True, stdout = PIPE, stderr=PIPE)
                res, err = process.communicate()
                if process.returncode != 0:
                    print 'result=FAILURE error=%s output= \"%s\"' % ( err, res.strip() )

                cmd = "iozone %s" % strargs
                process=Popen(cmd, shell=True, stdout = PIPE, stderr=PIPE)
                res, err = process.communicate()
                if process.returncode != 0:
                    print 'result=FAILURE error=%s output= \"%s\"' % ( err, res.strip() )
                    return
            else:
                # FIXME FIXME FIXME - need to mkfs on device or use the fsbase argument!
                raise Exception("Not implemented mkfs on dom0 device yet...")

        # ... or use a helper VM where we create & mount the FS?
        else:
            # then create a VM with the device attached
            (ip,conf) = start_node([device], memsize, '/onappstore/testinitrd.img')
            ID = os.path.basename(device)
            if diskutil._isXenHV():
                dev = "/dev/xvda"
            elif diskutil._isKVMHV():
                dev = "/dev/vda"
            else:
                raise Exception("HV not recognised\n")

            # In the VM case, the fsbase is where we need to mount the new FS INSIDE the VM !
            if fsbase == None or fsbase.find('.') != -1 or not fsbase.startswith('/mnt'):
                raise Exception("Invalid fsbase directory! (need to start with /mnt without any \'.\'")

            # ok, we have a plain device and need mkfs and mount to fsbase...
            #print 'Doing mkfs => mkfsbin=%s dev=%s fsbase=%s' % (mkfsbin,dev,fsbase)
            res = str(cmd_exec("mkdir -p %s && %s %s && mount %s %s" % (fsbase,mkfsbin,dev,dev,fsbase), ip)).strip()
            # make sure mkfs and mount succeeded
            if res.lower().find('error') != -1 or res.lower().find('fail') != -1 or res.lower().find('not found') != -1:
                raise Exception("mkfs or mount failed:\n" + res)

            mounted = cmd_exec("mount | grep %s" % dev,ip)
            if len(mounted) < 8 or mounted.find(dev) < 0:
                raise Exception('ERROR: failed to mount %s at %s !' % (dev,fsbase))

            dirs = " "
            for t in range(0, threads):
                dirs += "%s/t%d " % (fsbase, t)
            res = str(cmd_exec("mkdir -p %s && chmod a+rwx %s" % (dirs,dirs), ip)).strip()
            if res.lower().find('error') != -1 or res.lower().find('fail') != -1 or res.lower().find('not found') != -1:
                raise Exception("mkdir failed:\n" + res)

            # Do a single run of iozone using the args provided in the cmd line...
            if mintimesec < 10:
                starttime = int(time.time())
                #print 'OK, calling @ip: %s => iozone %s' % (str(ip),strargs)
                res = str(cmd_exec("iozone %s" % strargs, ip)).strip()
                if debugout:
                    print "res = %s" % res
                endtime = int(time.time())
                retVal += " reps=1 total_time=%s" % str(endtime-starttime)
            else:
                # Do multiple runs of workload until timeout expires...
                retVal += " mintimesec=%d" % mintimesec
                starttime = int(time.time())
                endtime = starttime + 1
                rep = 0
                while endtime <= starttime + mintimesec:
                    rep += 1
                    res = str(cmd_exec("iozone %s" % strargs, ip)).strip()
                    if debugout:
                        print "[%s] res = %s" % (rep,res)
                    cmd_exec("sync",ip)
                    time.sleep(5)
                    endtime = int(time.time())

                retVal += " reps=%s total_time=%s" % (rep,str(endtime-starttime))

            str(cmd_exec("umount %s" % fsbase, ip)).strip() # umount

        # process iozone internal error messages
        if res.lower().find('error') != -1 or res.lower().find('fail') != -1 or res.lower().find('segmentation fault') != -1:
            raise Exception("Iozone ERROR (benchmark failure): Output: \n" + res)

        # I/O error...
        if res.lower().find('failure') != -1:
            #dmesg_output = str(cmd_exec("dmesg", ip)).strip()
            #raise Exception("iozone failed with I/O error. Output: %s, Dmesg output: %s" % (res, dmesg_output))
            raise Exception("iozone failed with I/O error. Output: %s, Dmesg output: " % res)

        recsize = 'INVALID'
        workloads=[ 'write', 'rewrite', 'read', 'reread', 'stride_read', 'fwrite', 'fread', 'randmix' ]
        for workload in workloads:
            if workload == 'write':
                keyword="Initial write"
            elif workload == 'rewrite':
                keyword="Rewrite"
            elif workload == 'read':
                keyword="Read"
            elif workload == 'reread':
                keyword="Re-read"
            elif workload == 'stride_read':
                keyword="Stride read"
            elif workload == 'fwrite':
                keyword="Fwrite"
            elif workload == 'fread':
                keyword="Fread"
            elif workload == 'randmix':
                keyword="Mixed workload"
            else:
                continue
                # for now just ignore all other workloads...
                # raise Exception("Error parsing iozone output: %s" % res)

            recs = [ l.strip().strip('\"') for l in res.split('\n') if l.find('Record size =') > 0 ]
            recsize = recs[0].split()[3]

            resline = [ l.strip().strip('\"').lower() for l in res.split('\n') if l.find(keyword) > 0 ]
            #print 'resline=', resline
            if len(resline) != 1:
                #retVal = 'result=FAILURE error=\"Error parsing iozone output\"'
                #raise Exception("Error parsing iozone output (MUST have options -i 0 -i 1 -i 8): %s" % res )

                # if test does not exist in output, skip it...
                continue
            else:
                rate = resline[0].split()[-1]

            retVal += " %s_rate_KBps=%s" % (workload,rate)

        retVal += " recsize_kb=%s threads=%d total_data_mbytes=%d" % (recsize,threads,totsizemb)

    except Exception, e:
        retVal = 'result=FAILURE error=%s' % str(e)
    finally:
        # delete the created VM
        if not dom0 and ID != None:
            stop_node(ID, conf, silent=True)

    print retVal


# Example of usage and options (bonnie options passed through after device=X):
#
#  bonniewrapper device=/dev/mapper/uuid memsize=1024 -s 2g:1024k
#
# also working options: [memsize=1024] [mintimesec=60] [debugout=1]
#
def bonnie(args, dom0=True):
    retVal = 'result=SUCCESS '
    try:
        dom0 = False
        strargs = ''
        ID = None
        device = None
        fsbase = "/mnt"
        fstype = "xfs"
        threads = 1
        totsizemb = 1024
        memsize = 1024
        mintimesec = 0
        debugout = False
        for arg in args:
            if arg.startswith('device='):
                device = arg.split('=')[1]
                continue
            if arg.startswith('fsbase='):
                fsbase = arg.split('=')[1]
                continue
            if arg.startswith('fstype='):
                fstype = arg.split('=')[1]
                continue
            if arg.startswith('memsize='):
                msize = int(arg.split('=')[1])
                if msize >= 128 and msize <= 16384:
                    memsize = msize
                continue
            if arg.startswith('mintimesec='):
                timesec = int(arg.split('=')[1])
                if timesec >= 10 and timesec <= 1800:
                    mintimesec = timesec
                continue
            if arg.startswith('debugout='):
                if int(arg.split('=')[1]) != 0:
                    debugout = True
                continue
            if arg.startswith('dom0='):
                if arg.split('=')[1].lower() == 'true':
                    dom0 = True
                elif arg.split('=')[1].lower() == 'false':
                    dom0 = False
                else:
                    raise Exception("Invalid dom0 argument (need true or false).")
                continue
            if arg.strip() in ['-d', '-y']:
                print "Usage: bonniewrapper device=DEV [fsbase=BASEDIR] [memsize=1024] [mintimesec=60] [debugout=1] [dom0=true/false] [fstype=FS] [misc bonnie++ options]"
                raise Exception("Invalid argument! Bonnie++ arguments -d and -y are provided by the wrapper!")
            strargs += arg
            strargs += ' '
        strargs = strargs.strip()

        # Need at least a device or a mounted fs base name
        if device == None or fsbase == None:
            print "Usage: bonniewrapper device=DEV [fsbase=BASEDIR] [memsize=1024] [mintimesec=60] [debugout=1] [dom0=true/false] [fstype=FS] [misc bonnie++ options]"
            raise Exception("Invalid device or fsbase arguments (need device or fsbase).")
        elif not os.path.exists(device):
            raise Exception('ERROR: device %s does not exist!' % device)
        elif fstype not in ["ext2", "ext3", "ext4", "xfs"]:
            raise Exception("Invalid fstype %s argument (needed for mkfs)" % fstype)

        if fstype == "xfs":
            mkfsbin = "/onappstore/sbin/mkfs.xfs -f "
        elif fstype == "ext2":
            mkfsbin = "/sbin/mkfs.ext2"
        else:
            mkfsbin = "mkfs." + fstype
        retVal += "memsize=%s fstype=%s" % (memsize,fstype)

        # Example cmd line: bonnie++ -d /mnt/testfs/test -s 2g:1024k -u root -f

        #print 'dom0 = %s ' % str(dom0)
        strargs = " -d %s/ %s -f " % (fsbase,strargs)
        print 'strargs = ', strargs

        # do we use a mounted FS on dom0??
        if dom0:
            if fsbase != None:
                cmd = "bonnie++ %s" % strargs
                process=Popen(cmd, shell=True, stdout = PIPE, stderr=PIPE)
                res, err = process.communicate()
                if process.returncode != 0:
                    print 'result=FAILURE error=%s output= \"%s\"' % ( err, res.strip() )
                    return
            else:
                # FIXME FIXME FIXME - need to mkfs on device or use the fsbase argument!
                raise Exception("Not implemented mkfs on dom0 device yet...")

        # ... or use a helper VM where we create & mount the FS?
        else:

            # then create a VM with the device attached
            (ip,conf) = start_node([device], memsize, '/onappstore/testinitrd.img')
            ID = os.path.basename(device)
            if diskutil._isXenHV():
                dev = "/dev/xvda"
            elif diskutil._isKVMHV():
                dev = "/dev/vda"
            else:
                raise Exception("HV not recognised\n")

            # In the VM case, the fsbase is where we need to mount the new FS INSIDE the VM !
            if fsbase == None or fsbase.find('.') != -1 or not fsbase.startswith('/mnt'):
                raise Exception("Invalid fsbase directory! (need to start with /mnt/ without any \'.\'")

            # ok, we have a plain device and need mkfs and mount to fsbase...
            #print 'Doing mkfs => mkfsbin=%s dev=%s fsbase=%s' % (mkfsbin,dev,fsbase)
            res = str(cmd_exec("mkdir -p %s && %s %s && mount %s %s" % (fsbase,mkfsbin,dev,dev,fsbase), ip)).strip()
            # make sure mkfs and mount succeeded
            if res.lower().find('error') != -1 or res.lower().find('fail') != -1 or res.lower().find('not found') != -1:
                raise Exception("mkfs or mount failed:\n" + res)

            mounted = cmd_exec("mount | grep %s" % dev,ip)
            if len(mounted) < 8 or mounted.find(dev) < 0:
                raise Exception('ERROR: failed to mount %s at %s !' % (dev,fsbase))

            dirs = "%s/t0 " % fsbase
            res = str(cmd_exec("mkdir -p %s && chmod a+rwx %s" % (dirs,dirs), ip)).strip()
            if res.lower().find('error') != -1 or res.lower().find('fail') != -1 or res.lower().find('not found') != -1:
                raise Exception("mkdir failed:\n" + res)

            # Do a single run of bonnie++ using the args provided in the cmd line...
            if mintimesec < 10:
                starttime = int(time.time())
                #print 'OK, calling @ip: %s => bonnie++ %s' % (str(ip),strargs)
                res = str(cmd_exec("bonnie++ %s" % strargs, ip)).strip()
                if debugout:
                    print "res = %s" % res
                endtime = int(time.time())
                retVal += " reps=1 total_time=%s" % str(endtime-starttime)
            else:
                # Do multiple runs of workload until timeout expires...
                retVal += " mintimesec=%d" % mintimesec
                starttime = int(time.time())
                endtime = starttime + 1
                rep = 0
                while endtime <= starttime + mintimesec:
                    rep += 1
                    res = str(cmd_exec("bonnie++ %s" % strargs, ip)).strip()
                    if debugout:
                        print "[%s] res = %s" % (rep,res)
                    cmd_exec("sync",ip)
                    time.sleep(5)
                    endtime = int(time.time())

                retVal += " reps=%s total_time=%s" % (rep,str(endtime-starttime))

            str(cmd_exec("umount %s" % fsbase, ip)).strip() # umount

        # process bonnie++ internal error messages
        if res.lower().find('error') != -1 or res.lower().find('fail') != -1 or res.lower().find('segmentation fault') != -1:
            raise Exception("Bonnie++ ERROR (benchmark failure): Output: \n" + res)

        # FIXME: parse bonnie output here...
        print "res=", res
        rlines = [ l.strip().strip('+') for l in res.split('\n') ]
        recs = rlines[-1].split(',')

        if len(recs) < 5:
            retVal = 'result=FAILURE error=\"Error parsing bonnie++ output\"'
            raise Exception("Error parsing bonnie++ output: %s" % res )

        #resline = [ l.strip().strip('\"').lower() for l in res.split('\n') if l.find(keyword) > 0 ]
        #print 'resline=', resline

        #retVal += " %s_rate_KBps=%s" % (workload,rate)

        #retVal += " recsize_kb=%s" % recsize
        for (k,v) in enumerate(recs):
            retVal += "%s=%s" % (k,v)

    except Exception, e:
        retVal = 'result=FAILURE error=%s' % str(e)
    finally:
        # delete the created VM
        if not dom0 and ID != None:
            stop_node(ID, conf, silent=True)

    print retVal


# NOTE: for some tests (e.g. --fork) the VM will hang or fail external connections,
#       or the stress test will fail and return before the timeout!
#       -> In these cases stressng returns an error, briefly after a run!
#
# Example usage (cases with working options):
#  stressngwrapper device=/dev/mapper/uuid --cpu 4 --vm 4 --switch 2 --timeout 20s --metrics-brief
#  stressngwrapper device=/dev/mapper/uuid --cpu 4 --urandom 2 --timer 2 --timeout 20s --metrics-brief
#  stressngwrapper device=/dev/mapper/uuid --cpu 4 --pipe 2 --memcpy 2 --malloc 2 --timeout 20s --metrics-brief
#  stressngwrapper device=/dev/mapper/uuid --cpu 2 --aio 2 --mmap 2 --matrix 2 --timeout 20s --metrics-brief
#  stressngwrapper device=/dev/mapper/uuid --crypt 2 --sock 2 --dentry 2 --clock 2 --timeout 20s --metrics-brief
#  stressngwrapper device=/dev/mapper/uuid --cache 2 --bigheap 2 --msg 2 --pthread 2 --seek 2 --timeout 20s --metrics-brief
#  stressngwrapper device=/dev/mapper/uuid --sendfile 2 --shm-sysv 2 --sock 2 --sem 2 --stack 2 --timeout 20s --metrics-brief
#  stressngwrapper device=/dev/mapper/uuid --splice 2 --sigsegv 2 --sysinfo 2 --str 2 --tsearch 2 --timeout 20s --metrics-brief
#  stressngwrapper device=/dev/mapper/uuid --zero 2 --utime 2 --vfork 2 --zombie 2 --wait 2 --timeout 20s --metrics-brief
#
# with configurable memory size (of the VM), add the optional memsize= option to the examples above:
#  stressngwrapper device=/dev/mapper/uuid memsize=1024 --splice 2 --sysinfo 2 --str 2 --tsearch 2 --timeout 20s --metrics-brief
#
# also working options: [mintimesec=60] [debugout=1]
#
# NON-WORKING switches: --hdd, --fork, --xattr, --vm-rw, --vecmath
#
def stressng(args):
    retVal = 'result=SUCCESS'
    try:
        strargs = ''
        ID = None
        device = None
        memsize = 1024
        mintimesec = 0
        debugout = False

        for arg in args:
            if arg.startswith('device='):
                device = arg.split('=')[1]
                continue
            if arg.startswith('memsize='):
                msize = int(arg.split('=')[1])
                if msize >= 128 and msize <= 16384:
                    memsize = msize
                continue
            if arg.startswith('mintimesec='):
                timesec = int(arg.split('=')[1])
                if timesec >= 10 and timesec <= 1800:
                    mintimesec = timesec
                continue
            if arg.startswith('debugout='):
                if int(arg.split('=')[1]) != 0:
                    debugout = True
                continue
            if arg.strip() in ['-h', '--help']:
                print "Usage: stressngwrapper device=DEV [memsize=VM_RAM_MB] [misc stress-ng pass-through options]"
                return
            strargs += arg
            strargs += ' '
        strargs = strargs.strip()

        # NOTE: we don't really use the device here, but start_node needs it to work
        if not device:
            raise Exception("ERROR: No device provided!\n")
        elif not os.path.exists(device):
            raise Exception('ERROR: device %s does not exist!' % device)
        elif len(strargs) < 2:
            raise Exception('ERROR: empty stress-ng arguments!')

        # then create a VM with the device attached
        (ip,conf) = start_node([device], memsize, '/onappstore/testinitrd.img')
        ID = os.path.basename(device)
        #print "strargs = %s" % strargs

        # Do a single run of stress-ng using the args provided in the cmd line...
        if mintimesec < 10:
            starttime = int(time.time())
            res = str(cmd_exec("stress-ng %s" % strargs, ip)).strip()
            if debugout:
                print "res = %s" % res
            endtime = int(time.time())
            retVal += " reps=1 total_time=%s" % str(endtime-starttime)
        else:
            # Do multiple runs of workload until timeout expires...
            retVal += " mintimesec=%d" % mintimesec
            starttime = int(time.time())
            endtime = starttime + 1
            rep = 0
            while endtime <= starttime + mintimesec:
                rep += 1
                res = str(cmd_exec("stress-ng %s" % strargs, ip)).strip()
                if debugout:
                    print "[%s] res = %s" % (rep,res)
                cmd_exec("sync",ip)
                time.sleep(5)
                endtime = int(time.time())

            retVal += " reps=%s total_time=%s" % (rep,str(endtime-starttime))

        # process stress-ng internal error messages
        lres = res.lower()
        if lres.find('successful run completed') == -1 or lres.find('segmentation') != -1 or lres.find('error') != -1:
            raise Exception("stress-ng ERROR (failure): Output: \n" + res)

        stressout = [ l.strip() for l in lres.split('\r\n') if l.strip().find('successful run completed in') != -1 ]

        if len(stressout) != 1:
            #dmesg_output = str(cmd_exec("dmesg", ip)).strip()
            raise Exception("Error parsing stress output: %s, len: %d, stressout: %s" % (res, len(stressout), str(stressout)))

        retVal += " output=%s" % stressout[0].replace(' ', '_')

    except Exception, e:
        retVal = 'result=FAILURE error=%s' % str(e)
    finally:
        # delete the created VM
        if ID != None:
            stop_node(ID, conf, silent=True)

    print retVal


# Example usage (cases with working options):
#
#  workloadwrapper device=/dev/mapper/uuid workloadpath=/tmp/tarball.tar.bz2 [memsize=1024] [fstype=xfs] [mintimesec=60] [debugout=1]
#
# workloadwrapper device=/dev/mapper/uuid memsize=1024 workloadpath=/tmp/kernel_work1.tar.bz2 mintimesec=60 debugout=1
#
def runworkload(args):
    retVal = 'result=SUCCESS '
    try:
        ID = None
        device = None
        fsbase = "/mnt"
        fstype = "xfs"
        memsize = 1024
        mintimesec = 0
        debugout = False

        for arg in args:
            if arg.startswith('device='):
                device = arg.split('=')[1]
                continue
            if arg.startswith('fstype='):
                fstype = arg.split('=')[1]
                continue
            if arg.startswith('memsize='):
                msize = int(arg.split('=')[1])
                if msize >= 128 and msize <= 16384:
                    memsize = msize
                continue
            if arg.startswith('mintimesec='):
                timesec = int(arg.split('=')[1])
                if timesec >= 10 and timesec <= 1800:
                    mintimesec = timesec
                continue
            if arg.startswith('debugout='):
                if int(arg.split('=')[1]) != 0:
                    debugout = True
                continue
            if arg.startswith('workloadpath='):
                wlpath = arg.split('=')[1].replace(':', ' ')
                continue

            if arg.strip() in ['-h', '--help']:
                print "Usage: workloadwrapper device=DEV workloadpath=/path [memsize=VM_RAM_MB] [fstype=FS]"
                return

        if not device:
            raise Exception("ERROR: No device provided!\n")
        elif not os.path.exists(device):
            raise Exception('ERROR: device %s does not exist!' % device)
        elif fstype not in ["ext2", "xfs"]: # currently mkfs only for ext2, xfs
            raise Exception("Invalid fstype argument %s (needed for mkfs)." % fstype)
        elif not wlpath:
            raise Exception("ERROR: No workload path provided!\n")
        elif not os.path.exists(wlpath):
            raise Exception('ERROR: workload path %s does not exist!' % wlpath)

        if fstype == "xfs":
            mkfsbin = "/onappstore/sbin/mkfs.xfs -f "
        elif fstype == "ext2":
            mkfsbin = "/sbin/mkfs.ext2"
        else:
            mkfsbin = "mkfs." + fstype

        # then create a VM with the device attached
        (ip,conf) = start_node([device], memsize, '/onappstore/testinitrd.img')
        ID = os.path.basename(device)

        if diskutil._isXenHV():
            dev = "/dev/xvda"
        elif diskutil._isKVMHV():
            dev = "/dev/vda"
        else:
            raise Exception("HV type not recognised")

        wmkfs = cmd_exec("%s %s" % (mkfsbin,dev),ip)
        #print wmkfs
        cmd_exec("sync && mkdir -p %s" % fsbase,ip)

        cmd_exec("/bin/mount %s %s" % (dev,fsbase),ip)
        mounted = cmd_exec("/bin/mount | grep %s" % dev,ip)
        if len(mounted) < 8 or mounted.find(dev) < 0:
            raise Exception('ERROR: failed to mount %s at %s !' % (dev,fsbase))

        if not scp_file_to_node_tmp(wlpath,ip,fsbase):
            raise Exception('ERROR: scp failure for %s!' % wlpath)

        wlfname = os.path.basename(wlpath)
        wlremotepath = "%s/%s" % (fsbase,wlfname)
        if debugout:
            print cmd_exec("ls -la %s" % fsbase, ip)
        if wlfname.find('.tar.bz2') or wlfname.find('.tar.bz') or wlfname.fine('.tbz'):
            cmd_exec("cd %s && tar xjf %s && chmod a+rx %s/*" % (fsbase,wlremotepath,fsbase), ip)
        elif wlfname.find('.tar.gz') or wlfname.find('.tgz'):
            cmd_exec("cd %s && tar xzf %s && chmod a+rx %s/*" % (fsbase,wlremotepath,fsbase), ip)
        else:
            raise Exception('ERROR: unknown workload archive type for %s!' % wlfname)

        if debugout:
            print cmd_exec("ls -la %s" % fsbase, ip)

        if mintimesec < 10:
            starttime = int(time.time())
            res = str(cmd_exec("cd %s && ./runwork.sh" % fsbase,ip)).strip()
            if debugout:
                print "res = %s" % res
            endtime = int(time.time())
            retVal += " reps=1 total_time=%s" % str(endtime-starttime)
        else:
            # Do multiple runs of workload until timeout expires...
            retVal += " mintimesec=%d" % mintimesec
            starttime = int(time.time())
            endtime = starttime + 1
            rep = 0
            while endtime <= starttime + mintimesec:
                rep += 1
                res = str(cmd_exec("cd %s && ./runwork.sh" % fsbase,ip)).strip()
                if debugout:
                    print "[%s] res = %s" % (rep,res)
                cmd_exec("sync",ip)
                time.sleep(3)
                endtime = int(time.time())

            retVal += " reps=%s total_time=%s" % (rep,str(endtime-starttime))

        # process workload internal error messages
        lres = res.lower()
        if lres.find('done') == -1 or lres.find('segmentation') != -1 or lres.find('error') != -1:
            raise Exception("workload ERROR (failure): Output: \n" + res)

        retVal += ' result=SUCCESS'

    except Exception, e:
        retVal = 'result=FAILURE error=%s' % str(e)
    finally:
        # delete the created VM
        if ID != None:
            stop_node(ID, conf, silent=True)

    print retVal


# Example usage (cases with working options):
#  pmwrapper device=/dev/mapper/uuid [memsize=1024] [fstype=xfs] [mintimesec=60] [misc postmark-related options]
#
#  pmwrapper device=/dev/mapper/uuid memsize=1024 pm_sizes=600:1200000 pm_read=32768 pm_write=32768 pm_number=600 pm_biases=6:3 pm_transactions=800 pm_subdirs=25
#
# Example of all postmark options passed in args:
# pm_sizes=500:1100000 pm_read=65536 pm_write=65536 pm_number=300 pm_biases=7:4 pm_transactions=1000 pm_subdirs=20 mintimesec=60
#
def postmark(args):
    retVal = 'result=SUCCESS '
    try:
        ID = None
        device = None
        fsbase = "/mnt"
        fstype = "xfs"
        memsize = 1024
        mintimesec = 0

        # default postmark options
        pm_size = "4000 8100000"
        pm_read = 65536
        pm_write = 65536
        pm_number = 300
        pm_bias_read = 7
        pm_bias_create = 4
        pm_transactions = 500
        pm_subdirectories = 10

        for arg in args:
            if arg.startswith('device='):
                device = arg.split('=')[1]
                continue
            if arg.startswith('fstype='):
                fstype = arg.split('=')[1]
                continue
            if arg.startswith('memsize='):
                msize = int(arg.split('=')[1])
                if msize >= 128 and msize <= 16384:
                    memsize = msize
                continue
            if arg.startswith('mintimesec='):
                timesec = int(arg.split('=')[1])
                if timesec >= 10 and timesec <= 1800:
                    mintimesec = timesec
                continue

            if arg.startswith('pm_sizes='):
                pm_size = arg.split('=')[1].replace(':', ' ')
                continue
            if arg.startswith('pm_read='):
                pm_read = int(arg.split('=')[1])
                continue
            if arg.startswith('pm_write='):
                pm_write = int(arg.split('=')[1])
                continue
            if arg.startswith('pm_number='):
                pm_number = int(arg.split('=')[1])
                continue
            if arg.startswith('pm_biases='):
                pm_biases = arg.split('=')[1].replace(':', ' ')
                pm_bias_read = int(pm_biases.split()[0])
                pm_bias_create = int(pm_biases.split()[1])
                continue
            if arg.startswith('pm_transactions='):
                pm_transactions = int(arg.split('=')[1])
                continue
            if arg.startswith('pm_subdirs='):
                pm_subdirectories = int(arg.split('=')[1])
                continue

            if arg.strip() in ['-h', '--help']:
                print "Usage: pmwrapper device=DEV [memsize=VM_RAM_MB] [fstype=FS] [misc postmark options]"
                return

        if not device:
            raise Exception("ERROR: No device provided!\n")
        elif not os.path.exists(device):
            raise Exception('ERROR: device %s does not exist!' % device)
        elif fstype not in ["ext2", "xfs"]: # currently mkfs only for ext2, xfs
            raise Exception("Invalid fstype argument %s (needed for mkfs)." % fstype)

        if fstype == "xfs":
            mkfsbin = "/onappstore/sbin/mkfs.xfs -f "
        elif fstype == "ext2":
            mkfsbin = "/sbin/mkfs.ext2"
        else:
            mkfsbin = "mkfs." + fstype

        # then create a VM with the device attached
        (ip,conf) = start_node([device], memsize, '/onappstore/testinitrd.img')
        ID = os.path.basename(device)

        if diskutil._isXenHV():
            dev = "/dev/xvda"
        elif diskutil._isKVMHV():
            dev = "/dev/vda"
        else:
            raise Exception("HV type not recognised")

        #print cmd_exec("ls -la %s" % dev,ip)

        pmkfs = cmd_exec("%s %s" % (mkfsbin,dev),ip)
        #print pmkfs
        cmd_exec("sync",ip)
        cmd_exec("mkdir -p %s" % fsbase,ip)

        cmd_exec("/bin/mount %s %s" % (dev,fsbase),ip)
        mounted = cmd_exec("/bin/mount | grep %s" % dev,ip)
        if len(mounted) < 8 or mounted.find(dev) < 0:
            raise Exception('ERROR: failed to mount %s at %s !' % (dev,fsbase))

        pm_location = "%s/pm_dir" % fsbase
        cmd_exec("mkdir -p %s ; sync" % pm_location,ip)
        #print "Testdir: :", cmd_exec("ls -la %s" % pm_location,ip)

        cfgfile="/tmp/pmconfig"
        cmd_exec("rm -f %s ; touch %s" % (cfgfile,cfgfile),ip)
        #print "cfgfile: ", cmd_exec("ls -la %s" % cfgfile,ip)

        cmd_exec("echo \"set size %s\" >> %s" % (pm_size,cfgfile),ip)
        cmd_exec("echo \"set read %d\" >> %s" % (pm_read,cfgfile),ip)
        cmd_exec("echo \"set write %d\" >> %s" % (pm_write,cfgfile),ip)
        cmd_exec("echo \"set number %d\" >> %s" % (pm_number,cfgfile),ip)
        cmd_exec("echo \"set bias read %d\" >> %s" % (pm_bias_read,cfgfile),ip)
        cmd_exec("echo \"set bias create %d\" >> %s" % (pm_bias_create,cfgfile),ip)
        cmd_exec("echo \"set transactions %d\" >> %s" % (pm_transactions,cfgfile),ip)
        cmd_exec("echo \"set location %s\" >> %s" % (pm_location,cfgfile),ip)
        cmd_exec("echo \"set subdirectories %d\" >> %s" % (pm_subdirectories,cfgfile),ip)

        cmd_exec("echo \"set buffering false\" >> %s" % cfgfile,ip)
        cmd_exec("echo \"set report verbose\" >> %s" % cfgfile,ip)
        cmd_exec("echo \"show\" >> %s" % cfgfile,ip)
        cmd_exec("echo \"run\" >> %s" % cfgfile,ip)
        cmd_exec("echo \"quit\" >> %s" % cfgfile,ip)

        #print "Config: ", cmd_exec("cat %s" % cfgfile,ip)

        retVal += "pm_size=%s" % pm_size.replace(' ', '_')
        retVal += " pm_read=%d" % pm_read
        retVal += " pm_write=%d" % pm_write
        retVal += " pm_number=%d" % pm_number
        retVal += " pm_biases=%d:%d" % (pm_bias_read,pm_bias_create)
        retVal += " pm_transactions=%d" % pm_transactions
        retVal += " pm_subdirs=%s" % pm_subdirectories

        if mintimesec < 10:
            res = str(cmd_exec("run_postmark < %s" % cfgfile, ip)).strip()
            #print "res = %s" % res
        else:
            # Do multiple runs of postmark (using same config file)
            # until timeout expires...
            retVal += " mintimesec=%d" % mintimesec
            starttime = int(time.time())
            endtime = starttime + 1
            rep = 0
            while endtime <= starttime + mintimesec:
                rep += 1
                res = str(cmd_exec("run_postmark < %s" % cfgfile, ip)).strip()
                #print "[%s] res = %s" % (rep,res)
                cmd_exec("sync",ip)
                time.sleep(3)
                endtime = int(time.time())

            retVal += " reps=%s total_time=%s" % (rep,str(endtime-starttime))

        # process postmark internal error messages
        lres = res.lower()
        if lres.find('seconds of transactions') == -1 or lres.find('segmentation') != -1 or lres.find('error') != -1:
            raise Exception("postmark ERROR (failure): Output: \n" + res)

        pmres_xactions = [ l.strip() for l in lres.split('\r\n') if l.strip().find('seconds of transactions') != -1 ]
        pmres_reads = [ l.strip() for l in lres.split('\r\n') if l.strip().find('megabytes read') != -1 ]
        pmres_writes = [ l.strip() for l in lres.split('\r\n') if l.strip().find('megabytes written') != -1 ]

        if len(pmres_xactions) != 1 or len(pmres_reads) != 1 or len(pmres_writes) != 1:
            #dmesg_output = str(cmd_exec("dmesg", ip)).strip()
            raise Exception("Error parsing postmark output: %s" % res)

        res_tottime = [ l.strip() for l in lres.split('\r\n') if l.strip().find('seconds total') != -1 ]
        res_total_time = int(res_tottime[0].strip().split()[0])

        res_xact = pmres_xactions[0].strip().replace('(', ' ').split()
        res_xact_time = int(res_xact[0])
        res_xact_rate = int(res_xact[4])
        retVal += " res_total_time=%d res_xact_time=%d res_xact_rate=%d" % (res_total_time,res_xact_time,res_xact_rate)

        res_rd = pmres_reads[0].strip().replace('(', ' ').split()
        res_read_mb = float(res_rd[0])
        res_read_mbps = float(res_rd[3])
        retVal += " res_read_mb=%.2f res_read_mbps=%.2f" % (res_read_mb,res_read_mbps)

        res_wr = pmres_writes[0].strip().replace('(', ' ').split()
        res_write_mb = float(res_wr[0])
        res_write_mbps = float(res_wr[3])
        retVal += " res_write_mb=%.2f res_write_mbps=%.2f" % (res_write_mb,res_write_mbps)

    except Exception, e:
        retVal = 'result=FAILURE error=%s' % str(e)
    finally:
        # delete the created VM
        if ID != None:
            stop_node(ID, conf, silent=True)

    print retVal


def iperf(args):
    # FIXME
    retVal = 'result=FAILURE error=\'Not implemeted yet\''
    print retVal


def nttcp(args):
    # FIXME
    retVal = 'result=FAILURE error=\'Not implemeted yet\''
    print retVal


# Example of usage and options (dt options passed through after device=X):
#
#  dtwrapper device=/dev/mapper/uuid bs=8M pattern=iot flags=fsync
#
def dt(args):
    retVal = 'result=SUCCESS '
    try:
        strargs = ''
        ID = None
        for arg in args:
            if arg.find('device') != -1:
                device = arg.split('=')[1]
                continue
            strargs += arg
            strargs += ' '
        strargs = strargs.strip()

        # then create a VM with the device attached
        (ip, conf) = start_node([device], _getMemSizeFromParams(args), '/onappstore/testinitrd.img')
        ID = os.path.basename(device)

        if diskutil._isXenHV():
            dev = "/dev/xvda"
        elif diskutil._isKVMHV():
            dev = "/dev/vda"
        else:
            print "HV not recognised"
            return

        res = str(cmd_exec("/bin/dt of=%s %s" % (dev,strargs), ip)).strip()
        # make sure dt is installed
        if res.find('/bin/dt: not found') != -1:
            raise Exception("Data integrity toolkit not installed.")
        if res.find('Input/output error') != -1:
            dmesg_output = str(cmd_exec("dmesg", ip)).strip()
            retVal += 'dmesg_write=%s' % dmesg_output
            raise Exception("dt failed with I/O error. Output: %s, Dmesg output: %s" % (res, dmesg_output))

        for line in res.split('\n'):
            if line.find("Total errors detected") != -1:
                error_ratio = line.strip().split(':')[1].strip().split('/')
                if error_ratio[0] != '0':
                    raise Exception(res)
        retVal += 'results=%s ' % res

    except Exception, e:
        retVal = 'result=FAILURE error=%s' % str(e)
    finally:
        # delete the created VM
        if ID != None:
            stop_node(ID, conf, silent=True)

    print retVal

def tar(args):
    mntpoint = args[-1]
    dev = findMountDev(mntpoint)

    if diskutil._isXenHV():
        vdev = "/dev/xvda"
    elif diskutil._isKVMHV():
        vdev = "/dev/vda"
    else:
        print "HV not recognised"
        return

    #unmount local target
    util.runCommand(['umount',mntpoint])
    (ip, conf) = start_node([dev], _getMemSizeFromParams(args))
    cmd_exec("mount %s /DB" % vdev, ip)
    cmd_exec("cd /DB && nc -l -p 7000 | tar zx &", ip)
    sock = openSock(ip, 7000)
    netcat(sock)
    cmd_exec("sync", ip)
    cmd_exec("umount /DB", ip)
    ID =  os.path.basename(dev)
    stop_node(ID, conf)
    util.runCommand(['mount', dev, mntpoint])

def fio(args):
    try:
        bs = '4K'
        iodepth = '32'
        for arg in args:
            if arg.find('device') != -1:
                device = arg.split('=')[1]
                continue
            if arg.find('bs') != -1:
                bs = arg.split('=')[1]
                continue
            if arg.find('iodepth') != -1:
                iodepth = arg.split('=')[1]
                continue
        (ip, conf) = start_node([device], _getMemSizeFromParams(args), '/onappstore/testinitrd.img')
        ID = os.path.basename(device)

        if diskutil._isXenHV():
            vdev = "/dev/xvda"
        elif diskutil._isKVMHV():
            vdev = "/dev/vda"
        else:
            print "HV not recognised"
            return

        #unmount local target
        cmd_exec("cd /tmp && nc -l -p 7001 > fio.ini",ip)
        sock = openSock(ip,7001)
        fio_config = """
[readtest]
blocksize=%s
filename=%s
rw=read
direct=1
buffered=0
ioengine=libaio
iodepth=%s
[writetest]
blocksize=%s
filename=%s
rw=write
direct=1
buffered=0
ioengine=libaio
iodepth=%s
    """ % (bs, vdev, iodepth, bs, vdev, iodepth)
        sock.sendall(fio_config)
        sock.close()
        res = str(cmd_exec("/usr/bin/fio /tmp/fio.ini",ip)).strip()
        ID =  os.path.basename(vdev)
        stop_node(ID, conf)
        # make sure dt is installed
        retVal = ""
        if res.find(': not found') != -1:
            raise Exception("FIO toolkit not installed.")
        if res.find('Input/output error') != -1:
            dmesg_output = str(cmd_exec("dmesg", ip)).strip()
            retVal += 'dmesg_write=%s' % dmesg_output
            raise Exception("fio failed with I/O error. Output: %s, Dmesg output: %s" % (res, dmesg_output))
        for line in res.split('\n'):
            if line.find("Total errors detected") != -1:
                error_ratio = line.strip().split(':')[1].strip().split('/')
                if error_ratio[0] != '0':
                    raise Exception(res)
        retVal += 'results=%s ' % res
    except Exception, e:
        retVal = 'result=FAILURE error=%s' % str(e)
    finally:
        # delete the created VM
        if ID != None:
            stop_node(ID, conf, silent=True)

    print retVal

def testwrapper(args):
    retVal = 'FAILURE'
    try:
        if args[0].endswith(".sh"):
            args = ' '.join(args)
            process=Popen(args, shell=True, stdout = PIPE, stderr=STDOUT)
        else:
            process=Popen(args, shell=False, stdout = PIPE, stderr=STDOUT)
        out, err = process.communicate()
        if process.returncode != 0:
            print 'result=%s error=%s' % (retVal, out)
            return
    except Exception, e:
        print 'result=%s error=%s' % (retVal, str(e))
        return

    print 'result=SUCCESS out=%s' % out

def writepattern(args):
    params = util.parseNameValuePairs(args)
    if not params.has_key('pattern') and not params.has_key('offsets') and not params.has_key('device'):
        return 'result=FAILURE error=Missing parameters'
    print _writepattern(params['device'], params['offsets'].split(','), params['pattern'])

def verifypattern(args):
    params = util.parseNameValuePairs(args)
    if not params.has_key('pattern') and not params.has_key('offsets') and not params.has_key('device'):
        return 'result=FAILURE error=Missing parameters'
    print _verifypattern(params['device'], params['offsets'].split(','), params['pattern'])

def _writepattern(device, offsets, pattern):
    fd = os.open(device, os.O_RDWR | os.O_SYNC)
    for offset in offsets:
        os.lseek(fd, int(offset), 0)
        os.write(fd, pattern)
    os.close(fd)
    return "result=SUCCESS"

def _verifypattern(device, offsets, pattern):
    fd = os.open(device, os.O_RDONLY)
    for offset in offsets:
        os.lseek(fd, int(offset), 0)
        readpattern = os.read(fd, len(pattern))
        if readpattern != pattern:
            return "result=FAILURE error=Mismatching pattern %s at offset %d" % (readpattern, int(offset))
    return "result=SUCCESS"


def _getMemSizeFromParams(params):
    '''default RAM Size is 1 Gb'''
    memorySize = 1024
    try:
        for argument in params:
            if argument.startswith('memsize='):
                msize = int(argument.split('=')[1])
                if msize >= 128 and msize <= 16384:
                    memorySize = msize
                else:
                    rpclog("Failed to set ram size to %s because (128 <= size <= 16384) set to 1024", str(msize))
    except Exception, e:
        rpclog("Failed to get ramsize from parameters exception %s", str())

    return memorySize


rpclog(sys.argv)

cmd = os.path.basename(sys.argv[0])
if cmd == "rpc":
    cmd = os.path.basename(os.environ["_"])

rpclog("Handling command %s" % cmd)
if cmd == "mkfs.ext3" or cmd == "mkfs.ext2":
    mkfs(sys.argv[1:])
elif cmd == "tar":
    tar(sys.argv[1:])
elif cmd == "dccwrapper":
    dcc(sys.argv[1:])
elif cmd == "ddwrapper":
    dd(sys.argv[1:])
elif cmd == "dtwrapper":
    dt(sys.argv[1:])
elif cmd == "fiowrapper":
    fio(sys.argv[1:])
elif cmd == "xddwrapper":
    xdd(sys.argv[1:])
elif cmd == "iozwrapper":
    iozone(sys.argv[1:])
elif cmd == "bonniewrapper":
    bonnie(sys.argv[1:])
elif cmd == "pmwrapper":
    postmark(sys.argv[1:])
elif cmd == "stressngwrapper":
    stressng(sys.argv[1:])
elif cmd == "workloadwrapper":
    runworkload(sys.argv[1:])
elif cmd == "iperfwrapper":
    iperf(sys.argv[1:])
elif cmd == "nttcpwrapper":
    nttcp(sys.argv[1:])
elif cmd == "testwrapper":
    testwrapper(sys.argv[1:])
elif cmd == "launchHelper":
    launchHelper(sys.argv[1:])
elif cmd == "launchFakeSN":
    launchFakeSN(sys.argv[1:])
elif cmd == "replicationHelper":
    replicationHelper(sys.argv[1:])
elif cmd == "killHelper":
    killHelper(sys.argv[1:])
elif cmd == "writepattern":
    writepattern(sys.argv[1:])
elif cmd == "verifypattern":
    verifypattern(sys.argv[1:])
elif cmd == "scrun":
    storageControllerRun(sys.argv[1:])
elif cmd == "scrunall":
    storageControllerRunAll(sys.argv[1:])
