#!/usr/pythoncontroller/python3
import sys
import re
import os
import time
import subprocess
from threading import Thread, current_thread
from queue import Queue
from signal import SIGTERM

KVM_KERNEL_BASE = "2.6.32"
MIN_KVM_VERSION = 279

XEN3_KERNEL_BASE = "2.6.18"
MIN_XEN3_VERSION = 308

XEN4_KERNEL_BASE = "3.18.25"
MIN_XEN4_VERSION = 18

SSH_RETRY_MAX = 5

HOSTS = []

def usage():
    print("Usage:")
    print("\tliveUpdate listHVs [-nc]")
    print("\tliveUpdate updateToolstack <HV IP Addr> force")
    print("\nNext commands are dangerous and may cause data loss or disks degradation, please use with caution:")
    print("\tliveUpdate restartControllers <HV IP Addr> force <timeout_in_secs> <Controller ID>")
    print("\tliveUpdate refreshControllers <HV IP Addr>")
    print("\tliveUpdate updateDrivers <HV IP Addr>")
    print("\tliveUpdate liveRestartBackends <HV IP Addr>")
    sys.exit(0)

def getFormat(message):
    return time.strftime('%d-%m-%Y-%H:%M:%S ', time.gmtime()) + ': %s : ' % current_thread().name + message

def testPID(pid):
    try:
        os.kill(pid,0)
    except:
        return False
    return True

def filetransfer(src,dst,ip):
    cmdList = ["scp","-r",src,"%s:%s" % (ip,dst)]
    process = subprocess.Popen(cmdList, shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    out,err = process.communicate()

    if process.returncode != 0:
        print("Filetransfer failed! (%s,%s)" % (out,err))
        return False

    return True

def remoteExec(cmd, ip, notCheckHost=False, shell=False, verbouse=False):
    if shell == True:
        cmd_list = [f"ssh {'-o StrictHostKeyChecking no' if notCheckHost == True else ''} {ip} " + " ".join(cmd)]
    else:
        cmd_list = ["ssh"]
        if (notCheckHost == True):
            cmd_list.append("-o StrictHostKeyChecking no")

        cmd_list.append(ip)
        cmd_list.extend(cmd)
    
    if verbouse:
        print(f"shell: {str(shell)}\ncmd:{str(cmd_list)}\n")

    retries = SSH_RETRY_MAX
    retry = 0
    ret = -11
    while ret==-11 and retry < retries:
        process = subprocess.Popen(cmd_list, shell=shell,stdout=subprocess.PIPE,stderr=subprocess.PIPE, encoding='utf-8')
        out,err = process.communicate()
        ret = process.returncode
        if ret != -11:
            break
        else:
            print("remoteExec - retrying command %s, attempt: %d" % (cmd, retry))
        retry += 1
    
    if verbouse:
        print(f"result:{str({ 'return_code':ret, 'out':str(out), 'err':str(err), 'retry':retry })}")

    return { 'cmd':str(cmd), 'return_code':ret, 'out':str(out), 'err':str(err) }

def localExec(cmd, shell=False, verbouse=False):
    err = None
    out = None
    return_code = -1
    try:
        if verbouse:
            print(f"shell: {str(shell)}\ncmd:{str(cmd)}\n")

        process = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8')
        out, err = process.communicate()
        return_code = process.returncode

    except EnvironmentError as e:
        print("Failed to execute command %s exception: %s" % (str(cmd), str(e)))
        err = str(e)

    if verbouse:
        print(f"result:{str({ 'return_code':return_code, 'out':str(out), 'err':str(err) })}")

    return { 'cmd':str(cmd), 'return_code':return_code, 'out':str(out), 'err':str(err) }


def getHostid(ip):
    cmd = ['cat','/.rw/onappstore.conf']
    out = remoteExec(cmd,ip)
    lines = out['out'].split('\n')
    for line in lines:
        if re.match('^hostid',line):
            return line.split('=')[1]

def _isAlive(ip, notCheckHost = False):
    cmd = ['true']
    out = remoteExec(cmd, ip, notCheckHost)
    if out['return_code'] != 0:
        return False
    return True

def _isCentos6(ip):
    cmd = ['cat','/etc/redhat-release']
    out = remoteExec(cmd,ip)
    v = out['out'].split()[2]
    if re.match('^6',v):
        return True
    return False

def _isCentos7(ip):
    cmd = ['cat','/etc/redhat-release']
    out = remoteExec(cmd,ip)
    v = out['out'].split()[3]
    if re.match('^7',v):
        return True
    return False

def _needsToolstackRestart(ip):
    cmd = ['diff','-q','/onappstore/package-version.txt','/onappstore/toolstackversion.txt']
    out = remoteExec(cmd,ip)
    if out['return_code'] != 0:
        return True
    return False

def _needsControllerRestart(ip):
    cmd = ['diff','-q','/onappstore/package-version.txt','/onappstore/controllerversion.txt']
    out = remoteExec(cmd,ip)
    if out['return_code'] != 0:
        return True
    return False

def _isXenHV(ip):
    cmd = ['xm','info']
    out = remoteExec(cmd,ip)
    if out['return_code'] != 0:
        return False
    return True

# returns 3 or 4
def findXenVersion(ip):
    cmd = ['xm', 'info']
    out = remoteExec(cmd, ip)
    for line in out['out'].strip().split('\n'):
        if line.startswith('xen_major'):
            line = re.sub(r'\s+', '', line)
            return int(line.split(':')[1])

def _isKVMHV(ip):
    cmd = ['lsmod']
    out = remoteExec(cmd,ip)

    for line in out['out'].split('\n'):
        if re.match('^kvm',line):
            return True
    return False

def getHVIPs():
    if len(HOSTS):
        return HOSTS

    # first try with the new dhcp conf file path
    path = "/onapp/configuration/dhcp/dhcpd.conf"
    if not os.path.exists(path):
        # if not present, set to the pre-4.0 path
        path = "/home/onapp/dhcpd.conf"
    fd = open(path,'r')
    iplist = []
    for l in fd.readlines():
        if re.match("^fixed-address",l.strip()):
            iplist.append(l.strip()[:-1].split()[1])
    fd.close()
    return iplist

def parse_output(out):
    if out['return_code'] == 0:
        return "SUCCESS"
    
    return f"FAIL {out['out'] if len(out['out']) > 0 else ''} {out['err'] if len(out['err']) > 0 else ''}".strip()

def runPreScript(hvip):
    print(getFormat("Running the pre-script on HV: %s" % hvip))
    out = remoteExec(['/tmp/pre-%s.sh' % txn_id], hvip)
    if out['return_code'] != 0:
        print(getFormat("Failed to run pre-script on HV: %s. Output: %s Error: %s" % (hvip, out['out'], out['err'])))
        sys.exit(1)

def runPostScript(hvip):
    print(getFormat("Running the post-script on HV: %s" % hvip))
    out = remoteExec(['/tmp/post-%s.sh' % txn_id], hvip)
    if out['return_code'] != 0:
        print(getFormat("Failed to run post-script on HV: %s. Output: %s Error: %s" % (hvip, out['out'], out['err'])))
        sys.exit(1)

class runFnInthread(Thread):
    def __init__(self, queue, fn, *args):
        self.queue = queue
        self.fn = fn
        self.args = args
        Thread.__init__(self)
    def run(self):
        self.queue.put(self.fn(*self.args))

def waitForThreadsToFinish(threads, queues):
    results = []
    allover = False
    while not allover:
        allover = True
        for t,q in zip(threads, queues):
            if t.is_alive():
                allover = False
                break
            else:
                result = q.get()
                results.append(result)
                threads.remove(t)
                queues.remove(q)
                if len(threads):
                    allover = False
                break
        time.sleep(0.1)
    return results

class monitorPidInThreadAndKillOnTimeout(Thread):
    def __init__(self, pid, timeout):
        self.pid = pid
        self.timeout = int(timeout)
        Thread.__init__(self)
    def run(self):
        timeelapsed = 0
        while timeelapsed < self.timeout:
            if testPID(self.pid):
                if not timeelapsed % 10 and timeelapsed > 0:
                    print(getFormat("Pid %d still active, time elapsed %s, will timeout at %d seconds." % (int(self.pid), int(timeelapsed), self.timeout)))
                time.sleep(1)
                timeelapsed += 1
            else:
                return

        if timeelapsed == self.timeout:
            print(getFormat("Pid %d still active, time elapsed %s, timing it out." % (int(self.pid), int(timeelapsed))))
            os.kill(self.pid,SIGTERM)

def reconfigureSNMPD(ip):
    CP_IP = None
    HV_IP = None
    out = remoteExec(['cat','/etc/onapp.conf'], ip)
    try:
        for line in out['out'].splitlines():
            if line.startswith('HOST='):
                CP_IP = line.replace('"', '').split('HOST=')[1]

            if line.startswith('SERVER='):
                HV_IP = line.replace('"', '').split('SERVER=')[1]
    except Exception as e:
        print(f"failed to extract data from config /etc/onapp.conf {str(e)}")

    if HV_IP is None:
        # old version try to extract management IP address
        out = remoteExec(["ls -d /var/run/dhclient\-*\.pid 2>/dev/null | head -1 | cut -d '-' -f 2 | cut -d '.' -f 1"], ip, shell=True)
        try:
            interface = out['out'].replace('\n', '')
            out = remoteExec([f"/sbin/ip addr show {interface} scope global primary | grep -Ex '[[:blank:]]+inet.+scope global .* {interface}' | sed 's/^.*inet\ //;s/[\/\ ].*$//'"], ip, shell=True)
            HV_IP = out['out'].replace('\n', '')
        except Exception as e:
            print(f"failed to get management interface IP")

    if HV_IP is not None:
        remoteExec(['sed', '-i', '-e', f'"s/\<IP_OF_HV\>/{HV_IP}/g"', '/etc/snmp/snmpd.conf'], ip)
        remoteExec(['sed', '-i', '-e', f'"s/\<IP_OF_HV\>/{HV_IP}/g"', '/etc/snmp/snmptrapd.conf'], ip)
    else:
        print("Failed to configure snmp with Hypervisor IP")

    if CP_IP is not None:
        remoteExec(['sed', '-i', '-e', f'"s/\<IP_OF_CP\>/{CP_IP}/g"', '/etc/snmp/snmpd.conf'], ip)
        remoteExec(['sed', '-i', '-e', f'"s/\<IP_OF_CP\>/{CP_IP}/g"', '/etc/snmp/snmptrapd.conf'], ip)
    else:
        print("Failed to configure snmp with Controll Pannel IP")

    remoteExec(['service','snmptrapd','stop'], ip)
    remoteExec(['service','snmpd','stop'], ip)
    remoteExec(['service','snmpd','start'], ip)
    remoteExec(['service','snmptrapd','start'], ip)

def refreshControllers(ip):
    try:
        backends = list()
        out = remoteExec(['onappstore', 'getid'], ip)
        for option in out['out'].split():
            if option.startswith('backends='):
                backends = option.split('backends=')[1].split(',')
        
        for backend in backends:
            remoteExec([f"'cat /liveupdate-storagenode.tgz | ssh -o StrictHostKeyChecking=no {backend} tar -C / -zxf -'"], ip, shell=True)
            remoteExec(['ssh', '-o', 'StrictHostKeyChecking=no', backend, '/usr/pythoncontroller/refreshcontroller'], ip)

    except Exception as e:
        return f"Failed to refresh controller: {str(e)}"
    
    return "SUCCESS"

if len(sys.argv)<2:
    usage()

if sys.argv[1] == "listHVs":
    IPs = getHVIPs()
    notCheckHost = False
    if (len(sys.argv) == 3) and (sys.argv[2] == "-nc"):
        notCheckHost = True
    for ip in IPs:
        version = 5
        if not _isAlive(ip, notCheckHost):
            continue
        if _isCentos6(ip):
            version = 6
        if _isCentos7(ip):
            version = 7
        if _isKVMHV(ip):
            platform = "KVM"
            if version == 6:
                src = "/tftpboot/images/centos6/ramdisk-kvm/liveupdate.tgz"
            if version == 7:
                src = "/tftpboot/images/centos7/ramdisk-kvm/liveupdate.tgz"
        if _isXenHV(ip):
            platform = "XEN"
            xen_version = findXenVersion(ip)
            if xen_version == 3:
                src = "/tftpboot/images/centos5/ramdisk-xen/liveupdate.tgz"
            elif xen_version == 4:
                if version == 6:
                    src = "/tftpboot/images/centos6/ramdisk-xen/liveupdate.tgz"
                if version == 7:
                    src = "/tftpboot/images/centos7/ramdisk-xen/liveupdate.tgz"

        dst = "/."
        if not os.path.exists(src):
            print("Failed to find liveupdate archive. Please update cloudboot RPM on CP server.")
            sys.exit(1)
        filetransfer(src,dst,ip)

        # untar just the package version file
        remoteExec(['cd', '/',';','tar' ,'-xzf', '/liveupdate.tgz', './onappstore/package-version.txt'], ip)

        toolstack_upgradable = _needsToolstackRestart(ip)
        controller_upgradable = _needsControllerRestart(ip)
        if platform == "XEN":
            print("Node: %s\tCentOS Version: %d\tHV type: %s\tToolstack-upgradable: %s\tController-upgradable: %s\tXen Version: %d" % (ip, version, platform, toolstack_upgradable, controller_upgradable, xen_version))
        else:
            print("Node: %s\tCentOS Version: %d\tHV type: %s\tToolstack-upgradable: %s\tController-upgradable: %s" % (ip, version, platform, toolstack_upgradable, controller_upgradable))
    sys.exit(0)

if len(sys.argv)<3:
    usage()

if sys.argv[1] == "updateToolstack":
    ip = sys.argv[2]
    if not _isAlive(ip):
        print("Unable to contact host %s" % ip)
        sys.exit(1)
    version = 5
    if _isCentos6(ip):
        version = 6
    if _isCentos7(ip):
        version = 7
    if _isXenHV(ip):
        platform = "XEN"
        xen_version = findXenVersion(ip)
        if xen_version == 3:
            src = "/tftpboot/images/centos5/ramdisk-xen/liveupdate.tgz"
        elif xen_version == 4:
            if version == 6:
                src = "/tftpboot/images/centos6/ramdisk-xen/liveupdate.tgz"
            if version == 7:
                src = "/tftpboot/images/centos7/ramdisk-xen/liveupdate.tgz"
    else:
        platform = "KVM"
        if version == 6:
            src = "/tftpboot/images/centos6/ramdisk-kvm/liveupdate.tgz"
        elif version == 7:
            src = "/tftpboot/images/centos7/ramdisk-kvm/liveupdate.tgz"
    dst = "/."

    if not os.path.exists(src):
        print("Failed to find liveupdate archive. Please update cloudboot RPM on CP server.")
        sys.exit(1)

    filetransfer(src,dst,ip)
    print("STEP1 - Copied liveupdate archive onto HV")

    # untar just the package version file
    out = remoteExec(['cd', '/',';','tar' ,'-xzf', '/liveupdate.tgz', './onappstore/package-version.txt'], ip)
    print("STEP2 - Untarring toolstack version file from liveupdate tarball: %s" % (parse_output(out)))

    #Now check whether version has changed
    forcerestart = False
    if len(sys.argv) > 3:
        if sys.argv[3] == 'force':
            forcerestart = True

    restart = _needsToolstackRestart(ip)
    print("STEP3 - Check whether toolstack version has changed: %s" % str(restart))
    if not restart and not forcerestart:
        print("Exiting, no more work to do.")
        sys.exit(0)

    out = remoteExec(['service', 'crond', 'stop'], ip)
    print("STEP3.1 - Stop crond: %s" % parse_output(out))
    out = remoteExec(['tar','-zx', '--touch', '--no-overwrite-dir','-C','/', '-f', '/liveupdate.tgz'], ip)
    print("STEP4 - Untarring liveupdate archive: %s" % (parse_output(out)))

    out = remoteExec(['rm','/liveupdate.tgz'], ip)
    print("STEP5 - Removing liveupdate archive: %s" % (parse_output(out)))

    if version == 7:
        out = remoteExec(['/bin/cp', '-a', '/usr/pythoncontroller/storageAPI.service', '/usr/lib/systemd/system/storageAPI.service'], ip)
        print("STEP5.1 - Updating storageAPI.service unit file: %s" % (parse_output(out)))
        out = remoteExec(['systemctl', 'daemon-reload'], ip)
        print("STEP5.2 - Reloading systemd manager configuration: %s" % (parse_output(out)))

    out = remoteExec(['service', 'isd', 'stop'], ip)
    print("STEP6 - Stopping isd: %s" % (parse_output(out)))

    out = remoteExec(['service','storageAPI','stop'], ip)
    print("STEP7 - Stopping storageAPI: %s" % (parse_output(out)))

    print("STEP10 - Cleaning stale locks")
    remoteExec(['rm','-rf','/onappstore/DB/Datastore/*'], ip)
    remoteExec(['rm','-rf','/onappstore/DB/locks/*'], ip)
    remoteExec(['rm','-rf','/onappstore/DB/Node/*'], ip)
    remoteExec(['rm','-rf','/onappstore/DB/v2/*'], ip)
    remoteExec(['rm','-rf','/onappstore/DB/VDisk/*'], ip)

    print("STEP - Restarting SNMP services")
    reconfigureSNMPD(ip)

    mq_credentials = "/home/mq/onapp/messaging/credentials.yml"
    if os.path.exists(mq_credentials):
        print("STEP - Restarting onapp-messaging service")
        systemctl = "/usr/bin/systemctl"
        if os.path.exists(systemctl):
            remoteExec(['systemctl','daemon-reload'], ip)

        remoteExec(['service','onapp-messaging','stop'], ip)
        remoteExec(['service','onapp-messaging','start'], ip)

    out = remoteExec(['service', 'isd', 'start'], ip)
    print("STEP11 - Restarting isd: %s" % (parse_output(out)))

    remoteExec(['service', 'crond', 'start'], ip)
    remoteExec(['rm','-f','/onappstore/toolstackversion.txt'], ip)
    out = remoteExec(['cp','/onappstore/package-version.txt','/onappstore/toolstackversion.txt'], ip)
    print("STEP12 - Recording runtime toolstack version: %s" % (parse_output(out)))

    remoteExec(['/usr/pythoncontroller/convertVMConfigtoHotplug'], ip)
    print("STEP13 - Convert storage controllers to enable hotplug")

    time.sleep(2)

    print("STEP14 - wait for 3 minutes while Nodes DB gets re-populated")
    minute = 1
    for i in range(1,91):
        sys.stdout.write(".")
        sys.stdout.flush()
        time.sleep(2)
        if i>0 and i%30 == 0:
            print(" Minute %d complete" % minute)
            minute+=1

    # now that everything is up, start the storage API
    out = remoteExec(['service','storageAPI','start'], ip)
    print("Final step - Starting storageAPI: %s" % (parse_output(out)))

    sys.exit(0)

elif sys.argv[1] == "restartControllers":
    try:
        ip = sys.argv[2]
        if not _isAlive(ip):
            print(getFormat("Unable to contact host %s" % ip))
            sys.exit(1)
        if _isXenHV(ip):
            platform = "XEN"
        else:
            platform = "KVM"

        if not (len(sys.argv) > 3 and sys.argv[3] == 'force'):
            #Now check whether version has changed
            restart = _needsControllerRestart(ip)
            print(getFormat("Check whether controller version has changed: %s" % str(restart)))
            if not restart:
                print("Exiting, no more work to do.")
                sys.exit(0)

        timeout = 120
        if len(sys.argv) > 4:
            timeout = int(sys.argv[4])

        nodeid = None
        if len(sys.argv) > 5:
            nodeid = int(sys.argv[5])

        print(getFormat("Starting storage controllers restart process - PLEASE DO NOT INTERRUPT!!"))
        print(getFormat("1. Preparing HVs for the operation"))
        txn_id = time.strftime('%d-%m-%Y_%H-%M-%S', time.gmtime())
        host_id = getHostid(ip)
        IPs = getHVIPs()
        for hvip in IPs:
            print(getFormat("Preparing HV: %s" % hvip))
            out = localExec(['scp', '/usr/local/bin/prepareUpgradeScripts.sh', 'root@%s:/tmp/.' % hvip])
            if out['return_code']:
                print(getFormat("Failed to prepare HV: %s. Output: %s Error: %s" % (hvip, out['out'], out['err'])))
                sys.exit(1)

            out = remoteExec(['chmod 755 /tmp/prepareUpgradeScripts.sh'], hvip)
            if out['return_code']:
                print(getFormat("Failed to set permissions for prepare script on HV: %s. Output: %s Error: %s" % (hvip, out['out'], out['err'])))
                sys.exit(1)
            if nodeid != None:
                cmd = ['/tmp/prepareUpgradeScripts.sh', txn_id, str(host_id), str(nodeid + 1)]
            else:
                cmd = ['/tmp/prepareUpgradeScripts.sh', txn_id, str(host_id)]
            out = remoteExec(cmd, hvip)
            if out['return_code']:
                print(getFormat("Failed to prepare HV: %s. Output: %s Error: %s" % (hvip, out['out'], out['err'])))
                sys.exit(1)

        try:
            print(getFormat("2. Running the pre-script on the HVs"))
            threadlist = []
            queuelist = []
            for hvip in IPs:
                # run the pre-script on each hv in parallel
                q = Queue()
                t = runFnInthread(q, runPreScript, hvip)
                t.start()
                threadlist.append(t)
                queuelist.append(q)

            # wait for the threads to finish
            waitForThreadsToFinish(threadlist, queuelist)
            # run the controller restart script on the hv in question
            print(getFormat("3. Restarting the Storagecontrollers"))
            if nodeid != None:
                cmd = ['ssh', ip, '/usr/pythoncontroller/diskhotplug', 'restartController' , str(nodeid)]
            else:
                cmd = ['ssh', ip, '/usr/pythoncontroller/controllerRestart']
            q = Queue()
            restartctrlprocess = subprocess.Popen(cmd, shell=False,stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds = False)
            pid = restartctrlprocess.pid
            t = monitorPidInThreadAndKillOnTimeout(pid, timeout)
            t.start()
            out, err = restartctrlprocess.communicate()
            print(getFormat(f"Restarted Storagecontrollers: {parse_output({'out':out, 'err':err, 'return_code':restartctrlprocess.returncode})}"))
        finally:
            print(getFormat("4. Running the post-script on the HVs"))
            # run the post-script on each hv in parallel
            threadlist = []
            queuelist = []
            for hvip in IPs:
                # run the pre-script on each hv in turn
                q = Queue()
                t = runFnInthread(q, runPostScript, hvip)
                t.start()
                threadlist.append(t)
                queuelist.append(q)

            # wait for the threads to finish
            waitForThreadsToFinish(threadlist, queuelist)
            print(getFormat("5. Add SANController status attribute"))
            remoteExec(['touch','/var/lock/subsys/SANController'], ip)

        print(getFormat("Storage controllers restart complete!!"))
    except Exception as e:
        print(f"Failed to restart controllers reason:{str(e)}")
        sys.exit(1)
    
    sys.exit(0)
elif sys.argv[1] == "refreshControllers":
    ip = sys.argv[2]
    if not _isAlive(ip):
        print("Unable to contact host %s" % ip)
        sys.exit(1)
    version = 5
    if _isCentos6(ip):
        version = 6
    if _isCentos7(ip):
        version = 7
    if _isXenHV(ip):
        platform = "XEN"
        xen_version = findXenVersion(ip)
        if xen_version == 3:
            src = "/tftpboot/images/centos5/ramdisk-xen/liveupdate-storagenode.tgz"
        elif xen_version == 4:
            if version == 6:
                src = "/tftpboot/images/centos6/ramdisk-xen/liveupdate-storagenode.tgz"
            if version == 7:
                src = "/tftpboot/images/centos7/ramdisk-xen/liveupdate-storagenode.tgz"
    else:
        platform = "KVM"
        if version == 6:
            src = "/tftpboot/images/centos6/ramdisk-kvm/liveupdate-storagenode.tgz"
        if version == 7:
            src = "/tftpboot/images/centos7/ramdisk-kvm/liveupdate-storagenode.tgz"
    dst = "/."

    print("STEP1 - Copying liveupdate storagenode archive onto HV")
    if not os.path.exists(src):
        print("Failed to find liveupdate storagenode archive. Please update cloudboot RPM on CP server.")
        sys.exit(1)
    filetransfer(src,dst,ip)
    print("      - Copied liveupdate storagenode archive onto HV")

    # now refresh the controllers using the installed script
    print(f"STEP2 - Refreshing local controllers on the HV: {ip}")
    print(f"      - Refreshed local controllers on the HV: {refreshControllers(ip)}")

    #Remove the tar archive
    print("STEP3 - Removing liveupdate storagenode archive: %s" % ip)
    out = remoteExec(['rm','/liveupdate-storagenode.tgz'], ip)
    print("      - Removed liveupdate storagenode archive: %s" % (parse_output(out)))
    sys.exit(0)

elif sys.argv[1] == "liveRestartBackends":
    try:
        ip = sys.argv[2]
        if not _isAlive(ip):
            print("Unable to contact host %s" % ip)
            sys.exit(1)

        # now live restart the backends for this HV
        # first list active vdisks on this HV
        cmd = ['ps', 'ax', '|', 'grep', 'bdevclient' ,'|', 'grep', '-v', '\"grep bdevclient\"', '|', 'awk', '\'{print $12}\'', '|', 'sort', '|', 'uniq']
        out = remoteExec(cmd,ip)
        vdisks = out['out'].strip().split('\n')
        print("Live restarting the backends on the HV: %s, active VDisks: %s" % (ip, ','.join(vdisks)))
        for vdisk in vdisks:
            print("      - Live restarting the backends for the VDisk: %s" % vdisk)
            remoteExec(['/usr/bin/restartlivevdiskbackends', vdisk], ip)
    except Exception as e:
        print(f"Failed to Live restart backends on the HV: {ip} reason: {str(e)}")
        sys.exit(1)
    
    print("Live restarted the backends on the HV: %s" % ip)
    sys.exit(0)

elif sys.argv[1] == "updateDrivers":
    ip = sys.argv[2]
    if not _isAlive(ip):
        print("Unable to contact host %s" % ip)
        sys.exit(1)
    if _isXenHV(ip):
        platform = "XEN"
    else:
        platform = "KVM"
    
    print("STEP1 - Checking there are no active nbd devices on HV...")
    out = remoteExec(['ps gaux | grep nbd | grep -v grep | wc -l'], ip, shell=True)
    if out['out'] == 0:
        print("SUCCESS")
        out = remoteExec(['/sbin/lsmod | grep nbd | wc -l'], ip, shell=True)
        if int(out['out']) > 0:
            print(f"STEP2 - Unloading nbd module: {parse_output(remoteExec(['modprobe','-r','nbd'], ip))}")
        
        print(f"STEP3 - Reloading nbd module: {parse_output(remoteExec(['modprobe','nbd','nbds_max=4096'], ip))}")
    else:
        print("FAILURE please rebalance all online vdisk parts from HV. Or offline all disks")

    print("STEP4 - Checking there are no active device mapper devices on HV...")
    out = remoteExec(['ls','-1','/dev/mapper/*'], ip)
    try:
        if len(out['out'][:-1].splitlines()) <= 1:
            out = remoteExec(['/sbin/lsmod | grep dm_mirror_sync | wc -l'], ip, shell=True)
            if int(out['out']) > 0:
                print(f"STEP5 - Unloading dm_mirror_sync module: {parse_output(remoteExec(['modprobe','-r','dm_mirror_sync'], ip))}")

            print(f"STEP6 - Reloading dm_mirror_sync module: {parse_output(remoteExec(['modprobe','dm_mirror_sync'], ip))}")
        else:
            raise Exception("Please migrate all active VMs off this HV before re-running the updateDrivers utility. And stop SANController service")
    except Exception as e:
        print(f"FAIILURE device mapper devices present {str(e)}")
        sys.exit(1)

    sys.exit(0)

else:
    usage()
