<feature>[qga]: QGA Improvements IPv6 and Expanded OS Support

1.Add IPv6 vm config support
2.Read the IPv6 address from the VM.
3.Support additional guest operating systems, such as Centos6, Ubuntu16, debian, etc.
4.More bugfix

Resolves: ZSTAC-56243

Change-Id: I6868776c277a28fd38ad45eb9014d64a3931b6d0

Author: bustezero

kvmagent/kvmagent/plugins/qga_zwatch.py

@@ -30,6 +30,7 @@ class ZWatchMetricMonitor(kvmagent.KvmAgent):
     CONFIG_ZWATCH_METRIC_MONITOR = "/host/zwatchMetricMonitor/config"
 
     ZWATCH_RESTART_CMD = "/bin/systemctl restart zwatch-vm-agent.service"
+    ZWATCH_RESTART_CMD_EL6 = "service zwatch-vm-agent restart"
     ZWATCH_VM_INFO_PATH = "/var/log/zstack/vm.info"
     ZWATCH_VM_METRIC_PATH = "/var/log/zstack/vm_metrics.prom"
     ZWATCH_GET_NIC_INFO_PATH = "/usr/local/zstack/zs-tools/nic_info_linux.sh"
@@ -115,7 +116,10 @@ def qga_get_vm_nic(self, uuid, qga):
             nicInfoStatus = qga.guest_file_is_exist(zwatch_nic_info_path)
             if not nicInfoStatus:
                 return
-            nicInfo = qga.guest_exec_cmd_no_exitcode(zwatch_nic_info_path)
+            if is_windows_2008(qga):
+                nicInfo = get_nic_info_for_windows_2008(uuid, qga)
+            else:
+                nicInfo = qga.guest_exec_cmd_no_exitcode(zwatch_nic_info_path)
             nicInfo = str(nicInfo).strip()
             need_update = False
             if not self.vm_nic_info.get(uuid):
@@ -144,6 +148,9 @@ def zwatch_qga_monitor_vm(self, uuid, qga):
                     zwatch_vm_info_path = self.ZWATCH_VM_INFO_PATH
                     zwatch_vm_metric_path = self.ZWATCH_VM_METRIC_PATH
                     zwatch_restart_cmd = self.ZWATCH_RESTART_CMD
+                    # centos version 6.x need special cmd
+                    if qga.os_version == '6':
+                        zwatch_restart_cmd = self.ZWATCH_RESTART_CMD_EL6
                 dhcpStatus = not qga.guest_file_is_exist(zwatch_vm_info_path)
                 _, qgaZWatch = qga.guest_file_read(zwatch_vm_info_path)
                 # skip when dhcp enable
@@ -309,3 +316,50 @@ def push_metrics_to_gateway(url, uuid, metrics):
     }
     rsp = http.json_post(url, body=metrics, headers=headers)
     logger.debug('vm[%s] push metric with rsp[%s]' % (uuid, rsp))
+
+
+def is_windows_2008(qga):
+    return qga.os and 'mswindows' in qga.os and '2008r2' in qga.os_version
+
+
+def subnet_mask_to_prefix_length(mask):
+    return sum(bin(int(x)).count('1') for x in mask.split('.'))
+
+
+def get_nic_info_for_windows_2008(uuid, qga):
+    exitcode, ret_data = qga.guest_exec_wmic(
+        "nicconfig where IPEnabled=True get InterfaceIndex, IPaddress, IPSubnet, MACAddress /FORMAT:csv")
+    if exitcode != 0:
+        logger.debug('vm[%s] get nic info failed: %s' % (uuid, ret_data))
+        return None
+
+    lines = ret_data.replace('\r', '').strip().split('\n')
+    mac_to_ip = {}
+    for line in lines:
+        logger.debug('vm[%s] get nic info line: [%s]' % (uuid, line))
+        columns = line.split(',')
+        if len(columns) < 5:
+            logger.debug('vm[%s] skipping line: [%s]' % (uuid, line))
+            continue
+        else:
+            raw_ip_addresses = columns[2].strip('{}').split(';')
+            raw_ip_subnets = columns[3].strip('{}').split(';')
+            mac_address = columns[4].strip().lower()
+
+            if not len(mac_address.split(':')) == 6:
+                continue
+
+            ip_addresses_with_subnets = []
+            for ip, subnet in zip(raw_ip_addresses, raw_ip_subnets):
+                if '.' in subnet:  # Check if this is an IPv4 subnet mask
+                    prefix_length = subnet_mask_to_prefix_length(subnet)
+                    ip_addresses_with_subnets.append("{}/{}".format(ip, prefix_length))
+                else:  # Assume this is an IPv6 subnet in prefix length format
+                    ip_addresses_with_subnets.append("{}/{}".format(ip, subnet))
+
+            mac_to_ip[mac_address] = ip_addresses_with_subnets
+
+    mac_to_ip_json = json.dumps(mac_to_ip, indent=4)
+    logger.debug('vm[%s] get nic info all: [%s]' % (uuid, mac_to_ip_json))
+    return mac_to_ip_json
+

kvmagent/kvmagent/plugins/vm_config.py

@@ -142,17 +142,20 @@ class VmConfigPlugin(kvmagent.KvmAgent):
     VM_QGA_PARAM_FILE = "/usr/local/zstack/zs-nics.json"
     VM_QGA_CONFIG_LINUX_CMD = "/usr/local/zstack/zs-tools/config_linux.py"
     VM_QGA_SET_HOSTNAME = "/usr/local/zstack/zs-tools/set_hostname_linux.py"
+    VM_QGA_SET_HOSTNAME_EL6 = "/usr/local/zstack/zs-tools/set_hostname_linux_el6.py"
     VM_CONFIG_SYNC_OS_VERSION_SUPPORT = {
-        VmQga.VM_OS_LINUX_CENTOS: ("7", "8"),
-        VmQga.VM_OS_LINUX_KYLIN: ("v10",),
+        VmQga.VM_OS_LINUX_CENTOS: ("6", "7", "8"),
+        VmQga.VM_OS_LINUX_KYLIN: ("4", "v10",),
         VmQga.VM_OS_LINUX_UOS: ("20",),
         VmQga.VM_OS_LINUX_OPEN_SUSE: ("12", "15",),
         VmQga.VM_OS_LINUX_SUSE_S: ("12", "15",),
         VmQga.VM_OS_LINUX_SUSE_D: ("12", "15",),
         VmQga.VM_OS_LINUX_ORACLE: ("7",),
         VmQga.VM_OS_LINUX_REDHAT: ("7",),
-        VmQga.VM_OS_LINUX_UBUNTU: ("18",),
-        VmQga.VM_OS_WINDOWS: ("10", "2012", "2012r2", "2016", "2019",)
+        VmQga.VM_OS_LINUX_UBUNTU: ("14", "16", "18",),
+        VmQga.VM_OS_LINUX_DEBIAN: ("9", "10",),
+        VmQga.VM_OS_LINUX_FEDORA: ("30", "31",),
+        VmQga.VM_OS_WINDOWS: ("10", "2012", "2012r2", "2016", "2019", "2008r2",)
     }
 
     @lock.lock('config_vm_by_qga')
@@ -212,7 +215,10 @@ def set_vm_hostname_by_qga(self, domain, hostname, default_ip):
             return ret, msg
 
         # exec qga command
-        cmd_file = self.VM_QGA_SET_HOSTNAME
+        if qga.os_version == '6':
+            cmd_file = self.VM_QGA_SET_HOSTNAME_EL6
+        else:
+            cmd_file = self.VM_QGA_SET_HOSTNAME
         ret, msg = qga.guest_exec_python(cmd_file, [hostname, default_ip])
         if ret != 0:
             logger.debug("set vm hostname {} by qga failed: {}".format(vm_uuid, msg))

zstacklib/zstacklib/utils/qga.py

@@ -45,6 +45,17 @@
 qga_channel_state_connected = 'connected'
 qga_channel_state_disconnected = 'disconnected'
 
+encodings = ['utf-8', 'GB2312', 'ISO-8859-1']
+
+
+def decode_with_fallback(encoded_bytes):
+    for encoding in encodings:
+        try:
+            return encoded_bytes.decode(encoding).encode('utf-8')
+        except UnicodeDecodeError:
+            continue
+    raise UnicodeDecodeError("Unable to decode bytes using provided encodings")
+
 
 def get_qga_channel_state(vm_dom):
     xml_tree = ET.fromstring(vm_dom.XMLDesc())
@@ -61,9 +72,11 @@ def is_qga_connected(vm_dom):
     except:
         return False
 
+
 # windows zs-tools command wait 120s
 zs_tools_wait_retry = 120
 
+
 class QgaException(Exception):
     """ The base exception class for all exceptions this agent raises."""
 
@@ -85,6 +98,8 @@ class VmQga(object):
     VM_OS_LINUX_SUSE_D = "sled"
     VM_OS_LINUX_ORACLE = "ol"
     VM_OS_LINUX_REDHAT = "rhel"
+    VM_OS_LINUX_DEBIAN = "debian"
+    VM_OS_LINUX_FEDORA = "fedora"
     VM_OS_WINDOWS = "mswindows"
 
     ZS_TOOLS_PATN_WIN = "C:\Program Files\GuestTools\zs-tools\zs-tools.exe"
@@ -202,9 +217,9 @@ def guest_exec_bash(self, cmd, output=True, wait=qga_exec_wait_interval, retry=q
         exit_code = ret.get('exitcode')
         ret_data = None
         if 'out-data' in ret:
-            ret_data = ret['out-data']
+            ret_data = decode_with_fallback(ret['out-data'])
         elif 'err-data' in ret:
-            ret_data = ret['err-data']
+            ret_data = decode_with_fallback(ret['err-data'])
 
         return exit_code, ret_data
 
@@ -247,9 +262,9 @@ def guest_exec_python(self, file, params=None, output=True, wait=qga_exec_wait_i
         exit_code = ret.get('exitcode')
         ret_data = None
         if 'out-data' in ret:
-            ret_data = ret['out-data']
+            ret_data = decode_with_fallback(ret['out-data'])
         elif 'err-data' in ret:
-            ret_data = ret['err-data']
+            ret_data = decode_with_fallback(ret['err-data'])
 
         return exit_code, ret_data
 
@@ -264,9 +279,10 @@ def guest_exec_zs_tools(self, operate, config, output=True, wait=qga_exec_wait_i
             raise Exception('qga exec zs-tools unknow operate {} for vm {}'.format(operate, self.vm_uuid))
 
         if ret and "pid" in ret:
-                pid = ret["pid"]
+            pid = ret["pid"]
         else:
-            raise Exception('qga exec zs-tools operate {} config {} failed for vm {}'.format(operate, config, self.vm_uuid))
+            raise Exception(
+                'qga exec zs-tools operate {} config {} failed for vm {}'.format(operate, config, self.vm_uuid))
 
         ret = None
         for i in range(retry):
@@ -276,17 +292,52 @@ def guest_exec_zs_tools(self, operate, config, output=True, wait=qga_exec_wait_i
                 break
 
         if not ret or not ret.get('exited'):
-            raise Exception('qga exec zs-tools operate {} config {} timeout for vm {}'.format(operate, config, self.vm_uuid))
+            raise Exception(
+                'qga exec zs-tools operate {} config {} timeout for vm {}'.format(operate, config, self.vm_uuid))
 
         exit_code = ret.get('exitcode')
         ret_data = None
         if 'out-data' in ret:
-            ret_data = ret['out-data'].decode('utf-8').encode('utf-8')
+            ret_data = decode_with_fallback(ret['out-data'])
         elif 'err-data' in ret:
-            ret_data = ret['err-data'].decode('utf-8').encode('utf-8')
+            ret_data = decode_with_fallback(ret['err-data'])
 
         return exit_code, ret_data.replace('\r\n', '')
 
+    def guest_exec_wmic(self, cmd, output=True, wait=qga_exec_wait_interval, retry=qga_exec_wait_retry):
+        cmd_parts = cmd.split('|')
+        cmd = "{}".format(" ".join([part for part in cmd_parts]))
+
+        ret = self.guest_exec(
+            {"path": "wmic", "arg": cmd.split(" "), "capture-output": output})
+        if ret and "pid" in ret:
+            pid = ret["pid"]
+        else:
+            raise Exception('qga exec cmd {} failed for vm {}'.format(cmd, self.vm_uuid))
+
+        if not output:
+            logger.debug("run qga wmic: {} failed, no output".format(cmd))
+            return 0, None
+
+        ret = None
+        for i in range(retry):
+            time.sleep(wait)
+            ret = self.guest_exec_status(pid)
+            if ret['exited']:
+                break
+
+        if not ret or not ret.get('exited'):
+            raise Exception('qga exec cmd {} timeout for vm {}'.format(cmd, self.vm_uuid))
+
+        exit_code = ret.get('exitcode')
+        ret_data = None
+        if 'out-data' in ret:
+            ret_data = decode_with_fallback(ret['out-data'])
+        elif 'err-data' in ret:
+            ret_data = decode_with_fallback(ret['err-data'])
+
+        return exit_code, ret_data
+
     def guest_exec_powershell(self, cmd, output=True, wait=qga_exec_wait_interval, retry=qga_exec_wait_retry):
         cmd_parts = cmd.split('|')
         cmd = "& '{}'".format("' '".join([part for part in cmd_parts]))
@@ -315,9 +366,9 @@ def guest_exec_powershell(self, cmd, output=True, wait=qga_exec_wait_interval, r
         exit_code = ret.get('exitcode')
         ret_data = None
         if 'out-data' in ret:
-            ret_data = ret['out-data'].decode("GB2312")
+            ret_data = decode_with_fallback(ret['out-data'])
         elif 'err-data' in ret:
-            ret_data = ret['err-data'].decode("GB2312")
+            ret_data = decode_with_fallback(ret['err-data'])
 
         return exit_code, ret_data
 
@@ -438,18 +489,25 @@ def guest_get_os_id_like(self):
     def guest_get_os_info(self):
         ret = self.guest_exec_bash_no_exitcode('cat /etc/os-release')
         if not ret:
-            raise Exception('get os info failed')
-
-        lines = [line for line in ret.split('\n') if line != ""]
-        config = {}
-        for line in lines:
-            if line.startswith('#'):
-                continue
-
-            info = line.split('=')
-            if len(info) != 2:
-                continue
-            config[info[0].strip()] = info[1].strip().strip('"')
+            # Parse /etc/redhat-release for CentOS/RHEL 6
+            ret = self.guest_exec_bash_no_exitcode('cat /etc/redhat-release')
+            if not ret:
+                raise Exception('get os info failed')
+            parts = ret.split()
+            if len(parts) >= 3 and parts[1] == 'release':
+                config = {'ID': parts[0].lower(), 'VERSION_ID': parts[2]}
+        else:
+            # Parse /etc/os-release
+            lines = [line for line in ret.split('\n') if line != ""]
+            config = {}
+            for line in lines:
+                if line.startswith('#'):
+                    continue
+
+                info = line.split('=')
+                if len(info) != 2:
+                    continue
+                config[info[0].strip()] = info[1].strip().strip('"')
 
         vm_os = config.get('ID')
         version = config.get('VERSION_ID')