From 432b8d324007a239282f7cb9b28b34ad5d07eed8 Mon Sep 17 00:00:00 2001 From: "Houqi (Nick) Zuo" Date: Sat, 18 Nov 2023 17:48:29 +0800 Subject: [PATCH 1/5] qdevices: add QIOThreadVQbus add QIOThreadVQbus. Signed-off-by: Houqi (Nick) Zuo --- virttest/qemu_devices/qdevices.py | 50 +++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/virttest/qemu_devices/qdevices.py b/virttest/qemu_devices/qdevices.py index 5c697c522e..4c33995850 100644 --- a/virttest/qemu_devices/qdevices.py +++ b/virttest/qemu_devices/qdevices.py @@ -1497,7 +1497,9 @@ def __init__(self, iothread_id, params=None): super(QIOThread, self).__init__(**kwargs) self.set_aid(iothread_id) self.iothread_bus = QIOThreadBus(iothread_id) + self.iothread_vq_bus = QIOThreadVQBus(iothread_id + "_vq") self.add_child_bus(self.iothread_bus) + self.add_child_bus(self.iothread_vq_bus) @staticmethod def _query(monitor): @@ -3731,6 +3733,54 @@ def _update_device_props(self, device, addr): self._set_device_props(device, addr) +class QIOThreadVQBus(QSparseBus): + """IOThread, supporting vq, virtual bus.""" + + def __init__(self, iothread_id): + """ + iothread bus constructor. + + :param iothread_id: related QIOThread object id + """ + super(QIOThreadVQBus, self).__init__( + "iothread-vq-mapping", + [[], []], + "iothread_vq_bus_%s" % iothread_id, + "IOTHREAD", + iothread_id, + ) + + def _check_bus(self, device): + """ + Check if the device is pluggable. + Return true since iothread-vq-mapping can own more than one + iothread devices. + """ + return True + + def _dev2addr(self, device): + """Return the device id as address.""" + return [device.get_qid()] + + def get_free_slot(self, addr_pattern): + """Return the device id as unoccupied address.""" + return addr_pattern + + def _set_device_props(self, device, addr): + """Set device iothread param.""" + bus_item_vals = device.get_param(self.bus_item, []) + iothread = [] + for item in bus_item_vals: + iothread.append(item["iothread"]) + if self.get_device().get_qid() not in iothread: + bus_item_vals.append({"iothread": self.get_device().get_qid()}) + device.set_param(self.bus_item, bus_item_vals) + + def _update_device_props(self, device, addr): + """Always set device iothread param.""" + self._set_device_props(device, addr) + + class QUnixSocketBus(QSparseBus): """ Unix Socket pseudo bus. From a0318d9612c07bfae705ae3535dacd87334a72da Mon Sep 17 00:00:00 2001 From: "Houqi (Nick) Zuo" Date: Mon, 23 Oct 2023 14:12:59 +0800 Subject: [PATCH 2/5] qcontainer: New parameters add Add new support to the parameters: iothread_vqs_mapping. Signed-off-by: Houqi (Nick) Zuo --- virttest/qemu_devices/qcontainer.py | 3 +++ virttest/shared/cfg/base.cfg | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/virttest/qemu_devices/qcontainer.py b/virttest/qemu_devices/qcontainer.py index 4be4b0396b..674c21a443 100644 --- a/virttest/qemu_devices/qcontainer.py +++ b/virttest/qemu_devices/qcontainer.py @@ -2010,6 +2010,8 @@ def images_define_by_variables( :param image_auto_readonly: auto-read-only option in BlockdevOptions :param image_discard: discard option in BlockdevOptions :param image_copy_on_read: if support copy-on-read filter + :param image_iothread_vq_mapping: the mapping between iothread + and virt-queues """ def _get_access_tls_creds(image_access): @@ -2931,6 +2933,7 @@ def images_define_by_params( image_params.get("image_auto_readonly"), image_params.get("image_discard"), image_params.get("image_copy_on_read"), + image_params.get("image_iothread_vq_mapping"), ) def serials_define_by_variables( diff --git a/virttest/shared/cfg/base.cfg b/virttest/shared/cfg/base.cfg index 2e7dacbd8f..d5623db137 100644 --- a/virttest/shared/cfg/base.cfg +++ b/virttest/shared/cfg/base.cfg @@ -1032,6 +1032,30 @@ uuid_dimm = "" # qemu_cmdline_format_cfg = file:/home/user/libvirt-latest.json # qemu_cmdline_format_cfg = string:{"images":{"blockdev": "json"}} +# The key "image_iothread_vq_mapping" in QDevice. +# This key may have many structures of value. +# Example: +# iothread_vq_mapping = x:0,1,2 y:3 +# iothread_vq_mapping = x y z + +# New iothread schemes added: multipeerroundrobin, full +# multipeerroundrobin: allocate the iothread to each pci device until all +# iothreads are allocated completed. +# full: allocate the all iothreads to each pci device. + +# There are some conflict scenes due to the fact that setting the inappropriate +# parameters. The function __iothread_conflict_check is designed to detect the conflict. +# The following scenes cause the conflict: +# Scene#1: If iothread_scheme and image_iothread_vq_mapping both are set. +# If you want image_iothread_vq_mapping working well, suggest set +# iothread_scheme = "" +# Scene#2: If iothread_scheme is set to multipeerroundrobin or full and +# iothreads is set. +# Scene#3: If iothread_scheme and image_iothread_vq_mapping_xxx both are +# set. +# Scene#4: image_iothread_vq_mapping or num_queues may conflict depending on +# how to set drive_bus. + # # Secure guest params # From 787ca8df1d11402e62b7da15544d1a6b72b4656d Mon Sep 17 00:00:00 2001 From: "Houqi (Nick) Zuo" Date: Wed, 25 Oct 2023 18:26:27 +0800 Subject: [PATCH 3/5] qcontainer and qemu_vm: iothread scheme support Add new support to the parameter: iothread_scheme. Signed-off-by: Houqi (Nick) Zuo --- virttest/qemu_devices/qcontainer.py | 21 +++++++++++++++++++ virttest/qemu_vm.py | 31 +++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/virttest/qemu_devices/qcontainer.py b/virttest/qemu_devices/qcontainer.py index 674c21a443..209d43bdfb 100644 --- a/virttest/qemu_devices/qcontainer.py +++ b/virttest/qemu_devices/qcontainer.py @@ -225,6 +225,14 @@ def qemu_version(self): """:return: qemu version, e.g. 5.2.0""" return self.__qemu_ver + @property + def iothread_manager(self): + """ + :return: iothread_manager + :rtype: an object of classes in vt_iothread.py + """ + return self.__iothread_manager + def initialize_iothread_manager(self, params, guestcpuinfo, cmdline_format_cfg={}): """Initialize iothread manager. :param params: vt params @@ -238,6 +246,16 @@ def initialize_iothread_manager(self, params, guestcpuinfo, cmdline_format_cfg={ iothreads_lst = [] elif iothread_scheme == "rhv": iothreads_lst = iothreads_lst[:1] or ["iothread0"] + elif iothread_scheme and iothread_scheme.startswith( + ("multipeerroundrobin", "full") + ): + if len(iothread_scheme.split(":")) == 2: + iothread_scheme, num_iothreads = iothread_scheme.split(":") + else: + num_iothreads = len(params.objects("images")) + + for i in range(int(num_iothreads)): + iothreads_lst.append("iothread%s" % i) iothread_props = {"iothread_poll_max_ns": "poll-max-ns"} iothreads = [] @@ -257,6 +275,8 @@ def initialize_iothread_manager(self, params, guestcpuinfo, cmdline_format_cfg={ "roundrobin": vt_iothread.RoundRobinManager, "oto": vt_iothread.OTOManager, "rhv": vt_iothread.RoundRobinManager, + "multipeerroundrobin": vt_iothread.MultiPeerRoundRobinManager, + "full": vt_iothread.FullManager, } manager = scheme_to_manager.get(iothread_scheme, vt_iothread.PredefinedManager) @@ -1965,6 +1985,7 @@ def images_define_by_variables( image_auto_readonly=None, image_discard=None, image_copy_on_read=None, + image_iothread_vq_mapping=None, ): """ Creates related devices by variables diff --git a/virttest/qemu_vm.py b/virttest/qemu_vm.py index 8558b26b18..68bc480391 100644 --- a/virttest/qemu_vm.py +++ b/virttest/qemu_vm.py @@ -1677,6 +1677,36 @@ def add_secure_guest_descriptor(params): for k, v in backend_props.get(sectype, {}): machine_dev.set_param(k, v) + def __iothread_conflict_check(params): + """ + Based on the params, check if it's a conflict. + + :param params: A dict containing VM params + :type params: dict + """ + iothread_scheme = params.get("iothread_scheme") + image_iothread_vq_mapping = params.get("image_iothread_vq_mapping") + iothreads_lst = params.objects("iothreads") + # The legacy 'iothread_scheme' does NOT support the + # parameter 'iothread_vqs_mapping'. If you're going to use the legacy + # 'iothread_scheme', the 'iothread_vqs_mapping' must NOT be set. + if (iothread_scheme and image_iothread_vq_mapping) or ( + iothread_scheme in ("multipeerroundrobin", "full") and iothreads_lst + ): + raise ValueError( + "There's a conflict in the configuration! Once " + "'iothread_scheme' is set, 'image_iothread_vq_mapping' or" + " 'iothreads' can NOT be set!" + ) + for image_name in params.objects("images"): + image_params = params.object_params(image_name) + if image_params.get("image_iothread_vq_mapping") and iothread_scheme: + raise ValueError( + "There's a conflict in the configuration! Once " + "'iothread_scheme' is set, " + "'image_iothread_vq_mapping' can NOT be set!" + ) + # End of command line option wrappers # If nothing changed and devices exists, return immediately @@ -2408,6 +2438,7 @@ def add_secure_guest_descriptor(params): devices.insert(dev_usb) # initialize iothread manager + __iothread_conflict_check(params) devices.initialize_iothread_manager( params, self.cpuinfo, self._get_cmdline_format_cfg() ) From a5d0abcb64c4b9b56e79971ea7f93b42a4aa5b68 Mon Sep 17 00:00:00 2001 From: "Houqi (Nick) Zuo" Date: Tue, 14 Nov 2023 13:44:15 +0800 Subject: [PATCH 4/5] vt_iothread: New class added Add new support to the parameter: iothread_scheme. Signed-off-by: Houqi (Nick) Zuo --- virttest/vt_iothread.py | 79 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/virttest/vt_iothread.py b/virttest/vt_iothread.py index ae423c5164..8c27b46f90 100644 --- a/virttest/vt_iothread.py +++ b/virttest/vt_iothread.py @@ -101,3 +101,82 @@ def request_iothread(self, iothread): return iothread else: raise ValueError("Not support request specific iothread") + + +class MultiPeerRoundRobinManager(IOThreadManagerBase): + """ + Dispatch iothread object in new round-robin way. + Each iothreads will be allocated in round-robin way to the images. + """ + + def __init__(self, iothreads=None): + """ + Initialize iothread manager. + + :param iothreads: list of iothread objects, its id must conform with + ID_PATTERN. + :type iothreads: List + """ + super().__init__(iothreads) + self.__length = len(iothreads) if iothreads else 0 + self.__current_index = 0 + self.__pci_dev_iothread_vq_mapping = {} + + @property + def pci_dev_iothread_vq_mapping(self): + return self.__pci_dev_iothread_vq_mapping + + @pci_dev_iothread_vq_mapping.setter + def pci_dev_iothread_vq_mapping(self, mapping): + """ + Set the self.__pci_dev_iothread_vq_mapping + :param mapping: + :type mapping: dict, {string: iothread instance} + """ + for key, val in mapping.items(): + if key in self.__pci_dev_iothread_vq_mapping: + self.__pci_dev_iothread_vq_mapping[key].append(val) + else: + self.__pci_dev_iothread_vq_mapping[key] = [val] + + def request_iothread(self, iothread): + """Return iothread. + + :param iothread: iothread. + :type iothread: QIOThread + :return: iothread. + :rtype: QIOThread + """ + if iothread == "AUTO" or iothread == "auto": + iothread_aid = self.ID_PATTERN % (self.__current_index % self.__length) + iothread = self.find_iothread(iothread_aid) + self.__current_index += 1 + return iothread + raise ValueError("Not support request specific iothread!") + + +class FullManager(IOThreadManagerBase): + """Dispatch all iothread objects to an image device.""" + + def __init__(self, iothreads=None): + """ + Initialize iothread manager. + + :param iothreads: list of iothread objects, its id must conform with + ID_PATTERN. + :type iothreads: List + """ + super().__init__(iothreads) + self.__iothreads_list = iothreads + + def request_iothread(self, iothread): + """Return iothreads. + + :param iothread: iothread. + :type iothread: QIOThread + :return: the list of the iothreads. + :rtype: List + """ + if iothread == "AUTO" or iothread == "auto": + return self.__iothreads_list + raise ValueError("Not support request specific iothread!") From 51afff780e01cbf2686566b884350847c18280ed Mon Sep 17 00:00:00 2001 From: "Houqi (Nick) Zuo" Date: Wed, 29 Nov 2023 18:33:36 +0800 Subject: [PATCH 5/5] qemu_vm.py: implement the iothread scheme implement the iothread scheme. Signed-off-by: Houqi (Nick) Zuo --- virttest/qemu_devices/qcontainer.py | 100 +++++++++++++++++++++++++++- virttest/qemu_vm.py | 24 +++++++ 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/virttest/qemu_devices/qcontainer.py b/virttest/qemu_devices/qcontainer.py index 209d43bdfb..0c77b92d79 100644 --- a/virttest/qemu_devices/qcontainer.py +++ b/virttest/qemu_devices/qcontainer.py @@ -218,6 +218,7 @@ def get_qmp_cmds(qemu_binary, workaround_qemu_qmp_crash=False): self._probe_migration_parameters() self.__iothread_manager = None self.__iothread_supported_devices = set() + self.__iothread_vq_mapping_supported_devices = set() self.temporary_image_snapshots = set() @property @@ -300,11 +301,34 @@ def is_dev_iothread_supported(self, device): return True options = "--device %s,\\?" % device out = self.execute_qemu(options) - if "iothread" in out: + if re.findall("iothread=<[^>]+>+", out): self.__iothread_supported_devices.add(device) return True return False + def is_dev_iothread_vq_supported(self, device): + """Check if dev supports iothread-vq-mapping. + :param device: device to check + :type device: QDevice or string + """ + try: + device = device.get_param("driver") + except AttributeError: + if not isinstance(device, six.string_types): + raise TypeError("device: expected string or QDevice") + if not device: + return False + if device in self.__iothread_vq_mapping_supported_devices: + return True + options = "--device %s,\\?" % device + out = self.execute_qemu(options) + if re.findall("iothread-vq-mapping=<[^>]+>+", out) and ( + not device.startswith("virtio-scsi-pci") + ): + self.__iothread_vq_mapping_supported_devices.add(device) + return True + return False + def allocate_iothread(self, iothread, device): """ Allocate iothread for device to use. @@ -335,6 +359,44 @@ def allocate_iothread(self, iothread, device): ) raise TypeError(err_msg) + def allocate_iothread_vq(self, iothread, device): + """ + Allocate iothread( supporting vq ) for device to use. + + :param iothread: iothread specified in params + could be: + 'auto': allocate iothread based on schems specified by + 'iothread_scheme'. + iothread id: request specific iothread to use. + :param device: device object + :return: iothread object allocated + """ + if self.is_dev_iothread_vq_supported(device): + iothreads = self.iothread_manager.request_iothread(iothread) + iothreads_return = iothreads + if not isinstance(iothreads, Sequence): + iothreads = (iothreads,) + for iothread in iothreads: + dev_iothread_parent = {"busid": iothread.iothread_vq_bus.busid} + if device.parent_bus: + device.parent_bus += (dev_iothread_parent,) + else: + device.parent_bus = (dev_iothread_parent,) + + if isinstance( + self.iothread_manager, vt_iothread.MultiPeerRoundRobinManager + ): + self.iothread_manager.pci_dev_iothread_vq_mapping = { + device.get_qid(): iothreads[0] + } + return iothreads_return + else: + err_msg = "Device %s(%s) not support iothread-vq-mapping" % ( + device.get_aid(), + device.get_param("driver"), + ) + raise TypeError(err_msg) + def _probe_capabilities(self): """Probe capabilities.""" # -blockdev @@ -2818,6 +2880,42 @@ def define_hbas( ) for key, value in blk_extra_params: devices[-1].set_param(key, value) + if self.is_dev_iothread_vq_supported(devices[-1]): + if num_queues: + devices[-1].set_param("num-queues", num_queues) + # add iothread-vq-mapping if available + if image_iothread_vq_mapping: + val = [] + for item in image_iothread_vq_mapping.strip().split(" "): + allocated_iothread = self.allocate_iothread_vq( + item.split(":")[0], devices[-1] + ) + mapping = {"iothread": allocated_iothread.get_qid()} + if len(item.split(":")) == 2: + vqs = [int(_) for _ in item.split(":")[-1].split(",")] + mapping["vqs"] = vqs + val.append(mapping) + # FIXME: The reason using set_param() is that the format( + # Example: iothread0:0,1,2 ) can NOT be set by + # Devcontainer.insert() appropriately since the contents + # following after colon are lost. + if ":" in image_iothread_vq_mapping: + devices[-1].set_param("iothread-vq-mapping", val) + + if isinstance( + self.iothread_manager, vt_iothread.MultiPeerRoundRobinManager + ): + mapping = self.iothread_manager.pci_dev_iothread_vq_mapping + if devices[-1].get_qid() in mapping: + num_iothread = len(mapping[devices[-1].get_qid()]) + for i in range(num_iothread): + iothread = self.allocate_iothread_vq("auto", devices[-1]) + iothread.iothread_vq_bus.insert(devices[-1]) + elif isinstance(self.iothread_manager, vt_iothread.FullManager): + iothreads = self.allocate_iothread_vq("auto", devices[-1]) + if iothreads: + for ioth in iothreads: + ioth.iothread_vq_bus.insert(devices[-1]) return devices def images_define_by_params( diff --git a/virttest/qemu_vm.py b/virttest/qemu_vm.py index 68bc480391..b54bb3548c 100644 --- a/virttest/qemu_vm.py +++ b/virttest/qemu_vm.py @@ -46,6 +46,7 @@ utils_vdpa, utils_vsock, virt_vm, + vt_iothread, ) from virttest.qemu_capabilities import Flags from virttest.qemu_devices import qcontainer, qdevices @@ -2450,6 +2451,7 @@ def __iothread_conflict_check(params): set_cmdline_format_by_cfg(dev, self._get_cmdline_format_cfg(), "images") devices.insert(dev) + image_devs = [] # Add images (harddrives) for image_name in params.objects("images"): # FIXME: Use qemu_devices for handling indexes @@ -2497,6 +2499,28 @@ def __iothread_conflict_check(params): for _ in devs: set_cmdline_format_by_cfg(_, self._get_cmdline_format_cfg(), "images") devices.insert(_) + image_devs.extend(devs) + # FIXME: Here's a workaround solution about allocating the iothreads. + # Due to adapting the multipeerroundrobin iothread scheme, + # allocating the iothreads has to be executed after all the related + # image devices are created completely. + iothread_lst = [] + img_pci_mapping = [] + for dev in devices: + if isinstance(dev, qdevices.QIOThread): + iothread_lst.append(dev) + for dev in image_devs: + if dev and devices.is_dev_iothread_vq_supported(dev): + img_pci_mapping.append(dev) + + if len(img_pci_mapping) > 0: + if isinstance( + devices.iothread_manager, vt_iothread.MultiPeerRoundRobinManager + ): + for i in range(max(len(iothread_lst), len(img_pci_mapping))): + dev = img_pci_mapping[i % len(img_pci_mapping)] + iothread_assigned = devices.allocate_iothread_vq("auto", dev) + iothread_assigned.iothread_vq_bus.insert(dev) # Add filesystems for fs_name in params.objects("filesystems"):