diff --git a/virttest/qemu_devices/qcontainer.py b/virttest/qemu_devices/qcontainer.py index 4be4b0396b..0c77b92d79 100644 --- a/virttest/qemu_devices/qcontainer.py +++ b/virttest/qemu_devices/qcontainer.py @@ -218,6 +218,7 @@ def get_qmp_cmds(qemu_binary, workaround_qemu_qmp_crash=False): self._probe_migration_parameters() self.__iothread_manager = None self.__iothread_supported_devices = set() + self.__iothread_vq_mapping_supported_devices = set() self.temporary_image_snapshots = set() @property @@ -225,6 +226,14 @@ def qemu_version(self): """:return: qemu version, e.g. 5.2.0""" return self.__qemu_ver + @property + def iothread_manager(self): + """ + :return: iothread_manager + :rtype: an object of classes in vt_iothread.py + """ + return self.__iothread_manager + def initialize_iothread_manager(self, params, guestcpuinfo, cmdline_format_cfg={}): """Initialize iothread manager. :param params: vt params @@ -238,6 +247,16 @@ def initialize_iothread_manager(self, params, guestcpuinfo, cmdline_format_cfg={ iothreads_lst = [] elif iothread_scheme == "rhv": iothreads_lst = iothreads_lst[:1] or ["iothread0"] + elif iothread_scheme and iothread_scheme.startswith( + ("multipeerroundrobin", "full") + ): + if len(iothread_scheme.split(":")) == 2: + iothread_scheme, num_iothreads = iothread_scheme.split(":") + else: + num_iothreads = len(params.objects("images")) + + for i in range(int(num_iothreads)): + iothreads_lst.append("iothread%s" % i) iothread_props = {"iothread_poll_max_ns": "poll-max-ns"} iothreads = [] @@ -257,6 +276,8 @@ def initialize_iothread_manager(self, params, guestcpuinfo, cmdline_format_cfg={ "roundrobin": vt_iothread.RoundRobinManager, "oto": vt_iothread.OTOManager, "rhv": vt_iothread.RoundRobinManager, + "multipeerroundrobin": vt_iothread.MultiPeerRoundRobinManager, + "full": vt_iothread.FullManager, } manager = scheme_to_manager.get(iothread_scheme, vt_iothread.PredefinedManager) @@ -280,11 +301,34 @@ def is_dev_iothread_supported(self, device): return True options = "--device %s,\\?" % device out = self.execute_qemu(options) - if "iothread" in out: + if re.findall("iothread=<[^>]+>+", out): self.__iothread_supported_devices.add(device) return True return False + def is_dev_iothread_vq_supported(self, device): + """Check if dev supports iothread-vq-mapping. + :param device: device to check + :type device: QDevice or string + """ + try: + device = device.get_param("driver") + except AttributeError: + if not isinstance(device, six.string_types): + raise TypeError("device: expected string or QDevice") + if not device: + return False + if device in self.__iothread_vq_mapping_supported_devices: + return True + options = "--device %s,\\?" % device + out = self.execute_qemu(options) + if re.findall("iothread-vq-mapping=<[^>]+>+", out) and ( + not device.startswith("virtio-scsi-pci") + ): + self.__iothread_vq_mapping_supported_devices.add(device) + return True + return False + def allocate_iothread(self, iothread, device): """ Allocate iothread for device to use. @@ -315,6 +359,44 @@ def allocate_iothread(self, iothread, device): ) raise TypeError(err_msg) + def allocate_iothread_vq(self, iothread, device): + """ + Allocate iothread( supporting vq ) for device to use. + + :param iothread: iothread specified in params + could be: + 'auto': allocate iothread based on schems specified by + 'iothread_scheme'. + iothread id: request specific iothread to use. + :param device: device object + :return: iothread object allocated + """ + if self.is_dev_iothread_vq_supported(device): + iothreads = self.iothread_manager.request_iothread(iothread) + iothreads_return = iothreads + if not isinstance(iothreads, Sequence): + iothreads = (iothreads,) + for iothread in iothreads: + dev_iothread_parent = {"busid": iothread.iothread_vq_bus.busid} + if device.parent_bus: + device.parent_bus += (dev_iothread_parent,) + else: + device.parent_bus = (dev_iothread_parent,) + + if isinstance( + self.iothread_manager, vt_iothread.MultiPeerRoundRobinManager + ): + self.iothread_manager.pci_dev_iothread_vq_mapping = { + device.get_qid(): iothreads[0] + } + return iothreads_return + else: + err_msg = "Device %s(%s) not support iothread-vq-mapping" % ( + device.get_aid(), + device.get_param("driver"), + ) + raise TypeError(err_msg) + def _probe_capabilities(self): """Probe capabilities.""" # -blockdev @@ -1965,6 +2047,7 @@ def images_define_by_variables( image_auto_readonly=None, image_discard=None, image_copy_on_read=None, + image_iothread_vq_mapping=None, ): """ Creates related devices by variables @@ -2010,6 +2093,8 @@ def images_define_by_variables( :param image_auto_readonly: auto-read-only option in BlockdevOptions :param image_discard: discard option in BlockdevOptions :param image_copy_on_read: if support copy-on-read filter + :param image_iothread_vq_mapping: the mapping between iothread + and virt-queues """ def _get_access_tls_creds(image_access): @@ -2795,6 +2880,42 @@ def define_hbas( ) for key, value in blk_extra_params: devices[-1].set_param(key, value) + if self.is_dev_iothread_vq_supported(devices[-1]): + if num_queues: + devices[-1].set_param("num-queues", num_queues) + # add iothread-vq-mapping if available + if image_iothread_vq_mapping: + val = [] + for item in image_iothread_vq_mapping.strip().split(" "): + allocated_iothread = self.allocate_iothread_vq( + item.split(":")[0], devices[-1] + ) + mapping = {"iothread": allocated_iothread.get_qid()} + if len(item.split(":")) == 2: + vqs = [int(_) for _ in item.split(":")[-1].split(",")] + mapping["vqs"] = vqs + val.append(mapping) + # FIXME: The reason using set_param() is that the format( + # Example: iothread0:0,1,2 ) can NOT be set by + # Devcontainer.insert() appropriately since the contents + # following after colon are lost. + if ":" in image_iothread_vq_mapping: + devices[-1].set_param("iothread-vq-mapping", val) + + if isinstance( + self.iothread_manager, vt_iothread.MultiPeerRoundRobinManager + ): + mapping = self.iothread_manager.pci_dev_iothread_vq_mapping + if devices[-1].get_qid() in mapping: + num_iothread = len(mapping[devices[-1].get_qid()]) + for i in range(num_iothread): + iothread = self.allocate_iothread_vq("auto", devices[-1]) + iothread.iothread_vq_bus.insert(devices[-1]) + elif isinstance(self.iothread_manager, vt_iothread.FullManager): + iothreads = self.allocate_iothread_vq("auto", devices[-1]) + if iothreads: + for ioth in iothreads: + ioth.iothread_vq_bus.insert(devices[-1]) return devices def images_define_by_params( @@ -2931,6 +3052,7 @@ def images_define_by_params( image_params.get("image_auto_readonly"), image_params.get("image_discard"), image_params.get("image_copy_on_read"), + image_params.get("image_iothread_vq_mapping"), ) def serials_define_by_variables( diff --git a/virttest/qemu_devices/qdevices.py b/virttest/qemu_devices/qdevices.py index 5c697c522e..4c33995850 100644 --- a/virttest/qemu_devices/qdevices.py +++ b/virttest/qemu_devices/qdevices.py @@ -1497,7 +1497,9 @@ def __init__(self, iothread_id, params=None): super(QIOThread, self).__init__(**kwargs) self.set_aid(iothread_id) self.iothread_bus = QIOThreadBus(iothread_id) + self.iothread_vq_bus = QIOThreadVQBus(iothread_id + "_vq") self.add_child_bus(self.iothread_bus) + self.add_child_bus(self.iothread_vq_bus) @staticmethod def _query(monitor): @@ -3731,6 +3733,54 @@ def _update_device_props(self, device, addr): self._set_device_props(device, addr) +class QIOThreadVQBus(QSparseBus): + """IOThread, supporting vq, virtual bus.""" + + def __init__(self, iothread_id): + """ + iothread bus constructor. + + :param iothread_id: related QIOThread object id + """ + super(QIOThreadVQBus, self).__init__( + "iothread-vq-mapping", + [[], []], + "iothread_vq_bus_%s" % iothread_id, + "IOTHREAD", + iothread_id, + ) + + def _check_bus(self, device): + """ + Check if the device is pluggable. + Return true since iothread-vq-mapping can own more than one + iothread devices. + """ + return True + + def _dev2addr(self, device): + """Return the device id as address.""" + return [device.get_qid()] + + def get_free_slot(self, addr_pattern): + """Return the device id as unoccupied address.""" + return addr_pattern + + def _set_device_props(self, device, addr): + """Set device iothread param.""" + bus_item_vals = device.get_param(self.bus_item, []) + iothread = [] + for item in bus_item_vals: + iothread.append(item["iothread"]) + if self.get_device().get_qid() not in iothread: + bus_item_vals.append({"iothread": self.get_device().get_qid()}) + device.set_param(self.bus_item, bus_item_vals) + + def _update_device_props(self, device, addr): + """Always set device iothread param.""" + self._set_device_props(device, addr) + + class QUnixSocketBus(QSparseBus): """ Unix Socket pseudo bus. diff --git a/virttest/qemu_vm.py b/virttest/qemu_vm.py index 8558b26b18..b54bb3548c 100644 --- a/virttest/qemu_vm.py +++ b/virttest/qemu_vm.py @@ -46,6 +46,7 @@ utils_vdpa, utils_vsock, virt_vm, + vt_iothread, ) from virttest.qemu_capabilities import Flags from virttest.qemu_devices import qcontainer, qdevices @@ -1677,6 +1678,36 @@ def add_secure_guest_descriptor(params): for k, v in backend_props.get(sectype, {}): machine_dev.set_param(k, v) + def __iothread_conflict_check(params): + """ + Based on the params, check if it's a conflict. + + :param params: A dict containing VM params + :type params: dict + """ + iothread_scheme = params.get("iothread_scheme") + image_iothread_vq_mapping = params.get("image_iothread_vq_mapping") + iothreads_lst = params.objects("iothreads") + # The legacy 'iothread_scheme' does NOT support the + # parameter 'iothread_vqs_mapping'. If you're going to use the legacy + # 'iothread_scheme', the 'iothread_vqs_mapping' must NOT be set. + if (iothread_scheme and image_iothread_vq_mapping) or ( + iothread_scheme in ("multipeerroundrobin", "full") and iothreads_lst + ): + raise ValueError( + "There's a conflict in the configuration! Once " + "'iothread_scheme' is set, 'image_iothread_vq_mapping' or" + " 'iothreads' can NOT be set!" + ) + for image_name in params.objects("images"): + image_params = params.object_params(image_name) + if image_params.get("image_iothread_vq_mapping") and iothread_scheme: + raise ValueError( + "There's a conflict in the configuration! Once " + "'iothread_scheme' is set, " + "'image_iothread_vq_mapping' can NOT be set!" + ) + # End of command line option wrappers # If nothing changed and devices exists, return immediately @@ -2408,6 +2439,7 @@ def add_secure_guest_descriptor(params): devices.insert(dev_usb) # initialize iothread manager + __iothread_conflict_check(params) devices.initialize_iothread_manager( params, self.cpuinfo, self._get_cmdline_format_cfg() ) @@ -2419,6 +2451,7 @@ def add_secure_guest_descriptor(params): set_cmdline_format_by_cfg(dev, self._get_cmdline_format_cfg(), "images") devices.insert(dev) + image_devs = [] # Add images (harddrives) for image_name in params.objects("images"): # FIXME: Use qemu_devices for handling indexes @@ -2466,6 +2499,28 @@ def add_secure_guest_descriptor(params): for _ in devs: set_cmdline_format_by_cfg(_, self._get_cmdline_format_cfg(), "images") devices.insert(_) + image_devs.extend(devs) + # FIXME: Here's a workaround solution about allocating the iothreads. + # Due to adapting the multipeerroundrobin iothread scheme, + # allocating the iothreads has to be executed after all the related + # image devices are created completely. + iothread_lst = [] + img_pci_mapping = [] + for dev in devices: + if isinstance(dev, qdevices.QIOThread): + iothread_lst.append(dev) + for dev in image_devs: + if dev and devices.is_dev_iothread_vq_supported(dev): + img_pci_mapping.append(dev) + + if len(img_pci_mapping) > 0: + if isinstance( + devices.iothread_manager, vt_iothread.MultiPeerRoundRobinManager + ): + for i in range(max(len(iothread_lst), len(img_pci_mapping))): + dev = img_pci_mapping[i % len(img_pci_mapping)] + iothread_assigned = devices.allocate_iothread_vq("auto", dev) + iothread_assigned.iothread_vq_bus.insert(dev) # Add filesystems for fs_name in params.objects("filesystems"): diff --git a/virttest/shared/cfg/base.cfg b/virttest/shared/cfg/base.cfg index 2e7dacbd8f..d5623db137 100644 --- a/virttest/shared/cfg/base.cfg +++ b/virttest/shared/cfg/base.cfg @@ -1032,6 +1032,30 @@ uuid_dimm = "" # qemu_cmdline_format_cfg = file:/home/user/libvirt-latest.json # qemu_cmdline_format_cfg = string:{"images":{"blockdev": "json"}} +# The key "image_iothread_vq_mapping" in QDevice. +# This key may have many structures of value. +# Example: +# iothread_vq_mapping = x:0,1,2 y:3 +# iothread_vq_mapping = x y z + +# New iothread schemes added: multipeerroundrobin, full +# multipeerroundrobin: allocate the iothread to each pci device until all +# iothreads are allocated completed. +# full: allocate the all iothreads to each pci device. + +# There are some conflict scenes due to the fact that setting the inappropriate +# parameters. The function __iothread_conflict_check is designed to detect the conflict. +# The following scenes cause the conflict: +# Scene#1: If iothread_scheme and image_iothread_vq_mapping both are set. +# If you want image_iothread_vq_mapping working well, suggest set +# iothread_scheme = "" +# Scene#2: If iothread_scheme is set to multipeerroundrobin or full and +# iothreads is set. +# Scene#3: If iothread_scheme and image_iothread_vq_mapping_xxx both are +# set. +# Scene#4: image_iothread_vq_mapping or num_queues may conflict depending on +# how to set drive_bus. + # # Secure guest params # diff --git a/virttest/vt_iothread.py b/virttest/vt_iothread.py index ae423c5164..8c27b46f90 100644 --- a/virttest/vt_iothread.py +++ b/virttest/vt_iothread.py @@ -101,3 +101,82 @@ def request_iothread(self, iothread): return iothread else: raise ValueError("Not support request specific iothread") + + +class MultiPeerRoundRobinManager(IOThreadManagerBase): + """ + Dispatch iothread object in new round-robin way. + Each iothreads will be allocated in round-robin way to the images. + """ + + def __init__(self, iothreads=None): + """ + Initialize iothread manager. + + :param iothreads: list of iothread objects, its id must conform with + ID_PATTERN. + :type iothreads: List + """ + super().__init__(iothreads) + self.__length = len(iothreads) if iothreads else 0 + self.__current_index = 0 + self.__pci_dev_iothread_vq_mapping = {} + + @property + def pci_dev_iothread_vq_mapping(self): + return self.__pci_dev_iothread_vq_mapping + + @pci_dev_iothread_vq_mapping.setter + def pci_dev_iothread_vq_mapping(self, mapping): + """ + Set the self.__pci_dev_iothread_vq_mapping + :param mapping: + :type mapping: dict, {string: iothread instance} + """ + for key, val in mapping.items(): + if key in self.__pci_dev_iothread_vq_mapping: + self.__pci_dev_iothread_vq_mapping[key].append(val) + else: + self.__pci_dev_iothread_vq_mapping[key] = [val] + + def request_iothread(self, iothread): + """Return iothread. + + :param iothread: iothread. + :type iothread: QIOThread + :return: iothread. + :rtype: QIOThread + """ + if iothread == "AUTO" or iothread == "auto": + iothread_aid = self.ID_PATTERN % (self.__current_index % self.__length) + iothread = self.find_iothread(iothread_aid) + self.__current_index += 1 + return iothread + raise ValueError("Not support request specific iothread!") + + +class FullManager(IOThreadManagerBase): + """Dispatch all iothread objects to an image device.""" + + def __init__(self, iothreads=None): + """ + Initialize iothread manager. + + :param iothreads: list of iothread objects, its id must conform with + ID_PATTERN. + :type iothreads: List + """ + super().__init__(iothreads) + self.__iothreads_list = iothreads + + def request_iothread(self, iothread): + """Return iothreads. + + :param iothread: iothread. + :type iothread: QIOThread + :return: the list of the iothreads. + :rtype: List + """ + if iothread == "AUTO" or iothread == "auto": + return self.__iothreads_list + raise ValueError("Not support request specific iothread!")