qcontainer.py: add new support to the parameter

virttest/qemu_devices/qcontainer.py

@@ -218,13 +218,22 @@ def get_qmp_cmds(qemu_binary, workaround_qemu_qmp_crash=False):
         self._probe_migration_parameters()
         self.__iothread_manager = None
         self.__iothread_supported_devices = set()
+        self.__iothread_vq_mapping_supported_devices = set()
         self.temporary_image_snapshots = set()
 
     @property
     def qemu_version(self):
         """:return: qemu version, e.g. 5.2.0"""
         return self.__qemu_ver
 
+    @property
+    def iothread_manager(self):
+        """
+        :return: iothread_manager
+        :rtype: an object of classes in vt_iothread.py
+        """
+        return self.__iothread_manager
+
     def initialize_iothread_manager(self, params, guestcpuinfo, cmdline_format_cfg={}):
         """Initialize iothread manager.
         :param params: vt params
@@ -238,6 +247,16 @@ def initialize_iothread_manager(self, params, guestcpuinfo, cmdline_format_cfg={
             iothreads_lst = []
         elif iothread_scheme == "rhv":
             iothreads_lst = iothreads_lst[:1] or ["iothread0"]
+        elif iothread_scheme and iothread_scheme.startswith(
+            ("multipeerroundrobin", "full")
+        ):
+            if len(iothread_scheme.split(":")) == 2:
+                iothread_scheme, num_iothreads = iothread_scheme.split(":")
+            else:
+                num_iothreads = len(params.objects("images"))
+
+            for i in range(int(num_iothreads)):
+                iothreads_lst.append("iothread%s" % i)
 
         iothread_props = {"iothread_poll_max_ns": "poll-max-ns"}
         iothreads = []
@@ -257,6 +276,8 @@ def initialize_iothread_manager(self, params, guestcpuinfo, cmdline_format_cfg={
             "roundrobin": vt_iothread.RoundRobinManager,
             "oto": vt_iothread.OTOManager,
             "rhv": vt_iothread.RoundRobinManager,
+            "multipeerroundrobin": vt_iothread.MultiPeerRoundRobinManager,
+            "full": vt_iothread.FullManager,
         }
         manager = scheme_to_manager.get(iothread_scheme, vt_iothread.PredefinedManager)
 
@@ -280,11 +301,34 @@ def is_dev_iothread_supported(self, device):
             return True
         options = "--device %s,\\?" % device
         out = self.execute_qemu(options)
-        if "iothread" in out:
+        if re.findall("iothread=<[^>]+>+", out):
             self.__iothread_supported_devices.add(device)
             return True
         return False
 
+    def is_dev_iothread_vq_supported(self, device):
+        """Check if dev supports iothread-vq-mapping.
+        :param device: device to check
+        :type device: QDevice or string
+        """
+        try:
+            device = device.get_param("driver")
+        except AttributeError:
+            if not isinstance(device, six.string_types):
+                raise TypeError("device: expected string or QDevice")
+        if not device:
+            return False
+        if device in self.__iothread_vq_mapping_supported_devices:
+            return True
+        options = "--device %s,\\?" % device
+        out = self.execute_qemu(options)
+        if re.findall("iothread-vq-mapping=<[^>]+>+", out) and (
+            not device.startswith("virtio-scsi-pci")
+        ):
+            self.__iothread_vq_mapping_supported_devices.add(device)
+            return True
+        return False
+
     def allocate_iothread(self, iothread, device):
         """
         Allocate iothread for device to use.
@@ -315,6 +359,44 @@ def allocate_iothread(self, iothread, device):
             )
             raise TypeError(err_msg)
 
+    def allocate_iothread_vq(self, iothread, device):
+        """
+        Allocate iothread( supporting vq ) for device to use.
+
+        :param iothread: iothread specified in params
+                         could be:
+                         'auto': allocate iothread based on schems specified by
+                                 'iothread_scheme'.
+                         iothread id: request specific iothread to use.
+        :param device: device object
+        :return: iothread object allocated
+        """
+        if self.is_dev_iothread_vq_supported(device):
+            iothreads = self.iothread_manager.request_iothread(iothread)
+            iothreads_return = iothreads
+            if not isinstance(iothreads, Sequence):
+                iothreads = (iothreads,)
+            for iothread in iothreads:
+                dev_iothread_parent = {"busid": iothread.iothread_vq_bus.busid}
+                if device.parent_bus:
+                    device.parent_bus += (dev_iothread_parent,)
+                else:
+                    device.parent_bus = (dev_iothread_parent,)
+
+            if isinstance(
+                self.iothread_manager, vt_iothread.MultiPeerRoundRobinManager
+            ):
+                self.iothread_manager.pci_dev_iothread_vq_mapping = {
+                    device.get_qid(): iothreads[0]
+                }
+            return iothreads_return
+        else:
+            err_msg = "Device %s(%s) not support iothread-vq-mapping" % (
+                device.get_aid(),
+                device.get_param("driver"),
+            )
+            raise TypeError(err_msg)
+
     def _probe_capabilities(self):
         """Probe capabilities."""
         # -blockdev
@@ -1965,6 +2047,7 @@ def images_define_by_variables(
         image_auto_readonly=None,
         image_discard=None,
         image_copy_on_read=None,
+        image_iothread_vq_mapping=None,
     ):
         """
         Creates related devices by variables
@@ -2010,6 +2093,8 @@ def images_define_by_variables(
         :param image_auto_readonly: auto-read-only option in BlockdevOptions
         :param image_discard: discard option in BlockdevOptions
         :param image_copy_on_read: if support copy-on-read filter
+        :param image_iothread_vq_mapping: the mapping between iothread
+                                          and virt-queues
         """
 
         def _get_access_tls_creds(image_access):
@@ -2795,6 +2880,42 @@ def define_hbas(
             )
             for key, value in blk_extra_params:
                 devices[-1].set_param(key, value)
+        if self.is_dev_iothread_vq_supported(devices[-1]):
+            if num_queues:
+                devices[-1].set_param("num-queues", num_queues)
+            # add iothread-vq-mapping if available
+            if image_iothread_vq_mapping:
+                val = []
+                for item in image_iothread_vq_mapping.strip().split(" "):
+                    allocated_iothread = self.allocate_iothread_vq(
+                        item.split(":")[0], devices[-1]
+                    )
+                    mapping = {"iothread": allocated_iothread.get_qid()}
+                    if len(item.split(":")) == 2:
+                        vqs = [int(_) for _ in item.split(":")[-1].split(",")]
+                        mapping["vqs"] = vqs
+                    val.append(mapping)
+                # FIXME: The reason using set_param() is that the format(
+                #  Example: iothread0:0,1,2 ) can NOT be set by
+                #  Devcontainer.insert() appropriately since the contents
+                #  following after colon are lost.
+                if ":" in image_iothread_vq_mapping:
+                    devices[-1].set_param("iothread-vq-mapping", val)
+
+            if isinstance(
+                self.iothread_manager, vt_iothread.MultiPeerRoundRobinManager
+            ):
+                mapping = self.iothread_manager.pci_dev_iothread_vq_mapping
+                if devices[-1].get_qid() in mapping:
+                    num_iothread = len(mapping[devices[-1].get_qid()])
+                    for i in range(num_iothread):
+                        iothread = self.allocate_iothread_vq("auto", devices[-1])
+                        iothread.iothread_vq_bus.insert(devices[-1])
+            elif isinstance(self.iothread_manager, vt_iothread.FullManager):
+                iothreads = self.allocate_iothread_vq("auto", devices[-1])
+                if iothreads:
+                    for ioth in iothreads:
+                        ioth.iothread_vq_bus.insert(devices[-1])
         return devices
 
     def images_define_by_params(
@@ -2931,6 +3052,7 @@ def images_define_by_params(
             image_params.get("image_auto_readonly"),
             image_params.get("image_discard"),
             image_params.get("image_copy_on_read"),
+            image_params.get("image_iothread_vq_mapping"),
         )
 
     def serials_define_by_variables(

virttest/qemu_devices/qdevices.py

@@ -1497,7 +1497,9 @@ def __init__(self, iothread_id, params=None):
         super(QIOThread, self).__init__(**kwargs)
         self.set_aid(iothread_id)
         self.iothread_bus = QIOThreadBus(iothread_id)
+        self.iothread_vq_bus = QIOThreadVQBus(iothread_id + "_vq")
         self.add_child_bus(self.iothread_bus)
+        self.add_child_bus(self.iothread_vq_bus)
 
     @staticmethod
     def _query(monitor):
@@ -3731,6 +3733,54 @@ def _update_device_props(self, device, addr):
         self._set_device_props(device, addr)
 
 
+class QIOThreadVQBus(QSparseBus):
+    """IOThread, supporting vq, virtual bus."""
+
+    def __init__(self, iothread_id):
+        """
+        iothread bus constructor.
+
+        :param iothread_id: related QIOThread object id
+        """
+        super(QIOThreadVQBus, self).__init__(
+            "iothread-vq-mapping",
+            [[], []],
+            "iothread_vq_bus_%s" % iothread_id,
+            "IOTHREAD",
+            iothread_id,
+        )
+
+    def _check_bus(self, device):
+        """
+        Check if the device is pluggable.
+        Return true since iothread-vq-mapping can own more than one
+        iothread devices.
+        """
+        return True
+
+    def _dev2addr(self, device):
+        """Return the device id as address."""
+        return [device.get_qid()]
+
+    def get_free_slot(self, addr_pattern):
+        """Return the device id as unoccupied address."""
+        return addr_pattern
+
+    def _set_device_props(self, device, addr):
+        """Set device iothread param."""
+        bus_item_vals = device.get_param(self.bus_item, [])
+        iothread = []
+        for item in bus_item_vals:
+            iothread.append(item["iothread"])
+        if self.get_device().get_qid() not in iothread:
+            bus_item_vals.append({"iothread": self.get_device().get_qid()})
+            device.set_param(self.bus_item, bus_item_vals)
+
+    def _update_device_props(self, device, addr):
+        """Always set device iothread param."""
+        self._set_device_props(device, addr)
+
+
 class QUnixSocketBus(QSparseBus):
     """
     Unix Socket pseudo bus.

virttest/qemu_vm.py

@@ -46,6 +46,7 @@
     utils_vdpa,
     utils_vsock,
     virt_vm,
+    vt_iothread,
 )
 from virttest.qemu_capabilities import Flags
 from virttest.qemu_devices import qcontainer, qdevices
@@ -1677,6 +1678,36 @@ def add_secure_guest_descriptor(params):
             for k, v in backend_props.get(sectype, {}):
                 machine_dev.set_param(k, v)
 
+        def __iothread_conflict_check(params):
+            """
+            Based on the params, check if it's a conflict.
+
+            :param params: A dict containing VM params
+            :type params: dict
+            """
+            iothread_scheme = params.get("iothread_scheme")
+            image_iothread_vq_mapping = params.get("image_iothread_vq_mapping")
+            iothreads_lst = params.objects("iothreads")
+            # The legacy 'iothread_scheme' does NOT support the
+            # parameter 'iothread_vqs_mapping'. If you're going to use the legacy
+            # 'iothread_scheme', the 'iothread_vqs_mapping' must NOT be set.
+            if (iothread_scheme and image_iothread_vq_mapping) or (
+                iothread_scheme in ("multipeerroundrobin", "full") and iothreads_lst
+            ):
+                raise ValueError(
+                    "There's a conflict in the configuration! Once "
+                    "'iothread_scheme' is set, 'image_iothread_vq_mapping' or"
+                    " 'iothreads' can NOT be set!"
+                )
+            for image_name in params.objects("images"):
+                image_params = params.object_params(image_name)
+                if image_params.get("image_iothread_vq_mapping") and iothread_scheme:
+                    raise ValueError(
+                        "There's a conflict in the configuration! Once "
+                        "'iothread_scheme' is set, "
+                        "'image_iothread_vq_mapping' can NOT be set!"
+                    )
+
         # End of command line option wrappers
 
         # If nothing changed and devices exists, return immediately
@@ -2408,6 +2439,7 @@ def add_secure_guest_descriptor(params):
             devices.insert(dev_usb)
 
         # initialize iothread manager
+        __iothread_conflict_check(params)
         devices.initialize_iothread_manager(
             params, self.cpuinfo, self._get_cmdline_format_cfg()
         )
@@ -2419,6 +2451,7 @@ def add_secure_guest_descriptor(params):
             set_cmdline_format_by_cfg(dev, self._get_cmdline_format_cfg(), "images")
             devices.insert(dev)
 
+        image_devs = []
         # Add images (harddrives)
         for image_name in params.objects("images"):
             # FIXME: Use qemu_devices for handling indexes
@@ -2466,6 +2499,28 @@ def add_secure_guest_descriptor(params):
             for _ in devs:
                 set_cmdline_format_by_cfg(_, self._get_cmdline_format_cfg(), "images")
                 devices.insert(_)
+            image_devs.extend(devs)
+        # FIXME: Here's a workaround solution about allocating the iothreads.
+        #  Due to adapting the multipeerroundrobin iothread scheme,
+        #  allocating the iothreads has to be executed after all the related
+        #  image devices are created completely.
+        iothread_lst = []
+        img_pci_mapping = []
+        for dev in devices:
+            if isinstance(dev, qdevices.QIOThread):
+                iothread_lst.append(dev)
+        for dev in image_devs:
+            if dev and devices.is_dev_iothread_vq_supported(dev):
+                img_pci_mapping.append(dev)
+
+        if len(img_pci_mapping) > 0:
+            if isinstance(
+                devices.iothread_manager, vt_iothread.MultiPeerRoundRobinManager
+            ):
+                for i in range(max(len(iothread_lst), len(img_pci_mapping))):
+                    dev = img_pci_mapping[i % len(img_pci_mapping)]
+                    iothread_assigned = devices.allocate_iothread_vq("auto", dev)
+                    iothread_assigned.iothread_vq_bus.insert(dev)
 
         # Add filesystems
         for fs_name in params.objects("filesystems"):

virttest/shared/cfg/base.cfg

@@ -1032,6 +1032,30 @@ uuid_dimm = ""
 # qemu_cmdline_format_cfg = file:/home/user/libvirt-latest.json
 # qemu_cmdline_format_cfg = string:{"images":{"blockdev": "json"}}
 
+# The key "image_iothread_vq_mapping" in QDevice.
+# This key may have many structures of value.
+# Example:
+# iothread_vq_mapping = x:0,1,2 y:3
+# iothread_vq_mapping = x y z
+
+# New iothread schemes added: multipeerroundrobin, full
+# multipeerroundrobin: allocate the iothread to each pci device until all
+#                      iothreads are allocated completed.
+# full: allocate the all iothreads to each pci device.
+
+# There are some conflict scenes due to the fact that setting the inappropriate
+# parameters. The function __iothread_conflict_check is designed to detect the conflict.
+# The following scenes cause the conflict:
+# Scene#1: If iothread_scheme and image_iothread_vq_mapping both are set.
+#          If you want image_iothread_vq_mapping working well, suggest set
+#          iothread_scheme = ""
+# Scene#2: If iothread_scheme is set to multipeerroundrobin or full and
+#          iothreads is set.
+# Scene#3: If iothread_scheme and image_iothread_vq_mapping_xxx both are
+#          set.
+# Scene#4: image_iothread_vq_mapping or num_queues may conflict depending on
+#          how to set drive_bus.
+
 #
 # Secure guest params
 #