dm: add iothread support in virtio framework

Add ioeventfd and iothread to virtio framework. When a virtio device
claim to support iothread, virtio framework will register a ioeventfd
and add it to iothread's epoll. After that, the new notify will come
through the iothread instead of the vcpu thread. The notify handler will
be called to process the request.

Tracked-On: #7940
Signed-off-by: Conghui <conghui.chen@intel.com>
Acked-by: Wang, Yu1 <yu1.wang@intel.com>

devicemodel/hw/pci/virtio/vhost.c

@@ -181,7 +181,6 @@ vhost_vq_register_eventfd(struct vhost_dev *vdev,
 	struct virtio_base *base;
 	struct vhost_vq *vq;
 	struct virtio_vq_info *vqi;
-	struct pcibar *bar;
 	struct msix_table_entry *mte;
 	struct acrn_msi_entry msi;
 	int rc = -1;
@@ -194,56 +193,10 @@ vhost_vq_register_eventfd(struct vhost_dev *vdev,
 	vq = &vdev->vqs[idx];
 
 	if (!is_register) {
-		ioeventfd.flags = ACRN_IOEVENTFD_FLAG_DEASSIGN;
 		irqfd.flags = ACRN_IRQFD_FLAG_DEASSIGN;
 	}
 
-	/* register ioeventfd for kick */
+	virtio_register_ioeventfd(base, idx, is_register);
-	if (base->device_caps & (1UL << VIRTIO_F_VERSION_1)) {
-		/*
-		 * in the current implementation, if virtio 1.0 with pio
-		 * notity, its bar idx should be set to non-zero
-		 */
-		if (base->modern_pio_bar_idx) {
-			bar = &vdev->base->dev->bar[base->modern_pio_bar_idx];
-			ioeventfd.data = vdev->vq_idx + idx;
-			ioeventfd.addr = bar->addr;
-			ioeventfd.len = 2;
-			ioeventfd.flags |= (ACRN_IOEVENTFD_FLAG_DATAMATCH |
-				ACRN_IOEVENTFD_FLAG_PIO);
-		} else if (base->modern_mmio_bar_idx) {
-			bar = &vdev->base->dev->bar[base->modern_mmio_bar_idx];
-			ioeventfd.data = 0;
-			ioeventfd.addr = bar->addr + VIRTIO_CAP_NOTIFY_OFFSET
-				+ (vdev->vq_idx + idx) *
-				VIRTIO_MODERN_NOTIFY_OFF_MULT;
-			ioeventfd.len = 2;
-			/* no additional flag bit should be set for MMIO */
-		} else {
-			WPRINTF("invalid virtio 1.0 parameters, 0x%lx\n",
-				base->device_caps);
-			return -1;
-		}
-	} else {
-		bar = &vdev->base->dev->bar[base->legacy_pio_bar_idx];
-		ioeventfd.data = vdev->vq_idx + idx;
-		ioeventfd.addr = bar->addr + VIRTIO_PCI_QUEUE_NOTIFY;
-		ioeventfd.len = 2;
-		ioeventfd.flags |= (ACRN_IOEVENTFD_FLAG_DATAMATCH |
-			ACRN_IOEVENTFD_FLAG_PIO);
-	}
-
-	ioeventfd.fd = vq->kick_fd;
-	DPRINTF("[ioeventfd: %d][0x%lx@%d][flags: 0x%x][data: 0x%lx]\n",
-		ioeventfd.fd, ioeventfd.addr, ioeventfd.len,
-		ioeventfd.flags, ioeventfd.data);
-	rc = vm_ioeventfd(vdev->base->dev->vmctx, &ioeventfd);
-	if (rc < 0) {
-		WPRINTF("vm_ioeventfd failed rc = %d, errno = %d\n",
-			rc, errno);
-		return -1;
-	}
-
 	/* register irqfd for notify */
 	mte = &vdev->base->dev->msix.table[vqi->msix_idx];
 	msi.msi_addr = mte->addr;

devicemodel/hw/pci/virtio/virtio.c

@@ -30,12 +30,19 @@
 #include <pthread.h>
 #include <string.h>
 #include <stdlib.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <unistd.h>
 
 #include "dm.h"
 #include "pci_core.h"
 #include "virtio.h"
 #include "timer.h"
 #include <atomic.h>
+#include "hsm_ioctl_defs.h"
+#include "iothread.h"
+#include "vmmapi.h"
+#include <errno.h>
 
 /*
  * Functions for dealing with generalized "virtual devices" as
@@ -52,6 +59,68 @@
 static uint8_t virtio_poll_enabled;
 static size_t virtio_poll_interval;
 
+static
+void iothread_handler(void *arg)
+{
+	struct virtio_iothread *viothrd = arg;
+	struct virtio_base *base = viothrd->base;
+	int idx = viothrd->idx;
+	struct virtio_vq_info *vq = &base->queues[idx];
+
+	if (viothrd->iothread_run) {
+		if (base->mtx)
+			pthread_mutex_lock(base->mtx);
+		(*viothrd->iothread_run)(base, vq);
+		if (base->mtx)
+			pthread_mutex_unlock(base->mtx);
+	}
+}
+
+void
+virtio_set_iothread(struct virtio_base *base,
+			  bool is_register)
+{
+	struct virtio_vq_info *vq;
+	struct virtio_ops *vops;
+	int idx;
+
+	vops = base->vops;
+	for (idx = 0; idx < vops->nvq; idx++) {
+		vq = &base->queues[idx];
+
+		if ((vq->viothrd.ioevent_started && is_register) ||
+			(!vq->viothrd.ioevent_started && !is_register))
+				return;
+		if (is_register) {
+			vq->viothrd.kick_fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+
+			if (vops->qnotify)
+				vq->viothrd.iothread_run = vops->qnotify;
+			else if (vq->notify)
+				vq->viothrd.iothread_run = vq->notify;
+			else
+				vq->viothrd.iothread_run = NULL;
+			vq->viothrd.base = base;
+			vq->viothrd.idx = idx;
+			vq->viothrd.iomvt.arg = &vq->viothrd;
+			vq->viothrd.iomvt.run = iothread_handler;
+
+			if (!iothread_add(vq->viothrd.kick_fd, &vq->viothrd.iomvt))
+				if (!virtio_register_ioeventfd(base, idx, true))
+					vq->viothrd.ioevent_started = true;
+		} else {
+			if (!virtio_register_ioeventfd(base, idx, false))
+				if (!iothread_del(vq->viothrd.kick_fd)) {
+					vq->viothrd.ioevent_started = false;
+					if (vq->viothrd.kick_fd) {
+						close(vq->viothrd.kick_fd);
+						vq->viothrd.kick_fd = -1;
+					}
+				}
+		}
+	}
+}
+
 static void
 virtio_start_timer(struct acrn_timer *timer, time_t sec, time_t nsec)
 {
@@ -170,6 +239,8 @@ virtio_reset_dev(struct virtio_base *base)
 
 	acrn_timer_deinit(&base->polling_timer);
 	base->polling_in_progress = 0;
+	if (base->iothread)
+		virtio_set_iothread(base, false);
 
 	nvq = base->vops->nvq;
 	for (vq = base->queues, i = 0; i < nvq; vq++, i++) {
@@ -1021,6 +1092,15 @@ bad:
 			 */
 			virtio_start_timer(&base->polling_timer, 5, 0);
 		}
+		if (!virtio_poll_enabled &&
+			base->backend_type == BACKEND_VBSU &&
+			base->iothread) {
+			if (value & VIRTIO_CONFIG_S_DRIVER_OK) {
+				virtio_set_iothread(base, true);
+			} else {
+				virtio_set_iothread(base, false);
+			}
+		}
 		break;
 	case VIRTIO_MSI_CONFIG_VECTOR:
 		base->msix_cfg_idx = value;
@@ -1445,6 +1525,13 @@ virtio_common_cfg_write(struct pci_vdev *dev, uint64_t offset, int size,
 			(*vops->set_status)(DEV_STRUCT(base), value);
 		if ((base->status == 0) && (vops->reset))
 			(*vops->reset)(DEV_STRUCT(base));
+		if (base->backend_type == BACKEND_VBSU && base->iothread) {
+			if (value & VIRTIO_CONFIG_S_DRIVER_OK) {
+				virtio_set_iothread(base, true);
+			} else {
+				virtio_set_iothread(base, false);
+			}
+		}
 		/* TODO: virtio poll mode for modern devices */
 		break;
 	case VIRTIO_PCI_COMMON_Q_SELECT:
@@ -1902,3 +1989,67 @@ acrn_parse_virtio_poll_interval(const char *optarg)
 
 	return 0;
 }
+
+int virtio_register_ioeventfd(struct virtio_base *base, int idx, bool is_register)
+{
+	struct acrn_ioeventfd ioeventfd = {0};
+	struct virtio_vq_info *vq;
+	struct pcibar *bar;
+	int rc = 0;
+
+	if (!is_register)
+		ioeventfd.flags = ACRN_IOEVENTFD_FLAG_DEASSIGN;
+
+	/* register ioeventfd for kick */
+	if (base->device_caps & (1UL << VIRTIO_F_VERSION_1)) {
+		/*
+		 * in the current implementation, if virtio 1.0 with pio
+		 * notity, its bar idx should be set to non-zero
+		 */
+		if (base->modern_pio_bar_idx) {
+			bar = &base->dev->bar[base->modern_pio_bar_idx];
+			ioeventfd.data = idx;
+			ioeventfd.addr = bar->addr;
+			ioeventfd.len = 2;
+			ioeventfd.flags |= (ACRN_IOEVENTFD_FLAG_DATAMATCH |
+				ACRN_IOEVENTFD_FLAG_PIO);
+		} else if (base->modern_mmio_bar_idx) {
+			bar = &base->dev->bar[base->modern_mmio_bar_idx];
+			ioeventfd.data = 0;
+			ioeventfd.addr = bar->addr + VIRTIO_CAP_NOTIFY_OFFSET
+				+ idx *
+				VIRTIO_MODERN_NOTIFY_OFF_MULT;
+			ioeventfd.len = 2;
+			/* no additional flag bit should be set for MMIO */
+		} else {
+			pr_warn("invalid virtio 1.0 parameters, 0x%lx\n",
+				base->device_caps);
+			return -1;
+		}
+	} else {
+		bar = &base->dev->bar[base->legacy_pio_bar_idx];
+		ioeventfd.data = idx;
+		ioeventfd.addr = bar->addr + VIRTIO_PCI_QUEUE_NOTIFY;
+		ioeventfd.len = 2;
+		ioeventfd.flags |= (ACRN_IOEVENTFD_FLAG_DATAMATCH |
+			ACRN_IOEVENTFD_FLAG_PIO);
+	}
+
+	vq = &base->queues[idx];
+	ioeventfd.fd = vq->viothrd.kick_fd;
+	if (is_register)
+		pr_info("[ioeventfd: %d][0x%lx@%d][flags: 0x%x][data: 0x%lx]\r\n",
+			ioeventfd.fd, ioeventfd.addr, ioeventfd.len,
+			ioeventfd.flags, ioeventfd.data);
+	else
+		pr_info("[ioeventfd: %d][0x%lx@%d] unregister\r\n",
+				ioeventfd.fd, ioeventfd.addr, ioeventfd.len);
+
+	rc = vm_ioeventfd(base->dev->vmctx, &ioeventfd);
+	if (rc < 0) {
+		pr_err("vm_ioeventfd failed rc = %d, errno = %d\n",
+			rc, errno);
+		return -1;
+	}
+	return 0;
+}

devicemodel/include/virtio.h

@@ -132,6 +132,7 @@
 
 #include "types.h"
 #include "timer.h"
+#include "iothread.h"
 
 /**
  * @brief virtio API
@@ -345,6 +346,7 @@ struct virtio_vq_info;
 struct virtio_base {
 	struct virtio_ops *vops;	/**< virtio operations */
 	int	flags;			/**< VIRTIO_* flags from above */
+	bool	iothread;
 	pthread_mutex_t *mtx;		/**< POSIX mutex, if any */
 	struct pci_vdev *dev;		/**< PCI device instance */
 	uint64_t negotiated_caps;	/**< negotiated capabilities */
@@ -420,6 +422,15 @@ struct virtio_ops {
  * (but more easily) computable, and this time we'll compute them:
  * they're just XX_ring[N].
  */
+struct virtio_iothread {
+	struct virtio_base *base;
+	int idx;
+	int kick_fd;
+	bool	ioevent_started;
+	struct iothread_mevent iomvt;
+	void (*iothread_run)(void *, struct virtio_vq_info *);
+};
+
 struct virtio_vq_info {
 	uint16_t qsize;		/**< size of this queue (a power of 2) */
 	void	(*notify)(void *, struct virtio_vq_info *);
@@ -435,6 +446,7 @@ struct virtio_vq_info {
 	uint16_t msix_idx;	/**< MSI-X index, or VIRTIO_MSI_NO_VECTOR */
 
 	uint32_t pfn;		/**< PFN of virt queue (not shifted!) */
+	struct virtio_iothread viothrd;
 
 	volatile struct vring_desc *desc;
 				/**< descriptor array */
@@ -764,4 +776,5 @@ uint32_t virtio_device_cfg_read(
 int virtio_set_modern_pio_bar(
 		struct virtio_base *base, int barnum);
 
+int virtio_register_ioeventfd(struct virtio_base *base, int idx, bool is_register);
 #endif	/* _VIRTIO_H_ */