380 lines
12 KiB
Diff
380 lines
12 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: Gao Junhao <junhao.gao@intel.com>
|
|
Date: Thu, 6 Jun 2019 02:18:23 +0000
|
|
Subject: [PATCH] virtio: virtio polling mode POC
|
|
|
|
add virtio polling mode POC support
|
|
|
|
Changelog:
|
|
v2 -> v3:
|
|
modified function container_of usage.
|
|
use function ns_to_ktime to replace ktime_set.
|
|
|
|
Tracked-On: projectacrn/acrn-hypervisor#3237
|
|
Signed-off-by: Jian Jun Chen <jian.jun.chen@intel.com>
|
|
Signed-off-by: Gao Junhao <junhao.gao@intel.com>
|
|
Reviewed-by: Zhao Yakui <yakui.zhao@intel.com>
|
|
---
|
|
drivers/virtio/virtio.c | 104 ++++++++++++++++++++++++++++++++++
|
|
drivers/virtio/virtio_ring.c | 54 +++++++++++++++---
|
|
include/linux/virtio.h | 21 +++++++
|
|
include/linux/virtio_config.h | 17 ------
|
|
4 files changed, 170 insertions(+), 26 deletions(-)
|
|
|
|
diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
|
|
index 59e36ef4920f..7c2966260f98 100644
|
|
--- a/drivers/virtio/virtio.c
|
|
+++ b/drivers/virtio/virtio.c
|
|
@@ -68,6 +68,13 @@ static struct attribute *virtio_dev_attrs[] = {
|
|
};
|
|
ATTRIBUTE_GROUPS(virtio_dev);
|
|
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+static unsigned long virtio_polling_interval = 10000000UL;
|
|
+module_param(virtio_polling_interval, ulong, 0644);
|
|
+MODULE_PARM_DESC(virtio_polling_interval,
|
|
+ "virtio polling interval in ns. (default: 10000000)");
|
|
+#endif
|
|
+
|
|
static inline int virtio_id_match(const struct virtio_device *dev,
|
|
const struct virtio_device_id *id)
|
|
{
|
|
@@ -187,6 +194,74 @@ int virtio_finalize_features(struct virtio_device *dev)
|
|
}
|
|
EXPORT_SYMBOL_GPL(virtio_finalize_features);
|
|
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+static enum hrtimer_restart virtio_handle_polling_timer(struct hrtimer *t)
|
|
+{
|
|
+ struct virtio_device *dev =
|
|
+ container_of(t, struct virtio_device, hr_timer);
|
|
+
|
|
+ virtio_poll_virtqueues(dev);
|
|
+ /* virtio_config_changed(dev); */
|
|
+ hrtimer_start(&dev->hr_timer, ns_to_ktime(virtio_polling_interval),
|
|
+ HRTIMER_MODE_REL);
|
|
+ return HRTIMER_NORESTART;
|
|
+}
|
|
+
|
|
+static inline void virtio_init_polling_timer(struct virtio_device *dev)
|
|
+{
|
|
+ if (virtio_polling_mode_enabled(dev)) {
|
|
+ hrtimer_init(&dev->hr_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
|
+ (dev->hr_timer).function = virtio_handle_polling_timer;
|
|
+ }
|
|
+}
|
|
+
|
|
+static void virtio_start_polling_timer(struct virtio_device *dev)
|
|
+{
|
|
+ if (virtio_polling_mode_enabled(dev)) {
|
|
+ hrtimer_start(&dev->hr_timer,
|
|
+ ns_to_ktime(virtio_polling_interval), HRTIMER_MODE_REL);
|
|
+ dev_notice(&dev->dev, "start polling timer: %lu\n",
|
|
+ virtio_polling_interval);
|
|
+ }
|
|
+}
|
|
+
|
|
+static void virtio_stop_polling_timer(struct virtio_device *dev)
|
|
+{
|
|
+ if (virtio_polling_mode_enabled(dev)) {
|
|
+ hrtimer_cancel(&dev->hr_timer);
|
|
+ dev_notice(&dev->dev, "stop polling timer\n");
|
|
+ }
|
|
+}
|
|
+#endif
|
|
+
|
|
+/**
|
|
+ * virtio_device_ready - enable vq use in probe function
|
|
+ * @vdev: the device
|
|
+ *
|
|
+ * Driver must call this to use vqs in the probe function.
|
|
+ *
|
|
+ * Note: vqs are enabled automatically after probe returns.
|
|
+ */
|
|
+void virtio_device_ready(struct virtio_device *dev)
|
|
+{
|
|
+ unsigned int status = dev->config->get_status(dev);
|
|
+
|
|
+ WARN_ON(status & VIRTIO_CONFIG_S_DRIVER_OK);
|
|
+ dev->config->set_status(dev, status | VIRTIO_CONFIG_S_DRIVER_OK);
|
|
+
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+ /*
|
|
+ * In polling mode, virtqueue interrupts are disabled from the
|
|
+ * beginning. we must make sure the polling timer is started
|
|
+ * just after the virtqueue is ready. When vring_create_virtqueue
|
|
+ * is called the virtqueues are not ready. Start polling timer
|
|
+ * when status is changed to DRIVER_OK is a good chance then.
|
|
+ */
|
|
+ virtio_start_polling_timer(dev);
|
|
+#endif
|
|
+}
|
|
+EXPORT_SYMBOL_GPL(virtio_device_ready);
|
|
+
|
|
static int virtio_dev_probe(struct device *_d)
|
|
{
|
|
int err, i;
|
|
@@ -202,6 +277,13 @@ static int virtio_dev_probe(struct device *_d)
|
|
/* Figure out what features the device supports. */
|
|
device_features = dev->config->get_features(dev);
|
|
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+ if (virtio_polling_mode_enabled(dev)) {
|
|
+ device_features &= ~VIRTIO_F_NOTIFY_ON_EMPTY;
|
|
+ device_features &= ~VIRTIO_RING_F_EVENT_IDX;
|
|
+ }
|
|
+#endif
|
|
+
|
|
/* Figure out what features the driver supports. */
|
|
driver_features = 0;
|
|
for (i = 0; i < drv->feature_table_size; i++) {
|
|
@@ -242,6 +324,10 @@ static int virtio_dev_probe(struct device *_d)
|
|
if (err)
|
|
goto err;
|
|
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+ virtio_init_polling_timer(dev);
|
|
+#endif
|
|
+
|
|
err = drv->probe(dev);
|
|
if (err)
|
|
goto err;
|
|
@@ -257,6 +343,9 @@ static int virtio_dev_probe(struct device *_d)
|
|
|
|
return 0;
|
|
err:
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+ virtio_stop_polling_timer(dev);
|
|
+#endif
|
|
virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
|
|
return err;
|
|
|
|
@@ -269,6 +358,9 @@ static int virtio_dev_remove(struct device *_d)
|
|
|
|
virtio_config_disable(dev);
|
|
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+ virtio_stop_polling_timer(dev);
|
|
+#endif
|
|
drv->remove(dev);
|
|
|
|
/* Driver should have reset device. */
|
|
@@ -340,6 +432,10 @@ int register_virtio_device(struct virtio_device *dev)
|
|
|
|
INIT_LIST_HEAD(&dev->vqs);
|
|
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+ spin_lock_init(&dev->vq_lock);
|
|
+#endif
|
|
+
|
|
/*
|
|
* device_add() causes the bus infrastructure to look for a matching
|
|
* driver.
|
|
@@ -370,6 +466,10 @@ int virtio_device_freeze(struct virtio_device *dev)
|
|
|
|
virtio_config_disable(dev);
|
|
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+ virtio_stop_polling_timer(dev);
|
|
+#endif
|
|
+
|
|
dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;
|
|
|
|
if (drv && drv->freeze)
|
|
@@ -415,6 +515,10 @@ int virtio_device_restore(struct virtio_device *dev)
|
|
/* Finally, tell the device we're all set */
|
|
virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
|
|
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+ virtio_start_polling_timer(dev);
|
|
+#endif
|
|
+
|
|
virtio_config_enable(dev);
|
|
|
|
return 0;
|
|
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
|
|
index df7980aef927..feb3013f7f9a 100644
|
|
--- a/drivers/virtio/virtio_ring.c
|
|
+++ b/drivers/virtio/virtio_ring.c
|
|
@@ -804,11 +804,19 @@ unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)
|
|
/* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
|
|
* either clear the flags bit or point the event index at the next
|
|
* entry. Always do both to keep code simple. */
|
|
- if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
|
|
- vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
|
|
- if (!vq->event)
|
|
- vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+ if (!virtio_polling_mode_enabled(_vq->vdev)) {
|
|
+#endif
|
|
+ if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
|
|
+ vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
|
|
+ if (!vq->event)
|
|
+ vq->vring.avail->flags =
|
|
+ cpu_to_virtio16(_vq->vdev,
|
|
+ vq->avail_flags_shadow);
|
|
+ }
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
}
|
|
+#endif
|
|
vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx);
|
|
END_USE(vq);
|
|
return last_used_idx;
|
|
@@ -879,11 +887,19 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
|
|
/* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
|
|
* either clear the flags bit or point the event index at the next
|
|
* entry. Always update the event index to keep code simple. */
|
|
- if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
|
|
- vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
|
|
- if (!vq->event)
|
|
- vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+ if (!virtio_polling_mode_enabled(_vq->vdev)) {
|
|
+#endif
|
|
+ if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
|
|
+ vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
|
|
+ if (!vq->event)
|
|
+ vq->vring.avail->flags =
|
|
+ cpu_to_virtio16(_vq->vdev,
|
|
+ vq->avail_flags_shadow);
|
|
+ }
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
}
|
|
+#endif
|
|
/* TODO: tune this threshold */
|
|
bufs = (u16)(vq->avail_idx_shadow - vq->last_used_idx) * 3 / 4;
|
|
|
|
@@ -967,6 +983,7 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index,
|
|
{
|
|
unsigned int i;
|
|
struct vring_virtqueue *vq;
|
|
+ bool disable_cb = (callback == NULL);
|
|
|
|
vq = kmalloc(sizeof(*vq) + vring.num * sizeof(struct vring_desc_state),
|
|
GFP_KERNEL);
|
|
@@ -999,8 +1016,13 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index,
|
|
!context;
|
|
vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
|
|
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+ if (!disable_cb && virtio_polling_mode_enabled(vdev))
|
|
+ disable_cb = true;
|
|
+#endif
|
|
+
|
|
/* No callback? Tell other side not to bother us. */
|
|
- if (!callback) {
|
|
+ if (disable_cb) {
|
|
vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
|
|
if (!vq->event)
|
|
vq->vring.avail->flags = cpu_to_virtio16(vdev, vq->avail_flags_shadow);
|
|
@@ -1254,4 +1276,18 @@ const struct vring *virtqueue_get_vring(struct virtqueue *vq)
|
|
}
|
|
EXPORT_SYMBOL_GPL(virtqueue_get_vring);
|
|
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+void virtio_poll_virtqueues(struct virtio_device *dev)
|
|
+{
|
|
+ struct virtqueue *_vq;
|
|
+ unsigned long flags;
|
|
+
|
|
+ spin_lock_irqsave(&dev->vq_lock, flags);
|
|
+ list_for_each_entry(_vq, &dev->vqs, list)
|
|
+ vring_interrupt(0, _vq); /* parameter irq is not used */
|
|
+ spin_unlock_irqrestore(&dev->vq_lock, flags);
|
|
+}
|
|
+EXPORT_SYMBOL_GPL(virtio_poll_virtqueues);
|
|
+#endif
|
|
+
|
|
MODULE_LICENSE("GPL");
|
|
diff --git a/include/linux/virtio.h b/include/linux/virtio.h
|
|
index fa1b5da2804e..972d63b5675a 100644
|
|
--- a/include/linux/virtio.h
|
|
+++ b/include/linux/virtio.h
|
|
@@ -90,6 +90,10 @@ dma_addr_t virtqueue_get_desc_addr(struct virtqueue *vq);
|
|
dma_addr_t virtqueue_get_avail_addr(struct virtqueue *vq);
|
|
dma_addr_t virtqueue_get_used_addr(struct virtqueue *vq);
|
|
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+void virtio_poll_virtqueues(struct virtio_device *dev);
|
|
+#endif
|
|
+
|
|
/*
|
|
* Legacy accessors -- in almost all cases, these are the wrong functions
|
|
* to use.
|
|
@@ -135,6 +139,10 @@ struct virtio_device {
|
|
struct list_head vqs;
|
|
u64 features;
|
|
void *priv;
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+ spinlock_t vq_lock;
|
|
+ struct hrtimer hr_timer;
|
|
+#endif
|
|
};
|
|
|
|
static inline struct virtio_device *dev_to_virtio(struct device *_dev)
|
|
@@ -152,6 +160,7 @@ void virtio_config_changed(struct virtio_device *dev);
|
|
void virtio_config_disable(struct virtio_device *dev);
|
|
void virtio_config_enable(struct virtio_device *dev);
|
|
int virtio_finalize_features(struct virtio_device *dev);
|
|
+void virtio_device_ready(struct virtio_device *dev);
|
|
#ifdef CONFIG_PM_SLEEP
|
|
int virtio_device_freeze(struct virtio_device *dev);
|
|
int virtio_device_restore(struct virtio_device *dev);
|
|
@@ -184,6 +193,9 @@ struct virtio_driver {
|
|
unsigned int feature_table_size;
|
|
const unsigned int *feature_table_legacy;
|
|
unsigned int feature_table_size_legacy;
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+ bool polling_mode;
|
|
+#endif
|
|
int (*validate)(struct virtio_device *dev);
|
|
int (*probe)(struct virtio_device *dev);
|
|
void (*scan)(struct virtio_device *dev);
|
|
@@ -200,6 +212,15 @@ static inline struct virtio_driver *drv_to_virtio(struct device_driver *drv)
|
|
return container_of(drv, struct virtio_driver, driver);
|
|
}
|
|
|
|
+#ifdef CONFIG_VIRTIO_PMD
|
|
+static inline bool virtio_polling_mode_enabled(struct virtio_device *dev)
|
|
+{
|
|
+ struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
|
|
+
|
|
+ return drv->polling_mode;
|
|
+}
|
|
+#endif
|
|
+
|
|
int register_virtio_driver(struct virtio_driver *drv);
|
|
void unregister_virtio_driver(struct virtio_driver *drv);
|
|
|
|
diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
|
|
index 32baf8e26735..c85a4ae611c9 100644
|
|
--- a/include/linux/virtio_config.h
|
|
+++ b/include/linux/virtio_config.h
|
|
@@ -202,23 +202,6 @@ int virtio_find_vqs_ctx(struct virtio_device *vdev, unsigned nvqs,
|
|
desc);
|
|
}
|
|
|
|
-/**
|
|
- * virtio_device_ready - enable vq use in probe function
|
|
- * @vdev: the device
|
|
- *
|
|
- * Driver must call this to use vqs in the probe function.
|
|
- *
|
|
- * Note: vqs are enabled automatically after probe returns.
|
|
- */
|
|
-static inline
|
|
-void virtio_device_ready(struct virtio_device *dev)
|
|
-{
|
|
- unsigned status = dev->config->get_status(dev);
|
|
-
|
|
- BUG_ON(status & VIRTIO_CONFIG_S_DRIVER_OK);
|
|
- dev->config->set_status(dev, status | VIRTIO_CONFIG_S_DRIVER_OK);
|
|
-}
|
|
-
|
|
static inline
|
|
const char *virtio_bus_name(struct virtio_device *vdev)
|
|
{
|
|
--
|
|
https://clearlinux.org
|
|
|