From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Gao Junhao Date: Thu, 6 Jun 2019 02:18:23 +0000 Subject: [PATCH] virtio: virtio polling mode POC add virtio polling mode POC support Changelog: v2 -> v3: modified function container_of usage. use function ns_to_ktime to replace ktime_set. Tracked-On: projectacrn/acrn-hypervisor#3237 Signed-off-by: Jian Jun Chen Signed-off-by: Gao Junhao Reviewed-by: Zhao Yakui --- drivers/virtio/virtio.c | 104 ++++++++++++++++++++++++++++++++++ drivers/virtio/virtio_ring.c | 54 +++++++++++++++--- include/linux/virtio.h | 21 +++++++ include/linux/virtio_config.h | 17 ------ 4 files changed, 170 insertions(+), 26 deletions(-) diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 59e36ef4920f..7c2966260f98 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -68,6 +68,13 @@ static struct attribute *virtio_dev_attrs[] = { }; ATTRIBUTE_GROUPS(virtio_dev); +#ifdef CONFIG_VIRTIO_PMD +static unsigned long virtio_polling_interval = 10000000UL; +module_param(virtio_polling_interval, ulong, 0644); +MODULE_PARM_DESC(virtio_polling_interval, + "virtio polling interval in ns. (default: 10000000)"); +#endif + static inline int virtio_id_match(const struct virtio_device *dev, const struct virtio_device_id *id) { @@ -187,6 +194,74 @@ int virtio_finalize_features(struct virtio_device *dev) } EXPORT_SYMBOL_GPL(virtio_finalize_features); +#ifdef CONFIG_VIRTIO_PMD +static enum hrtimer_restart virtio_handle_polling_timer(struct hrtimer *t) +{ + struct virtio_device *dev = + container_of(t, struct virtio_device, hr_timer); + + virtio_poll_virtqueues(dev); + /* virtio_config_changed(dev); */ + hrtimer_start(&dev->hr_timer, ns_to_ktime(virtio_polling_interval), + HRTIMER_MODE_REL); + return HRTIMER_NORESTART; +} + +static inline void virtio_init_polling_timer(struct virtio_device *dev) +{ + if (virtio_polling_mode_enabled(dev)) { + hrtimer_init(&dev->hr_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + (dev->hr_timer).function = virtio_handle_polling_timer; + } +} + +static void virtio_start_polling_timer(struct virtio_device *dev) +{ + if (virtio_polling_mode_enabled(dev)) { + hrtimer_start(&dev->hr_timer, + ns_to_ktime(virtio_polling_interval), HRTIMER_MODE_REL); + dev_notice(&dev->dev, "start polling timer: %lu\n", + virtio_polling_interval); + } +} + +static void virtio_stop_polling_timer(struct virtio_device *dev) +{ + if (virtio_polling_mode_enabled(dev)) { + hrtimer_cancel(&dev->hr_timer); + dev_notice(&dev->dev, "stop polling timer\n"); + } +} +#endif + +/** + * virtio_device_ready - enable vq use in probe function + * @vdev: the device + * + * Driver must call this to use vqs in the probe function. + * + * Note: vqs are enabled automatically after probe returns. + */ +void virtio_device_ready(struct virtio_device *dev) +{ + unsigned int status = dev->config->get_status(dev); + + WARN_ON(status & VIRTIO_CONFIG_S_DRIVER_OK); + dev->config->set_status(dev, status | VIRTIO_CONFIG_S_DRIVER_OK); + +#ifdef CONFIG_VIRTIO_PMD + /* + * In polling mode, virtqueue interrupts are disabled from the + * beginning. we must make sure the polling timer is started + * just after the virtqueue is ready. When vring_create_virtqueue + * is called the virtqueues are not ready. Start polling timer + * when status is changed to DRIVER_OK is a good chance then. + */ + virtio_start_polling_timer(dev); +#endif +} +EXPORT_SYMBOL_GPL(virtio_device_ready); + static int virtio_dev_probe(struct device *_d) { int err, i; @@ -202,6 +277,13 @@ static int virtio_dev_probe(struct device *_d) /* Figure out what features the device supports. */ device_features = dev->config->get_features(dev); +#ifdef CONFIG_VIRTIO_PMD + if (virtio_polling_mode_enabled(dev)) { + device_features &= ~VIRTIO_F_NOTIFY_ON_EMPTY; + device_features &= ~VIRTIO_RING_F_EVENT_IDX; + } +#endif + /* Figure out what features the driver supports. */ driver_features = 0; for (i = 0; i < drv->feature_table_size; i++) { @@ -242,6 +324,10 @@ static int virtio_dev_probe(struct device *_d) if (err) goto err; +#ifdef CONFIG_VIRTIO_PMD + virtio_init_polling_timer(dev); +#endif + err = drv->probe(dev); if (err) goto err; @@ -257,6 +343,9 @@ static int virtio_dev_probe(struct device *_d) return 0; err: +#ifdef CONFIG_VIRTIO_PMD + virtio_stop_polling_timer(dev); +#endif virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED); return err; @@ -269,6 +358,9 @@ static int virtio_dev_remove(struct device *_d) virtio_config_disable(dev); +#ifdef CONFIG_VIRTIO_PMD + virtio_stop_polling_timer(dev); +#endif drv->remove(dev); /* Driver should have reset device. */ @@ -340,6 +432,10 @@ int register_virtio_device(struct virtio_device *dev) INIT_LIST_HEAD(&dev->vqs); +#ifdef CONFIG_VIRTIO_PMD + spin_lock_init(&dev->vq_lock); +#endif + /* * device_add() causes the bus infrastructure to look for a matching * driver. @@ -370,6 +466,10 @@ int virtio_device_freeze(struct virtio_device *dev) virtio_config_disable(dev); +#ifdef CONFIG_VIRTIO_PMD + virtio_stop_polling_timer(dev); +#endif + dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED; if (drv && drv->freeze) @@ -415,6 +515,10 @@ int virtio_device_restore(struct virtio_device *dev) /* Finally, tell the device we're all set */ virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK); +#ifdef CONFIG_VIRTIO_PMD + virtio_start_polling_timer(dev); +#endif + virtio_config_enable(dev); return 0; diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index df7980aef927..feb3013f7f9a 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -804,11 +804,19 @@ unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq) /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to * either clear the flags bit or point the event index at the next * entry. Always do both to keep code simple. */ - if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { - vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; - if (!vq->event) - vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); +#ifdef CONFIG_VIRTIO_PMD + if (!virtio_polling_mode_enabled(_vq->vdev)) { +#endif + if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { + vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; + if (!vq->event) + vq->vring.avail->flags = + cpu_to_virtio16(_vq->vdev, + vq->avail_flags_shadow); + } +#ifdef CONFIG_VIRTIO_PMD } +#endif vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx); END_USE(vq); return last_used_idx; @@ -879,11 +887,19 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq) /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to * either clear the flags bit or point the event index at the next * entry. Always update the event index to keep code simple. */ - if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { - vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; - if (!vq->event) - vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); +#ifdef CONFIG_VIRTIO_PMD + if (!virtio_polling_mode_enabled(_vq->vdev)) { +#endif + if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { + vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; + if (!vq->event) + vq->vring.avail->flags = + cpu_to_virtio16(_vq->vdev, + vq->avail_flags_shadow); + } +#ifdef CONFIG_VIRTIO_PMD } +#endif /* TODO: tune this threshold */ bufs = (u16)(vq->avail_idx_shadow - vq->last_used_idx) * 3 / 4; @@ -967,6 +983,7 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, { unsigned int i; struct vring_virtqueue *vq; + bool disable_cb = (callback == NULL); vq = kmalloc(sizeof(*vq) + vring.num * sizeof(struct vring_desc_state), GFP_KERNEL); @@ -999,8 +1016,13 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, !context; vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); +#ifdef CONFIG_VIRTIO_PMD + if (!disable_cb && virtio_polling_mode_enabled(vdev)) + disable_cb = true; +#endif + /* No callback? Tell other side not to bother us. */ - if (!callback) { + if (disable_cb) { vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; if (!vq->event) vq->vring.avail->flags = cpu_to_virtio16(vdev, vq->avail_flags_shadow); @@ -1254,4 +1276,18 @@ const struct vring *virtqueue_get_vring(struct virtqueue *vq) } EXPORT_SYMBOL_GPL(virtqueue_get_vring); +#ifdef CONFIG_VIRTIO_PMD +void virtio_poll_virtqueues(struct virtio_device *dev) +{ + struct virtqueue *_vq; + unsigned long flags; + + spin_lock_irqsave(&dev->vq_lock, flags); + list_for_each_entry(_vq, &dev->vqs, list) + vring_interrupt(0, _vq); /* parameter irq is not used */ + spin_unlock_irqrestore(&dev->vq_lock, flags); +} +EXPORT_SYMBOL_GPL(virtio_poll_virtqueues); +#endif + MODULE_LICENSE("GPL"); diff --git a/include/linux/virtio.h b/include/linux/virtio.h index fa1b5da2804e..972d63b5675a 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -90,6 +90,10 @@ dma_addr_t virtqueue_get_desc_addr(struct virtqueue *vq); dma_addr_t virtqueue_get_avail_addr(struct virtqueue *vq); dma_addr_t virtqueue_get_used_addr(struct virtqueue *vq); +#ifdef CONFIG_VIRTIO_PMD +void virtio_poll_virtqueues(struct virtio_device *dev); +#endif + /* * Legacy accessors -- in almost all cases, these are the wrong functions * to use. @@ -135,6 +139,10 @@ struct virtio_device { struct list_head vqs; u64 features; void *priv; +#ifdef CONFIG_VIRTIO_PMD + spinlock_t vq_lock; + struct hrtimer hr_timer; +#endif }; static inline struct virtio_device *dev_to_virtio(struct device *_dev) @@ -152,6 +160,7 @@ void virtio_config_changed(struct virtio_device *dev); void virtio_config_disable(struct virtio_device *dev); void virtio_config_enable(struct virtio_device *dev); int virtio_finalize_features(struct virtio_device *dev); +void virtio_device_ready(struct virtio_device *dev); #ifdef CONFIG_PM_SLEEP int virtio_device_freeze(struct virtio_device *dev); int virtio_device_restore(struct virtio_device *dev); @@ -184,6 +193,9 @@ struct virtio_driver { unsigned int feature_table_size; const unsigned int *feature_table_legacy; unsigned int feature_table_size_legacy; +#ifdef CONFIG_VIRTIO_PMD + bool polling_mode; +#endif int (*validate)(struct virtio_device *dev); int (*probe)(struct virtio_device *dev); void (*scan)(struct virtio_device *dev); @@ -200,6 +212,15 @@ static inline struct virtio_driver *drv_to_virtio(struct device_driver *drv) return container_of(drv, struct virtio_driver, driver); } +#ifdef CONFIG_VIRTIO_PMD +static inline bool virtio_polling_mode_enabled(struct virtio_device *dev) +{ + struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); + + return drv->polling_mode; +} +#endif + int register_virtio_driver(struct virtio_driver *drv); void unregister_virtio_driver(struct virtio_driver *drv); diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index 32baf8e26735..c85a4ae611c9 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -202,23 +202,6 @@ int virtio_find_vqs_ctx(struct virtio_device *vdev, unsigned nvqs, desc); } -/** - * virtio_device_ready - enable vq use in probe function - * @vdev: the device - * - * Driver must call this to use vqs in the probe function. - * - * Note: vqs are enabled automatically after probe returns. - */ -static inline -void virtio_device_ready(struct virtio_device *dev) -{ - unsigned status = dev->config->get_status(dev); - - BUG_ON(status & VIRTIO_CONFIG_S_DRIVER_OK); - dev->config->set_status(dev, status | VIRTIO_CONFIG_S_DRIVER_OK); -} - static inline const char *virtio_bus_name(struct virtio_device *vdev) { -- https://clearlinux.org