OrgACRN
/
acrn-kernel
mirror of https://github.com/projectacrn/acrn-kernel.git
1
0
Fork 0
Code Issues Releases Wiki Activity

io_uring: sendzc with fixed buffers 

Allow zerocopy sends to use fixed buffers. There is an optimisation for
this case, the network layer don't need to reference the pages, see
SKBFL_MANAGED_FRAG_REFS, so io_uring have to ensure validity of fixed
buffers until the notifier is released.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/e1d8bd1b5934e541d90c1824eb4020ae3f5f43f3.1657643355.git.asml.silence@gmail.com
[axboe: fold in 32-bit pointer cast warning fix]
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2022-07-12 21:52:46 +01:00 committed by Jens Axboe
parent 092aeedb75
commit 10c7d33ecd
2 changed files with 29 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/uapi/linux/io_uring.h
View File

@ -272,9 +272,13 @@ enum io_uring_op {
* IORING_RECV_MULTISHOT Multishot recv. Sets IORING_CQE_F_MORE if * IORING_RECV_MULTISHOT Multishot recv. Sets IORING_CQE_F_MORE if
* the handler will continue to report * the handler will continue to report
* CQEs on behalf of the same SQE. * CQEs on behalf of the same SQE.
*
* IORING_RECVSEND_FIXED_BUF Use registered buffers, the index is stored in
* the buf_index field.
*/ */
#define IORING_RECVSEND_POLL_FIRST (1U << 0) #define IORING_RECVSEND_POLL_FIRST (1U << 0)
#define IORING_RECV_MULTISHOT (1U << 1) #define IORING_RECV_MULTISHOT (1U << 1)
#define IORING_RECVSEND_FIXED_BUF (1U << 2)
/* /*
* accept flags stored in sqe->ioprio * accept flags stored in sqe->ioprio

23
io_uring/net.c
View File

@ -15,6 +15,7 @@
#include "alloc_cache.h" #include "alloc_cache.h"
#include "net.h" #include "net.h"
#include "notif.h" #include "notif.h"
#include "rsrc.h"
#if defined(CONFIG_NET) #if defined(CONFIG_NET)
struct io_shutdown { struct io_shutdown {
@ -849,13 +850,23 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{ {
struct io_sendzc *zc = io_kiocb_to_cmd(req); struct io_sendzc *zc = io_kiocb_to_cmd(req);
struct io_ring_ctx *ctx = req->ctx;
if (READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3)) if (READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3))
return -EINVAL; return -EINVAL;
zc->flags = READ_ONCE(sqe->ioprio); zc->flags = READ_ONCE(sqe->ioprio);
if (zc->flags & ~IORING_RECVSEND_POLL_FIRST) if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF))
return -EINVAL; return -EINVAL;
if (zc->flags & IORING_RECVSEND_FIXED_BUF) {
unsigned idx = READ_ONCE(sqe->buf_index);
if (unlikely(idx >= ctx->nr_user_bufs))
return -EFAULT;
idx = array_index_nospec(idx, ctx->nr_user_bufs);
req->imu = READ_ONCE(ctx->user_bufs[idx]);
io_req_set_rsrc_node(req, ctx, 0);
}
zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
zc->len = READ_ONCE(sqe->len); zc->len = READ_ONCE(sqe->len);
@ -909,10 +920,18 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)
msg.msg_controllen = 0; msg.msg_controllen = 0;
msg.msg_namelen = 0; msg.msg_namelen = 0;
ret = import_single_range(WRITE, zc->buf, zc->len, &iov, &msg.msg_iter); if (zc->flags & IORING_RECVSEND_FIXED_BUF) {
ret = io_import_fixed(WRITE, &msg.msg_iter, req->imu,
(u64)(uintptr_t)zc->buf, zc->len);
if (unlikely(ret))
return ret;
} else {
ret = import_single_range(WRITE, zc->buf, zc->len, &iov,
&msg.msg_iter);
if (unlikely(ret)) if (unlikely(ret))
return ret; return ret;
mm_account_pinned_pages(&notif->uarg.mmp, zc->len); mm_account_pinned_pages(&notif->uarg.mmp, zc->len);
}
if (zc->addr) { if (zc->addr) {
ret = move_addr_to_kernel(zc->addr, zc->addr_len, &address); ret = move_addr_to_kernel(zc->addr, zc->addr_len, &address);