p71924506
/
linux_old1
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

io_uring: utilize the io batching infrastructure for more efficient polled IO 

Wire up using an io_comp_batch for f_op->iopoll(). If the lower stack
supports it, we can handle high rates of polled IO more efficiently.

This raises the single core efficiency on my system from ~6.1M IOPS to
~6.6M IOPS running a random read workload at depth 128 on two gen2
Optane drives.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2021-10-12 09:28:46 -06:00
parent c234a65392
commit b688f11e86
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
fs/io_uring.c
View File

@ -2458,6 +2458,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
{
struct io_kiocb *req, *tmp;
unsigned int poll_flags = BLK_POLL_NOSLEEP;
DEFINE_IO_COMP_BATCH(iob);
LIST_HEAD(done);
/*
@ -2483,17 +2484,20 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
if (!list_empty(&done))
break;
ret = kiocb->ki_filp->f_op->iopoll(kiocb, NULL, poll_flags);
ret = kiocb->ki_filp->f_op->iopoll(kiocb, &iob, poll_flags);
if (unlikely(ret < 0))
return ret;
else if (ret)
poll_flags |= BLK_POLL_ONESHOT;
/* iopoll may have completed current req */
if (READ_ONCE(req->iopoll_completed))
if (!rq_list_empty(iob.req_list) ||
READ_ONCE(req->iopoll_completed))
list_move_tail(&req->inflight_entry, &done);
}
if (!rq_list_empty(iob.req_list))
iob.complete(&iob);
if (!list_empty(&done))
io_iopoll_complete(ctx, nr_events, &done);