From 3a81fd02045c329f25e5900fa61f613c9b317644 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 10 Dec 2020 12:25:36 -0700 Subject: [PATCH 001/103] io_uring: enable LOOKUP_CACHED path resolution for filename lookups Instead of being pessimistic and assume that path lookup will block, use LOOKUP_CACHED to attempt just a cached lookup. This ensures that the fast path is always done inline, and we only punt to async context if IO is needed to satisfy the lookup. For forced nonblock open attempts, mark the file O_NONBLOCK over the actual ->open() call as well. We can safely clear this again before doing fd_install(), so it'll never be user visible that we fiddled with it. This greatly improves the performance of file open where the dentry is already cached: ached 5.10-git 5.10-git+LOOKUP_CACHED Speedup --------------------------------------------------------------- 33% 1,014,975 900,474 1.1x 89% 545,466 292,937 1.9x 100% 435,636 151,475 2.9x The more cache hot we are, the faster the inline LOOKUP_CACHED optimization helps. This is unsurprising and expected, as a thread offload becomes a more dominant part of the total overhead. If we look at io_uring tracing, doing an IORING_OP_OPENAT on a file that isn't in the dentry cache will yield: 275.550481: io_uring_create: ring 00000000ddda6278, fd 3 sq size 8, cq size 16, flags 0 275.550491: io_uring_submit_sqe: ring 00000000ddda6278, op 18, data 0x0, non block 1, sq_thread 0 275.550498: io_uring_queue_async_work: ring 00000000ddda6278, request 00000000c0267d17, flags 69760, normal queue, work 000000003d683991 275.550502: io_uring_cqring_wait: ring 00000000ddda6278, min_events 1 275.550556: io_uring_complete: ring 00000000ddda6278, user_data 0x0, result 4 which shows a failed nonblock lookup, then punt to worker, and then we complete with fd == 4. This takes 65 usec in total. Re-running the same test case again: 281.253956: io_uring_create: ring 0000000008207252, fd 3 sq size 8, cq size 16, flags 0 281.253967: io_uring_submit_sqe: ring 0000000008207252, op 18, data 0x0, non block 1, sq_thread 0 281.253973: io_uring_complete: ring 0000000008207252, user_data 0x0, result 4 shows the same request completing inline, also returning fd == 4. This takes 6 usec. Signed-off-by: Jens Axboe --- fs/io_uring.c | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 38c6cbe1ab38..fbc6d2fb7c1d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -489,7 +489,6 @@ struct io_sr_msg { struct io_open { struct file *file; int dfd; - bool ignore_nonblock; struct filename *filename; struct open_how how; unsigned long nofile; @@ -4054,7 +4053,6 @@ static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe return ret; } req->open.nofile = rlimit(RLIMIT_NOFILE); - req->open.ignore_nonblock = false; req->flags |= REQ_F_NEED_CLEANUP; return 0; } @@ -4096,39 +4094,48 @@ static int io_openat2(struct io_kiocb *req, bool force_nonblock) { struct open_flags op; struct file *file; + bool nonblock_set; + bool resolve_nonblock; int ret; - if (force_nonblock && !req->open.ignore_nonblock) - return -EAGAIN; - ret = build_open_flags(&req->open.how, &op); if (ret) goto err; + nonblock_set = op.open_flag & O_NONBLOCK; + resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED; + if (force_nonblock) { + /* + * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open, + * it'll always -EAGAIN + */ + if (req->open.how.flags & (O_TRUNC | O_CREAT | O_TMPFILE)) + return -EAGAIN; + op.lookup_flags |= LOOKUP_CACHED; + op.open_flag |= O_NONBLOCK; + } ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile); if (ret < 0) goto err; file = do_filp_open(req->open.dfd, req->open.filename, &op); + /* only retry if RESOLVE_CACHED wasn't already set by application */ + if ((!resolve_nonblock && force_nonblock) && file == ERR_PTR(-EAGAIN)) { + /* + * We could hang on to this 'fd', but seems like marginal + * gain for something that is now known to be a slower path. + * So just put it, and we'll get a new one when we retry. + */ + put_unused_fd(ret); + return -EAGAIN; + } + if (IS_ERR(file)) { put_unused_fd(ret); ret = PTR_ERR(file); - /* - * A work-around to ensure that /proc/self works that way - * that it should - if we get -EOPNOTSUPP back, then assume - * that proc_self_get_link() failed us because we're in async - * context. We should be safe to retry this from the task - * itself with force_nonblock == false set, as it should not - * block on lookup. Would be nice to know this upfront and - * avoid the async dance, but doesn't seem feasible. - */ - if (ret == -EOPNOTSUPP && io_wq_current_is_worker()) { - req->open.ignore_nonblock = true; - refcount_inc(&req->refs); - io_req_task_queue(req); - return 0; - } } else { + if (force_nonblock && !nonblock_set) + file->f_flags &= ~O_NONBLOCK; fsnotify_open(file); fd_install(ret, file); } From 0a96bbe49994a46c1fea34619a501ead46aa7584 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Wed, 6 Jan 2021 12:39:10 -0800 Subject: [PATCH 002/103] io_uring: modularize io_sqe_buffer_register Split io_sqe_buffer_register into two routines: - io_sqe_buffer_register() registers a single buffer - io_sqe_buffers_register iterates over all user specified buffers Reviewed-by: Pavel Begunkov Signed-off-by: Bijan Mottahedeh Signed-off-by: Jens Axboe --- fs/io_uring.c | 210 +++++++++++++++++++++++++------------------------- 1 file changed, 107 insertions(+), 103 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fbc6d2fb7c1d..ec70ba064774 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8370,7 +8370,7 @@ static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries) return pages; } -static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx) +static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx) { int i, j; @@ -8488,14 +8488,103 @@ static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages, return ret; } -static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg, - unsigned nr_args) +static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, + struct io_mapped_ubuf *imu, + struct page **last_hpage) { struct vm_area_struct **vmas = NULL; struct page **pages = NULL; + unsigned long off, start, end, ubuf; + size_t size; + int ret, pret, nr_pages, i; + + ubuf = (unsigned long) iov->iov_base; + end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT; + start = ubuf >> PAGE_SHIFT; + nr_pages = end - start; + + ret = -ENOMEM; + + pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL); + if (!pages) + goto done; + + vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *), + GFP_KERNEL); + if (!vmas) + goto done; + + imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec), + GFP_KERNEL); + if (!imu->bvec) + goto done; + + ret = 0; + mmap_read_lock(current->mm); + pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM, + pages, vmas); + if (pret == nr_pages) { + /* don't support file backed memory */ + for (i = 0; i < nr_pages; i++) { + struct vm_area_struct *vma = vmas[i]; + + if (vma->vm_file && + !is_file_hugepages(vma->vm_file)) { + ret = -EOPNOTSUPP; + break; + } + } + } else { + ret = pret < 0 ? pret : -EFAULT; + } + mmap_read_unlock(current->mm); + if (ret) { + /* + * if we did partial map, or found file backed vmas, + * release any pages we did get + */ + if (pret > 0) + unpin_user_pages(pages, pret); + kvfree(imu->bvec); + goto done; + } + + ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage); + if (ret) { + unpin_user_pages(pages, pret); + kvfree(imu->bvec); + goto done; + } + + off = ubuf & ~PAGE_MASK; + size = iov->iov_len; + for (i = 0; i < nr_pages; i++) { + size_t vec_len; + + vec_len = min_t(size_t, size, PAGE_SIZE - off); + imu->bvec[i].bv_page = pages[i]; + imu->bvec[i].bv_len = vec_len; + imu->bvec[i].bv_offset = off; + off = 0; + size -= vec_len; + } + /* store original address for later verification */ + imu->ubuf = ubuf; + imu->len = iov->iov_len; + imu->nr_bvecs = nr_pages; + ret = 0; +done: + kvfree(pages); + kvfree(vmas); + return ret; +} + +static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, + unsigned int nr_args) +{ + int i, ret; + struct iovec iov; struct page *last_hpage = NULL; - int i, j, got_pages = 0; - int ret = -EINVAL; if (ctx->user_bufs) return -EBUSY; @@ -8509,14 +8598,10 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg, for (i = 0; i < nr_args; i++) { struct io_mapped_ubuf *imu = &ctx->user_bufs[i]; - unsigned long off, start, end, ubuf; - int pret, nr_pages; - struct iovec iov; - size_t size; ret = io_copy_iov(ctx, &iov, arg, i); if (ret) - goto err; + break; /* * Don't impose further limits on the size and buffer @@ -8525,103 +8610,22 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg, */ ret = -EFAULT; if (!iov.iov_base || !iov.iov_len) - goto err; + break; /* arbitrary limit, but we need something */ if (iov.iov_len > SZ_1G) - goto err; + break; - ubuf = (unsigned long) iov.iov_base; - end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT; - start = ubuf >> PAGE_SHIFT; - nr_pages = end - start; - - ret = 0; - if (!pages || nr_pages > got_pages) { - kvfree(vmas); - kvfree(pages); - pages = kvmalloc_array(nr_pages, sizeof(struct page *), - GFP_KERNEL); - vmas = kvmalloc_array(nr_pages, - sizeof(struct vm_area_struct *), - GFP_KERNEL); - if (!pages || !vmas) { - ret = -ENOMEM; - goto err; - } - got_pages = nr_pages; - } - - imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec), - GFP_KERNEL); - ret = -ENOMEM; - if (!imu->bvec) - goto err; - - ret = 0; - mmap_read_lock(current->mm); - pret = pin_user_pages(ubuf, nr_pages, - FOLL_WRITE | FOLL_LONGTERM, - pages, vmas); - if (pret == nr_pages) { - /* don't support file backed memory */ - for (j = 0; j < nr_pages; j++) { - struct vm_area_struct *vma = vmas[j]; - - if (vma->vm_file && - !is_file_hugepages(vma->vm_file)) { - ret = -EOPNOTSUPP; - break; - } - } - } else { - ret = pret < 0 ? pret : -EFAULT; - } - mmap_read_unlock(current->mm); - if (ret) { - /* - * if we did partial map, or found file backed vmas, - * release any pages we did get - */ - if (pret > 0) - unpin_user_pages(pages, pret); - kvfree(imu->bvec); - goto err; - } - - ret = io_buffer_account_pin(ctx, pages, pret, imu, &last_hpage); - if (ret) { - unpin_user_pages(pages, pret); - kvfree(imu->bvec); - goto err; - } - - off = ubuf & ~PAGE_MASK; - size = iov.iov_len; - for (j = 0; j < nr_pages; j++) { - size_t vec_len; - - vec_len = min_t(size_t, size, PAGE_SIZE - off); - imu->bvec[j].bv_page = pages[j]; - imu->bvec[j].bv_len = vec_len; - imu->bvec[j].bv_offset = off; - off = 0; - size -= vec_len; - } - /* store original address for later verification */ - imu->ubuf = ubuf; - imu->len = iov.iov_len; - imu->nr_bvecs = nr_pages; + ret = io_sqe_buffer_register(ctx, &iov, imu, &last_hpage); + if (ret) + break; ctx->nr_user_bufs++; } - kvfree(pages); - kvfree(vmas); - return 0; -err: - kvfree(pages); - kvfree(vmas); - io_sqe_buffer_unregister(ctx); + + if (ret) + io_sqe_buffers_unregister(ctx); + return ret; } @@ -8675,7 +8679,7 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) static void io_ring_ctx_free(struct io_ring_ctx *ctx) { io_finish_async(ctx); - io_sqe_buffer_unregister(ctx); + io_sqe_buffers_unregister(ctx); if (ctx->sqo_task) { put_task_struct(ctx->sqo_task); @@ -10057,13 +10061,13 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, switch (opcode) { case IORING_REGISTER_BUFFERS: - ret = io_sqe_buffer_register(ctx, arg, nr_args); + ret = io_sqe_buffers_register(ctx, arg, nr_args); break; case IORING_UNREGISTER_BUFFERS: ret = -EINVAL; if (arg || nr_args) break; - ret = io_sqe_buffer_unregister(ctx); + ret = io_sqe_buffers_unregister(ctx); break; case IORING_REGISTER_FILES: ret = io_sqe_files_register(ctx, arg, nr_args); From 2b358604aa6e8c12d7efa14777fcc66c377682b0 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Wed, 6 Jan 2021 12:39:11 -0800 Subject: [PATCH 003/103] io_uring: modularize io_sqe_buffers_register Move allocation of buffer management structures, and validation of buffers into separate routines. Reviewed-by: Pavel Begunkov Signed-off-by: Bijan Mottahedeh Signed-off-by: Jens Axboe --- fs/io_uring.c | 51 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index ec70ba064774..4cdf0f906f12 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8579,13 +8579,8 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, return ret; } -static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, - unsigned int nr_args) +static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args) { - int i, ret; - struct iovec iov; - struct page *last_hpage = NULL; - if (ctx->user_bufs) return -EBUSY; if (!nr_args || nr_args > UIO_MAXIOV) @@ -8596,6 +8591,37 @@ static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, if (!ctx->user_bufs) return -ENOMEM; + return 0; +} + +static int io_buffer_validate(struct iovec *iov) +{ + /* + * Don't impose further limits on the size and buffer + * constraints here, we'll -EINVAL later when IO is + * submitted if they are wrong. + */ + if (!iov->iov_base || !iov->iov_len) + return -EFAULT; + + /* arbitrary limit, but we need something */ + if (iov->iov_len > SZ_1G) + return -EFAULT; + + return 0; +} + +static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, + unsigned int nr_args) +{ + int i, ret; + struct iovec iov; + struct page *last_hpage = NULL; + + ret = io_buffers_map_alloc(ctx, nr_args); + if (ret) + return ret; + for (i = 0; i < nr_args; i++) { struct io_mapped_ubuf *imu = &ctx->user_bufs[i]; @@ -8603,17 +8629,8 @@ static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, if (ret) break; - /* - * Don't impose further limits on the size and buffer - * constraints here, we'll -EINVAL later when IO is - * submitted if they are wrong. - */ - ret = -EFAULT; - if (!iov.iov_base || !iov.iov_len) - break; - - /* arbitrary limit, but we need something */ - if (iov.iov_len > SZ_1G) + ret = io_buffer_validate(&iov); + if (ret) break; ret = io_sqe_buffer_register(ctx, &iov, imu, &last_hpage); From 269bbe5fd4d2fdd3b0d3a82a3c3c1dd1209aa8b8 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:44 +0000 Subject: [PATCH 004/103] io_uring: rename file related variables to rsrc This is a prep rename patch for subsequent patches to generalize file registration. [io_uring_rsrc_update:: rename fds -> data] Reviewed-by: Pavel Begunkov Signed-off-by: Bijan Mottahedeh [leave io_uring_files_update as struct] Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 228 +++++++++++++++++----------------- include/uapi/linux/io_uring.h | 7 ++ 2 files changed, 124 insertions(+), 111 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 4cdf0f906f12..95b3b8747a65 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -195,24 +195,29 @@ struct io_mapped_ubuf { unsigned long acct_pages; }; -struct fixed_file_table { +struct io_rsrc_put { + struct list_head list; + struct file *file; +}; + +struct fixed_rsrc_table { struct file **files; }; -struct fixed_file_ref_node { +struct fixed_rsrc_ref_node { struct percpu_ref refs; struct list_head node; - struct list_head file_list; - struct fixed_file_data *file_data; + struct list_head rsrc_list; + struct fixed_rsrc_data *rsrc_data; struct llist_node llist; bool done; }; -struct fixed_file_data { - struct fixed_file_table *table; +struct fixed_rsrc_data { + struct fixed_rsrc_table *table; struct io_ring_ctx *ctx; - struct fixed_file_ref_node *node; + struct fixed_rsrc_ref_node *node; struct percpu_ref refs; struct completion done; struct list_head ref_list; @@ -319,7 +324,7 @@ struct io_ring_ctx { * readers must ensure that ->refs is alive as long as the file* is * used. Only updated through io_uring_register(2). */ - struct fixed_file_data *file_data; + struct fixed_rsrc_data *file_data; unsigned nr_user_files; /* if used, fixed mapped user buffers */ @@ -384,8 +389,8 @@ struct io_ring_ctx { struct list_head inflight_list; } ____cacheline_aligned_in_smp; - struct delayed_work file_put_work; - struct llist_head file_put_llist; + struct delayed_work rsrc_put_work; + struct llist_head rsrc_put_llist; struct work_struct exit_work; struct io_restriction restrictions; @@ -494,7 +499,7 @@ struct io_open { unsigned long nofile; }; -struct io_files_update { +struct io_rsrc_update { struct file *file; u64 arg; u32 nr_args; @@ -688,7 +693,7 @@ struct io_kiocb { struct io_sr_msg sr_msg; struct io_open open; struct io_close close; - struct io_files_update files_update; + struct io_rsrc_update rsrc_update; struct io_fadvise fadvise; struct io_madvise madvise; struct io_epoll epoll; @@ -718,7 +723,7 @@ struct io_kiocb { u64 user_data; struct io_kiocb *link; - struct percpu_ref *fixed_file_refs; + struct percpu_ref *fixed_rsrc_refs; /* * 1. used with ctx->iopoll_list with reads/writes @@ -996,8 +1001,8 @@ enum io_mem_account { static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx, struct task_struct *task); -static void destroy_fixed_file_ref_node(struct fixed_file_ref_node *ref_node); -static struct fixed_file_ref_node *alloc_fixed_file_ref_node( +static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node); +static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( struct io_ring_ctx *ctx); static void __io_complete_rw(struct io_kiocb *req, long res, long res2, @@ -1010,13 +1015,13 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req); static void __io_queue_linked_timeout(struct io_kiocb *req); static void io_queue_linked_timeout(struct io_kiocb *req); static int __io_sqe_files_update(struct io_ring_ctx *ctx, - struct io_uring_files_update *ip, + struct io_uring_rsrc_update *ip, unsigned nr_args); static void __io_clean_op(struct io_kiocb *req); static struct file *io_file_get(struct io_submit_state *state, struct io_kiocb *req, int fd, bool fixed); static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs); -static void io_file_put_work(struct work_struct *work); +static void io_rsrc_put_work(struct work_struct *work); static ssize_t io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, struct iov_iter *iter, @@ -1057,9 +1062,9 @@ static inline void io_set_resource_node(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; - if (!req->fixed_file_refs) { - req->fixed_file_refs = &ctx->file_data->node->refs; - percpu_ref_get(req->fixed_file_refs); + if (!req->fixed_rsrc_refs) { + req->fixed_rsrc_refs = &ctx->file_data->node->refs; + percpu_ref_get(req->fixed_rsrc_refs); } } @@ -1330,8 +1335,8 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->timeout_list); spin_lock_init(&ctx->inflight_lock); INIT_LIST_HEAD(&ctx->inflight_list); - INIT_DELAYED_WORK(&ctx->file_put_work, io_file_put_work); - init_llist_head(&ctx->file_put_llist); + INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); + init_llist_head(&ctx->rsrc_put_llist); return ctx; err: if (ctx->fallback_req) @@ -2011,8 +2016,8 @@ static void io_dismantle_req(struct io_kiocb *req) kfree(req->async_data); if (req->file) io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE)); - if (req->fixed_file_refs) - percpu_ref_put(req->fixed_file_refs); + if (req->fixed_rsrc_refs) + percpu_ref_put(req->fixed_rsrc_refs); io_req_clean_work(req); } @@ -5988,7 +5993,7 @@ static int io_async_cancel(struct io_kiocb *req) return 0; } -static int io_files_update_prep(struct io_kiocb *req, +static int io_rsrc_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { if (unlikely(req->ctx->flags & IORING_SETUP_SQPOLL)) @@ -5998,11 +6003,11 @@ static int io_files_update_prep(struct io_kiocb *req, if (sqe->ioprio || sqe->rw_flags) return -EINVAL; - req->files_update.offset = READ_ONCE(sqe->off); - req->files_update.nr_args = READ_ONCE(sqe->len); - if (!req->files_update.nr_args) + req->rsrc_update.offset = READ_ONCE(sqe->off); + req->rsrc_update.nr_args = READ_ONCE(sqe->len); + if (!req->rsrc_update.nr_args) return -EINVAL; - req->files_update.arg = READ_ONCE(sqe->addr); + req->rsrc_update.arg = READ_ONCE(sqe->addr); return 0; } @@ -6010,17 +6015,17 @@ static int io_files_update(struct io_kiocb *req, bool force_nonblock, struct io_comp_state *cs) { struct io_ring_ctx *ctx = req->ctx; - struct io_uring_files_update up; + struct io_uring_rsrc_update up; int ret; if (force_nonblock) return -EAGAIN; - up.offset = req->files_update.offset; - up.fds = req->files_update.arg; + up.offset = req->rsrc_update.offset; + up.data = req->rsrc_update.arg; mutex_lock(&ctx->uring_lock); - ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args); + ret = __io_sqe_files_update(ctx, &up, req->rsrc_update.nr_args); mutex_unlock(&ctx->uring_lock); if (ret < 0) @@ -6075,7 +6080,7 @@ static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) case IORING_OP_CLOSE: return io_close_prep(req, sqe); case IORING_OP_FILES_UPDATE: - return io_files_update_prep(req, sqe); + return io_rsrc_update_prep(req, sqe); case IORING_OP_STATX: return io_statx_prep(req, sqe); case IORING_OP_FADVISE: @@ -6444,7 +6449,7 @@ static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work) static inline struct file *io_file_from_index(struct io_ring_ctx *ctx, int index) { - struct fixed_file_table *table; + struct fixed_rsrc_table *table; table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT]; return table->files[index & IORING_FILE_TABLE_MASK]; @@ -6840,7 +6845,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->ctx = ctx; req->flags = 0; req->link = NULL; - req->fixed_file_refs = NULL; + req->fixed_rsrc_refs = NULL; /* one is dropped after submission, the other at completion */ refcount_set(&req->refs, 2); req->task = current; @@ -7328,28 +7333,28 @@ static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) #endif } -static void io_file_ref_kill(struct percpu_ref *ref) +static void io_rsrc_ref_kill(struct percpu_ref *ref) { - struct fixed_file_data *data; + struct fixed_rsrc_data *data; - data = container_of(ref, struct fixed_file_data, refs); + data = container_of(ref, struct fixed_rsrc_data, refs); complete(&data->done); } -static void io_sqe_files_set_node(struct fixed_file_data *file_data, - struct fixed_file_ref_node *ref_node) +static void io_sqe_rsrc_set_node(struct fixed_rsrc_data *rsrc_data, + struct fixed_rsrc_ref_node *ref_node) { - spin_lock_bh(&file_data->lock); - file_data->node = ref_node; - list_add_tail(&ref_node->node, &file_data->ref_list); - spin_unlock_bh(&file_data->lock); - percpu_ref_get(&file_data->refs); + spin_lock_bh(&rsrc_data->lock); + rsrc_data->node = ref_node; + list_add_tail(&ref_node->node, &rsrc_data->ref_list); + spin_unlock_bh(&rsrc_data->lock); + percpu_ref_get(&rsrc_data->refs); } static int io_sqe_files_unregister(struct io_ring_ctx *ctx) { - struct fixed_file_data *data = ctx->file_data; - struct fixed_file_ref_node *backup_node, *ref_node = NULL; + struct fixed_rsrc_data *data = ctx->file_data; + struct fixed_rsrc_ref_node *backup_node, *ref_node = NULL; unsigned nr_tables, i; int ret; @@ -7368,7 +7373,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) percpu_ref_kill(&data->refs); /* wait for all refs nodes to complete */ - flush_delayed_work(&ctx->file_put_work); + flush_delayed_work(&ctx->rsrc_put_work); do { ret = wait_for_completion_interruptible(&data->done); if (!ret) @@ -7377,7 +7382,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) if (ret < 0) { percpu_ref_resurrect(&data->refs); reinit_completion(&data->done); - io_sqe_files_set_node(data, backup_node); + io_sqe_rsrc_set_node(data, backup_node); return ret; } } while (1); @@ -7391,7 +7396,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) kfree(data); ctx->file_data = NULL; ctx->nr_user_files = 0; - destroy_fixed_file_ref_node(backup_node); + destroy_fixed_rsrc_ref_node(backup_node); return 0; } @@ -7614,13 +7619,13 @@ static int io_sqe_files_scm(struct io_ring_ctx *ctx) } #endif -static int io_sqe_alloc_file_tables(struct fixed_file_data *file_data, +static int io_sqe_alloc_file_tables(struct fixed_rsrc_data *file_data, unsigned nr_tables, unsigned nr_files) { int i; for (i = 0; i < nr_tables; i++) { - struct fixed_file_table *table = &file_data->table[i]; + struct fixed_rsrc_table *table = &file_data->table[i]; unsigned this_files; this_files = min(nr_files, IORING_MAX_FILES_TABLE); @@ -7635,7 +7640,7 @@ static int io_sqe_alloc_file_tables(struct fixed_file_data *file_data, return 0; for (i = 0; i < nr_tables; i++) { - struct fixed_file_table *table = &file_data->table[i]; + struct fixed_rsrc_table *table = &file_data->table[i]; kfree(table->files); } return 1; @@ -7703,56 +7708,51 @@ static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file) #endif } -struct io_file_put { - struct list_head list; - struct file *file; -}; - -static void __io_file_put_work(struct fixed_file_ref_node *ref_node) +static void __io_rsrc_put_work(struct fixed_rsrc_ref_node *ref_node) { - struct fixed_file_data *file_data = ref_node->file_data; - struct io_ring_ctx *ctx = file_data->ctx; - struct io_file_put *pfile, *tmp; + struct fixed_rsrc_data *rsrc_data = ref_node->rsrc_data; + struct io_ring_ctx *ctx = rsrc_data->ctx; + struct io_rsrc_put *prsrc, *tmp; - list_for_each_entry_safe(pfile, tmp, &ref_node->file_list, list) { - list_del(&pfile->list); - io_ring_file_put(ctx, pfile->file); - kfree(pfile); + list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) { + list_del(&prsrc->list); + io_ring_file_put(ctx, prsrc->file); + kfree(prsrc); } percpu_ref_exit(&ref_node->refs); kfree(ref_node); - percpu_ref_put(&file_data->refs); + percpu_ref_put(&rsrc_data->refs); } -static void io_file_put_work(struct work_struct *work) +static void io_rsrc_put_work(struct work_struct *work) { struct io_ring_ctx *ctx; struct llist_node *node; - ctx = container_of(work, struct io_ring_ctx, file_put_work.work); - node = llist_del_all(&ctx->file_put_llist); + ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work); + node = llist_del_all(&ctx->rsrc_put_llist); while (node) { - struct fixed_file_ref_node *ref_node; + struct fixed_rsrc_ref_node *ref_node; struct llist_node *next = node->next; - ref_node = llist_entry(node, struct fixed_file_ref_node, llist); - __io_file_put_work(ref_node); + ref_node = llist_entry(node, struct fixed_rsrc_ref_node, llist); + __io_rsrc_put_work(ref_node); node = next; } } -static void io_file_data_ref_zero(struct percpu_ref *ref) +static void io_rsrc_data_ref_zero(struct percpu_ref *ref) { - struct fixed_file_ref_node *ref_node; - struct fixed_file_data *data; + struct fixed_rsrc_ref_node *ref_node; + struct fixed_rsrc_data *data; struct io_ring_ctx *ctx; bool first_add = false; int delay = HZ; - ref_node = container_of(ref, struct fixed_file_ref_node, refs); - data = ref_node->file_data; + ref_node = container_of(ref, struct fixed_rsrc_ref_node, refs); + data = ref_node->rsrc_data; ctx = data->ctx; spin_lock_bh(&data->lock); @@ -7760,12 +7760,12 @@ static void io_file_data_ref_zero(struct percpu_ref *ref) while (!list_empty(&data->ref_list)) { ref_node = list_first_entry(&data->ref_list, - struct fixed_file_ref_node, node); + struct fixed_rsrc_ref_node, node); /* recycle ref nodes in order */ if (!ref_node->done) break; list_del(&ref_node->node); - first_add |= llist_add(&ref_node->llist, &ctx->file_put_llist); + first_add |= llist_add(&ref_node->llist, &ctx->rsrc_put_llist); } spin_unlock_bh(&data->lock); @@ -7773,33 +7773,33 @@ static void io_file_data_ref_zero(struct percpu_ref *ref) delay = 0; if (!delay) - mod_delayed_work(system_wq, &ctx->file_put_work, 0); + mod_delayed_work(system_wq, &ctx->rsrc_put_work, 0); else if (first_add) - queue_delayed_work(system_wq, &ctx->file_put_work, delay); + queue_delayed_work(system_wq, &ctx->rsrc_put_work, delay); } -static struct fixed_file_ref_node *alloc_fixed_file_ref_node( +static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( struct io_ring_ctx *ctx) { - struct fixed_file_ref_node *ref_node; + struct fixed_rsrc_ref_node *ref_node; ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL); if (!ref_node) return NULL; - if (percpu_ref_init(&ref_node->refs, io_file_data_ref_zero, + if (percpu_ref_init(&ref_node->refs, io_rsrc_data_ref_zero, 0, GFP_KERNEL)) { kfree(ref_node); return NULL; } INIT_LIST_HEAD(&ref_node->node); - INIT_LIST_HEAD(&ref_node->file_list); - ref_node->file_data = ctx->file_data; + INIT_LIST_HEAD(&ref_node->rsrc_list); + ref_node->rsrc_data = ctx->file_data; ref_node->done = false; return ref_node; } -static void destroy_fixed_file_ref_node(struct fixed_file_ref_node *ref_node) +static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node) { percpu_ref_exit(&ref_node->refs); kfree(ref_node); @@ -7812,8 +7812,8 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_tables, i; struct file *file; int fd, ret = -ENOMEM; - struct fixed_file_ref_node *ref_node; - struct fixed_file_data *file_data; + struct fixed_rsrc_ref_node *ref_node; + struct fixed_rsrc_data *file_data; if (ctx->file_data) return -EBUSY; @@ -7836,7 +7836,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, if (!file_data->table) goto out_free; - if (percpu_ref_init(&file_data->refs, io_file_ref_kill, + if (percpu_ref_init(&file_data->refs, io_rsrc_ref_kill, PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) goto out_free; @@ -7845,7 +7845,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, ctx->file_data = file_data; for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { - struct fixed_file_table *table; + struct fixed_rsrc_table *table; unsigned index; if (copy_from_user(&fd, &fds[i], sizeof(fd))) { @@ -7889,7 +7889,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, return -ENOMEM; } - io_sqe_files_set_node(file_data, ref_node); + io_sqe_rsrc_set_node(file_data, ref_node); return ret; out_fput: for (i = 0; i < ctx->nr_user_files; i++) { @@ -7952,28 +7952,34 @@ static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file, #endif } -static int io_queue_file_removal(struct fixed_file_data *data, - struct file *file) +static int io_queue_rsrc_removal(struct fixed_rsrc_data *data, + struct file *rsrc) { - struct io_file_put *pfile; - struct fixed_file_ref_node *ref_node = data->node; + struct io_rsrc_put *prsrc; + struct fixed_rsrc_ref_node *ref_node = data->node; - pfile = kzalloc(sizeof(*pfile), GFP_KERNEL); - if (!pfile) + prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL); + if (!prsrc) return -ENOMEM; - pfile->file = file; - list_add(&pfile->list, &ref_node->file_list); + prsrc->file = rsrc; + list_add(&prsrc->list, &ref_node->rsrc_list); return 0; } +static inline int io_queue_file_removal(struct fixed_rsrc_data *data, + struct file *file) +{ + return io_queue_rsrc_removal(data, file); +} + static int __io_sqe_files_update(struct io_ring_ctx *ctx, - struct io_uring_files_update *up, + struct io_uring_rsrc_update *up, unsigned nr_args) { - struct fixed_file_data *data = ctx->file_data; - struct fixed_file_ref_node *ref_node; + struct fixed_rsrc_data *data = ctx->file_data; + struct fixed_rsrc_ref_node *ref_node; struct file *file; __s32 __user *fds; int fd, i, err; @@ -7990,9 +7996,9 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, return -ENOMEM; done = 0; - fds = u64_to_user_ptr(up->fds); + fds = u64_to_user_ptr(up->data); while (nr_args) { - struct fixed_file_table *table; + struct fixed_rsrc_table *table; unsigned index; err = 0; @@ -8045,9 +8051,9 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, if (needs_switch) { percpu_ref_kill(&data->node->refs); - io_sqe_files_set_node(data, ref_node); + io_sqe_rsrc_set_node(data, ref_node); } else - destroy_fixed_file_ref_node(ref_node); + destroy_fixed_rsrc_ref_node(ref_node); return done ? done : err; } @@ -8055,7 +8061,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) { - struct io_uring_files_update up; + struct io_uring_rsrc_update up; if (!ctx->file_data) return -ENXIO; @@ -9482,7 +9488,7 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1); seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files); for (i = 0; has_lock && i < ctx->nr_user_files; i++) { - struct fixed_file_table *table; + struct fixed_rsrc_table *table; struct file *f; table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index d31a2a1e8ef9..f9f106c54d90 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -285,12 +285,19 @@ enum { IORING_REGISTER_LAST }; +/* deprecated, see struct io_uring_rsrc_update */ struct io_uring_files_update { __u32 offset; __u32 resv; __aligned_u64 /* __s32 * */ fds; }; +struct io_uring_rsrc_update { + __u32 offset; + __u32 resv; + __aligned_u64 data; +}; + #define IO_URING_OP_SUPPORTED (1U << 0) struct io_uring_probe_op { From 5023853183699dd1e3e47622c03d7ae11343837a Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:45 +0000 Subject: [PATCH 005/103] io_uring: generalize io_queue_rsrc_removal Generalize io_queue_rsrc_removal to handle both files and buffers. Reviewed-by: Pavel Begunkov Signed-off-by: Bijan Mottahedeh [remove io_mapped_ubuf from rsrc tables/etc. for now] Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 95b3b8747a65..e52800e19c60 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -195,9 +195,14 @@ struct io_mapped_ubuf { unsigned long acct_pages; }; +struct io_ring_ctx; + struct io_rsrc_put { struct list_head list; - struct file *file; + union { + void *rsrc; + struct file *file; + }; }; struct fixed_rsrc_table { @@ -209,6 +214,8 @@ struct fixed_rsrc_ref_node { struct list_head node; struct list_head rsrc_list; struct fixed_rsrc_data *rsrc_data; + void (*rsrc_put)(struct io_ring_ctx *ctx, + struct io_rsrc_put *prsrc); struct llist_node llist; bool done; }; @@ -7646,8 +7653,9 @@ static int io_sqe_alloc_file_tables(struct fixed_rsrc_data *file_data, return 1; } -static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file) +static void io_ring_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc) { + struct file *file = prsrc->file; #if defined(CONFIG_UNIX) struct sock *sock = ctx->ring_sock->sk; struct sk_buff_head list, *head = &sock->sk_receive_queue; @@ -7716,7 +7724,7 @@ static void __io_rsrc_put_work(struct fixed_rsrc_ref_node *ref_node) list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) { list_del(&prsrc->list); - io_ring_file_put(ctx, prsrc->file); + ref_node->rsrc_put(ctx, prsrc); kfree(prsrc); } @@ -7795,6 +7803,7 @@ static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( INIT_LIST_HEAD(&ref_node->node); INIT_LIST_HEAD(&ref_node->rsrc_list); ref_node->rsrc_data = ctx->file_data; + ref_node->rsrc_put = io_ring_file_put; ref_node->done = false; return ref_node; } @@ -7952,8 +7961,7 @@ static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file, #endif } -static int io_queue_rsrc_removal(struct fixed_rsrc_data *data, - struct file *rsrc) +static int io_queue_rsrc_removal(struct fixed_rsrc_data *data, void *rsrc) { struct io_rsrc_put *prsrc; struct fixed_rsrc_ref_node *ref_node = data->node; @@ -7962,7 +7970,7 @@ static int io_queue_rsrc_removal(struct fixed_rsrc_data *data, if (!prsrc) return -ENOMEM; - prsrc->file = rsrc; + prsrc->rsrc = rsrc; list_add(&prsrc->list, &ref_node->rsrc_list); return 0; @@ -7971,7 +7979,7 @@ static int io_queue_rsrc_removal(struct fixed_rsrc_data *data, static inline int io_queue_file_removal(struct fixed_rsrc_data *data, struct file *file) { - return io_queue_rsrc_removal(data, file); + return io_queue_rsrc_removal(data, (void *)file); } static int __io_sqe_files_update(struct io_ring_ctx *ctx, From d67d2263fb2350a68074f2cb4dd78549aeebbfae Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:46 +0000 Subject: [PATCH 006/103] io_uring: separate ref_list from fixed_rsrc_data Uplevel ref_list and make it common to all resources. This is to allow one common ref_list to be used for both files, and buffers in upcoming patches. Signed-off-by: Bijan Mottahedeh Reviewed-by: Pavel Begunkov Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index e52800e19c60..fcc7a3ed800a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -227,8 +227,6 @@ struct fixed_rsrc_data { struct fixed_rsrc_ref_node *node; struct percpu_ref refs; struct completion done; - struct list_head ref_list; - spinlock_t lock; }; struct io_buffer { @@ -398,6 +396,8 @@ struct io_ring_ctx { struct delayed_work rsrc_put_work; struct llist_head rsrc_put_llist; + struct list_head rsrc_ref_list; + spinlock_t rsrc_ref_lock; struct work_struct exit_work; struct io_restriction restrictions; @@ -1342,6 +1342,8 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->timeout_list); spin_lock_init(&ctx->inflight_lock); INIT_LIST_HEAD(&ctx->inflight_list); + spin_lock_init(&ctx->rsrc_ref_lock); + INIT_LIST_HEAD(&ctx->rsrc_ref_list); INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); init_llist_head(&ctx->rsrc_put_llist); return ctx; @@ -7348,13 +7350,14 @@ static void io_rsrc_ref_kill(struct percpu_ref *ref) complete(&data->done); } -static void io_sqe_rsrc_set_node(struct fixed_rsrc_data *rsrc_data, +static void io_sqe_rsrc_set_node(struct io_ring_ctx *ctx, + struct fixed_rsrc_data *rsrc_data, struct fixed_rsrc_ref_node *ref_node) { - spin_lock_bh(&rsrc_data->lock); + spin_lock_bh(&ctx->rsrc_ref_lock); rsrc_data->node = ref_node; - list_add_tail(&ref_node->node, &rsrc_data->ref_list); - spin_unlock_bh(&rsrc_data->lock); + list_add_tail(&ref_node->node, &ctx->rsrc_ref_list); + spin_unlock_bh(&ctx->rsrc_ref_lock); percpu_ref_get(&rsrc_data->refs); } @@ -7371,9 +7374,9 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) if (!backup_node) return -ENOMEM; - spin_lock_bh(&data->lock); + spin_lock_bh(&ctx->rsrc_ref_lock); ref_node = data->node; - spin_unlock_bh(&data->lock); + spin_unlock_bh(&ctx->rsrc_ref_lock); if (ref_node) percpu_ref_kill(&ref_node->refs); @@ -7389,7 +7392,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) if (ret < 0) { percpu_ref_resurrect(&data->refs); reinit_completion(&data->done); - io_sqe_rsrc_set_node(data, backup_node); + io_sqe_rsrc_set_node(ctx, data, backup_node); return ret; } } while (1); @@ -7763,11 +7766,11 @@ static void io_rsrc_data_ref_zero(struct percpu_ref *ref) data = ref_node->rsrc_data; ctx = data->ctx; - spin_lock_bh(&data->lock); + spin_lock_bh(&ctx->rsrc_ref_lock); ref_node->done = true; - while (!list_empty(&data->ref_list)) { - ref_node = list_first_entry(&data->ref_list, + while (!list_empty(&ctx->rsrc_ref_list)) { + ref_node = list_first_entry(&ctx->rsrc_ref_list, struct fixed_rsrc_ref_node, node); /* recycle ref nodes in order */ if (!ref_node->done) @@ -7775,7 +7778,7 @@ static void io_rsrc_data_ref_zero(struct percpu_ref *ref) list_del(&ref_node->node); first_add |= llist_add(&ref_node->llist, &ctx->rsrc_put_llist); } - spin_unlock_bh(&data->lock); + spin_unlock_bh(&ctx->rsrc_ref_lock); if (percpu_ref_is_dying(&data->refs)) delay = 0; @@ -7836,8 +7839,6 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, return -ENOMEM; file_data->ctx = ctx; init_completion(&file_data->done); - INIT_LIST_HEAD(&file_data->ref_list); - spin_lock_init(&file_data->lock); nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE); file_data->table = kcalloc(nr_tables, sizeof(*file_data->table), @@ -7898,7 +7899,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, return -ENOMEM; } - io_sqe_rsrc_set_node(file_data, ref_node); + io_sqe_rsrc_set_node(ctx, file_data, ref_node); return ret; out_fput: for (i = 0; i < ctx->nr_user_files; i++) { @@ -8059,7 +8060,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, if (needs_switch) { percpu_ref_kill(&data->node->refs); - io_sqe_rsrc_set_node(data, ref_node); + io_sqe_rsrc_set_node(ctx, data, ref_node); } else destroy_fixed_rsrc_ref_node(ref_node); From 2a63b2d9c30b2029892c368d11ede1434de6c565 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:47 +0000 Subject: [PATCH 007/103] io_uring: add rsrc_ref locking routines Encapsulate resource reference locking into separate routines. Signed-off-by: Bijan Mottahedeh Signed-off-by: Pavel Begunkov Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fcc7a3ed800a..a129192c20d3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7350,14 +7350,24 @@ static void io_rsrc_ref_kill(struct percpu_ref *ref) complete(&data->done); } +static inline void io_rsrc_ref_lock(struct io_ring_ctx *ctx) +{ + spin_lock_bh(&ctx->rsrc_ref_lock); +} + +static inline void io_rsrc_ref_unlock(struct io_ring_ctx *ctx) +{ + spin_unlock_bh(&ctx->rsrc_ref_lock); +} + static void io_sqe_rsrc_set_node(struct io_ring_ctx *ctx, struct fixed_rsrc_data *rsrc_data, struct fixed_rsrc_ref_node *ref_node) { - spin_lock_bh(&ctx->rsrc_ref_lock); + io_rsrc_ref_lock(ctx); rsrc_data->node = ref_node; list_add_tail(&ref_node->node, &ctx->rsrc_ref_list); - spin_unlock_bh(&ctx->rsrc_ref_lock); + io_rsrc_ref_unlock(ctx); percpu_ref_get(&rsrc_data->refs); } @@ -7374,9 +7384,9 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) if (!backup_node) return -ENOMEM; - spin_lock_bh(&ctx->rsrc_ref_lock); + io_rsrc_ref_lock(ctx); ref_node = data->node; - spin_unlock_bh(&ctx->rsrc_ref_lock); + io_rsrc_ref_unlock(ctx); if (ref_node) percpu_ref_kill(&ref_node->refs); @@ -7766,7 +7776,7 @@ static void io_rsrc_data_ref_zero(struct percpu_ref *ref) data = ref_node->rsrc_data; ctx = data->ctx; - spin_lock_bh(&ctx->rsrc_ref_lock); + io_rsrc_ref_lock(ctx); ref_node->done = true; while (!list_empty(&ctx->rsrc_ref_list)) { @@ -7778,7 +7788,7 @@ static void io_rsrc_data_ref_zero(struct percpu_ref *ref) list_del(&ref_node->node); first_add |= llist_add(&ref_node->llist, &ctx->rsrc_put_llist); } - spin_unlock_bh(&ctx->rsrc_ref_lock); + io_rsrc_ref_unlock(ctx); if (percpu_ref_is_dying(&data->refs)) delay = 0; From 6802535df7bf807c94de32a9d0bf0401d3109671 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:48 +0000 Subject: [PATCH 008/103] io_uring: split alloc_fixed_file_ref_node Split alloc_fixed_file_ref_node into resource generic/specific parts, to be leveraged for fixed buffers. Signed-off-by: Bijan Mottahedeh Signed-off-by: Pavel Begunkov Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index a129192c20d3..ab5bf1bf0779 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7799,7 +7799,7 @@ static void io_rsrc_data_ref_zero(struct percpu_ref *ref) queue_delayed_work(system_wq, &ctx->rsrc_put_work, delay); } -static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( +static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( struct io_ring_ctx *ctx) { struct fixed_rsrc_ref_node *ref_node; @@ -7815,9 +7815,21 @@ static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( } INIT_LIST_HEAD(&ref_node->node); INIT_LIST_HEAD(&ref_node->rsrc_list); + ref_node->done = false; + return ref_node; +} + +static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( + struct io_ring_ctx *ctx) +{ + struct fixed_rsrc_ref_node *ref_node; + + ref_node = alloc_fixed_rsrc_ref_node(ctx); + if (!ref_node) + return NULL; + ref_node->rsrc_data = ctx->file_data; ref_node->rsrc_put = io_ring_file_put; - ref_node->done = false; return ref_node; } From bc9744cd162b2f6c38d75dc49c310677dc13afa8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Jan 2021 17:37:49 +0000 Subject: [PATCH 009/103] io_uring: split ref_node alloc and init A simple prep patch allowing to set refnode callbacks after it was allocated. This needed to 1) keep ourself off of hi-level functions where it's not pretty and they are not necessary 2) amortise ref_node allocation in the future, e.g. for updates. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index ab5bf1bf0779..bb51f2abd009 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1009,8 +1009,10 @@ static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx, struct task_struct *task); static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node); -static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( +static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( struct io_ring_ctx *ctx); +static void init_fixed_file_ref_node(struct io_ring_ctx *ctx, + struct fixed_rsrc_ref_node *ref_node); static void __io_complete_rw(struct io_kiocb *req, long res, long res2, struct io_comp_state *cs); @@ -7380,9 +7382,10 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) if (!data) return -ENXIO; - backup_node = alloc_fixed_file_ref_node(ctx); + backup_node = alloc_fixed_rsrc_ref_node(ctx); if (!backup_node) return -ENOMEM; + init_fixed_file_ref_node(ctx, backup_node); io_rsrc_ref_lock(ctx); ref_node = data->node; @@ -7819,18 +7822,11 @@ static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( return ref_node; } -static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( - struct io_ring_ctx *ctx) +static void init_fixed_file_ref_node(struct io_ring_ctx *ctx, + struct fixed_rsrc_ref_node *ref_node) { - struct fixed_rsrc_ref_node *ref_node; - - ref_node = alloc_fixed_rsrc_ref_node(ctx); - if (!ref_node) - return NULL; - ref_node->rsrc_data = ctx->file_data; ref_node->rsrc_put = io_ring_file_put; - return ref_node; } static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node) @@ -7915,11 +7911,12 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, return ret; } - ref_node = alloc_fixed_file_ref_node(ctx); + ref_node = alloc_fixed_rsrc_ref_node(ctx); if (!ref_node) { io_sqe_files_unregister(ctx); return -ENOMEM; } + init_fixed_file_ref_node(ctx, ref_node); io_sqe_rsrc_set_node(ctx, file_data, ref_node); return ret; @@ -8022,9 +8019,10 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, if (done > ctx->nr_user_files) return -EINVAL; - ref_node = alloc_fixed_file_ref_node(ctx); + ref_node = alloc_fixed_rsrc_ref_node(ctx); if (!ref_node) return -ENOMEM; + init_fixed_file_ref_node(ctx, ref_node); done = 0; fds = u64_to_user_ptr(up->data); From d7954b2ba94639b7f5b08760d36e54c28544730f Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:50 +0000 Subject: [PATCH 010/103] io_uring: create common fixed_rsrc_ref_node handling routines Create common routines to be used for both files/buffers registration. [remove io_sqe_rsrc_set_node substitution] Reviewed-by: Pavel Begunkov Signed-off-by: Bijan Mottahedeh [merge, quiesce only for files] Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index bb51f2abd009..727d0d3cdbcc 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7373,20 +7373,13 @@ static void io_sqe_rsrc_set_node(struct io_ring_ctx *ctx, percpu_ref_get(&rsrc_data->refs); } -static int io_sqe_files_unregister(struct io_ring_ctx *ctx) +static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, + struct io_ring_ctx *ctx, + struct fixed_rsrc_ref_node *backup_node) { - struct fixed_rsrc_data *data = ctx->file_data; - struct fixed_rsrc_ref_node *backup_node, *ref_node = NULL; - unsigned nr_tables, i; + struct fixed_rsrc_ref_node *ref_node; int ret; - if (!data) - return -ENXIO; - backup_node = alloc_fixed_rsrc_ref_node(ctx); - if (!backup_node) - return -ENOMEM; - init_fixed_file_ref_node(ctx, backup_node); - io_rsrc_ref_lock(ctx); ref_node = data->node; io_rsrc_ref_unlock(ctx); @@ -7410,6 +7403,28 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) } } while (1); + destroy_fixed_rsrc_ref_node(backup_node); + return 0; +} + +static int io_sqe_files_unregister(struct io_ring_ctx *ctx) +{ + struct fixed_rsrc_data *data = ctx->file_data; + struct fixed_rsrc_ref_node *backup_node; + unsigned nr_tables, i; + int ret; + + if (!data) + return -ENXIO; + backup_node = alloc_fixed_rsrc_ref_node(ctx); + if (!backup_node) + return -ENOMEM; + init_fixed_file_ref_node(ctx, backup_node); + + ret = io_rsrc_ref_quiesce(data, ctx, backup_node); + if (ret) + return ret; + __io_sqe_files_unregister(ctx); nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE); for (i = 0; i < nr_tables; i++) @@ -7419,7 +7434,6 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) kfree(data); ctx->file_data = NULL; ctx->nr_user_files = 0; - destroy_fixed_rsrc_ref_node(backup_node); return 0; } From 1ad555c6ae6e28ec7b1acaa2af72a9904e6ba96a Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:51 +0000 Subject: [PATCH 011/103] io_uring: create common fixed_rsrc_data allocation routines Create common alloc/free fixed_rsrc_data routines for both files and buffers. Reviewed-by: Pavel Begunkov Signed-off-by: Bijan Mottahedeh [remove buffer part] Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 727d0d3cdbcc..8f7d95e0d240 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7407,6 +7407,31 @@ static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, return 0; } +static struct fixed_rsrc_data *alloc_fixed_rsrc_data(struct io_ring_ctx *ctx) +{ + struct fixed_rsrc_data *data; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return NULL; + + if (percpu_ref_init(&data->refs, io_rsrc_ref_kill, + PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) { + kfree(data); + return NULL; + } + data->ctx = ctx; + init_completion(&data->done); + return data; +} + +static void free_fixed_rsrc_data(struct fixed_rsrc_data *data) +{ + percpu_ref_exit(&data->refs); + kfree(data->table); + kfree(data); +} + static int io_sqe_files_unregister(struct io_ring_ctx *ctx) { struct fixed_rsrc_data *data = ctx->file_data; @@ -7429,9 +7454,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE); for (i = 0; i < nr_tables; i++) kfree(data->table[i].files); - kfree(data->table); - percpu_ref_exit(&data->refs); - kfree(data); + free_fixed_rsrc_data(data); ctx->file_data = NULL; ctx->nr_user_files = 0; return 0; @@ -7866,11 +7889,9 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, if (nr_args > IORING_MAX_FIXED_FILES) return -EMFILE; - file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL); + file_data = alloc_fixed_rsrc_data(ctx); if (!file_data) return -ENOMEM; - file_data->ctx = ctx; - init_completion(&file_data->done); nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE); file_data->table = kcalloc(nr_tables, sizeof(*file_data->table), @@ -7878,12 +7899,8 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, if (!file_data->table) goto out_free; - if (percpu_ref_init(&file_data->refs, io_rsrc_ref_kill, - PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) - goto out_free; - if (io_sqe_alloc_file_tables(file_data, nr_tables, nr_args)) - goto out_ref; + goto out_free; ctx->file_data = file_data; for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { @@ -7943,11 +7960,8 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, for (i = 0; i < nr_tables; i++) kfree(file_data->table[i].files); ctx->nr_user_files = 0; -out_ref: - percpu_ref_exit(&file_data->refs); out_free: - kfree(file_data->table); - kfree(file_data); + free_fixed_rsrc_data(ctx->file_data); ctx->file_data = NULL; return ret; } From 00835dce1406e746fe5ab8c522cceb9594c78acb Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:52 +0000 Subject: [PATCH 012/103] io_uring: make percpu_ref_release names consistent Make the percpu ref release function names consistent between rsrc data and nodes. Signed-off-by: Bijan Mottahedeh Reviewed-by: Pavel Begunkov Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 8f7d95e0d240..98789fece715 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7344,7 +7344,7 @@ static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) #endif } -static void io_rsrc_ref_kill(struct percpu_ref *ref) +static void io_rsrc_data_ref_zero(struct percpu_ref *ref) { struct fixed_rsrc_data *data; @@ -7415,7 +7415,7 @@ static struct fixed_rsrc_data *alloc_fixed_rsrc_data(struct io_ring_ctx *ctx) if (!data) return NULL; - if (percpu_ref_init(&data->refs, io_rsrc_ref_kill, + if (percpu_ref_init(&data->refs, io_rsrc_data_ref_zero, PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) { kfree(data); return NULL; @@ -7804,7 +7804,7 @@ static void io_rsrc_put_work(struct work_struct *work) } } -static void io_rsrc_data_ref_zero(struct percpu_ref *ref) +static void io_rsrc_node_ref_zero(struct percpu_ref *ref) { struct fixed_rsrc_ref_node *ref_node; struct fixed_rsrc_data *data; @@ -7848,7 +7848,7 @@ static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( if (!ref_node) return NULL; - if (percpu_ref_init(&ref_node->refs, io_rsrc_data_ref_zero, + if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero, 0, GFP_KERNEL)) { kfree(ref_node); return NULL; From bf6182b6d46e28c3e59b9c0d6097b379cae56b94 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:34 +0000 Subject: [PATCH 013/103] io_uring: optimise io_rw_reissue() The hot path is IO completing on the first try. Reshuffle io_rw_reissue() so it's checked first. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 98789fece715..4a8900d480c5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2738,12 +2738,13 @@ static bool io_resubmit_prep(struct io_kiocb *req, int error) static bool io_rw_reissue(struct io_kiocb *req, long res) { #ifdef CONFIG_BLOCK - umode_t mode = file_inode(req->file)->i_mode; + umode_t mode; int ret; - if (!S_ISBLK(mode) && !S_ISREG(mode)) + if (res != -EAGAIN && res != -EOPNOTSUPP) return false; - if ((res != -EAGAIN && res != -EOPNOTSUPP) || io_wq_current_is_worker()) + mode = file_inode(req->file)->i_mode; + if ((!S_ISBLK(mode) && !S_ISREG(mode)) || io_wq_current_is_worker()) return false; lockdep_assert_held(&req->ctx->uring_lock); From dc2a6e9aa9c349d76c318d22bbe26006fda1ce97 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:35 +0000 Subject: [PATCH 014/103] io_uring: refactor io_resubmit_prep() It's awkward to pass return a value into a function for it to return it back. Check it at the caller site and clean up io_resubmit_prep() a bit. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 4a8900d480c5..be2760ae6c23 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2689,17 +2689,16 @@ static void io_complete_rw_common(struct kiocb *kiocb, long res, } #ifdef CONFIG_BLOCK -static bool io_resubmit_prep(struct io_kiocb *req, int error) +static bool io_resubmit_prep(struct io_kiocb *req) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; ssize_t ret = -ECANCELED; struct iov_iter iter; int rw; - if (error) { - ret = error; - goto end_req; - } + /* already prepared */ + if (req->async_data) + return true; switch (req->opcode) { case IORING_OP_READV: @@ -2715,22 +2714,16 @@ static bool io_resubmit_prep(struct io_kiocb *req, int error) default: printk_once(KERN_WARNING "io_uring: bad opcode in resubmit %d\n", req->opcode); - goto end_req; + return false; } - if (!req->async_data) { - ret = io_import_iovec(rw, req, &iovec, &iter, false); - if (ret < 0) - goto end_req; - ret = io_setup_async_rw(req, iovec, inline_vecs, &iter, false); - if (!ret) - return true; - kfree(iovec); - } else { + ret = io_import_iovec(rw, req, &iovec, &iter, false); + if (ret < 0) + return false; + ret = io_setup_async_rw(req, iovec, inline_vecs, &iter, false); + if (!ret) return true; - } -end_req: - req_set_fail_links(req); + kfree(iovec); return false; } #endif @@ -2751,12 +2744,12 @@ static bool io_rw_reissue(struct io_kiocb *req, long res) ret = io_sq_thread_acquire_mm_files(req->ctx, req); - if (io_resubmit_prep(req, ret)) { + if (!ret && io_resubmit_prep(req)) { refcount_inc(&req->refs); io_queue_async_work(req); return true; } - + req_set_fail_links(req); #endif return false; } From 5c766a908d06e96d30e0ec2511a24fa311553d2c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:36 +0000 Subject: [PATCH 015/103] io_uring: cleanup personalities under uring_lock personality_idr is usually synchronised by uring_lock, the exception would be removing personalities in io_ring_ctx_wait_and_kill(), which is legit as refs are killed by that point but still would be more resilient to do it under the lock. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index be2760ae6c23..5e576878efd9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8867,6 +8867,7 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) ctx->cq_overflow_flushed = 1; if (ctx->rings) __io_cqring_overflow_flush(ctx, true, NULL, NULL); + idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx); mutex_unlock(&ctx->uring_lock); io_kill_timeouts(ctx, NULL, NULL); @@ -8877,7 +8878,6 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) /* if we failed setting up the ctx, we might not have any rings */ io_iopoll_try_reap_events(ctx); - idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx); /* * Do this upfront, so we won't have a grace period where the ring From 2d7e935809b7f740442ce79fc6f53e94a1f0b874 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:37 +0000 Subject: [PATCH 016/103] io_uring: inline io_async_submit() The name is confusing and it's used only in one place. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 5e576878efd9..6eb4c25fa18b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1278,11 +1278,6 @@ static inline void io_req_init_async(struct io_kiocb *req) refcount_inc(&req->work.identity->count); } -static inline bool io_async_submit(struct io_ring_ctx *ctx) -{ - return ctx->flags & IORING_SETUP_SQPOLL; -} - static void io_ring_ctx_ref_free(struct percpu_ref *ref) { struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); @@ -6969,7 +6964,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) } trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, - true, io_async_submit(ctx)); + true, ctx->flags & IORING_SETUP_SQPOLL); err = io_submit_sqe(req, sqe, &link, &state.comp); if (err) goto fail_req; From ec30e04ba4a5c265f52482092a5f5f5232947c48 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:38 +0000 Subject: [PATCH 017/103] io_uring: inline __io_commit_cqring() Inline it in its only user, that's cleaner Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 6eb4c25fa18b..347bdcd2c0fe 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1364,14 +1364,6 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq) return false; } -static void __io_commit_cqring(struct io_ring_ctx *ctx) -{ - struct io_rings *rings = ctx->rings; - - /* order cqe stores with ring update */ - smp_store_release(&rings->cq.tail, ctx->cached_cq_tail); -} - static void io_put_identity(struct io_uring_task *tctx, struct io_kiocb *req) { if (req->work.identity == &tctx->__identity) @@ -1693,7 +1685,9 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx) static void io_commit_cqring(struct io_ring_ctx *ctx) { io_flush_timeouts(ctx); - __io_commit_cqring(ctx); + + /* order cqe stores with ring update */ + smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); if (unlikely(!list_empty(&ctx->defer_list))) __io_queue_deferred(ctx); From 888aae2eeddfe1d6c9731cf4af1a1b2605af6470 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:39 +0000 Subject: [PATCH 018/103] io_uring: further deduplicate #CQ events calc Apparently, there is one more place hand coded calculation of number of CQ events in the ring. Use __io_cqring_events() helper in io_get_cqring() as well. Naturally, assembly stays identical. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 347bdcd2c0fe..0a578c40b854 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1700,21 +1700,25 @@ static inline bool io_sqring_full(struct io_ring_ctx *ctx) return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == r->sq_ring_entries; } +static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) +{ + return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); +} + static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx) { struct io_rings *rings = ctx->rings; unsigned tail; - tail = ctx->cached_cq_tail; /* * writes to the cq entry need to come after reading head; the * control dependency is enough as we're using WRITE_ONCE to * fill the cq entry */ - if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries) + if (__io_cqring_events(ctx) == rings->cq_ring_entries) return NULL; - ctx->cached_cq_tail++; + tail = ctx->cached_cq_tail++; return &rings->cqes[tail & ctx->cq_mask]; } @@ -1729,11 +1733,6 @@ static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx) return io_wq_current_is_worker(); } -static inline unsigned __io_cqring_events(struct io_ring_ctx *ctx) -{ - return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); -} - static void io_cqring_ev_posted(struct io_ring_ctx *ctx) { /* see waitqueue_active() comment */ From 85bcb6c67ea145b8032089db891218e3339cbdb8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:40 +0000 Subject: [PATCH 019/103] io_uring: simplify io_alloc_req() Get rid of a label in io_alloc_req(), it's cleaner to do return directly. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 0a578c40b854..9ff84ceff4f9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1988,7 +1988,7 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx, if (unlikely(ret <= 0)) { state->reqs[0] = kmem_cache_alloc(req_cachep, gfp); if (!state->reqs[0]) - goto fallback; + return io_get_fallback_req(ctx); ret = 1; } state->free_reqs = ret; @@ -1996,8 +1996,6 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx, state->free_reqs--; return state->reqs[state->free_reqs]; -fallback: - return io_get_fallback_req(ctx); } static inline void io_put_file(struct io_kiocb *req, struct file *file, From 02b23a9af5ba4db0a85ebb81c8b376b2fe860d0f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:41 +0000 Subject: [PATCH 020/103] io_uring: remove __io_state_file_put The check in io_state_file_put() is optimised pretty well when called from __io_file_get(). Don't pollute the code with all these variants. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 9ff84ceff4f9..c3e0d6246d71 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2815,16 +2815,12 @@ static void io_iopoll_req_issued(struct io_kiocb *req, bool in_async) wake_up(&ctx->sq_data->wait); } -static inline void __io_state_file_put(struct io_submit_state *state) -{ - fput_many(state->file, state->file_refs); - state->file_refs = 0; -} - static inline void io_state_file_put(struct io_submit_state *state) { - if (state->file_refs) - __io_state_file_put(state); + if (state->file_refs) { + fput_many(state->file, state->file_refs); + state->file_refs = 0; + } } /* @@ -2842,7 +2838,7 @@ static struct file *__io_file_get(struct io_submit_state *state, int fd) state->file_refs--; return state->file; } - __io_state_file_put(state); + io_state_file_put(state); } state->file = fget_many(fd, state->ios_left); if (unlikely(!state->file)) From eab30c4d20dc761d463445e5130421863ff81505 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:42 +0000 Subject: [PATCH 021/103] io_uring: deduplicate failing task_work_add When io_req_task_work_add() fails, the request will be cancelled by enqueueing via task_works of io-wq. Extract a function for that. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 46 +++++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index c3e0d6246d71..90c3cad1723b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2171,6 +2171,16 @@ static int io_req_task_work_add(struct io_kiocb *req) return ret; } +static void io_req_task_work_add_fallback(struct io_kiocb *req, + void (*cb)(struct callback_head *)) +{ + struct task_struct *tsk = io_wq_get_task(req->ctx->io_wq); + + init_task_work(&req->task_work, cb); + task_work_add(tsk, &req->task_work, TWA_NONE); + wake_up_process(tsk); +} + static void __io_req_task_cancel(struct io_kiocb *req, int error) { struct io_ring_ctx *ctx = req->ctx; @@ -2225,14 +2235,8 @@ static void io_req_task_queue(struct io_kiocb *req) percpu_ref_get(&req->ctx->refs); ret = io_req_task_work_add(req); - if (unlikely(ret)) { - struct task_struct *tsk; - - init_task_work(&req->task_work, io_req_task_cancel); - tsk = io_wq_get_task(req->ctx->io_wq); - task_work_add(tsk, &req->task_work, TWA_NONE); - wake_up_process(tsk); - } + if (unlikely(ret)) + io_req_task_work_add_fallback(req, io_req_task_cancel); } static inline void io_queue_next(struct io_kiocb *req) @@ -2350,13 +2354,8 @@ static void io_free_req_deferred(struct io_kiocb *req) init_task_work(&req->task_work, io_put_req_deferred_cb); ret = io_req_task_work_add(req); - if (unlikely(ret)) { - struct task_struct *tsk; - - tsk = io_wq_get_task(req->ctx->io_wq); - task_work_add(tsk, &req->task_work, TWA_NONE); - wake_up_process(tsk); - } + if (unlikely(ret)) + io_req_task_work_add_fallback(req, io_put_req_deferred_cb); } static inline void io_put_req_deferred(struct io_kiocb *req, int refs) @@ -3425,15 +3424,8 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, /* submit ref gets dropped, acquire a new one */ refcount_inc(&req->refs); ret = io_req_task_work_add(req); - if (unlikely(ret)) { - struct task_struct *tsk; - - /* queue just for cancelation */ - init_task_work(&req->task_work, io_req_task_cancel); - tsk = io_wq_get_task(req->ctx->io_wq); - task_work_add(tsk, &req->task_work, TWA_NONE); - wake_up_process(tsk); - } + if (unlikely(ret)) + io_req_task_work_add_fallback(req, io_req_task_cancel); return 1; } @@ -5153,12 +5145,8 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll, */ ret = io_req_task_work_add(req); if (unlikely(ret)) { - struct task_struct *tsk; - WRITE_ONCE(poll->canceled, true); - tsk = io_wq_get_task(req->ctx->io_wq); - task_work_add(tsk, &req->task_work, TWA_NONE); - wake_up_process(tsk); + io_req_task_work_add_fallback(req, func); } return 1; } From 8662daec09edcdba2659799040aee1ba575c4799 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:44 +0000 Subject: [PATCH 022/103] io_uring: add a helper timeout mode calculation Deduplicates translation of timeout flags into hrtimer_mode. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 90c3cad1723b..4e167217c898 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5771,6 +5771,12 @@ static int io_timeout_remove_prep(struct io_kiocb *req, return 0; } +static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) +{ + return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS + : HRTIMER_MODE_REL; +} + /* * Remove or update an existing timeout command */ @@ -5781,14 +5787,11 @@ static int io_timeout_remove(struct io_kiocb *req) int ret; spin_lock_irq(&ctx->completion_lock); - if (req->timeout_rem.flags & IORING_TIMEOUT_UPDATE) { - enum hrtimer_mode mode = (tr->flags & IORING_TIMEOUT_ABS) - ? HRTIMER_MODE_ABS : HRTIMER_MODE_REL; - - ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode); - } else { + if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE)) ret = io_timeout_cancel(ctx, tr->addr); - } + else + ret = io_timeout_update(ctx, tr->addr, &tr->ts, + io_translate_timeout_mode(tr->flags)); io_cqring_fill_event(req, ret); io_commit_cqring(ctx); @@ -5828,11 +5831,7 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr))) return -EFAULT; - if (flags & IORING_TIMEOUT_ABS) - data->mode = HRTIMER_MODE_ABS; - else - data->mode = HRTIMER_MODE_REL; - + data->mode = io_translate_timeout_mode(flags); hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode); return 0; } From a38d68db6742c19a74141c0f56785ef67f51c504 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:45 +0000 Subject: [PATCH 023/103] io_uring: help inlining of io_req_complete() __io_req_complete() inlining is a bit weird, some compilers don't optimise out the non-NULL branch of it even when called as io_req_complete(). Help it a bit by extracting state and stateless helpers out of __io_req_complete(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 4e167217c898..f676b198ee1b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1886,7 +1886,8 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) __io_cqring_fill_event(req, res, 0); } -static void io_cqring_add_event(struct io_kiocb *req, long res, long cflags) +static void io_req_complete_nostate(struct io_kiocb *req, long res, + unsigned int cflags) { struct io_ring_ctx *ctx = req->ctx; unsigned long flags; @@ -1897,6 +1898,7 @@ static void io_cqring_add_event(struct io_kiocb *req, long res, long cflags) spin_unlock_irqrestore(&ctx->completion_lock, flags); io_cqring_ev_posted(ctx); + io_put_req(req); } static void io_submit_flush_completions(struct io_comp_state *cs) @@ -1932,23 +1934,27 @@ static void io_submit_flush_completions(struct io_comp_state *cs) cs->nr = 0; } -static void __io_req_complete(struct io_kiocb *req, long res, unsigned cflags, - struct io_comp_state *cs) +static void io_req_complete_state(struct io_kiocb *req, long res, + unsigned int cflags, struct io_comp_state *cs) { - if (!cs) { - io_cqring_add_event(req, res, cflags); - io_put_req(req); - } else { - io_clean_op(req); - req->result = res; - req->compl.cflags = cflags; - list_add_tail(&req->compl.list, &cs->list); - if (++cs->nr >= 32) - io_submit_flush_completions(cs); - } + io_clean_op(req); + req->result = res; + req->compl.cflags = cflags; + list_add_tail(&req->compl.list, &cs->list); + if (++cs->nr >= 32) + io_submit_flush_completions(cs); } -static void io_req_complete(struct io_kiocb *req, long res) +static inline void __io_req_complete(struct io_kiocb *req, long res, + unsigned cflags, struct io_comp_state *cs) +{ + if (!cs) + io_req_complete_nostate(req, res, cflags); + else + io_req_complete_state(req, res, cflags, cs); +} + +static inline void io_req_complete(struct io_kiocb *req, long res) { __io_req_complete(req, res, 0, NULL); } From 9affd664f0e0512d8997dbdddb1448a4faf9bc82 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:46 +0000 Subject: [PATCH 024/103] io_uring: don't flush CQEs deep down the stack io_submit_flush_completions() is called down the stack in the _state version of io_req_complete(), that's ok because is only called by io_uring opcode handler functions directly. Move it up to __io_queue_sqe() as preparation. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index f676b198ee1b..935a16a682a2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1941,8 +1941,7 @@ static void io_req_complete_state(struct io_kiocb *req, long res, req->result = res; req->compl.cflags = cflags; list_add_tail(&req->compl.list, &cs->list); - if (++cs->nr >= 32) - io_submit_flush_completions(cs); + cs->nr++; } static inline void __io_req_complete(struct io_kiocb *req, long res, @@ -6577,7 +6576,15 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) io_queue_linked_timeout(linked_timeout); } else if (likely(!ret)) { /* drop submission reference */ - req = io_put_req_find_next(req); + if (cs) { + io_put_req(req); + if (cs->nr >= 32) + io_submit_flush_completions(cs); + req = NULL; + } else { + req = io_put_req_find_next(req); + } + if (linked_timeout) io_queue_linked_timeout(linked_timeout); From e342c807f556dbcee1370ab78af1d8faf497d771 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:47 +0000 Subject: [PATCH 025/103] io_uring: save atomic dec for inline executed reqs When a request is completed with comp_state, its completion reference put is deferred to io_submit_flush_completions(), but the submission is put not far from there, so do it together to save one atomic dec per request. That targets requests that complete inline, e.g. buffered rw, send/recv. Proper benchmarking haven't been conducted but for nops(batch=32) it was around 7901 vs 8117 KIOPS (~2.7%), or ~4% per perf profiling. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 935a16a682a2..3f6d055eb6d4 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -629,6 +629,7 @@ enum { REQ_F_NO_FILE_TABLE_BIT, REQ_F_WORK_INITIALIZED_BIT, REQ_F_LTIMEOUT_ACTIVE_BIT, + REQ_F_COMPLETE_INLINE_BIT, /* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT, @@ -672,6 +673,8 @@ enum { REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT), /* linked timeout is active, i.e. prepared by link's head */ REQ_F_LTIMEOUT_ACTIVE = BIT(REQ_F_LTIMEOUT_ACTIVE_BIT), + /* completion is deferred through io_comp_state */ + REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT), }; struct async_poll { @@ -1917,14 +1920,15 @@ static void io_submit_flush_completions(struct io_comp_state *cs) * io_free_req() doesn't care about completion_lock unless one * of these flags is set. REQ_F_WORK_INITIALIZED is in the list * because of a potential deadlock with req->work.fs->lock + * We defer both, completion and submission refs. */ if (req->flags & (REQ_F_FAIL_LINK|REQ_F_LINK_TIMEOUT |REQ_F_WORK_INITIALIZED)) { spin_unlock_irq(&ctx->completion_lock); - io_put_req(req); + io_double_put_req(req); spin_lock_irq(&ctx->completion_lock); } else { - io_put_req(req); + io_double_put_req(req); } } io_commit_cqring(ctx); @@ -1940,8 +1944,7 @@ static void io_req_complete_state(struct io_kiocb *req, long res, io_clean_op(req); req->result = res; req->compl.cflags = cflags; - list_add_tail(&req->compl.list, &cs->list); - cs->nr++; + req->flags |= REQ_F_COMPLETE_INLINE; } static inline void __io_req_complete(struct io_kiocb *req, long res, @@ -6576,9 +6579,9 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) io_queue_linked_timeout(linked_timeout); } else if (likely(!ret)) { /* drop submission reference */ - if (cs) { - io_put_req(req); - if (cs->nr >= 32) + if (req->flags & REQ_F_COMPLETE_INLINE) { + list_add_tail(&req->compl.list, &cs->list); + if (++cs->nr >= 32) io_submit_flush_completions(cs); req = NULL; } else { From 53dec2ea74f2ef360e8455439be96a780baa6097 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 19 Jan 2021 15:41:52 -0700 Subject: [PATCH 026/103] fs: provide locked helper variant of close_fd_get_file() Assumes current->files->file_lock is already held on invocation. Helps the caller check the file before removing the fd, if it needs to. Signed-off-by: Jens Axboe --- fs/file.c | 48 +++++++++++++++++++++++++++++++----------------- fs/internal.h | 1 + 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/fs/file.c b/fs/file.c index dab120b71e44..f3a4bac2cbe9 100644 --- a/fs/file.c +++ b/fs/file.c @@ -22,6 +22,8 @@ #include #include +#include "internal.h" + unsigned int sysctl_nr_open __read_mostly = 1024*1024; unsigned int sysctl_nr_open_min = BITS_PER_LONG; /* our min() is unusable in constant expressions ;-/ */ @@ -731,6 +733,32 @@ int __close_range(unsigned fd, unsigned max_fd, unsigned int flags) return 0; } +/* + * See close_fd_get_file() below, this variant assumes current->files->file_lock + * is held. + */ +int __close_fd_get_file(unsigned int fd, struct file **res) +{ + struct files_struct *files = current->files; + struct file *file; + struct fdtable *fdt; + + fdt = files_fdtable(files); + if (fd >= fdt->max_fds) + goto out_err; + file = fdt->fd[fd]; + if (!file) + goto out_err; + rcu_assign_pointer(fdt->fd[fd], NULL); + __put_unused_fd(files, fd); + get_file(file); + *res = file; + return 0; +out_err: + *res = NULL; + return -ENOENT; +} + /* * variant of close_fd that gets a ref on the file for later fput. * The caller must ensure that filp_close() called on the file, and then @@ -739,27 +767,13 @@ int __close_range(unsigned fd, unsigned max_fd, unsigned int flags) int close_fd_get_file(unsigned int fd, struct file **res) { struct files_struct *files = current->files; - struct file *file; - struct fdtable *fdt; + int ret; spin_lock(&files->file_lock); - fdt = files_fdtable(files); - if (fd >= fdt->max_fds) - goto out_unlock; - file = fdt->fd[fd]; - if (!file) - goto out_unlock; - rcu_assign_pointer(fdt->fd[fd], NULL); - __put_unused_fd(files, fd); + ret = __close_fd_get_file(fd, res); spin_unlock(&files->file_lock); - get_file(file); - *res = file; - return 0; -out_unlock: - spin_unlock(&files->file_lock); - *res = NULL; - return -ENOENT; + return ret; } void do_close_on_exec(struct files_struct *files) diff --git a/fs/internal.h b/fs/internal.h index 77c50befbfbe..c6c85f6ad598 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -132,6 +132,7 @@ extern struct file *do_file_open_root(struct dentry *, struct vfsmount *, const char *, const struct open_flags *); extern struct open_how build_open_how(int flags, umode_t mode); extern int build_open_flags(const struct open_how *how, struct open_flags *op); +extern int __close_fd_get_file(unsigned int fd, struct file **res); long do_sys_ftruncate(unsigned int fd, loff_t length, int small); int chmod_common(const struct path *path, umode_t mode); From 9eac1904d3364254d622bf2c771c4f85cd435fc2 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 19 Jan 2021 15:50:37 -0700 Subject: [PATCH 027/103] io_uring: get rid of intermediate IORING_OP_CLOSE stage We currently split the close into two, in case we have a ->flush op that we can't safely handle from non-blocking context. This requires us to flag the op as uncancelable if we do need to punt it async, and that means special handling for just this op type. Use __close_fd_get_file() and grab the files lock so we can get the file and check if we need to go async in one atomic operation. That gets rid of the need for splitting this into two steps, and hence the need for IO_WQ_WORK_NO_CANCEL. Signed-off-by: Jens Axboe --- fs/io_uring.c | 64 ++++++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 3f6d055eb6d4..4dd18c81789c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -423,7 +423,6 @@ struct io_poll_remove { struct io_close { struct file *file; - struct file *put_file; int fd; }; @@ -920,8 +919,6 @@ static const struct io_op_def io_op_defs[] = { IO_WQ_WORK_FS | IO_WQ_WORK_MM, }, [IORING_OP_CLOSE] = { - .needs_file = 1, - .needs_file_no_error = 1, .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_BLKCG, }, [IORING_OP_FILES_UPDATE] = { @@ -4475,13 +4472,6 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock) static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - /* - * If we queue this for async, it must not be cancellable. That would - * leave the 'file' in an undeterminate state, and here need to modify - * io_wq_work.flags, so initialize io_wq_work firstly. - */ - io_req_init_async(req); - if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; if (sqe->ioprio || sqe->off || sqe->addr || sqe->len || @@ -4491,43 +4481,59 @@ static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EBADF; req->close.fd = READ_ONCE(sqe->fd); - if ((req->file && req->file->f_op == &io_uring_fops)) - return -EBADF; - - req->close.put_file = NULL; return 0; } static int io_close(struct io_kiocb *req, bool force_nonblock, struct io_comp_state *cs) { + struct files_struct *files = current->files; struct io_close *close = &req->close; + struct fdtable *fdt; + struct file *file; int ret; - /* might be already done during nonblock submission */ - if (!close->put_file) { - ret = close_fd_get_file(close->fd, &close->put_file); - if (ret < 0) - return (ret == -ENOENT) ? -EBADF : ret; + file = NULL; + ret = -EBADF; + spin_lock(&files->file_lock); + fdt = files_fdtable(files); + if (close->fd >= fdt->max_fds) { + spin_unlock(&files->file_lock); + goto err; + } + file = fdt->fd[close->fd]; + if (!file) { + spin_unlock(&files->file_lock); + goto err; + } + + if (file->f_op == &io_uring_fops) { + spin_unlock(&files->file_lock); + file = NULL; + goto err; } /* if the file has a flush method, be safe and punt to async */ - if (close->put_file->f_op->flush && force_nonblock) { - /* not safe to cancel at this point */ - req->work.flags |= IO_WQ_WORK_NO_CANCEL; - /* was never set, but play safe */ - req->flags &= ~REQ_F_NOWAIT; - /* avoid grabbing files - we don't need the files */ - req->flags |= REQ_F_NO_FILE_TABLE; + if (file->f_op->flush && force_nonblock) { + spin_unlock(&files->file_lock); return -EAGAIN; } + ret = __close_fd_get_file(close->fd, &file); + spin_unlock(&files->file_lock); + if (ret < 0) { + if (ret == -ENOENT) + ret = -EBADF; + goto err; + } + /* No ->flush() or already async, safely close from here */ - ret = filp_close(close->put_file, req->work.identity->files); + ret = filp_close(file, current->files); +err: if (ret < 0) req_set_fail_links(req); - fput(close->put_file); - close->put_file = NULL; + if (file) + fput(file); __io_req_complete(req, ret, 0, cs); return 0; } From 4014d943cb62db892eb023d385a966a3fce5ee4c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 19 Jan 2021 15:53:54 -0700 Subject: [PATCH 028/103] io_uring/io-wq: kill off now unused IO_WQ_WORK_NO_CANCEL It's no longer used as IORING_OP_CLOSE got rid for the need of flagging it as uncancelable, kill it of. Signed-off-by: Jens Axboe --- fs/io-wq.c | 1 - fs/io-wq.h | 1 - fs/io_uring.c | 5 +---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index a564f36e260c..2e2f14f42bf2 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -944,7 +944,6 @@ static bool io_wq_worker_cancel(struct io_worker *worker, void *data) */ spin_lock_irqsave(&worker->lock, flags); if (worker->cur_work && - !(worker->cur_work->flags & IO_WQ_WORK_NO_CANCEL) && match->fn(worker->cur_work, match->data)) { send_sig(SIGINT, worker->task, 1); match->nr_running++; diff --git a/fs/io-wq.h b/fs/io-wq.h index b158f8addcf3..e1ffb80a4a1d 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -9,7 +9,6 @@ enum { IO_WQ_WORK_CANCEL = 1, IO_WQ_WORK_HASHED = 2, IO_WQ_WORK_UNBOUND = 4, - IO_WQ_WORK_NO_CANCEL = 8, IO_WQ_WORK_CONCURRENT = 16, IO_WQ_WORK_FILES = 32, diff --git a/fs/io_uring.c b/fs/io_uring.c index 4dd18c81789c..be73f6ddbd9e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6386,11 +6386,8 @@ static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work) if (timeout) io_queue_linked_timeout(timeout); - /* if NO_CANCEL is set, we must still run the work */ - if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) == - IO_WQ_WORK_CANCEL) { + if (work->flags & IO_WQ_WORK_CANCEL) ret = -ECANCELED; - } if (!ret) { do { From 0bead8cd39b9c9c7c4e902018ccf129107ac50ef Mon Sep 17 00:00:00 2001 From: Yejune Deng Date: Thu, 24 Dec 2020 11:02:20 +0800 Subject: [PATCH 029/103] io_uring: simplify io_remove_personalities() The function io_remove_personalities() is very similar to io_unregister_personality(),so implement io_remove_personalities() calling io_unregister_personality(). Signed-off-by: Yejune Deng Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index be73f6ddbd9e..b05d37431c12 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8805,9 +8805,8 @@ static int io_uring_fasync(int fd, struct file *file, int on) return fasync_helper(fd, file, on, &ctx->cq_fasync); } -static int io_remove_personalities(int id, void *p, void *data) +static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) { - struct io_ring_ctx *ctx = data; struct io_identity *iod; iod = idr_remove(&ctx->personality_idr, id); @@ -8815,7 +8814,17 @@ static int io_remove_personalities(int id, void *p, void *data) put_cred(iod->creds); if (refcount_dec_and_test(&iod->count)) kfree(iod); + return 0; } + + return -EINVAL; +} + +static int io_remove_personalities(int id, void *p, void *data) +{ + struct io_ring_ctx *ctx = data; + + io_unregister_personality(ctx, id); return 0; } @@ -9951,21 +9960,6 @@ static int io_register_personality(struct io_ring_ctx *ctx) return ret; } -static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) -{ - struct io_identity *iod; - - iod = idr_remove(&ctx->personality_idr, id); - if (iod) { - put_cred(iod->creds); - if (refcount_dec_and_test(&iod->count)) - kfree(iod); - return 0; - } - - return -EINVAL; -} - static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg, unsigned int nr_args) { From ecfc8492820732be652146280912554ced62c32b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 25 Jan 2021 11:42:20 +0000 Subject: [PATCH 030/103] io_uring: ensure only sqo_task has file notes For SQPOLL io_uring we want to have only one file note held by sqo_task. Add a warning to make sure it holds. It's deep in io_uring_add_task_file() out of hot path, so shouldn't hurt. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index b05d37431c12..68bf2c8c23a9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9099,6 +9099,10 @@ static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) fput(file); return ret; } + + /* one and only SQPOLL file note, held by sqo_task */ + WARN_ON_ONCE((ctx->flags & IORING_SETUP_SQPOLL) && + current != ctx->sqo_task); } tctx->last = file; } From 7c6607313f032b73638a6f752cb4adf50ba947cf Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 25 Jan 2021 11:42:21 +0000 Subject: [PATCH 031/103] io_uring: consolidate putting reqs task We grab a task for each request and while putting it it also have to do extra work like inflight accounting and waking up that task. This sequence is duplicated several time, it's good time to add a helper. More to that, the helper generates better code due to better locality and so not failing alias analysis. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 68bf2c8c23a9..6d45a0975d9c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2023,17 +2023,22 @@ static void io_dismantle_req(struct io_kiocb *req) io_req_clean_work(req); } +static inline void io_put_task(struct task_struct *task, int nr) +{ + struct io_uring_task *tctx = task->io_uring; + + percpu_counter_sub(&tctx->inflight, nr); + if (unlikely(atomic_read(&tctx->in_idle))) + wake_up(&tctx->wait); + put_task_struct_many(task, nr); +} + static void __io_free_req(struct io_kiocb *req) { - struct io_uring_task *tctx = req->task->io_uring; struct io_ring_ctx *ctx = req->ctx; io_dismantle_req(req); - - percpu_counter_dec(&tctx->inflight); - if (atomic_read(&tctx->in_idle)) - wake_up(&tctx->wait); - put_task_struct(req->task); + io_put_task(req->task, 1); if (likely(!io_is_fallback_req(req))) kmem_cache_free(req_cachep, req); @@ -2287,12 +2292,7 @@ static void io_req_free_batch_finish(struct io_ring_ctx *ctx, if (rb->to_free) __io_req_free_batch_flush(ctx, rb); if (rb->task) { - struct io_uring_task *tctx = rb->task->io_uring; - - percpu_counter_sub(&tctx->inflight, rb->task_refs); - if (atomic_read(&tctx->in_idle)) - wake_up(&tctx->wait); - put_task_struct_many(rb->task, rb->task_refs); + io_put_task(rb->task, rb->task_refs); rb->task = NULL; } } @@ -2306,14 +2306,8 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) io_queue_next(req); if (req->task != rb->task) { - if (rb->task) { - struct io_uring_task *tctx = rb->task->io_uring; - - percpu_counter_sub(&tctx->inflight, rb->task_refs); - if (atomic_read(&tctx->in_idle)) - wake_up(&tctx->wait); - put_task_struct_many(rb->task, rb->task_refs); - } + if (rb->task) + io_put_task(rb->task, rb->task_refs); rb->task = req->task; rb->task_refs = 0; } From 67973b933e347c38478b591d6c9dc076bea7c9dc Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 26 Jan 2021 13:51:09 +0000 Subject: [PATCH 032/103] io_uring: cleanup files_update looping Replace a while with a simple for loop, that looks way more natural, and enables us to use "continue" as indexes are no more updated by hand in the end of the loop. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 6d45a0975d9c..0ca99bd5c316 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8028,9 +8028,8 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, return -ENOMEM; init_fixed_file_ref_node(ctx, ref_node); - done = 0; fds = u64_to_user_ptr(up->data); - while (nr_args) { + for (done = 0; done < nr_args; done++) { struct fixed_rsrc_table *table; unsigned index; @@ -8039,7 +8038,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, err = -EFAULT; break; } - i = array_index_nospec(up->offset, ctx->nr_user_files); + i = array_index_nospec(up->offset + done, ctx->nr_user_files); table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; index = i & IORING_FILE_TABLE_MASK; if (table->files[index]) { @@ -8077,9 +8076,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, break; } } - nr_args--; - done++; - up->offset++; } if (needs_switch) { From 4e0377a1c5c633852f443a562ec55f7dfea65350 Mon Sep 17 00:00:00 2001 From: noah Date: Tue, 26 Jan 2021 15:23:28 -0500 Subject: [PATCH 033/103] io_uring: Add skip option for __io_sqe_files_update This patch adds support for skipping a file descriptor when using IORING_REGISTER_FILES_UPDATE. __io_sqe_files_update will skip fds set to IORING_REGISTER_FILES_SKIP. IORING_REGISTER_FILES_SKIP is inturn added as a #define in io_uring.h Signed-off-by: noah Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 +++ include/uapi/linux/io_uring.h | 3 +++ 2 files changed, 6 insertions(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index 0ca99bd5c316..dd83a64ba709 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8038,6 +8038,9 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, err = -EFAULT; break; } + if (fd == IORING_REGISTER_FILES_SKIP) + continue; + i = array_index_nospec(up->offset + done, ctx->nr_user_files); table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; index = i & IORING_FILE_TABLE_MASK; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index f9f106c54d90..ac4e1738a9af 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -298,6 +298,9 @@ struct io_uring_rsrc_update { __aligned_u64 data; }; +/* Skip updating fd indexes set to this value in the fd table */ +#define IORING_REGISTER_FILES_SKIP (-2) + #define IO_URING_OP_SUPPORTED (1U << 0) struct io_uring_probe_op { From 090da7d52fe2aeabb73bf300154278e411cd069e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 27 Jan 2021 01:25:04 +0000 Subject: [PATCH 034/103] MAINTAINERS: update io_uring section Add the missing kernel io_uring header, add Pavel as a reviewer, and exclude io_uring from the FILESYSTEMS section to avoid keep spamming Al (mainly) with bug reports, patches, etc. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- MAINTAINERS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index d3e847f7f3dc..363e7aa3b79c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6863,6 +6863,9 @@ F: include/linux/fs.h F: include/linux/fs_types.h F: include/uapi/linux/fs.h F: include/uapi/linux/openat2.h +X: fs/io-wq.c +X: fs/io-wq.h +X: fs/io_uring.c FINTEK F75375S HARDWARE MONITOR AND FAN CONTROLLER DRIVER M: Riku Voipio @@ -9295,6 +9298,7 @@ F: include/uapi/linux/iommu.h IO_URING M: Jens Axboe +R: Pavel Begunkov L: io-uring@vger.kernel.org S: Maintained T: git git://git.kernel.dk/linux-block @@ -9302,6 +9306,7 @@ T: git git://git.kernel.dk/liburing F: fs/io-wq.c F: fs/io-wq.h F: fs/io_uring.c +F: include/linux/io_uring.h F: include/uapi/linux/io_uring.h IPMI SUBSYSTEM From 8b28fdf21193d35d6ec5a8430f0241f5f977c6ac Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Sun, 31 Jan 2021 22:39:04 +0800 Subject: [PATCH 035/103] io_uring: check kthread parked flag before sqthread goes to sleep Abaci reported this issue: #[ 605.170872] INFO: task kworker/u4:1:53 blocked for more than 143 seconds. [ 605.172123] Not tainted 5.10.0+ #1 [ 605.172811] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 605.173915] task:kworker/u4:1 state:D stack: 0 pid: 53 ppid: 2 flags:0x00004000 [ 605.175130] Workqueue: events_unbound io_ring_exit_work [ 605.175931] Call Trace: [ 605.176334] __schedule+0xe0e/0x25a0 [ 605.176971] ? firmware_map_remove+0x1a1/0x1a1 [ 605.177631] ? write_comp_data+0x2a/0x80 [ 605.178272] schedule+0xd0/0x270 [ 605.178811] schedule_timeout+0x6b6/0x940 [ 605.179415] ? mark_lock.part.0+0xca/0x1420 [ 605.180062] ? usleep_range+0x170/0x170 [ 605.180684] ? wait_for_completion+0x16d/0x280 [ 605.181392] ? mark_held_locks+0x9e/0xe0 [ 605.182079] ? rwlock_bug.part.0+0x90/0x90 [ 605.182853] ? lockdep_hardirqs_on_prepare+0x286/0x400 [ 605.183817] wait_for_completion+0x175/0x280 [ 605.184713] ? wait_for_completion_interruptible+0x340/0x340 [ 605.185611] ? _raw_spin_unlock_irq+0x24/0x30 [ 605.186307] ? migrate_swap_stop+0x9c0/0x9c0 [ 605.187046] kthread_park+0x127/0x1c0 [ 605.187738] io_sq_thread_stop+0xd5/0x530 [ 605.188459] io_ring_exit_work+0xb1/0x970 [ 605.189207] process_one_work+0x92c/0x1510 [ 605.189947] ? pwq_dec_nr_in_flight+0x360/0x360 [ 605.190682] ? rwlock_bug.part.0+0x90/0x90 [ 605.191430] ? write_comp_data+0x2a/0x80 [ 605.192207] worker_thread+0x9b/0xe20 [ 605.192900] ? process_one_work+0x1510/0x1510 [ 605.193599] kthread+0x353/0x460 [ 605.194154] ? _raw_spin_unlock_irq+0x24/0x30 [ 605.194910] ? kthread_create_on_node+0x100/0x100 [ 605.195821] ret_from_fork+0x1f/0x30 [ 605.196605] [ 605.196605] Showing all locks held in the system: [ 605.197598] 1 lock held by khungtaskd/25: [ 605.198301] #0: ffffffff8b5f76a0 (rcu_read_lock){....}-{1:2}, at: rcu_lock_acquire.constprop.0+0x0/0x30 [ 605.199914] 3 locks held by kworker/u4:1/53: [ 605.200609] #0: ffff888100109938 ((wq_completion)events_unbound){+.+.}-{0:0}, at: process_one_work+0x82a/0x1510 [ 605.202108] #1: ffff888100e47dc0 ((work_completion)(&ctx->exit_work)){+.+.}-{0:0}, at: process_one_work+0x85e/0x1510 [ 605.203681] #2: ffff888116931870 (&sqd->lock){+.+.}-{3:3}, at: io_sq_thread_park.part.0+0x19/0x50 [ 605.205183] 3 locks held by systemd-journal/161: [ 605.206037] 1 lock held by syslog-ng/254: [ 605.206674] 2 locks held by agetty/311: [ 605.207292] #0: ffff888101097098 (&tty->ldisc_sem){++++}-{0:0}, at: tty_ldisc_ref_wait+0x27/0x80 [ 605.208715] #1: ffffc900000332e8 (&ldata->atomic_read_lock){+.+.}-{3:3}, at: n_tty_read+0x222/0x1bb0 [ 605.210131] 2 locks held by bash/677: [ 605.210723] #0: ffff88810419a098 (&tty->ldisc_sem){++++}-{0:0}, at: tty_ldisc_ref_wait+0x27/0x80 [ 605.212105] #1: ffffc900000512e8 (&ldata->atomic_read_lock){+.+.}-{3:3}, at: n_tty_read+0x222/0x1bb0 [ 605.213777] [ 605.214151] ============================================= I believe this is caused by the follow race: (ctx_list is empty now) => io_put_sq_data | ==> kthread_park(sqd->thread); | ====> set KTHREAD_SHOULD_PARK | ====> wake_up_process(k) | sq thread is running | | | needs_sched is true since no ctx, | so TASK_INTERRUPTIBLE set and schedule | out then never wake up again | ====> wait_for_completion | (stuck here) So check if sqthread gets park flag right before schedule(). since ctx_list is always empty when this problem happens, here I put kthread_should_park() before setting the wakeup flag(ctx_list is empty so this for loop is fast), where is close enough to schedule(). The problem doesn't show again in my repro testing after this fix. Reported-by: Abaci Signed-off-by: Hao Xu Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index dd83a64ba709..a8bf867b6cf2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7117,9 +7117,6 @@ static int io_sq_thread(void *data) continue; } - if (kthread_should_park()) - continue; - needs_sched = true; prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { @@ -7134,7 +7131,7 @@ static int io_sq_thread(void *data) } } - if (needs_sched) { + if (needs_sched && !kthread_should_park()) { list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_set_wakeup_flag(ctx); From 13770a71ed35512cc73c6b350297a797f0b27880 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 1 Feb 2021 15:23:42 +0300 Subject: [PATCH 036/103] io_uring: Fix NULL dereference in error in io_sqe_files_register() If we hit a "goto out_free;" before the "ctx->file_data" pointer has been assigned then it leads to a NULL derefence when we call: free_fixed_rsrc_data(ctx->file_data); We can fix this by moving the assignment earlier. Fixes: 1ad555c6ae6e ("io_uring: create common fixed_rsrc_data allocation routines") Signed-off-by: Dan Carpenter Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index a8bf867b6cf2..6711200ece22 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7865,6 +7865,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, file_data = alloc_fixed_rsrc_data(ctx); if (!file_data) return -ENOMEM; + ctx->file_data = file_data; nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE); file_data->table = kcalloc(nr_tables, sizeof(*file_data->table), @@ -7874,7 +7875,6 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, if (io_sqe_alloc_file_tables(file_data, nr_tables, nr_args)) goto out_free; - ctx->file_data = file_data; for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { struct fixed_rsrc_table *table; From 9ae1f8dd372e0e4c020b345cf9e09f519265e981 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Feb 2021 18:59:51 +0000 Subject: [PATCH 037/103] io_uring: fix inconsistent lock state WARNING: inconsistent lock state inconsistent {HARDIRQ-ON-W} -> {IN-HARDIRQ-W} usage. syz-executor217/8450 [HC1[1]:SC0[0]:HE0:SE1] takes: ffff888023d6e620 (&fs->lock){?.+.}-{2:2}, at: spin_lock include/linux/spinlock.h:354 [inline] ffff888023d6e620 (&fs->lock){?.+.}-{2:2}, at: io_req_clean_work fs/io_uring.c:1398 [inline] ffff888023d6e620 (&fs->lock){?.+.}-{2:2}, at: io_dismantle_req+0x66f/0xf60 fs/io_uring.c:2029 other info that might help us debug this: Possible unsafe locking scenario: CPU0 ---- lock(&fs->lock); lock(&fs->lock); *** DEADLOCK *** 1 lock held by syz-executor217/8450: #0: ffff88802417c3e8 (&ctx->uring_lock){+.+.}-{3:3}, at: __do_sys_io_uring_enter+0x1071/0x1f30 fs/io_uring.c:9442 stack backtrace: CPU: 1 PID: 8450 Comm: syz-executor217 Not tainted 5.11.0-rc5-next-20210129-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: [...] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:151 spin_lock include/linux/spinlock.h:354 [inline] io_req_clean_work fs/io_uring.c:1398 [inline] io_dismantle_req+0x66f/0xf60 fs/io_uring.c:2029 __io_free_req+0x3d/0x2e0 fs/io_uring.c:2046 io_free_req fs/io_uring.c:2269 [inline] io_double_put_req fs/io_uring.c:2392 [inline] io_put_req+0xf9/0x570 fs/io_uring.c:2388 io_link_timeout_fn+0x30c/0x480 fs/io_uring.c:6497 __run_hrtimer kernel/time/hrtimer.c:1519 [inline] __hrtimer_run_queues+0x609/0xe40 kernel/time/hrtimer.c:1583 hrtimer_interrupt+0x334/0x940 kernel/time/hrtimer.c:1645 local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1085 [inline] __sysvec_apic_timer_interrupt+0x146/0x540 arch/x86/kernel/apic/apic.c:1102 asm_call_irq_on_stack+0xf/0x20 __run_sysvec_on_irqstack arch/x86/include/asm/irq_stack.h:37 [inline] run_sysvec_on_irqstack_cond arch/x86/include/asm/irq_stack.h:89 [inline] sysvec_apic_timer_interrupt+0xbd/0x100 arch/x86/kernel/apic/apic.c:1096 asm_sysvec_apic_timer_interrupt+0x12/0x20 arch/x86/include/asm/idtentry.h:629 RIP: 0010:__raw_spin_unlock_irq include/linux/spinlock_api_smp.h:169 [inline] RIP: 0010:_raw_spin_unlock_irq+0x25/0x40 kernel/locking/spinlock.c:199 spin_unlock_irq include/linux/spinlock.h:404 [inline] io_queue_linked_timeout+0x194/0x1f0 fs/io_uring.c:6525 __io_queue_sqe+0x328/0x1290 fs/io_uring.c:6594 io_queue_sqe+0x631/0x10d0 fs/io_uring.c:6639 io_queue_link_head fs/io_uring.c:6650 [inline] io_submit_sqe fs/io_uring.c:6697 [inline] io_submit_sqes+0x19b5/0x2720 fs/io_uring.c:6960 __do_sys_io_uring_enter+0x107d/0x1f30 fs/io_uring.c:9443 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xa9 Don't free requests from under hrtimer context (softirq) as it may sleep or take spinlocks improperly (e.g. non-irq versions). Cc: stable@vger.kernel.org # 5.6+ Reported-by: syzbot+81d17233a2b02eafba33@syzkaller.appspotmail.com Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 6711200ece22..1310c074f4cc 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1886,8 +1886,8 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) __io_cqring_fill_event(req, res, 0); } -static void io_req_complete_nostate(struct io_kiocb *req, long res, - unsigned int cflags) +static void io_req_complete_post(struct io_kiocb *req, long res, + unsigned int cflags) { struct io_ring_ctx *ctx = req->ctx; unsigned long flags; @@ -1898,6 +1898,12 @@ static void io_req_complete_nostate(struct io_kiocb *req, long res, spin_unlock_irqrestore(&ctx->completion_lock, flags); io_cqring_ev_posted(ctx); +} + +static inline void io_req_complete_nostate(struct io_kiocb *req, long res, + unsigned int cflags) +{ + io_req_complete_post(req, res, cflags); io_put_req(req); } @@ -6489,9 +6495,10 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) if (prev) { req_set_fail_links(prev); io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME); - io_put_req(prev); + io_put_req_deferred(prev, 1); } else { - io_req_complete(req, -ETIME); + io_req_complete_post(req, -ETIME, 0); + io_put_req_deferred(req, 1); } return HRTIMER_NORESTART; } From ba13e23f37c795bdd993523a6749d7afbf5ff7fb Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Feb 2021 18:59:52 +0000 Subject: [PATCH 038/103] io_uring: kill not used needs_file_no_error We have no request types left using needs_file_no_error, remove it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 1310c074f4cc..66d9f3f4e43b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -788,8 +788,6 @@ struct io_submit_state { struct io_op_def { /* needs req->file assigned */ unsigned needs_file : 1; - /* don't fail if file grab fails */ - unsigned needs_file_no_error : 1; /* hash wq insertion if file is a regular file */ unsigned hash_reg_file : 1; /* unbound wq insertion if file is a non-regular file */ @@ -6896,8 +6894,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, bool fixed = req->flags & REQ_F_FIXED_FILE; req->file = io_file_get(state, req, READ_ONCE(sqe->fd), fixed); - if (unlikely(!req->file && - !io_op_defs[req->opcode].needs_file_no_error)) + if (unlikely(!req->file)) ret = -EBADF; } From 34e08fed2c1cc67df88d85fedde1d05fec62e5ca Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Feb 2021 18:59:53 +0000 Subject: [PATCH 039/103] io_uring: inline io_req_drop_files() req->files now have same lifetime as all other iowq-work resources, inline io_req_drop_files() for consistency. Moreover, since REQ_F_INFLIGHT is no more files specific, the function name became very confusing. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 66d9f3f4e43b..9354e61243d9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1036,7 +1036,6 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, const struct iovec *fast_iov, struct iov_iter *iter, bool force); -static void io_req_drop_files(struct io_kiocb *req); static void io_req_task_queue(struct io_kiocb *req); static struct kmem_cache *req_cachep; @@ -1402,8 +1401,23 @@ static void io_req_clean_work(struct io_kiocb *req) free_fs_struct(fs); req->work.flags &= ~IO_WQ_WORK_FS; } - if (req->flags & REQ_F_INFLIGHT) - io_req_drop_files(req); + if (req->work.flags & IO_WQ_WORK_FILES) { + put_files_struct(req->work.identity->files); + put_nsproxy(req->work.identity->nsproxy); + req->work.flags &= ~IO_WQ_WORK_FILES; + } + if (req->flags & REQ_F_INFLIGHT) { + struct io_ring_ctx *ctx = req->ctx; + struct io_uring_task *tctx = req->task->io_uring; + unsigned long flags; + + spin_lock_irqsave(&ctx->inflight_lock, flags); + list_del(&req->inflight_entry); + spin_unlock_irqrestore(&ctx->inflight_lock, flags); + req->flags &= ~REQ_F_INFLIGHT; + if (atomic_read(&tctx->in_idle)) + wake_up(&tctx->wait); + } io_put_identity(req->task->io_uring, req); } @@ -6164,25 +6178,6 @@ static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EIOCBQUEUED; } -static void io_req_drop_files(struct io_kiocb *req) -{ - struct io_ring_ctx *ctx = req->ctx; - struct io_uring_task *tctx = req->task->io_uring; - unsigned long flags; - - if (req->work.flags & IO_WQ_WORK_FILES) { - put_files_struct(req->work.identity->files); - put_nsproxy(req->work.identity->nsproxy); - } - spin_lock_irqsave(&ctx->inflight_lock, flags); - list_del(&req->inflight_entry); - spin_unlock_irqrestore(&ctx->inflight_lock, flags); - req->flags &= ~REQ_F_INFLIGHT; - req->work.flags &= ~IO_WQ_WORK_FILES; - if (atomic_read(&tctx->in_idle)) - wake_up(&tctx->wait); -} - static void __io_clean_op(struct io_kiocb *req) { if (req->flags & REQ_F_BUFFER_SELECTED) { From e86d004729ae9ce7d16ff3fad3708e1601eec0d2 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Feb 2021 18:59:54 +0000 Subject: [PATCH 040/103] io_uring: remove work flags after cleanup Shouldn't be a problem now, but it's better to clean REQ_F_WORK_INITIALIZED and work->flags only after relevant resources are killed, so cancellation see them. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 9354e61243d9..9b1f919b05c9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1374,22 +1374,14 @@ static void io_req_clean_work(struct io_kiocb *req) if (!(req->flags & REQ_F_WORK_INITIALIZED)) return; - req->flags &= ~REQ_F_WORK_INITIALIZED; - - if (req->work.flags & IO_WQ_WORK_MM) { + if (req->work.flags & IO_WQ_WORK_MM) mmdrop(req->work.identity->mm); - req->work.flags &= ~IO_WQ_WORK_MM; - } #ifdef CONFIG_BLK_CGROUP - if (req->work.flags & IO_WQ_WORK_BLKCG) { + if (req->work.flags & IO_WQ_WORK_BLKCG) css_put(req->work.identity->blkcg_css); - req->work.flags &= ~IO_WQ_WORK_BLKCG; - } #endif - if (req->work.flags & IO_WQ_WORK_CREDS) { + if (req->work.flags & IO_WQ_WORK_CREDS) put_cred(req->work.identity->creds); - req->work.flags &= ~IO_WQ_WORK_CREDS; - } if (req->work.flags & IO_WQ_WORK_FS) { struct fs_struct *fs = req->work.identity->fs; @@ -1399,12 +1391,10 @@ static void io_req_clean_work(struct io_kiocb *req) spin_unlock(&req->work.identity->fs->lock); if (fs) free_fs_struct(fs); - req->work.flags &= ~IO_WQ_WORK_FS; } if (req->work.flags & IO_WQ_WORK_FILES) { put_files_struct(req->work.identity->files); put_nsproxy(req->work.identity->nsproxy); - req->work.flags &= ~IO_WQ_WORK_FILES; } if (req->flags & REQ_F_INFLIGHT) { struct io_ring_ctx *ctx = req->ctx; @@ -1419,6 +1409,9 @@ static void io_req_clean_work(struct io_kiocb *req) wake_up(&tctx->wait); } + req->flags &= ~REQ_F_WORK_INITIALIZED; + req->work.flags &= ~(IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | IO_WQ_WORK_FS | + IO_WQ_WORK_CREDS | IO_WQ_WORK_FILES); io_put_identity(req->task->io_uring, req); } From ce3d5aae331fa0eb1e88199e0380f517ed0c58f6 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Feb 2021 18:59:55 +0000 Subject: [PATCH 041/103] io_uring: deduplicate adding to REQ_F_INFLIGHT We don't know for how long REQ_F_INFLIGHT is going to stay, cleaner to extract a helper for marking requests as so. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 9b1f919b05c9..77878274fcb1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1460,11 +1460,24 @@ static bool io_identity_cow(struct io_kiocb *req) return true; } +static void io_req_track_inflight(struct io_kiocb *req) +{ + struct io_ring_ctx *ctx = req->ctx; + + if (!(req->flags & REQ_F_INFLIGHT)) { + io_req_init_async(req); + req->flags |= REQ_F_INFLIGHT; + + spin_lock_irq(&ctx->inflight_lock); + list_add(&req->inflight_entry, &ctx->inflight_list); + spin_unlock_irq(&ctx->inflight_lock); + } +} + static bool io_grab_identity(struct io_kiocb *req) { const struct io_op_def *def = &io_op_defs[req->opcode]; struct io_identity *id = req->work.identity; - struct io_ring_ctx *ctx = req->ctx; if (def->work_flags & IO_WQ_WORK_FSIZE) { if (id->fsize != rlimit(RLIMIT_FSIZE)) @@ -1520,15 +1533,8 @@ static bool io_grab_identity(struct io_kiocb *req) return false; atomic_inc(&id->files->count); get_nsproxy(id->nsproxy); - - if (!(req->flags & REQ_F_INFLIGHT)) { - req->flags |= REQ_F_INFLIGHT; - - spin_lock_irq(&ctx->inflight_lock); - list_add(&req->inflight_entry, &ctx->inflight_list); - spin_unlock_irq(&ctx->inflight_lock); - } req->work.flags |= IO_WQ_WORK_FILES; + io_req_track_inflight(req); } if (!(req->work.flags & IO_WQ_WORK_MM) && (def->work_flags & IO_WQ_WORK_MM)) { @@ -6443,16 +6449,8 @@ static struct file *io_file_get(struct io_submit_state *state, file = __io_file_get(state, fd); } - if (file && file->f_op == &io_uring_fops && - !(req->flags & REQ_F_INFLIGHT)) { - io_req_init_async(req); - req->flags |= REQ_F_INFLIGHT; - - spin_lock_irq(&ctx->inflight_lock); - list_add(&req->inflight_entry, &ctx->inflight_list); - spin_unlock_irq(&ctx->inflight_lock); - } - + if (file && unlikely(file->f_op == &io_uring_fops)) + io_req_track_inflight(req); return file; } From 57cd657b8272a66277c139e7bbdc8b86057cb415 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Feb 2021 18:59:56 +0000 Subject: [PATCH 042/103] io_uring: simplify do_read return parsing do_read() returning 0 bytes read (not -EAGAIN/etc.) is not an important enough of a case to prioritise it. Fold it into ret < 0 check, so we get rid of an extra if and make it a bit more readable. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 77878274fcb1..24ad36d71289 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3526,7 +3526,6 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, else kiocb->ki_flags |= IOCB_NOWAIT; - /* If the file doesn't support async, just async punt */ no_async = force_nonblock && !io_file_supports_async(req->file, READ); if (no_async) @@ -3538,9 +3537,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, ret = io_iter_do_read(req, iter); - if (!ret) { - goto done; - } else if (ret == -EIOCBQUEUED) { + if (ret == -EIOCBQUEUED) { ret = 0; goto out_free; } else if (ret == -EAGAIN) { @@ -3554,7 +3551,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, iov_iter_revert(iter, io_size - iov_iter_count(iter)); ret = 0; goto copy_iov; - } else if (ret < 0) { + } else if (ret <= 0) { /* make sure -ERESTARTSYS -> -EINTR is done */ goto done; } From 9936c7c2bc76a0b2276f6d19de6d1d92f03deeab Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:51:56 +0000 Subject: [PATCH 043/103] io_uring: deduplicate core cancellations sequence Files and task cancellations go over same steps trying to cancel requests in io-wq, poll, etc. Deduplicate it with a helper. note: new io_uring_try_cancel_requests() is former __io_uring_cancel_task_requests() with files passed as an agrument and flushing overflowed requests. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 85 ++++++++++++++++++++++++--------------------------- 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 24ad36d71289..a750c504366d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1003,9 +1003,9 @@ enum io_mem_account { ACCT_PINNED, }; -static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx, - struct task_struct *task); - +static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, + struct task_struct *task, + struct files_struct *files); static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node); static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( struct io_ring_ctx *ctx); @@ -8817,7 +8817,7 @@ static void io_ring_exit_work(struct work_struct *work) * as nobody else will be looking for them. */ do { - __io_uring_cancel_task_requests(ctx, NULL); + io_uring_try_cancel_requests(ctx, NULL, NULL); } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)); io_ring_ctx_free(ctx); } @@ -8931,6 +8931,40 @@ static void io_cancel_defer_files(struct io_ring_ctx *ctx, } } +static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, + struct task_struct *task, + struct files_struct *files) +{ + struct io_task_cancel cancel = { .task = task, .files = files, }; + + while (1) { + enum io_wq_cancel cret; + bool ret = false; + + if (ctx->io_wq) { + cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, + &cancel, true); + ret |= (cret != IO_WQ_CANCEL_NOTFOUND); + } + + /* SQPOLL thread does its own polling */ + if (!(ctx->flags & IORING_SETUP_SQPOLL) && !files) { + while (!list_empty_careful(&ctx->iopoll_list)) { + io_iopoll_try_reap_events(ctx); + ret = true; + } + } + + ret |= io_poll_remove_all(ctx, task, files); + ret |= io_kill_timeouts(ctx, task, files); + ret |= io_run_task_work(); + io_cqring_overflow_flush(ctx, true, task, files); + if (!ret) + break; + cond_resched(); + } +} + static int io_uring_count_inflight(struct io_ring_ctx *ctx, struct task_struct *task, struct files_struct *files) @@ -8950,7 +8984,6 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, struct files_struct *files) { while (!list_empty_careful(&ctx->inflight_list)) { - struct io_task_cancel cancel = { .task = task, .files = files }; DEFINE_WAIT(wait); int inflight; @@ -8958,13 +8991,7 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, if (!inflight) break; - io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, &cancel, true); - io_poll_remove_all(ctx, task, files); - io_kill_timeouts(ctx, task, files); - io_cqring_overflow_flush(ctx, true, task, files); - /* cancellations _may_ trigger task work */ - io_run_task_work(); - + io_uring_try_cancel_requests(ctx, task, files); prepare_to_wait(&task->io_uring->wait, &wait, TASK_UNINTERRUPTIBLE); if (inflight == io_uring_count_inflight(ctx, task, files)) @@ -8973,37 +9000,6 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, } } -static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx, - struct task_struct *task) -{ - while (1) { - struct io_task_cancel cancel = { .task = task, .files = NULL, }; - enum io_wq_cancel cret; - bool ret = false; - - if (ctx->io_wq) { - cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, - &cancel, true); - ret |= (cret != IO_WQ_CANCEL_NOTFOUND); - } - - /* SQPOLL thread does its own polling */ - if (!(ctx->flags & IORING_SETUP_SQPOLL)) { - while (!list_empty_careful(&ctx->iopoll_list)) { - io_iopoll_try_reap_events(ctx); - ret = true; - } - } - - ret |= io_poll_remove_all(ctx, task, NULL); - ret |= io_kill_timeouts(ctx, task, NULL); - ret |= io_run_task_work(); - if (!ret) - break; - cond_resched(); - } -} - static void io_disable_sqo_submit(struct io_ring_ctx *ctx) { mutex_lock(&ctx->uring_lock); @@ -9033,11 +9029,10 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, } io_cancel_defer_files(ctx, task, files); - io_cqring_overflow_flush(ctx, true, task, files); io_uring_cancel_files(ctx, task, files); if (!files) - __io_uring_cancel_task_requests(ctx, task); + io_uring_try_cancel_requests(ctx, task, NULL); if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { atomic_dec(&task->io_uring->in_idle); From c1d5a224683b333ddbe278e455d639ccd4f5ca2b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:51:57 +0000 Subject: [PATCH 044/103] io_uring: refactor scheduling in io_cqring_wait schedule_timeout() with timeout=MAX_SCHEDULE_TIMEOUT is guaranteed to work just as schedule(), so instead of hand-coding it based on arguments always use the timeout version and simplify code. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index a750c504366d..5b735635b8f0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7213,9 +7213,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, .to_wait = min_events, }; struct io_rings *rings = ctx->rings; - struct timespec64 ts; - signed long timeout = 0; - int ret = 0; + signed long timeout = MAX_SCHEDULE_TIMEOUT; + int ret; do { io_cqring_overflow_flush(ctx, false, NULL, NULL); @@ -7239,6 +7238,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, } if (uts) { + struct timespec64 ts; + if (get_timespec64(&ts, uts)) return -EFAULT; timeout = timespec64_to_jiffies(&ts); @@ -7264,14 +7265,10 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, finish_wait(&ctx->wait, &iowq.wq); continue; } - if (uts) { - timeout = schedule_timeout(timeout); - if (timeout == 0) { - ret = -ETIME; - break; - } - } else { - schedule(); + timeout = schedule_timeout(timeout); + if (timeout == 0) { + ret = -ETIME; + break; } } while (1); finish_wait(&ctx->wait, &iowq.wq); From eeb60b9ab4000d20261973642dfc9fb0e4b5d073 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:51:58 +0000 Subject: [PATCH 045/103] io_uring: refactor io_cqring_wait It's easy to make a mistake in io_cqring_wait() because for all break/continue clauses we need to watch for prepare/finish_wait to be used correctly. Extract all those into a new helper io_cqring_wait_schedule(), and transforming the loop into simple series of func calls: prepare(); check_and_schedule(); finish(); Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 5b735635b8f0..dcb9e937daa3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7195,6 +7195,25 @@ static int io_run_task_work_sig(void) return -EINTR; } +/* when returns >0, the caller should retry */ +static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, + struct io_wait_queue *iowq, + signed long *timeout) +{ + int ret; + + /* make sure we run task_work before checking for signals */ + ret = io_run_task_work_sig(); + if (ret || io_should_wake(iowq)) + return ret; + /* let the caller flush overflows, retry */ + if (test_bit(0, &ctx->cq_check_overflow)) + return 1; + + *timeout = schedule_timeout(*timeout); + return !*timeout ? -ETIME : 1; +} + /* * Wait until events become available, if we don't already have some. The * application must reap them itself, as they reside on the shared cq ring. @@ -7251,27 +7270,9 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, io_cqring_overflow_flush(ctx, false, NULL, NULL); prepare_to_wait_exclusive(&ctx->wait, &iowq.wq, TASK_INTERRUPTIBLE); - /* make sure we run task_work before checking for signals */ - ret = io_run_task_work_sig(); - if (ret > 0) { - finish_wait(&ctx->wait, &iowq.wq); - continue; - } - else if (ret < 0) - break; - if (io_should_wake(&iowq)) - break; - if (test_bit(0, &ctx->cq_check_overflow)) { - finish_wait(&ctx->wait, &iowq.wq); - continue; - } - timeout = schedule_timeout(timeout); - if (timeout == 0) { - ret = -ETIME; - break; - } - } while (1); - finish_wait(&ctx->wait, &iowq.wq); + ret = io_cqring_wait_schedule(ctx, &iowq, &timeout); + finish_wait(&ctx->wait, &iowq.wq); + } while (ret > 0); restore_saved_sigmask_unless(ret == -EINTR); From 6713e7a6145a4b5a61e33a37f0b4d06ca6d2c6d8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:51:59 +0000 Subject: [PATCH 046/103] io_uring: refactor io_read for unsupported nowait !io_file_supports_async() case of io_read() is hard to read, it jumps somewhere in the middle of the function just to do async setup and fail on a similar check. Call io_setup_async_rw() directly for this case, it's much easier to follow. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index dcb9e937daa3..866e0ea83dbe 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3506,7 +3506,6 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, struct iov_iter __iter, *iter = &__iter; struct io_async_rw *rw = req->async_data; ssize_t io_size, ret, ret2; - bool no_async; if (rw) { iter = &rw->iter; @@ -3527,9 +3526,12 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, kiocb->ki_flags |= IOCB_NOWAIT; /* If the file doesn't support async, just async punt */ - no_async = force_nonblock && !io_file_supports_async(req->file, READ); - if (no_async) - goto copy_iov; + if (force_nonblock && !io_file_supports_async(req->file, READ)) { + ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true); + if (!ret) + return -EAGAIN; + goto out_free; + } ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size); if (unlikely(ret)) @@ -3568,8 +3570,6 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, ret = ret2; goto out_free; } - if (no_async) - return -EAGAIN; rw = req->async_data; /* it's copied and will be cleaned with ->io */ iovec = NULL; From 1a2cc0ce8d18c9e5592733cb6381e9ff5c23d916 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:00 +0000 Subject: [PATCH 047/103] io_uring: further simplify do_read error parsing First, instead of checking iov_iter_count(iter) for 0 to find out that all needed bytes were read, just compare returned code against io_size. It's more reliable and arguably cleaner. Also, place the half-read case into an else branch and delete an extra label. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 866e0ea83dbe..1d1fa1f77332 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3552,19 +3552,18 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, /* some cases will consume bytes even on error returns */ iov_iter_revert(iter, io_size - iov_iter_count(iter)); ret = 0; - goto copy_iov; - } else if (ret <= 0) { + } else if (ret <= 0 || ret == io_size) { /* make sure -ERESTARTSYS -> -EINTR is done */ goto done; + } else { + /* we did blocking attempt. no retry. */ + if (!force_nonblock || (req->file->f_flags & O_NONBLOCK) || + !(req->flags & REQ_F_ISREG)) + goto done; + + io_size -= ret; } - /* read it all, or we did blocking attempt. no retry. */ - if (!iov_iter_count(iter) || !force_nonblock || - (req->file->f_flags & O_NONBLOCK) || !(req->flags & REQ_F_ISREG)) - goto done; - - io_size -= ret; -copy_iov: ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true); if (ret2) { ret = ret2; From 6bf985dc50dd882a95fffa9c7eef0d1416f512e6 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:01 +0000 Subject: [PATCH 048/103] io_uring: let io_setup_async_rw take care of iovec Now we give out ownership of iovec into io_setup_async_rw(), so it either sets request's context right or frees the iovec on error itself. Makes our life a bit easier at call sites. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 1d1fa1f77332..f8492d62b6a1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2721,11 +2721,7 @@ static bool io_resubmit_prep(struct io_kiocb *req) ret = io_import_iovec(rw, req, &iovec, &iter, false); if (ret < 0) return false; - ret = io_setup_async_rw(req, iovec, inline_vecs, &iter, false); - if (!ret) - return true; - kfree(iovec); - return false; + return !io_setup_async_rw(req, iovec, inline_vecs, &iter, false); } #endif @@ -3366,8 +3362,10 @@ static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, if (!force && !io_op_defs[req->opcode].needs_async_data) return 0; if (!req->async_data) { - if (__io_alloc_async_data(req)) + if (__io_alloc_async_data(req)) { + kfree(iovec); return -ENOMEM; + } io_req_map_rw(req, iovec, fast_iov, iter); } @@ -3528,9 +3526,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, /* If the file doesn't support async, just async punt */ if (force_nonblock && !io_file_supports_async(req->file, READ)) { ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true); - if (!ret) - return -EAGAIN; - goto out_free; + return ret ?: -EAGAIN; } ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size); @@ -3565,10 +3561,9 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, } ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true); - if (ret2) { - ret = ret2; - goto out_free; - } + if (ret2) + return ret2; + rw = req->async_data; /* it's copied and will be cleaned with ->io */ iovec = NULL; @@ -3703,8 +3698,7 @@ static int io_write(struct io_kiocb *req, bool force_nonblock, /* some cases will consume bytes even on error returns */ iov_iter_revert(iter, io_size - iov_iter_count(iter)); ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false); - if (!ret) - return -EAGAIN; + return ret ?: -EAGAIN; } out_free: /* it's reportedly faster than delegating the null check to kfree() */ From 7335e3bf9d0a92be09bb4f38d06ab22c40f0fead Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:02 +0000 Subject: [PATCH 049/103] io_uring: don't forget to adjust io_size We have invariant in io_read() of how much we're trying to read spilled into an iter and io_size variable. The last one controls decision making about whether to do read-retries. However, io_size is modified only after the first read attempt, so if we happen to go for a third retry in a single call to io_read(), we will get io_size greater than in the iterator, so may lead to various side effects up to live-locking. Modify io_size each time. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index f8492d62b6a1..25fffff27c76 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3548,16 +3548,11 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, /* some cases will consume bytes even on error returns */ iov_iter_revert(iter, io_size - iov_iter_count(iter)); ret = 0; - } else if (ret <= 0 || ret == io_size) { - /* make sure -ERESTARTSYS -> -EINTR is done */ + } else if (ret <= 0 || ret == io_size || !force_nonblock || + (req->file->f_flags & O_NONBLOCK) || + !(req->flags & REQ_F_ISREG)) { + /* read all, failed, already did sync or don't want to retry */ goto done; - } else { - /* we did blocking attempt. no retry. */ - if (!force_nonblock || (req->file->f_flags & O_NONBLOCK) || - !(req->flags & REQ_F_ISREG)) - goto done; - - io_size -= ret; } ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true); @@ -3570,6 +3565,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, /* now use our persistent iterator, if we aren't already */ iter = &rw->iter; retry: + io_size -= ret; rw->bytes_done += ret; /* if we can retry, do so with the callbacks armed */ if (!io_rw_should_retry(req)) { From 5ea5dd45844d1b727ab2a76f47d6e9aa65d1e921 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:03 +0000 Subject: [PATCH 050/103] io_uring: inline io_read()'s iovec freeing io_read() has not the simpliest control flow with a lot of jumps and it's hard to read. One of those is a out_free: label, which frees iovec. However, from the middle of io_read() iovec is NULL'ed and so kfree(iovec) is no-op, it leaves us with two place where we can inline it and further clean up the code. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 25fffff27c76..35ad889afaec 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3530,14 +3530,18 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, } ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size); - if (unlikely(ret)) - goto out_free; + if (unlikely(ret)) { + kfree(iovec); + return ret; + } ret = io_iter_do_read(req, iter); if (ret == -EIOCBQUEUED) { - ret = 0; - goto out_free; + /* it's faster to check here then delegate to kfree */ + if (iovec) + kfree(iovec); + return 0; } else if (ret == -EAGAIN) { /* IOPOLL retry should happen for io-wq threads */ if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) @@ -3560,8 +3564,6 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, return ret2; rw = req->async_data; - /* it's copied and will be cleaned with ->io */ - iovec = NULL; /* now use our persistent iterator, if we aren't already */ iter = &rw->iter; retry: @@ -3580,21 +3582,14 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, * do, then just retry at the new offset. */ ret = io_iter_do_read(req, iter); - if (ret == -EIOCBQUEUED) { - ret = 0; - goto out_free; - } else if (ret > 0 && ret < io_size) { - /* we got some bytes, but not all. retry. */ + if (ret == -EIOCBQUEUED) + return 0; + /* we got some bytes, but not all. retry. */ + if (ret > 0 && ret < io_size) goto retry; - } done: kiocb_done(kiocb, ret, cs); - ret = 0; -out_free: - /* it's reportedly faster than delegating the null check to kfree() */ - if (iovec) - kfree(iovec); - return ret; + return 0; } static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) From b23df91bff954ebd8aee39eb22e5028f41cd9e56 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:04 +0000 Subject: [PATCH 051/103] io_uring: highlight read-retry loop We already have implicit do-while for read-retries but with goto in the end. Convert it to an actual do-while, it highlights it so making a bit more understandable and is cleaner in general. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 35ad889afaec..bbf8ea8370d6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3566,27 +3566,27 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, rw = req->async_data; /* now use our persistent iterator, if we aren't already */ iter = &rw->iter; -retry: - io_size -= ret; - rw->bytes_done += ret; - /* if we can retry, do so with the callbacks armed */ - if (!io_rw_should_retry(req)) { - kiocb->ki_flags &= ~IOCB_WAITQ; - return -EAGAIN; - } - /* - * Now retry read with the IOCB_WAITQ parts set in the iocb. If we - * get -EIOCBQUEUED, then we'll get a notification when the desired - * page gets unlocked. We can also get a partial read here, and if we - * do, then just retry at the new offset. - */ - ret = io_iter_do_read(req, iter); - if (ret == -EIOCBQUEUED) - return 0; - /* we got some bytes, but not all. retry. */ - if (ret > 0 && ret < io_size) - goto retry; + do { + io_size -= ret; + rw->bytes_done += ret; + /* if we can retry, do so with the callbacks armed */ + if (!io_rw_should_retry(req)) { + kiocb->ki_flags &= ~IOCB_WAITQ; + return -EAGAIN; + } + + /* + * Now retry read with the IOCB_WAITQ parts set in the iocb. If + * we get -EIOCBQUEUED, then we'll get a notification when the + * desired page gets unlocked. We can also get a partial read + * here, and if we do, then just retry at the new offset. + */ + ret = io_iter_do_read(req, iter); + if (ret == -EIOCBQUEUED) + return 0; + /* we got some bytes, but not all. retry. */ + } while (ret > 0 && ret < io_size); done: kiocb_done(kiocb, ret, cs); return 0; From 75c668cdd6ca05dd9c7138a5a080c0088d72cf51 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:05 +0000 Subject: [PATCH 052/103] io_uring: treat NONBLOCK and RWF_NOWAIT similarly Make decision making of whether we need to retry read/write similar for O_NONBLOCK and RWF_NOWAIT. Set REQ_F_NOWAIT when either is specified and use it for all relevant checks. Also fix resubmitting NOWAIT requests via io_rw_reissue(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index bbf8ea8370d6..ce2ea3f55f65 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2734,7 +2734,9 @@ static bool io_rw_reissue(struct io_kiocb *req, long res) if (res != -EAGAIN && res != -EOPNOTSUPP) return false; mode = file_inode(req->file)->i_mode; - if ((!S_ISBLK(mode) && !S_ISREG(mode)) || io_wq_current_is_worker()) + if (!S_ISBLK(mode) && !S_ISREG(mode)) + return false; + if ((req->flags & REQ_F_NOWAIT) || io_wq_current_is_worker()) return false; lockdep_assert_held(&req->ctx->uring_lock); @@ -2907,16 +2909,17 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_ring_ctx *ctx = req->ctx; struct kiocb *kiocb = &req->rw.kiocb; + struct file *file = req->file; unsigned ioprio; int ret; - if (S_ISREG(file_inode(req->file)->i_mode)) + if (S_ISREG(file_inode(file)->i_mode)) req->flags |= REQ_F_ISREG; kiocb->ki_pos = READ_ONCE(sqe->off); - if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) { + if (kiocb->ki_pos == -1 && !(file->f_mode & FMODE_STREAM)) { req->flags |= REQ_F_CUR_POS; - kiocb->ki_pos = req->file->f_pos; + kiocb->ki_pos = file->f_pos; } kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp)); kiocb->ki_flags = iocb_flags(kiocb->ki_filp); @@ -2924,6 +2927,10 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (unlikely(ret)) return ret; + /* don't allow async punt for O_NONBLOCK or RWF_NOWAIT */ + if ((kiocb->ki_flags & IOCB_NOWAIT) || (file->f_flags & O_NONBLOCK)) + req->flags |= REQ_F_NOWAIT; + ioprio = READ_ONCE(sqe->ioprio); if (ioprio) { ret = ioprio_check_cap(ioprio); @@ -2934,10 +2941,6 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) } else kiocb->ki_ioprio = get_current_ioprio(); - /* don't allow async punt if RWF_NOWAIT was requested */ - if (kiocb->ki_flags & IOCB_NOWAIT) - req->flags |= REQ_F_NOWAIT; - if (ctx->flags & IORING_SETUP_IOPOLL) { if (!(kiocb->ki_flags & IOCB_DIRECT) || !kiocb->ki_filp->f_op->iopoll) @@ -3546,15 +3549,14 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, /* IOPOLL retry should happen for io-wq threads */ if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) goto done; - /* no retry on NONBLOCK marked file */ - if (req->file->f_flags & O_NONBLOCK) + /* no retry on NONBLOCK nor RWF_NOWAIT */ + if (req->flags & REQ_F_NOWAIT) goto done; /* some cases will consume bytes even on error returns */ iov_iter_revert(iter, io_size - iov_iter_count(iter)); ret = 0; } else if (ret <= 0 || ret == io_size || !force_nonblock || - (req->file->f_flags & O_NONBLOCK) || - !(req->flags & REQ_F_ISREG)) { + (req->flags & REQ_F_NOWAIT) || !(req->flags & REQ_F_ISREG)) { /* read all, failed, already did sync or don't want to retry */ goto done; } @@ -3675,8 +3677,8 @@ static int io_write(struct io_kiocb *req, bool force_nonblock, */ if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT)) ret2 = -EAGAIN; - /* no retry on NONBLOCK marked file */ - if (ret2 == -EAGAIN && (req->file->f_flags & O_NONBLOCK)) + /* no retry on NONBLOCK nor RWF_NOWAIT */ + if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT)) goto done; if (!force_nonblock || ret2 != -EAGAIN) { /* IOPOLL retry should happen for io-wq threads */ From 847595de1732a6e928f241929d24dde2e9ffaf15 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:06 +0000 Subject: [PATCH 053/103] io_uring: io_import_iovec return type cleanup io_import_iovec() doesn't return IO size anymore, only error code. Make it more apparent by returning int instead of ssize and clean up leftovers. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index ce2ea3f55f65..24cc00ff7155 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1030,9 +1030,8 @@ static struct file *io_file_get(struct io_submit_state *state, static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs); static void io_rsrc_put_work(struct work_struct *work); -static ssize_t io_import_iovec(int rw, struct io_kiocb *req, - struct iovec **iovec, struct iov_iter *iter, - bool needs_lock); +static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, + struct iov_iter *iter, bool needs_lock); static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, const struct iovec *fast_iov, struct iov_iter *iter, bool force); @@ -2693,9 +2692,8 @@ static void io_complete_rw_common(struct kiocb *kiocb, long res, static bool io_resubmit_prep(struct io_kiocb *req) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; - ssize_t ret = -ECANCELED; + int rw, ret = -ECANCELED; struct iov_iter iter; - int rw; /* already prepared */ if (req->async_data) @@ -3004,8 +3002,7 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret, io_rw_done(kiocb, ret); } -static ssize_t io_import_fixed(struct io_kiocb *req, int rw, - struct iov_iter *iter) +static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter) { struct io_ring_ctx *ctx = req->ctx; size_t len = req->rw.len; @@ -3069,7 +3066,7 @@ static ssize_t io_import_fixed(struct io_kiocb *req, int rw, } } - return len; + return 0; } static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock) @@ -3210,16 +3207,14 @@ static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, return __io_iov_buffer_select(req, iov, needs_lock); } -static ssize_t io_import_iovec(int rw, struct io_kiocb *req, - struct iovec **iovec, struct iov_iter *iter, - bool needs_lock) +static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, + struct iov_iter *iter, bool needs_lock) { void __user *buf = u64_to_user_ptr(req->rw.addr); size_t sqe_len = req->rw.len; + u8 opcode = req->opcode; ssize_t ret; - u8 opcode; - opcode = req->opcode; if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) { *iovec = NULL; return io_import_fixed(req, rw, iter); @@ -3244,10 +3239,8 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, if (req->flags & REQ_F_BUFFER_SELECT) { ret = io_iov_buffer_select(req, *iovec, needs_lock); - if (!ret) { - ret = (*iovec)->iov_len; - iov_iter_init(iter, rw, *iovec, 1, ret); - } + if (!ret) + iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len); *iovec = NULL; return ret; } @@ -3379,7 +3372,7 @@ static inline int io_rw_prep_async(struct io_kiocb *req, int rw) { struct io_async_rw *iorw = req->async_data; struct iovec *iov = iorw->fast_iov; - ssize_t ret; + int ret; ret = io_import_iovec(rw, req, &iov, &iorw->iter, false); if (unlikely(ret < 0)) @@ -3518,7 +3511,6 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, } io_size = iov_iter_count(iter); req->result = io_size; - ret = 0; /* Ensure we clear previously set non-block flag */ if (!force_nonblock) From ea64ec02b31d5b05ae94ac4d57e38f8a02117c76 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:07 +0000 Subject: [PATCH 054/103] io_uring: deduplicate file table slot calculation Extract a helper io_fixed_file_slot() returning a place in our fixed files table, so we don't hand-code it three times in the code. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 24cc00ff7155..5ee6a9273fca 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7740,6 +7740,15 @@ static void io_rsrc_put_work(struct work_struct *work) } } +static struct file **io_fixed_file_slot(struct fixed_rsrc_data *file_data, + unsigned i) +{ + struct fixed_rsrc_table *table; + + table = &file_data->table[i >> IORING_FILE_TABLE_SHIFT]; + return &table->files[i & IORING_FILE_TABLE_MASK]; +} + static void io_rsrc_node_ref_zero(struct percpu_ref *ref) { struct fixed_rsrc_ref_node *ref_node; @@ -7808,6 +7817,7 @@ static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node) kfree(ref_node); } + static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) { @@ -7840,9 +7850,6 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, goto out_free; for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { - struct fixed_rsrc_table *table; - unsigned index; - if (copy_from_user(&fd, &fds[i], sizeof(fd))) { ret = -EFAULT; goto out_fput; @@ -7867,9 +7874,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, fput(file); goto out_fput; } - table = &file_data->table[i >> IORING_FILE_TABLE_SHIFT]; - index = i & IORING_FILE_TABLE_MASK; - table->files[index] = file; + *io_fixed_file_slot(file_data, i) = file; } ret = io_sqe_files_scm(ctx); @@ -7972,7 +7977,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, { struct fixed_rsrc_data *data = ctx->file_data; struct fixed_rsrc_ref_node *ref_node; - struct file *file; + struct file *file, **file_slot; __s32 __user *fds; int fd, i, err; __u32 done; @@ -7990,9 +7995,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, fds = u64_to_user_ptr(up->data); for (done = 0; done < nr_args; done++) { - struct fixed_rsrc_table *table; - unsigned index; - err = 0; if (copy_from_user(&fd, &fds[done], sizeof(fd))) { err = -EFAULT; @@ -8002,14 +8004,13 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, continue; i = array_index_nospec(up->offset + done, ctx->nr_user_files); - table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; - index = i & IORING_FILE_TABLE_MASK; - if (table->files[index]) { - file = table->files[index]; - err = io_queue_file_removal(data, file); + file_slot = io_fixed_file_slot(ctx->file_data, i); + + if (*file_slot) { + err = io_queue_file_removal(data, *file_slot); if (err) break; - table->files[index] = NULL; + *file_slot = NULL; needs_switch = true; } if (fd != -1) { @@ -8031,13 +8032,12 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, err = -EBADF; break; } - table->files[index] = file; err = io_sqe_file_register(ctx, file, i); if (err) { - table->files[index] = NULL; fput(file); break; } + *file_slot = file; } } @@ -9488,11 +9488,8 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1); seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files); for (i = 0; has_lock && i < ctx->nr_user_files; i++) { - struct fixed_rsrc_table *table; - struct file *f; + struct file *f = *io_fixed_file_slot(ctx->file_data, i); - table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; - f = table->files[i & IORING_FILE_TABLE_MASK]; if (f) seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname); else From 5280f7e530f71ba85baf90169393196976ad0e52 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:08 +0000 Subject: [PATCH 055/103] io_uring/io-wq: return 2-step work swap scheme Saving one lock/unlock for io-wq is not super important, but adds some ugliness in the code. More important, atomic decs not turning it to zero for some archs won't give the right ordering/barriers so the io_steal_work() may pretty easily get subtly and completely broken. Return back 2-step io-wq work exchange and clean it up. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io-wq.c | 16 ++++++---------- fs/io-wq.h | 4 ++-- fs/io_uring.c | 26 ++++---------------------- 3 files changed, 12 insertions(+), 34 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index 2e2f14f42bf2..63ef195b1acb 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -555,23 +555,21 @@ static void io_worker_handle_work(struct io_worker *worker) /* handle a whole dependent link */ do { - struct io_wq_work *old_work, *next_hashed, *linked; + struct io_wq_work *next_hashed, *linked; unsigned int hash = io_get_work_hash(work); next_hashed = wq_next_work(work); io_impersonate_work(worker, work); + wq->do_work(work); + io_assign_current_work(worker, NULL); - old_work = work; - linked = wq->do_work(work); - + linked = wq->free_work(work); work = next_hashed; if (!work && linked && !io_wq_is_hashed(linked)) { work = linked; linked = NULL; } io_assign_current_work(worker, work); - wq->free_work(old_work); - if (linked) io_wqe_enqueue(wqe, linked); @@ -850,11 +848,9 @@ static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe) struct io_wq *wq = wqe->wq; do { - struct io_wq_work *old_work = work; - work->flags |= IO_WQ_WORK_CANCEL; - work = wq->do_work(work); - wq->free_work(old_work); + wq->do_work(work); + work = wq->free_work(work); } while (work); } diff --git a/fs/io-wq.h b/fs/io-wq.h index e1ffb80a4a1d..e37a0f217cc8 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -106,8 +106,8 @@ static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) return container_of(work->list.next, struct io_wq_work, list); } -typedef void (free_work_fn)(struct io_wq_work *); -typedef struct io_wq_work *(io_wq_work_fn)(struct io_wq_work *); +typedef struct io_wq_work *(free_work_fn)(struct io_wq_work *); +typedef void (io_wq_work_fn)(struct io_wq_work *); struct io_wq_data { struct user_struct *user; diff --git a/fs/io_uring.c b/fs/io_uring.c index 5ee6a9273fca..b740a39110d6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2379,22 +2379,6 @@ static inline void io_put_req_deferred(struct io_kiocb *req, int refs) io_free_req_deferred(req); } -static struct io_wq_work *io_steal_work(struct io_kiocb *req) -{ - struct io_kiocb *nxt; - - /* - * A ref is owned by io-wq in which context we're. So, if that's the - * last one, it's safe to steal next work. False negatives are Ok, - * it just will be re-punted async in io_put_work() - */ - if (refcount_read(&req->refs) != 1) - return NULL; - - nxt = io_req_find_next(req); - return nxt ? &nxt->work : NULL; -} - static void io_double_put_req(struct io_kiocb *req) { /* drop both submit and complete references */ @@ -6343,7 +6327,7 @@ static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock, return 0; } -static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work) +static void io_wq_submit_work(struct io_wq_work *work) { struct io_kiocb *req = container_of(work, struct io_kiocb, work); struct io_kiocb *timeout; @@ -6394,8 +6378,6 @@ static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work) if (lock_ctx) mutex_unlock(&lock_ctx->uring_lock); } - - return io_steal_work(req); } static inline struct file *io_file_from_index(struct io_ring_ctx *ctx, @@ -8067,12 +8049,12 @@ static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg, return __io_sqe_files_update(ctx, &up, nr_args); } -static void io_free_work(struct io_wq_work *work) +static struct io_wq_work *io_free_work(struct io_wq_work *work) { struct io_kiocb *req = container_of(work, struct io_kiocb, work); - /* Consider that io_steal_work() relies on this ref */ - io_put_req(req); + req = io_put_req_find_next(req); + return req ? &req->work : NULL; } static int io_init_wq_offload(struct io_ring_ctx *ctx, From 2a7808024b195a342779fb5d7b7df1c4af45cc71 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 5 Feb 2021 00:57:58 +0000 Subject: [PATCH 056/103] io_uring: set msg_name on msg fixup io_setup_async_msg() should fully prepare io_async_msghdr, let it also handle assigning msg_name and don't hand code it in [send,recv]msg(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index b740a39110d6..39bc1df9bb64 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4558,6 +4558,7 @@ static int io_setup_async_msg(struct io_kiocb *req, async_msg = req->async_data; req->flags |= REQ_F_NEED_CLEANUP; memcpy(async_msg, kmsg, sizeof(*kmsg)); + async_msg->msg.msg_name = &async_msg->addr; return -EAGAIN; } @@ -4610,7 +4611,6 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, if (req->async_data) { kmsg = req->async_data; - kmsg->msg.msg_name = &kmsg->addr; /* if iov is set, it's allocated already */ if (!kmsg->iov) kmsg->iov = kmsg->fast_iov; @@ -4839,7 +4839,6 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, if (req->async_data) { kmsg = req->async_data; - kmsg->msg.msg_name = &kmsg->addr; /* if iov is set, it's allocated already */ if (!kmsg->iov) kmsg->iov = kmsg->fast_iov; From 5476dfed29ad9b19d4e187685ab71bb9c496f965 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 5 Feb 2021 00:57:59 +0000 Subject: [PATCH 057/103] io_uring: clean iov usage for recvmsg buf select Don't pretend we don't know that REQ_F_BUFFER_SELECT for recvmsg always uses fast_iov -- clean up confusing intermixing kmsg->iov and kmsg->fast_iov for buffer select. Also don't init iter with garbage in __io_recvmsg_copy_hdr() only for it to be set shortly after in io_recvmsg(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 39bc1df9bb64..e07a7fa15cfa 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4701,11 +4701,9 @@ static int __io_recvmsg_copy_hdr(struct io_kiocb *req, if (req->flags & REQ_F_BUFFER_SELECT) { if (iov_len > 1) return -EINVAL; - if (copy_from_user(iomsg->iov, uiov, sizeof(*uiov))) + if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov))) return -EFAULT; - sr->len = iomsg->iov[0].iov_len; - iov_iter_init(&iomsg->msg.msg_iter, READ, iomsg->iov, 1, - sr->len); + sr->len = iomsg->fast_iov[0].iov_len; iomsg->iov = NULL; } else { ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV, @@ -4748,7 +4746,6 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, if (clen < 0) return -EINVAL; sr->len = clen; - iomsg->iov[0].iov_len = clen; iomsg->iov = NULL; } else { ret = __import_iovec(READ, (struct iovec __user *)uiov, len, @@ -4855,7 +4852,8 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, if (IS_ERR(kbuf)) return PTR_ERR(kbuf); kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr); - iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov, + kmsg->fast_iov[0].iov_len = req->sr_msg.len; + iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1, req->sr_msg.len); } From 257e84a5377fbbc336ff563833a8712619acce56 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 5 Feb 2021 00:58:00 +0000 Subject: [PATCH 058/103] io_uring: refactor sendmsg/recvmsg iov managing Current iov handling with recvmsg/sendmsg may be confusing. First make a rule for msg->iov: either it points to an allocated iov that have to be kfree()'d later, or it's NULL and we use fast_iov. That's much better than current 3-state (also can point to fast_iov). And rename it into free_iov for uniformity with read/write. Also, instead of after struct io_async_msghdr copy fixing up of msg.msg_iter.iov has been happening in io_recvmsg()/io_sendmsg(). Move it into io_setup_async_msg(), that's the right place. Signed-off-by: Pavel Begunkov [axboe: add comment on NULL check before kfree()] Signed-off-by: Jens Axboe --- fs/io_uring.c | 57 ++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index e07a7fa15cfa..7242cc48e97b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -594,7 +594,8 @@ struct io_async_connect { struct io_async_msghdr { struct iovec fast_iov[UIO_FASTIOV]; - struct iovec *iov; + /* points to an allocated iov, if NULL we use fast_iov instead */ + struct iovec *free_iov; struct sockaddr __user *uaddr; struct msghdr msg; struct sockaddr_storage addr; @@ -4551,24 +4552,27 @@ static int io_setup_async_msg(struct io_kiocb *req, if (async_msg) return -EAGAIN; if (io_alloc_async_data(req)) { - if (kmsg->iov != kmsg->fast_iov) - kfree(kmsg->iov); + kfree(kmsg->free_iov); return -ENOMEM; } async_msg = req->async_data; req->flags |= REQ_F_NEED_CLEANUP; memcpy(async_msg, kmsg, sizeof(*kmsg)); async_msg->msg.msg_name = &async_msg->addr; + /* if were using fast_iov, set it to the new one */ + if (!async_msg->free_iov) + async_msg->msg.msg_iter.iov = async_msg->fast_iov; + return -EAGAIN; } static int io_sendmsg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg) { - iomsg->iov = iomsg->fast_iov; iomsg->msg.msg_name = &iomsg->addr; + iomsg->free_iov = iomsg->fast_iov; return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg, - req->sr_msg.msg_flags, &iomsg->iov); + req->sr_msg.msg_flags, &iomsg->free_iov); } static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) @@ -4609,13 +4613,8 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, if (unlikely(!sock)) return -ENOTSOCK; - if (req->async_data) { - kmsg = req->async_data; - /* if iov is set, it's allocated already */ - if (!kmsg->iov) - kmsg->iov = kmsg->fast_iov; - kmsg->msg.msg_iter.iov = kmsg->iov; - } else { + kmsg = req->async_data; + if (!kmsg) { ret = io_sendmsg_copy_hdr(req, &iomsg); if (ret) return ret; @@ -4634,8 +4633,9 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, if (ret == -ERESTARTSYS) ret = -EINTR; - if (kmsg->iov != kmsg->fast_iov) - kfree(kmsg->iov); + /* fast path, check for non-NULL to avoid function call */ + if (kmsg->free_iov) + kfree(kmsg->free_iov); req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < 0) req_set_fail_links(req); @@ -4704,10 +4704,11 @@ static int __io_recvmsg_copy_hdr(struct io_kiocb *req, if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov))) return -EFAULT; sr->len = iomsg->fast_iov[0].iov_len; - iomsg->iov = NULL; + iomsg->free_iov = NULL; } else { + iomsg->free_iov = iomsg->fast_iov; ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV, - &iomsg->iov, &iomsg->msg.msg_iter, + &iomsg->free_iov, &iomsg->msg.msg_iter, false); if (ret > 0) ret = 0; @@ -4746,10 +4747,11 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, if (clen < 0) return -EINVAL; sr->len = clen; - iomsg->iov = NULL; + iomsg->free_iov = NULL; } else { + iomsg->free_iov = iomsg->fast_iov; ret = __import_iovec(READ, (struct iovec __user *)uiov, len, - UIO_FASTIOV, &iomsg->iov, + UIO_FASTIOV, &iomsg->free_iov, &iomsg->msg.msg_iter, true); if (ret < 0) return ret; @@ -4763,7 +4765,6 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg) { iomsg->msg.msg_name = &iomsg->addr; - iomsg->iov = iomsg->fast_iov; #ifdef CONFIG_COMPAT if (req->ctx->compat) @@ -4834,13 +4835,8 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, if (unlikely(!sock)) return -ENOTSOCK; - if (req->async_data) { - kmsg = req->async_data; - /* if iov is set, it's allocated already */ - if (!kmsg->iov) - kmsg->iov = kmsg->fast_iov; - kmsg->msg.msg_iter.iov = kmsg->iov; - } else { + kmsg = req->async_data; + if (!kmsg) { ret = io_recvmsg_copy_hdr(req, &iomsg); if (ret) return ret; @@ -4872,8 +4868,9 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, if (req->flags & REQ_F_BUFFER_SELECTED) cflags = io_put_recv_kbuf(req); - if (kmsg->iov != kmsg->fast_iov) - kfree(kmsg->iov); + /* fast path, check for non-NULL to avoid function call */ + if (kmsg->free_iov) + kfree(kmsg->free_iov); req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < 0) req_set_fail_links(req); @@ -6166,8 +6163,8 @@ static void __io_clean_op(struct io_kiocb *req) case IORING_OP_RECVMSG: case IORING_OP_SENDMSG: { struct io_async_msghdr *io = req->async_data; - if (io->iov != io->fast_iov) - kfree(io->iov); + + kfree(io->free_iov); break; } case IORING_OP_SPLICE: From 0e9ddb39b7d964d716cddd6e6bd1aab3f800066e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 7 Feb 2021 22:34:26 +0000 Subject: [PATCH 059/103] io_uring: cleanup up cancel SQPOLL reqs across exec For SQPOLL rings tctx_inflight() always returns zero, so it might skip doing full cancelation. It's fine because we jam all sqpoll submissions in any case and do go through files cancel for them, but not nice. Do the intended full cancellation, by mimicking __io_uring_task_cancel() waiting but impersonating SQPOLL task. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 57 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 7242cc48e97b..9c77fbc0c395 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9083,29 +9083,39 @@ void __io_uring_files_cancel(struct files_struct *files) static s64 tctx_inflight(struct io_uring_task *tctx) { - unsigned long index; - struct file *file; + return percpu_counter_sum(&tctx->inflight); +} + +static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx) +{ + struct io_uring_task *tctx; s64 inflight; + DEFINE_WAIT(wait); - inflight = percpu_counter_sum(&tctx->inflight); - if (!tctx->sqpoll) - return inflight; + if (!ctx->sq_data) + return; + tctx = ctx->sq_data->thread->io_uring; + io_disable_sqo_submit(ctx); - /* - * If we have SQPOLL rings, then we need to iterate and find them, and - * add the pending count for those. - */ - xa_for_each(&tctx->xa, index, file) { - struct io_ring_ctx *ctx = file->private_data; + atomic_inc(&tctx->in_idle); + do { + /* read completions before cancelations */ + inflight = tctx_inflight(tctx); + if (!inflight) + break; + io_uring_cancel_task_requests(ctx, NULL); - if (ctx->flags & IORING_SETUP_SQPOLL) { - struct io_uring_task *__tctx = ctx->sqo_task->io_uring; - - inflight += percpu_counter_sum(&__tctx->inflight); - } - } - - return inflight; + prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE); + /* + * If we've seen completions, retry without waiting. This + * avoids a race where a completion comes in before we did + * prepare_to_wait(). + */ + if (inflight == tctx_inflight(tctx)) + schedule(); + finish_wait(&tctx->wait, &wait); + } while (1); + atomic_dec(&tctx->in_idle); } /* @@ -9122,8 +9132,13 @@ void __io_uring_task_cancel(void) atomic_inc(&tctx->in_idle); /* trigger io_disable_sqo_submit() */ - if (tctx->sqpoll) - __io_uring_files_cancel(NULL); + if (tctx->sqpoll) { + struct file *file; + unsigned long index; + + xa_for_each(&tctx->xa, index, file) + io_uring_cancel_sqpoll(file->private_data); + } do { /* read completions before cancelations */ From 45d189c6062922ffe272e98013ba464b355dede7 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:07 +0000 Subject: [PATCH 060/103] io_uring: replace force_nonblock with flags Replace bool force_nonblock with flags. It has a long standing goal of differentiating context from which we execute. Currently we have some subtle places where some invariants, like holding of uring_lock, are subtly inferred. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 178 +++++++++++++++++++++++++++----------------------- 1 file changed, 96 insertions(+), 82 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 9c77fbc0c395..862121c48cee 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -187,6 +187,10 @@ struct io_rings { struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp; }; +enum io_uring_cmd_flags { + IO_URING_F_NONBLOCK = 1, +}; + struct io_mapped_ubuf { u64 ubuf; size_t len; @@ -3477,7 +3481,7 @@ static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter) return -EINVAL; } -static int io_read(struct io_kiocb *req, bool force_nonblock, +static int io_read(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; @@ -3485,6 +3489,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, struct iov_iter __iter, *iter = &__iter; struct io_async_rw *rw = req->async_data; ssize_t io_size, ret, ret2; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; if (rw) { iter = &rw->iter; @@ -3588,7 +3593,7 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return io_rw_prep_async(req, WRITE); } -static int io_write(struct io_kiocb *req, bool force_nonblock, +static int io_write(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; @@ -3596,6 +3601,7 @@ static int io_write(struct io_kiocb *req, bool force_nonblock, struct iov_iter __iter, *iter = &__iter; struct io_async_rw *rw = req->async_data; ssize_t ret, ret2, io_size; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; if (rw) { iter = &rw->iter; @@ -3706,12 +3712,12 @@ static int io_renameat_prep(struct io_kiocb *req, return 0; } -static int io_renameat(struct io_kiocb *req, bool force_nonblock) +static int io_renameat(struct io_kiocb *req, unsigned int issue_flags) { struct io_rename *ren = &req->rename; int ret; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd, @@ -3748,12 +3754,12 @@ static int io_unlinkat_prep(struct io_kiocb *req, return 0; } -static int io_unlinkat(struct io_kiocb *req, bool force_nonblock) +static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags) { struct io_unlink *un = &req->unlink; int ret; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; if (un->flags & AT_REMOVEDIR) @@ -3785,13 +3791,13 @@ static int io_shutdown_prep(struct io_kiocb *req, #endif } -static int io_shutdown(struct io_kiocb *req, bool force_nonblock) +static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) { #if defined(CONFIG_NET) struct socket *sock; int ret; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; sock = sock_from_file(req->file); @@ -3850,7 +3856,7 @@ static int io_tee_prep(struct io_kiocb *req, return __io_splice_prep(req, sqe); } -static int io_tee(struct io_kiocb *req, bool force_nonblock) +static int io_tee(struct io_kiocb *req, unsigned int issue_flags) { struct io_splice *sp = &req->splice; struct file *in = sp->file_in; @@ -3858,7 +3864,7 @@ static int io_tee(struct io_kiocb *req, bool force_nonblock) unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; long ret = 0; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; if (sp->len) ret = do_tee(in, out, sp->len, flags); @@ -3881,7 +3887,7 @@ static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return __io_splice_prep(req, sqe); } -static int io_splice(struct io_kiocb *req, bool force_nonblock) +static int io_splice(struct io_kiocb *req, unsigned int issue_flags) { struct io_splice *sp = &req->splice; struct file *in = sp->file_in; @@ -3890,7 +3896,7 @@ static int io_splice(struct io_kiocb *req, bool force_nonblock) loff_t *poff_in, *poff_out; long ret = 0; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; poff_in = (sp->off_in == -1) ? NULL : &sp->off_in; @@ -3943,13 +3949,13 @@ static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_fsync(struct io_kiocb *req, bool force_nonblock) +static int io_fsync(struct io_kiocb *req, unsigned int issue_flags) { loff_t end = req->sync.off + req->sync.len; int ret; /* fsync always requires a blocking context */ - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; ret = vfs_fsync_range(req->file, req->sync.off, @@ -3975,12 +3981,12 @@ static int io_fallocate_prep(struct io_kiocb *req, return 0; } -static int io_fallocate(struct io_kiocb *req, bool force_nonblock) +static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags) { int ret; /* fallocate always requiring blocking context */ - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off, req->sync.len); @@ -4050,7 +4056,7 @@ static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return __io_openat_prep(req, sqe); } -static int io_openat2(struct io_kiocb *req, bool force_nonblock) +static int io_openat2(struct io_kiocb *req, unsigned int issue_flags) { struct open_flags op; struct file *file; @@ -4063,7 +4069,7 @@ static int io_openat2(struct io_kiocb *req, bool force_nonblock) goto err; nonblock_set = op.open_flag & O_NONBLOCK; resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED; - if (force_nonblock) { + if (issue_flags & IO_URING_F_NONBLOCK) { /* * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open, * it'll always -EAGAIN @@ -4080,7 +4086,8 @@ static int io_openat2(struct io_kiocb *req, bool force_nonblock) file = do_filp_open(req->open.dfd, req->open.filename, &op); /* only retry if RESOLVE_CACHED wasn't already set by application */ - if ((!resolve_nonblock && force_nonblock) && file == ERR_PTR(-EAGAIN)) { + if ((!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)) && + file == ERR_PTR(-EAGAIN)) { /* * We could hang on to this 'fd', but seems like marginal * gain for something that is now known to be a slower path. @@ -4094,7 +4101,7 @@ static int io_openat2(struct io_kiocb *req, bool force_nonblock) put_unused_fd(ret); ret = PTR_ERR(file); } else { - if (force_nonblock && !nonblock_set) + if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set) file->f_flags &= ~O_NONBLOCK; fsnotify_open(file); fd_install(ret, file); @@ -4108,9 +4115,9 @@ static int io_openat2(struct io_kiocb *req, bool force_nonblock) return 0; } -static int io_openat(struct io_kiocb *req, bool force_nonblock) +static int io_openat(struct io_kiocb *req, unsigned int issue_flags) { - return io_openat2(req, force_nonblock); + return io_openat2(req, issue_flags & IO_URING_F_NONBLOCK); } static int io_remove_buffers_prep(struct io_kiocb *req, @@ -4158,13 +4165,14 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf, return i; } -static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock, +static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_provide_buf *p = &req->pbuf; struct io_ring_ctx *ctx = req->ctx; struct io_buffer *head; int ret = 0; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; io_ring_submit_lock(ctx, !force_nonblock); @@ -4242,13 +4250,14 @@ static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head) return i ? i : -ENOMEM; } -static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock, +static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_provide_buf *p = &req->pbuf; struct io_ring_ctx *ctx = req->ctx; struct io_buffer *head, *list; int ret = 0; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; io_ring_submit_lock(ctx, !force_nonblock); @@ -4310,12 +4319,13 @@ static int io_epoll_ctl_prep(struct io_kiocb *req, #endif } -static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock, +static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { #if defined(CONFIG_EPOLL) struct io_epoll *ie = &req->epoll; int ret; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock); if (force_nonblock && ret == -EAGAIN) @@ -4347,13 +4357,13 @@ static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) #endif } -static int io_madvise(struct io_kiocb *req, bool force_nonblock) +static int io_madvise(struct io_kiocb *req, unsigned int issue_flags) { #if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) struct io_madvise *ma = &req->madvise; int ret; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice); @@ -4379,12 +4389,12 @@ static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_fadvise(struct io_kiocb *req, bool force_nonblock) +static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags) { struct io_fadvise *fa = &req->fadvise; int ret; - if (force_nonblock) { + if (issue_flags & IO_URING_F_NONBLOCK) { switch (fa->advice) { case POSIX_FADV_NORMAL: case POSIX_FADV_RANDOM: @@ -4420,12 +4430,12 @@ static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_statx(struct io_kiocb *req, bool force_nonblock) +static int io_statx(struct io_kiocb *req, unsigned int issue_flags) { struct io_statx *ctx = &req->statx; int ret; - if (force_nonblock) { + if (issue_flags & IO_URING_F_NONBLOCK) { /* only need file table for an actual valid fd */ if (ctx->dfd == -1 || ctx->dfd == AT_FDCWD) req->flags |= REQ_F_NO_FILE_TABLE; @@ -4455,7 +4465,7 @@ static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_close(struct io_kiocb *req, bool force_nonblock, +static int io_close(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct files_struct *files = current->files; @@ -4485,7 +4495,7 @@ static int io_close(struct io_kiocb *req, bool force_nonblock, } /* if the file has a flush method, be safe and punt to async */ - if (file->f_op->flush && force_nonblock) { + if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) { spin_unlock(&files->file_lock); return -EAGAIN; } @@ -4527,12 +4537,12 @@ static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock) +static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags) { int ret; /* sync_file_range always requires a blocking context */ - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; ret = sync_file_range(req->file, req->sync.off, req->sync.len, @@ -4601,7 +4611,7 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return ret; } -static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, +static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_async_msghdr iomsg, *kmsg; @@ -4624,11 +4634,11 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, flags = req->sr_msg.msg_flags; if (flags & MSG_DONTWAIT) req->flags |= REQ_F_NOWAIT; - else if (force_nonblock) + else if (issue_flags & IO_URING_F_NONBLOCK) flags |= MSG_DONTWAIT; ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); - if (force_nonblock && ret == -EAGAIN) + if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN) return io_setup_async_msg(req, kmsg); if (ret == -ERESTARTSYS) ret = -EINTR; @@ -4643,7 +4653,7 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, return 0; } -static int io_send(struct io_kiocb *req, bool force_nonblock, +static int io_send(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_sr_msg *sr = &req->sr_msg; @@ -4669,12 +4679,12 @@ static int io_send(struct io_kiocb *req, bool force_nonblock, flags = req->sr_msg.msg_flags; if (flags & MSG_DONTWAIT) req->flags |= REQ_F_NOWAIT; - else if (force_nonblock) + else if (issue_flags & IO_URING_F_NONBLOCK) flags |= MSG_DONTWAIT; msg.msg_flags = flags; ret = sock_sendmsg(sock, &msg); - if (force_nonblock && ret == -EAGAIN) + if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN) return -EAGAIN; if (ret == -ERESTARTSYS) ret = -EINTR; @@ -4822,7 +4832,7 @@ static int io_recvmsg_prep(struct io_kiocb *req, return ret; } -static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, +static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_async_msghdr iomsg, *kmsg; @@ -4830,6 +4840,7 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, struct io_buffer *kbuf; unsigned flags; int ret, cflags = 0; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; sock = sock_from_file(req->file); if (unlikely(!sock)) @@ -4878,7 +4889,7 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, return 0; } -static int io_recv(struct io_kiocb *req, bool force_nonblock, +static int io_recv(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_buffer *kbuf; @@ -4889,6 +4900,7 @@ static int io_recv(struct io_kiocb *req, bool force_nonblock, struct iovec iov; unsigned flags; int ret, cflags = 0; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; sock = sock_from_file(req->file); if (unlikely(!sock)) @@ -4948,10 +4960,11 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_accept(struct io_kiocb *req, bool force_nonblock, +static int io_accept(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_accept *accept = &req->accept; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; int ret; @@ -4992,12 +5005,13 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) &io->address); } -static int io_connect(struct io_kiocb *req, bool force_nonblock, +static int io_connect(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_async_connect __io, *io; unsigned file_flags; int ret; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; if (req->async_data) { io = req->async_data; @@ -5039,13 +5053,13 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EOPNOTSUPP; } -static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, +static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { return -EOPNOTSUPP; } -static int io_send(struct io_kiocb *req, bool force_nonblock, +static int io_send(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { return -EOPNOTSUPP; @@ -5057,13 +5071,13 @@ static int io_recvmsg_prep(struct io_kiocb *req, return -EOPNOTSUPP; } -static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, +static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { return -EOPNOTSUPP; } -static int io_recv(struct io_kiocb *req, bool force_nonblock, +static int io_recv(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { return -EOPNOTSUPP; @@ -5074,7 +5088,7 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EOPNOTSUPP; } -static int io_accept(struct io_kiocb *req, bool force_nonblock, +static int io_accept(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { return -EOPNOTSUPP; @@ -5085,7 +5099,7 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EOPNOTSUPP; } -static int io_connect(struct io_kiocb *req, bool force_nonblock, +static int io_connect(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { return -EOPNOTSUPP; @@ -5963,14 +5977,14 @@ static int io_rsrc_update_prep(struct io_kiocb *req, return 0; } -static int io_files_update(struct io_kiocb *req, bool force_nonblock, +static int io_files_update(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_ring_ctx *ctx = req->ctx; struct io_uring_rsrc_update up; int ret; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; up.offset = req->rsrc_update.offset; @@ -6189,7 +6203,7 @@ static void __io_clean_op(struct io_kiocb *req) } } -static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock, +static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_ring_ctx *ctx = req->ctx; @@ -6202,15 +6216,15 @@ static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock, case IORING_OP_READV: case IORING_OP_READ_FIXED: case IORING_OP_READ: - ret = io_read(req, force_nonblock, cs); + ret = io_read(req, issue_flags, cs); break; case IORING_OP_WRITEV: case IORING_OP_WRITE_FIXED: case IORING_OP_WRITE: - ret = io_write(req, force_nonblock, cs); + ret = io_write(req, issue_flags, cs); break; case IORING_OP_FSYNC: - ret = io_fsync(req, force_nonblock); + ret = io_fsync(req, issue_flags); break; case IORING_OP_POLL_ADD: ret = io_poll_add(req); @@ -6219,19 +6233,19 @@ static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock, ret = io_poll_remove(req); break; case IORING_OP_SYNC_FILE_RANGE: - ret = io_sync_file_range(req, force_nonblock); + ret = io_sync_file_range(req, issue_flags); break; case IORING_OP_SENDMSG: - ret = io_sendmsg(req, force_nonblock, cs); + ret = io_sendmsg(req, issue_flags, cs); break; case IORING_OP_SEND: - ret = io_send(req, force_nonblock, cs); + ret = io_send(req, issue_flags, cs); break; case IORING_OP_RECVMSG: - ret = io_recvmsg(req, force_nonblock, cs); + ret = io_recvmsg(req, issue_flags, cs); break; case IORING_OP_RECV: - ret = io_recv(req, force_nonblock, cs); + ret = io_recv(req, issue_flags, cs); break; case IORING_OP_TIMEOUT: ret = io_timeout(req); @@ -6240,61 +6254,61 @@ static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock, ret = io_timeout_remove(req); break; case IORING_OP_ACCEPT: - ret = io_accept(req, force_nonblock, cs); + ret = io_accept(req, issue_flags, cs); break; case IORING_OP_CONNECT: - ret = io_connect(req, force_nonblock, cs); + ret = io_connect(req, issue_flags, cs); break; case IORING_OP_ASYNC_CANCEL: ret = io_async_cancel(req); break; case IORING_OP_FALLOCATE: - ret = io_fallocate(req, force_nonblock); + ret = io_fallocate(req, issue_flags); break; case IORING_OP_OPENAT: - ret = io_openat(req, force_nonblock); + ret = io_openat(req, issue_flags); break; case IORING_OP_CLOSE: - ret = io_close(req, force_nonblock, cs); + ret = io_close(req, issue_flags, cs); break; case IORING_OP_FILES_UPDATE: - ret = io_files_update(req, force_nonblock, cs); + ret = io_files_update(req, issue_flags, cs); break; case IORING_OP_STATX: - ret = io_statx(req, force_nonblock); + ret = io_statx(req, issue_flags); break; case IORING_OP_FADVISE: - ret = io_fadvise(req, force_nonblock); + ret = io_fadvise(req, issue_flags); break; case IORING_OP_MADVISE: - ret = io_madvise(req, force_nonblock); + ret = io_madvise(req, issue_flags); break; case IORING_OP_OPENAT2: - ret = io_openat2(req, force_nonblock); + ret = io_openat2(req, issue_flags); break; case IORING_OP_EPOLL_CTL: - ret = io_epoll_ctl(req, force_nonblock, cs); + ret = io_epoll_ctl(req, issue_flags, cs); break; case IORING_OP_SPLICE: - ret = io_splice(req, force_nonblock); + ret = io_splice(req, issue_flags); break; case IORING_OP_PROVIDE_BUFFERS: - ret = io_provide_buffers(req, force_nonblock, cs); + ret = io_provide_buffers(req, issue_flags, cs); break; case IORING_OP_REMOVE_BUFFERS: - ret = io_remove_buffers(req, force_nonblock, cs); + ret = io_remove_buffers(req, issue_flags, cs); break; case IORING_OP_TEE: - ret = io_tee(req, force_nonblock); + ret = io_tee(req, issue_flags); break; case IORING_OP_SHUTDOWN: - ret = io_shutdown(req, force_nonblock); + ret = io_shutdown(req, issue_flags); break; case IORING_OP_RENAMEAT: - ret = io_renameat(req, force_nonblock); + ret = io_renameat(req, issue_flags); break; case IORING_OP_UNLINKAT: - ret = io_unlinkat(req, force_nonblock); + ret = io_unlinkat(req, issue_flags); break; default: ret = -EINVAL; @@ -6336,7 +6350,7 @@ static void io_wq_submit_work(struct io_wq_work *work) if (!ret) { do { - ret = io_issue_sqe(req, false, NULL); + ret = io_issue_sqe(req, 0, NULL); /* * We can get EAGAIN for polled IO even though we're * forcing a sync submission from here, since we can't @@ -6499,7 +6513,7 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) old_creds = override_creds(req->work.identity->creds); } - ret = io_issue_sqe(req, true, cs); + ret = io_issue_sqe(req, IO_URING_F_NONBLOCK, cs); /* * We async punt it if the file wasn't marked NOWAIT, or if the file From 61e98203047983fd959cfef889b328a57315847c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:08 +0000 Subject: [PATCH 061/103] io_uring: make op handlers always take issue flags Make opcode handler interfaces a bit more consistent by always passing in issue flags. Bulky but pretty easy and mechanical change. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 862121c48cee..ac233d04ee71 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3917,7 +3917,8 @@ static int io_splice(struct io_kiocb *req, unsigned int issue_flags) /* * IORING_OP_NOP just posts a completion event, nothing else. */ -static int io_nop(struct io_kiocb *req, struct io_comp_state *cs) +static int io_nop(struct io_kiocb *req, unsigned int issue_flags, + struct io_comp_state *cs) { struct io_ring_ctx *ctx = req->ctx; @@ -5581,7 +5582,7 @@ static int io_poll_remove_prep(struct io_kiocb *req, * Find a running poll command that matches one specified in sqe->addr, * and remove it if found. */ -static int io_poll_remove(struct io_kiocb *req) +static int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; int ret; @@ -5632,7 +5633,7 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe return 0; } -static int io_poll_add(struct io_kiocb *req) +static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) { struct io_poll_iocb *poll = &req->poll; struct io_ring_ctx *ctx = req->ctx; @@ -5772,7 +5773,7 @@ static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) /* * Remove or update an existing timeout command */ -static int io_timeout_remove(struct io_kiocb *req) +static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) { struct io_timeout_rem *tr = &req->timeout_rem; struct io_ring_ctx *ctx = req->ctx; @@ -5828,7 +5829,7 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, return 0; } -static int io_timeout(struct io_kiocb *req) +static int io_timeout(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; struct io_timeout_data *data = req->async_data; @@ -5951,7 +5952,7 @@ static int io_async_cancel_prep(struct io_kiocb *req, return 0; } -static int io_async_cancel(struct io_kiocb *req) +static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; @@ -6211,7 +6212,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, switch (req->opcode) { case IORING_OP_NOP: - ret = io_nop(req, cs); + ret = io_nop(req, issue_flags, cs); break; case IORING_OP_READV: case IORING_OP_READ_FIXED: @@ -6227,10 +6228,10 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_fsync(req, issue_flags); break; case IORING_OP_POLL_ADD: - ret = io_poll_add(req); + ret = io_poll_add(req, issue_flags); break; case IORING_OP_POLL_REMOVE: - ret = io_poll_remove(req); + ret = io_poll_remove(req, issue_flags); break; case IORING_OP_SYNC_FILE_RANGE: ret = io_sync_file_range(req, issue_flags); @@ -6248,10 +6249,10 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_recv(req, issue_flags, cs); break; case IORING_OP_TIMEOUT: - ret = io_timeout(req); + ret = io_timeout(req, issue_flags); break; case IORING_OP_TIMEOUT_REMOVE: - ret = io_timeout_remove(req); + ret = io_timeout_remove(req, issue_flags); break; case IORING_OP_ACCEPT: ret = io_accept(req, issue_flags, cs); @@ -6260,7 +6261,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_connect(req, issue_flags, cs); break; case IORING_OP_ASYNC_CANCEL: - ret = io_async_cancel(req); + ret = io_async_cancel(req, issue_flags); break; case IORING_OP_FALLOCATE: ret = io_fallocate(req, issue_flags); From 889fca73287b0ae21c9d8712379c9ae5a3b27d08 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:09 +0000 Subject: [PATCH 062/103] io_uring: don't propagate io_comp_state There is no reason to drag io_comp_state into opcode handlers, we just need a flag and the actual work will be done in __io_queue_sqe(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 164 ++++++++++++++++++++++---------------------------- 1 file changed, 73 insertions(+), 91 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index ac233d04ee71..273ebbac0654 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -189,6 +189,7 @@ struct io_rings { enum io_uring_cmd_flags { IO_URING_F_NONBLOCK = 1, + IO_URING_F_COMPLETE_DEFER = 2, }; struct io_mapped_ubuf { @@ -1018,7 +1019,7 @@ static void init_fixed_file_ref_node(struct io_ring_ctx *ctx, struct fixed_rsrc_ref_node *ref_node); static void __io_complete_rw(struct io_kiocb *req, long res, long res2, - struct io_comp_state *cs); + unsigned int issue_flags); static void io_cqring_fill_event(struct io_kiocb *req, long res); static void io_put_req(struct io_kiocb *req); static void io_put_req_deferred(struct io_kiocb *req, int nr); @@ -1957,7 +1958,7 @@ static void io_submit_flush_completions(struct io_comp_state *cs) } static void io_req_complete_state(struct io_kiocb *req, long res, - unsigned int cflags, struct io_comp_state *cs) + unsigned int cflags) { io_clean_op(req); req->result = res; @@ -1965,18 +1966,18 @@ static void io_req_complete_state(struct io_kiocb *req, long res, req->flags |= REQ_F_COMPLETE_INLINE; } -static inline void __io_req_complete(struct io_kiocb *req, long res, - unsigned cflags, struct io_comp_state *cs) +static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags, + long res, unsigned cflags) { - if (!cs) - io_req_complete_nostate(req, res, cflags); + if (issue_flags & IO_URING_F_COMPLETE_DEFER) + io_req_complete_state(req, res, cflags); else - io_req_complete_state(req, res, cflags, cs); + io_req_complete_nostate(req, res, cflags); } static inline void io_req_complete(struct io_kiocb *req, long res) { - __io_req_complete(req, res, 0, NULL); + __io_req_complete(req, 0, res, 0); } static inline bool io_is_fallback_req(struct io_kiocb *req) @@ -2449,7 +2450,7 @@ static void io_iopoll_queue(struct list_head *again) do { req = list_first_entry(again, struct io_kiocb, inflight_entry); list_del(&req->inflight_entry); - __io_complete_rw(req, -EAGAIN, 0, NULL); + __io_complete_rw(req, -EAGAIN, 0, 0); } while (!list_empty(again)); } @@ -2662,7 +2663,7 @@ static void kiocb_end_write(struct io_kiocb *req) } static void io_complete_rw_common(struct kiocb *kiocb, long res, - struct io_comp_state *cs) + unsigned int issue_flags) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); int cflags = 0; @@ -2674,7 +2675,7 @@ static void io_complete_rw_common(struct kiocb *kiocb, long res, req_set_fail_links(req); if (req->flags & REQ_F_BUFFER_SELECTED) cflags = io_put_rw_kbuf(req); - __io_req_complete(req, res, cflags, cs); + __io_req_complete(req, issue_flags, res, cflags); } #ifdef CONFIG_BLOCK @@ -2741,17 +2742,17 @@ static bool io_rw_reissue(struct io_kiocb *req, long res) } static void __io_complete_rw(struct io_kiocb *req, long res, long res2, - struct io_comp_state *cs) + unsigned int issue_flags) { if (!io_rw_reissue(req, res)) - io_complete_rw_common(&req->rw.kiocb, res, cs); + io_complete_rw_common(&req->rw.kiocb, res, issue_flags); } static void io_complete_rw(struct kiocb *kiocb, long res, long res2) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); - __io_complete_rw(req, res, res2, NULL); + __io_complete_rw(req, res, res2, 0); } static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) @@ -2970,7 +2971,7 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) } static void kiocb_done(struct kiocb *kiocb, ssize_t ret, - struct io_comp_state *cs) + unsigned int issue_flags) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); struct io_async_rw *io = req->async_data; @@ -2986,7 +2987,7 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret, if (req->flags & REQ_F_CUR_POS) req->file->f_pos = kiocb->ki_pos; if (ret >= 0 && kiocb->ki_complete == io_complete_rw) - __io_complete_rw(req, ret, 0, cs); + __io_complete_rw(req, ret, 0, issue_flags); else io_rw_done(kiocb, ret); } @@ -3481,8 +3482,7 @@ static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter) return -EINVAL; } -static int io_read(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_read(struct io_kiocb *req, unsigned int issue_flags) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct kiocb *kiocb = &req->rw.kiocb; @@ -3572,7 +3572,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags, /* we got some bytes, but not all. retry. */ } while (ret > 0 && ret < io_size); done: - kiocb_done(kiocb, ret, cs); + kiocb_done(kiocb, ret, issue_flags); return 0; } @@ -3593,8 +3593,7 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return io_rw_prep_async(req, WRITE); } -static int io_write(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_write(struct io_kiocb *req, unsigned int issue_flags) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct kiocb *kiocb = &req->rw.kiocb; @@ -3668,7 +3667,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags, if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN) goto copy_iov; done: - kiocb_done(kiocb, ret2, cs); + kiocb_done(kiocb, ret2, issue_flags); } else { copy_iov: /* some cases will consume bytes even on error returns */ @@ -3917,15 +3916,14 @@ static int io_splice(struct io_kiocb *req, unsigned int issue_flags) /* * IORING_OP_NOP just posts a completion event, nothing else. */ -static int io_nop(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_nop(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; - __io_req_complete(req, 0, 0, cs); + __io_req_complete(req, issue_flags, 0, 0); return 0; } @@ -4166,8 +4164,7 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf, return i; } -static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) { struct io_provide_buf *p = &req->pbuf; struct io_ring_ctx *ctx = req->ctx; @@ -4188,11 +4185,11 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags, /* need to hold the lock to complete IOPOLL requests */ if (ctx->flags & IORING_SETUP_IOPOLL) { - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); io_ring_submit_unlock(ctx, !force_nonblock); } else { io_ring_submit_unlock(ctx, !force_nonblock); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); } return 0; } @@ -4251,8 +4248,7 @@ static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head) return i ? i : -ENOMEM; } -static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags) { struct io_provide_buf *p = &req->pbuf; struct io_ring_ctx *ctx = req->ctx; @@ -4284,11 +4280,11 @@ static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags, /* need to hold the lock to complete IOPOLL requests */ if (ctx->flags & IORING_SETUP_IOPOLL) { - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); io_ring_submit_unlock(ctx, !force_nonblock); } else { io_ring_submit_unlock(ctx, !force_nonblock); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); } return 0; } @@ -4320,8 +4316,7 @@ static int io_epoll_ctl_prep(struct io_kiocb *req, #endif } -static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags) { #if defined(CONFIG_EPOLL) struct io_epoll *ie = &req->epoll; @@ -4334,7 +4329,7 @@ static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags, if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; #else return -EOPNOTSUPP; @@ -4466,8 +4461,7 @@ static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_close(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_close(struct io_kiocb *req, unsigned int issue_flags) { struct files_struct *files = current->files; struct io_close *close = &req->close; @@ -4516,7 +4510,7 @@ static int io_close(struct io_kiocb *req, unsigned int issue_flags, req_set_fail_links(req); if (file) fput(file); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -4612,8 +4606,7 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return ret; } -static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) { struct io_async_msghdr iomsg, *kmsg; struct socket *sock; @@ -4650,12 +4643,11 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags, req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; } -static int io_send(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_send(struct io_kiocb *req, unsigned int issue_flags) { struct io_sr_msg *sr = &req->sr_msg; struct msghdr msg; @@ -4692,7 +4684,7 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags, if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -4833,8 +4825,7 @@ static int io_recvmsg_prep(struct io_kiocb *req, return ret; } -static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) { struct io_async_msghdr iomsg, *kmsg; struct socket *sock; @@ -4886,12 +4877,11 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags, req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, cflags, cs); + __io_req_complete(req, issue_flags, ret, cflags); return 0; } -static int io_recv(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_recv(struct io_kiocb *req, unsigned int issue_flags) { struct io_buffer *kbuf; struct io_sr_msg *sr = &req->sr_msg; @@ -4941,7 +4931,7 @@ static int io_recv(struct io_kiocb *req, unsigned int issue_flags, cflags = io_put_recv_kbuf(req); if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, cflags, cs); + __io_req_complete(req, issue_flags, ret, cflags); return 0; } @@ -4961,8 +4951,7 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_accept(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_accept(struct io_kiocb *req, unsigned int issue_flags) { struct io_accept *accept = &req->accept; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; @@ -4982,7 +4971,7 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags, ret = -EINTR; req_set_fail_links(req); } - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -5006,8 +4995,7 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) &io->address); } -static int io_connect(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_connect(struct io_kiocb *req, unsigned int issue_flags) { struct io_async_connect __io, *io; unsigned file_flags; @@ -5045,7 +5033,7 @@ static int io_connect(struct io_kiocb *req, unsigned int issue_flags, out: if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; } #else /* !CONFIG_NET */ @@ -5054,14 +5042,12 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EOPNOTSUPP; } -static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) { return -EOPNOTSUPP; } -static int io_send(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_send(struct io_kiocb *req, unsigned int issue_flags) { return -EOPNOTSUPP; } @@ -5072,14 +5058,12 @@ static int io_recvmsg_prep(struct io_kiocb *req, return -EOPNOTSUPP; } -static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) { return -EOPNOTSUPP; } -static int io_recv(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_recv(struct io_kiocb *req, unsigned int issue_flags) { return -EOPNOTSUPP; } @@ -5089,8 +5073,7 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EOPNOTSUPP; } -static int io_accept(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_accept(struct io_kiocb *req, unsigned int issue_flags) { return -EOPNOTSUPP; } @@ -5100,8 +5083,7 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EOPNOTSUPP; } -static int io_connect(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_connect(struct io_kiocb *req, unsigned int issue_flags) { return -EOPNOTSUPP; } @@ -5978,8 +5960,7 @@ static int io_rsrc_update_prep(struct io_kiocb *req, return 0; } -static int io_files_update(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; struct io_uring_rsrc_update up; @@ -5997,7 +5978,7 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags, if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -6204,25 +6185,24 @@ static void __io_clean_op(struct io_kiocb *req) } } -static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; int ret; switch (req->opcode) { case IORING_OP_NOP: - ret = io_nop(req, issue_flags, cs); + ret = io_nop(req, issue_flags); break; case IORING_OP_READV: case IORING_OP_READ_FIXED: case IORING_OP_READ: - ret = io_read(req, issue_flags, cs); + ret = io_read(req, issue_flags); break; case IORING_OP_WRITEV: case IORING_OP_WRITE_FIXED: case IORING_OP_WRITE: - ret = io_write(req, issue_flags, cs); + ret = io_write(req, issue_flags); break; case IORING_OP_FSYNC: ret = io_fsync(req, issue_flags); @@ -6237,16 +6217,16 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_sync_file_range(req, issue_flags); break; case IORING_OP_SENDMSG: - ret = io_sendmsg(req, issue_flags, cs); + ret = io_sendmsg(req, issue_flags); break; case IORING_OP_SEND: - ret = io_send(req, issue_flags, cs); + ret = io_send(req, issue_flags); break; case IORING_OP_RECVMSG: - ret = io_recvmsg(req, issue_flags, cs); + ret = io_recvmsg(req, issue_flags); break; case IORING_OP_RECV: - ret = io_recv(req, issue_flags, cs); + ret = io_recv(req, issue_flags); break; case IORING_OP_TIMEOUT: ret = io_timeout(req, issue_flags); @@ -6255,10 +6235,10 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_timeout_remove(req, issue_flags); break; case IORING_OP_ACCEPT: - ret = io_accept(req, issue_flags, cs); + ret = io_accept(req, issue_flags); break; case IORING_OP_CONNECT: - ret = io_connect(req, issue_flags, cs); + ret = io_connect(req, issue_flags); break; case IORING_OP_ASYNC_CANCEL: ret = io_async_cancel(req, issue_flags); @@ -6270,10 +6250,10 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_openat(req, issue_flags); break; case IORING_OP_CLOSE: - ret = io_close(req, issue_flags, cs); + ret = io_close(req, issue_flags); break; case IORING_OP_FILES_UPDATE: - ret = io_files_update(req, issue_flags, cs); + ret = io_files_update(req, issue_flags); break; case IORING_OP_STATX: ret = io_statx(req, issue_flags); @@ -6288,16 +6268,16 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_openat2(req, issue_flags); break; case IORING_OP_EPOLL_CTL: - ret = io_epoll_ctl(req, issue_flags, cs); + ret = io_epoll_ctl(req, issue_flags); break; case IORING_OP_SPLICE: ret = io_splice(req, issue_flags); break; case IORING_OP_PROVIDE_BUFFERS: - ret = io_provide_buffers(req, issue_flags, cs); + ret = io_provide_buffers(req, issue_flags); break; case IORING_OP_REMOVE_BUFFERS: - ret = io_remove_buffers(req, issue_flags, cs); + ret = io_remove_buffers(req, issue_flags); break; case IORING_OP_TEE: ret = io_tee(req, issue_flags); @@ -6351,7 +6331,7 @@ static void io_wq_submit_work(struct io_wq_work *work) if (!ret) { do { - ret = io_issue_sqe(req, 0, NULL); + ret = io_issue_sqe(req, 0); /* * We can get EAGAIN for polled IO even though we're * forcing a sync submission from here, since we can't @@ -6498,8 +6478,10 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) { struct io_kiocb *linked_timeout; const struct cred *old_creds = NULL; - int ret; + int ret, issue_flags = IO_URING_F_NONBLOCK; + if (cs) + issue_flags |= IO_URING_F_COMPLETE_DEFER; again: linked_timeout = io_prep_linked_timeout(req); @@ -6514,7 +6496,7 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) old_creds = override_creds(req->work.identity->creds); } - ret = io_issue_sqe(req, IO_URING_F_NONBLOCK, cs); + ret = io_issue_sqe(req, issue_flags); /* * We async punt it if the file wasn't marked NOWAIT, or if the file From 258b29a93bfe74a57c01e1b10b698d5b62e173fe Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:10 +0000 Subject: [PATCH 063/103] io_uring: don't keep submit_state on stack struct io_submit_state is quite big (168 bytes) and going to grow. It's better to not keep it on stack as it is now. Move it to context, it's always protected by uring_lock, so it's fine to have only one instance of it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 90 ++++++++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 273ebbac0654..743090364890 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -264,6 +264,39 @@ struct io_sq_data { unsigned sq_thread_idle; }; +#define IO_IOPOLL_BATCH 8 + +struct io_comp_state { + unsigned int nr; + struct list_head list; + struct io_ring_ctx *ctx; +}; + +struct io_submit_state { + struct blk_plug plug; + + /* + * io_kiocb alloc cache + */ + void *reqs[IO_IOPOLL_BATCH]; + unsigned int free_reqs; + + bool plug_started; + + /* + * Batch completion logic + */ + struct io_comp_state comp; + + /* + * File reference cache + */ + struct file *file; + unsigned int fd; + unsigned int file_refs; + unsigned int ios_left; +}; + struct io_ring_ctx { struct { struct percpu_ref refs; @@ -406,6 +439,7 @@ struct io_ring_ctx { struct work_struct exit_work; struct io_restriction restrictions; + struct io_submit_state submit_state; }; /* @@ -758,39 +792,6 @@ struct io_defer_entry { u32 seq; }; -#define IO_IOPOLL_BATCH 8 - -struct io_comp_state { - unsigned int nr; - struct list_head list; - struct io_ring_ctx *ctx; -}; - -struct io_submit_state { - struct blk_plug plug; - - /* - * io_kiocb alloc cache - */ - void *reqs[IO_IOPOLL_BATCH]; - unsigned int free_reqs; - - bool plug_started; - - /* - * Batch completion logic - */ - struct io_comp_state comp; - - /* - * File reference cache - */ - struct file *file; - unsigned int fd; - unsigned int file_refs; - unsigned int ios_left; -}; - struct io_op_def { /* needs req->file assigned */ unsigned needs_file : 1; @@ -1997,9 +1998,10 @@ static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx) return NULL; } -static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx, - struct io_submit_state *state) +static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) { + struct io_submit_state *state = &ctx->submit_state; + if (!state->free_reqs) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; size_t sz; @@ -6758,9 +6760,9 @@ static inline bool io_check_restriction(struct io_ring_ctx *ctx, IOSQE_BUFFER_SELECT) static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, - const struct io_uring_sqe *sqe, - struct io_submit_state *state) + const struct io_uring_sqe *sqe) { + struct io_submit_state *state; unsigned int sqe_flags; int id, ret; @@ -6812,6 +6814,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, /* same numerical values with corresponding REQ_F_*, safe to copy */ req->flags |= sqe_flags; + state = &ctx->submit_state; /* * Plug now if we have more than 1 IO left after this, and the target @@ -6838,7 +6841,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) { - struct io_submit_state state; struct io_submit_link link; int i, submitted = 0; @@ -6857,7 +6859,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) percpu_counter_add(¤t->io_uring->inflight, nr); refcount_add(nr, ¤t->usage); - io_submit_state_start(&state, ctx, nr); + io_submit_state_start(&ctx->submit_state, ctx, nr); link.head = NULL; for (i = 0; i < nr; i++) { @@ -6870,7 +6872,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) io_consume_sqe(ctx); break; } - req = io_alloc_req(ctx, &state); + req = io_alloc_req(ctx); if (unlikely(!req)) { if (!submitted) submitted = -EAGAIN; @@ -6880,7 +6882,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) /* will complete beyond this point, count as submitted */ submitted++; - err = io_init_req(ctx, req, sqe, &state); + err = io_init_req(ctx, req, sqe); if (unlikely(err)) { fail_req: io_put_req(req); @@ -6890,7 +6892,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, true, ctx->flags & IORING_SETUP_SQPOLL); - err = io_submit_sqe(req, sqe, &link, &state.comp); + err = io_submit_sqe(req, sqe, &link, &ctx->submit_state.comp); if (err) goto fail_req; } @@ -6905,8 +6907,8 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) put_task_struct_many(current, unused); } if (link.head) - io_queue_link_head(link.head, &state.comp); - io_submit_state_end(&state); + io_queue_link_head(link.head, &ctx->submit_state.comp); + io_submit_state_end(&ctx->submit_state); /* Commit SQ ring head once we've consumed and submitted all SQEs */ io_commit_sqring(ctx); From ba88ff112bdfde8103a8143f867bcdc46bc0e50f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:11 +0000 Subject: [PATCH 064/103] io_uring: remove ctx from comp_state completion state is closely bound to ctx, we don't need to store ctx inside as we always have it around to pass to flush. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 743090364890..6e800f9df292 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -269,7 +269,6 @@ struct io_sq_data { struct io_comp_state { unsigned int nr; struct list_head list; - struct io_ring_ctx *ctx; }; struct io_submit_state { @@ -1924,10 +1923,9 @@ static inline void io_req_complete_nostate(struct io_kiocb *req, long res, io_put_req(req); } -static void io_submit_flush_completions(struct io_comp_state *cs) +static void io_submit_flush_completions(struct io_comp_state *cs, + struct io_ring_ctx *ctx) { - struct io_ring_ctx *ctx = cs->ctx; - spin_lock_irq(&ctx->completion_lock); while (!list_empty(&cs->list)) { struct io_kiocb *req; @@ -6520,7 +6518,7 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) if (req->flags & REQ_F_COMPLETE_INLINE) { list_add_tail(&req->compl.list, &cs->list); if (++cs->nr >= 32) - io_submit_flush_completions(cs); + io_submit_flush_completions(cs, req->ctx); req = NULL; } else { req = io_put_req_find_next(req); @@ -6655,10 +6653,11 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, /* * Batched submission is done, ensure local IO is flushed out. */ -static void io_submit_state_end(struct io_submit_state *state) +static void io_submit_state_end(struct io_submit_state *state, + struct io_ring_ctx *ctx) { if (!list_empty(&state->comp.list)) - io_submit_flush_completions(&state->comp); + io_submit_flush_completions(&state->comp, ctx); if (state->plug_started) blk_finish_plug(&state->plug); io_state_file_put(state); @@ -6670,12 +6669,11 @@ static void io_submit_state_end(struct io_submit_state *state) * Start submission side cache. */ static void io_submit_state_start(struct io_submit_state *state, - struct io_ring_ctx *ctx, unsigned int max_ios) + unsigned int max_ios) { state->plug_started = false; state->comp.nr = 0; INIT_LIST_HEAD(&state->comp.list); - state->comp.ctx = ctx; state->free_reqs = 0; state->file_refs = 0; state->ios_left = max_ios; @@ -6859,7 +6857,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) percpu_counter_add(¤t->io_uring->inflight, nr); refcount_add(nr, ¤t->usage); - io_submit_state_start(&ctx->submit_state, ctx, nr); + io_submit_state_start(&ctx->submit_state, nr); link.head = NULL; for (i = 0; i < nr; i++) { @@ -6908,7 +6906,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) } if (link.head) io_queue_link_head(link.head, &ctx->submit_state.comp); - io_submit_state_end(&ctx->submit_state); + io_submit_state_end(&ctx->submit_state, ctx); /* Commit SQ ring head once we've consumed and submitted all SQEs */ io_commit_sqring(ctx); From 5087275dba02943179720bd95d1d6c7047007550 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:12 +0000 Subject: [PATCH 065/103] io_uring: don't reinit submit state every time As now submit_state is retained across syscalls, we can save ourself from initialising it from ground up for each io_submit_sqes(). Set some fields during ctx allocation, and just keep them always consistent. Signed-off-by: Pavel Begunkov [axboe: remove unnecessary zeroing of ctx members] Signed-off-by: Jens Axboe --- fs/io_uring.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 6e800f9df292..6bce9094280c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1345,6 +1345,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->rsrc_ref_list); INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); init_llist_head(&ctx->rsrc_put_llist); + INIT_LIST_HEAD(&submit_state->comp.list); return ctx; err: if (ctx->fallback_req) @@ -6661,8 +6662,10 @@ static void io_submit_state_end(struct io_submit_state *state, if (state->plug_started) blk_finish_plug(&state->plug); io_state_file_put(state); - if (state->free_reqs) + if (state->free_reqs) { kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs); + state->free_reqs = 0; + } } /* @@ -6672,10 +6675,6 @@ static void io_submit_state_start(struct io_submit_state *state, unsigned int max_ios) { state->plug_started = false; - state->comp.nr = 0; - INIT_LIST_HEAD(&state->comp.list); - state->free_reqs = 0; - state->file_refs = 0; state->ios_left = max_ios; } From 6dd0be1e2481b32c39870e187840ade6c2a11a72 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:13 +0000 Subject: [PATCH 066/103] io_uring: replace list with array for compl batch Reincarnation of an old patch that replaces a list in struct io_compl_batch with an array. It's needed to avoid hooking requests via their compl.list, because it won't be always available in the future. It's also nice to split io_submit_flush_completions() to avoid free under locks and remove unlock/lock with a long comment describing when it can be done. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 6bce9094280c..aef640616edb 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -265,10 +265,11 @@ struct io_sq_data { }; #define IO_IOPOLL_BATCH 8 +#define IO_COMPL_BATCH 32 struct io_comp_state { unsigned int nr; - struct list_head list; + struct io_kiocb *reqs[IO_COMPL_BATCH]; }; struct io_submit_state { @@ -1345,7 +1346,6 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->rsrc_ref_list); INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); init_llist_head(&ctx->rsrc_put_llist); - INIT_LIST_HEAD(&submit_state->comp.list); return ctx; err: if (ctx->fallback_req) @@ -1927,33 +1927,20 @@ static inline void io_req_complete_nostate(struct io_kiocb *req, long res, static void io_submit_flush_completions(struct io_comp_state *cs, struct io_ring_ctx *ctx) { + int i, nr = cs->nr; + spin_lock_irq(&ctx->completion_lock); - while (!list_empty(&cs->list)) { - struct io_kiocb *req; + for (i = 0; i < nr; i++) { + struct io_kiocb *req = cs->reqs[i]; - req = list_first_entry(&cs->list, struct io_kiocb, compl.list); - list_del(&req->compl.list); __io_cqring_fill_event(req, req->result, req->compl.cflags); - - /* - * io_free_req() doesn't care about completion_lock unless one - * of these flags is set. REQ_F_WORK_INITIALIZED is in the list - * because of a potential deadlock with req->work.fs->lock - * We defer both, completion and submission refs. - */ - if (req->flags & (REQ_F_FAIL_LINK|REQ_F_LINK_TIMEOUT - |REQ_F_WORK_INITIALIZED)) { - spin_unlock_irq(&ctx->completion_lock); - io_double_put_req(req); - spin_lock_irq(&ctx->completion_lock); - } else { - io_double_put_req(req); - } } io_commit_cqring(ctx); spin_unlock_irq(&ctx->completion_lock); io_cqring_ev_posted(ctx); + for (i = 0; i < nr; i++) + io_double_put_req(cs->reqs[i]); cs->nr = 0; } @@ -6517,8 +6504,8 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) } else if (likely(!ret)) { /* drop submission reference */ if (req->flags & REQ_F_COMPLETE_INLINE) { - list_add_tail(&req->compl.list, &cs->list); - if (++cs->nr >= 32) + cs->reqs[cs->nr++] = req; + if (cs->nr == IO_COMPL_BATCH) io_submit_flush_completions(cs, req->ctx); req = NULL; } else { @@ -6657,7 +6644,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, static void io_submit_state_end(struct io_submit_state *state, struct io_ring_ctx *ctx) { - if (!list_empty(&state->comp.list)) + if (state->comp.nr) io_submit_flush_completions(&state->comp, ctx); if (state->plug_started) blk_finish_plug(&state->plug); From 905c172f32c56f0740630b639ca5c10ba3689da0 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:14 +0000 Subject: [PATCH 067/103] io_uring: submit-completion free batching io_submit_flush_completions() does completion batching, but may also use free batching as iopoll does. The main beneficiaries should be buffered reads/writes and send/recv. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 49 +++++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index aef640616edb..885dd16893c0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1924,26 +1924,6 @@ static inline void io_req_complete_nostate(struct io_kiocb *req, long res, io_put_req(req); } -static void io_submit_flush_completions(struct io_comp_state *cs, - struct io_ring_ctx *ctx) -{ - int i, nr = cs->nr; - - spin_lock_irq(&ctx->completion_lock); - for (i = 0; i < nr; i++) { - struct io_kiocb *req = cs->reqs[i]; - - __io_cqring_fill_event(req, req->result, req->compl.cflags); - } - io_commit_cqring(ctx); - spin_unlock_irq(&ctx->completion_lock); - - io_cqring_ev_posted(ctx); - for (i = 0; i < nr; i++) - io_double_put_req(cs->reqs[i]); - cs->nr = 0; -} - static void io_req_complete_state(struct io_kiocb *req, long res, unsigned int cflags) { @@ -2329,6 +2309,35 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) __io_req_free_batch_flush(req->ctx, rb); } +static void io_submit_flush_completions(struct io_comp_state *cs, + struct io_ring_ctx *ctx) +{ + int i, nr = cs->nr; + struct io_kiocb *req; + struct req_batch rb; + + io_init_req_batch(&rb); + spin_lock_irq(&ctx->completion_lock); + for (i = 0; i < nr; i++) { + req = cs->reqs[i]; + __io_cqring_fill_event(req, req->result, req->compl.cflags); + } + io_commit_cqring(ctx); + spin_unlock_irq(&ctx->completion_lock); + + io_cqring_ev_posted(ctx); + for (i = 0; i < nr; i++) { + req = cs->reqs[i]; + + /* submission and completion refs */ + if (refcount_sub_and_test(2, &req->refs)) + io_req_free_batch(&rb, req); + } + + io_req_free_batch_finish(ctx, &rb); + cs->nr = 0; +} + /* * Drop reference to request, return next in chain (if there is one) if this * was the last reference to this request. From 3893f39f2245eec04b8052cd441c2cb8a9ea3447 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:15 +0000 Subject: [PATCH 068/103] io_uring: remove fallback_req Remove fallback_req for now, it gets in the way of other changes. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 38 ++------------------------------------ 1 file changed, 2 insertions(+), 36 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 885dd16893c0..a0b5f2c6d8ea 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -386,9 +386,6 @@ struct io_ring_ctx { struct completion ref_comp; struct completion sq_thread_comp; - /* if all else fails... */ - struct io_kiocb *fallback_req; - #if defined(CONFIG_UNIX) struct socket *ring_sock; #endif @@ -1302,10 +1299,6 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) if (!ctx) return NULL; - ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL); - if (!ctx->fallback_req) - goto err; - /* * Use 5 bits less than the max cq entries, that should give us around * 32 entries per hash list if totally full and uniformly spread. @@ -1348,8 +1341,6 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) init_llist_head(&ctx->rsrc_put_llist); return ctx; err: - if (ctx->fallback_req) - kmem_cache_free(req_cachep, ctx->fallback_req); kfree(ctx->cancel_hash); kfree(ctx); return NULL; @@ -1947,23 +1938,6 @@ static inline void io_req_complete(struct io_kiocb *req, long res) __io_req_complete(req, 0, res, 0); } -static inline bool io_is_fallback_req(struct io_kiocb *req) -{ - return req == (struct io_kiocb *) - ((unsigned long) req->ctx->fallback_req & ~1UL); -} - -static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx) -{ - struct io_kiocb *req; - - req = ctx->fallback_req; - if (!test_and_set_bit_lock(0, (unsigned long *) &ctx->fallback_req)) - return req; - - return NULL; -} - static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) { struct io_submit_state *state = &ctx->submit_state; @@ -1983,7 +1957,7 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) if (unlikely(ret <= 0)) { state->reqs[0] = kmem_cache_alloc(req_cachep, gfp); if (!state->reqs[0]) - return io_get_fallback_req(ctx); + return NULL; ret = 1; } state->free_reqs = ret; @@ -2030,10 +2004,7 @@ static void __io_free_req(struct io_kiocb *req) io_dismantle_req(req); io_put_task(req->task, 1); - if (likely(!io_is_fallback_req(req))) - kmem_cache_free(req_cachep, req); - else - clear_bit_unlock(0, (unsigned long *) &ctx->fallback_req); + kmem_cache_free(req_cachep, req); percpu_ref_put(&ctx->refs); } @@ -2289,10 +2260,6 @@ static void io_req_free_batch_finish(struct io_ring_ctx *ctx, static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) { - if (unlikely(io_is_fallback_req(req))) { - io_free_req(req); - return; - } io_queue_next(req); if (req->task != rb->task) { @@ -8695,7 +8662,6 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) free_uid(ctx->user); put_cred(ctx->creds); kfree(ctx->cancel_hash); - kmem_cache_free(req_cachep, ctx->fallback_req); kfree(ctx); } From 9ae7246321d2b735867f6767e0fab96dd248c555 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:16 +0000 Subject: [PATCH 069/103] io_uring: count ctx refs separately from reqs Currently batch free handles request memory freeing and ctx ref putting together. Separate them and use different counters, that will be needed for reusing reqs memory. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index a0b5f2c6d8ea..9b84d6314c11 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2227,6 +2227,7 @@ static void io_free_req(struct io_kiocb *req) struct req_batch { void *reqs[IO_IOPOLL_BATCH]; int to_free; + int ctx_refs; struct task_struct *task; int task_refs; @@ -2236,6 +2237,7 @@ static inline void io_init_req_batch(struct req_batch *rb) { rb->to_free = 0; rb->task_refs = 0; + rb->ctx_refs = 0; rb->task = NULL; } @@ -2243,7 +2245,6 @@ static void __io_req_free_batch_flush(struct io_ring_ctx *ctx, struct req_batch *rb) { kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs); - percpu_ref_put_many(&ctx->refs, rb->to_free); rb->to_free = 0; } @@ -2256,6 +2257,8 @@ static void io_req_free_batch_finish(struct io_ring_ctx *ctx, io_put_task(rb->task, rb->task_refs); rb->task = NULL; } + if (rb->ctx_refs) + percpu_ref_put_many(&ctx->refs, rb->ctx_refs); } static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) @@ -2269,6 +2272,7 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) rb->task_refs = 0; } rb->task_refs++; + rb->ctx_refs++; io_dismantle_req(req); rb->reqs[rb->to_free++] = req; From bf019da7fcbe7e42372582cc339fd1fb8e1e4fa5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:17 +0000 Subject: [PATCH 070/103] io_uring: persistent req cache Don't free batch-allocated requests across syscalls. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 9b84d6314c11..1f0b3b332d32 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -266,6 +266,8 @@ struct io_sq_data { #define IO_IOPOLL_BATCH 8 #define IO_COMPL_BATCH 32 +#define IO_REQ_CACHE_SIZE 8 +#define IO_REQ_ALLOC_BATCH 8 struct io_comp_state { unsigned int nr; @@ -278,7 +280,7 @@ struct io_submit_state { /* * io_kiocb alloc cache */ - void *reqs[IO_IOPOLL_BATCH]; + void *reqs[IO_REQ_CACHE_SIZE]; unsigned int free_reqs; bool plug_started; @@ -1942,13 +1944,14 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) { struct io_submit_state *state = &ctx->submit_state; + BUILD_BUG_ON(IO_REQ_ALLOC_BATCH > ARRAY_SIZE(state->reqs)); + if (!state->free_reqs) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; - size_t sz; int ret; - sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs)); - ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs); + ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH, + state->reqs); /* * Bulk alloc is all-or-nothing. If we fail to get a batch, @@ -6629,10 +6632,6 @@ static void io_submit_state_end(struct io_submit_state *state, if (state->plug_started) blk_finish_plug(&state->plug); io_state_file_put(state); - if (state->free_reqs) { - kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs); - state->free_reqs = 0; - } } /* @@ -8632,6 +8631,8 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) static void io_ring_ctx_free(struct io_ring_ctx *ctx) { + struct io_submit_state *submit_state = &ctx->submit_state; + io_finish_async(ctx); io_sqe_buffers_unregister(ctx); @@ -8642,6 +8643,10 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) ctx->mm_account = NULL; } + if (submit_state->free_reqs) + kmem_cache_free_bulk(req_cachep, submit_state->free_reqs, + submit_state->reqs); + #ifdef CONFIG_BLK_CGROUP if (ctx->sqo_blkcg_css) css_put(ctx->sqo_blkcg_css); From 6ff119a6e4c3fe900e75e6667930dc086f185f2b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:18 +0000 Subject: [PATCH 071/103] io_uring: feed reqs back into alloc cache Make io_req_free_batch(), which is used for inline executed requests and IOPOLL, to return requests back into the allocation cache, so avoid most of kmalloc()/kfree() for those cases. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 1f0b3b332d32..fe07af756186 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -266,7 +266,7 @@ struct io_sq_data { #define IO_IOPOLL_BATCH 8 #define IO_COMPL_BATCH 32 -#define IO_REQ_CACHE_SIZE 8 +#define IO_REQ_CACHE_SIZE 32 #define IO_REQ_ALLOC_BATCH 8 struct io_comp_state { @@ -2264,7 +2264,8 @@ static void io_req_free_batch_finish(struct io_ring_ctx *ctx, percpu_ref_put_many(&ctx->refs, rb->ctx_refs); } -static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) +static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, + struct io_submit_state *state) { io_queue_next(req); @@ -2278,9 +2279,13 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) rb->ctx_refs++; io_dismantle_req(req); - rb->reqs[rb->to_free++] = req; - if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs))) - __io_req_free_batch_flush(req->ctx, rb); + if (state->free_reqs != ARRAY_SIZE(state->reqs)) { + state->reqs[state->free_reqs++] = req; + } else { + rb->reqs[rb->to_free++] = req; + if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs))) + __io_req_free_batch_flush(req->ctx, rb); + } } static void io_submit_flush_completions(struct io_comp_state *cs, @@ -2305,7 +2310,7 @@ static void io_submit_flush_completions(struct io_comp_state *cs, /* submission and completion refs */ if (refcount_sub_and_test(2, &req->refs)) - io_req_free_batch(&rb, req); + io_req_free_batch(&rb, req, &ctx->submit_state); } io_req_free_batch_finish(ctx, &rb); @@ -2458,7 +2463,7 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, (*nr_events)++; if (refcount_dec_and_test(&req->refs)) - io_req_free_batch(&rb, req); + io_req_free_batch(&rb, req, &ctx->submit_state); } io_commit_cqring(ctx); From 1b4c351f6eb7467c77fc19e0cd7e5f0083ecd847 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 10 Feb 2021 00:03:19 +0000 Subject: [PATCH 072/103] io_uring: use persistent request cache Now that we have the submit_state in the ring itself, we can have io_kiocb allocations that are persistent across invocations. This reduces the time spent doing slab allocations and frees. [sil: rebased] Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fe07af756186..87a4b727fe1c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -270,8 +270,9 @@ struct io_sq_data { #define IO_REQ_ALLOC_BATCH 8 struct io_comp_state { - unsigned int nr; struct io_kiocb *reqs[IO_COMPL_BATCH]; + unsigned int nr; + struct list_head free_list; }; struct io_submit_state { @@ -1341,6 +1342,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->rsrc_ref_list); INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); init_llist_head(&ctx->rsrc_put_llist); + INIT_LIST_HEAD(&ctx->submit_state.comp.free_list); return ctx; err: kfree(ctx->cancel_hash); @@ -1946,6 +1948,15 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) BUILD_BUG_ON(IO_REQ_ALLOC_BATCH > ARRAY_SIZE(state->reqs)); + if (!list_empty(&state->comp.free_list)) { + struct io_kiocb *req; + + req = list_first_entry(&state->comp.free_list, struct io_kiocb, + compl.list); + list_del(&req->compl.list); + return req; + } + if (!state->free_reqs) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; int ret; @@ -2228,34 +2239,21 @@ static void io_free_req(struct io_kiocb *req) } struct req_batch { - void *reqs[IO_IOPOLL_BATCH]; - int to_free; - int ctx_refs; - struct task_struct *task; int task_refs; + int ctx_refs; }; static inline void io_init_req_batch(struct req_batch *rb) { - rb->to_free = 0; rb->task_refs = 0; rb->ctx_refs = 0; rb->task = NULL; } -static void __io_req_free_batch_flush(struct io_ring_ctx *ctx, - struct req_batch *rb) -{ - kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs); - rb->to_free = 0; -} - static void io_req_free_batch_finish(struct io_ring_ctx *ctx, struct req_batch *rb) { - if (rb->to_free) - __io_req_free_batch_flush(ctx, rb); if (rb->task) { io_put_task(rb->task, rb->task_refs); rb->task = NULL; @@ -2282,9 +2280,9 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, if (state->free_reqs != ARRAY_SIZE(state->reqs)) { state->reqs[state->free_reqs++] = req; } else { - rb->reqs[rb->to_free++] = req; - if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs))) - __io_req_free_batch_flush(req->ctx, rb); + struct io_comp_state *cs = &req->ctx->submit_state.comp; + + list_add(&req->compl.list, &cs->free_list); } } @@ -8634,6 +8632,19 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) idr_destroy(&ctx->io_buffer_idr); } +static void io_req_cache_free(struct io_ring_ctx *ctx) +{ + struct io_comp_state *cs = &ctx->submit_state.comp; + + while (!list_empty(&cs->free_list)) { + struct io_kiocb *req; + + req = list_first_entry(&cs->free_list, struct io_kiocb, compl.list); + list_del(&req->compl.list); + kmem_cache_free(req_cachep, req); + } +} + static void io_ring_ctx_free(struct io_ring_ctx *ctx) { struct io_submit_state *submit_state = &ctx->submit_state; @@ -8676,6 +8687,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) free_uid(ctx->user); put_cred(ctx->creds); kfree(ctx->cancel_hash); + io_req_cache_free(ctx); kfree(ctx); } From 7cbf1722d5fc5779946ee8f338e9e38b5de15856 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 10 Feb 2021 00:03:20 +0000 Subject: [PATCH 073/103] io_uring: provide FIFO ordering for task_work task_work is a LIFO list, due to how it's implemented as a lockless list. For long chains of task_work, this can be problematic as the first entry added is the last one processed. Similarly, we'd waste a lot of CPU cycles reversing this list. Wrap the task_work so we have a single task_work entry per task per ctx, and use that to run it in the right order. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io-wq.h | 9 ---- fs/io_uring.c | 101 ++++++++++++++++++++++++++++++++++++--- include/linux/io_uring.h | 14 ++++++ 3 files changed, 108 insertions(+), 16 deletions(-) diff --git a/fs/io-wq.h b/fs/io-wq.h index e37a0f217cc8..096f1021018e 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -27,15 +27,6 @@ enum io_wq_cancel { IO_WQ_CANCEL_NOTFOUND, /* work not found */ }; -struct io_wq_work_node { - struct io_wq_work_node *next; -}; - -struct io_wq_work_list { - struct io_wq_work_node *first; - struct io_wq_work_node *last; -}; - static inline void wq_list_add_after(struct io_wq_work_node *node, struct io_wq_work_node *pos, struct io_wq_work_list *list) diff --git a/fs/io_uring.c b/fs/io_uring.c index 87a4b727fe1c..bfc8fcd93504 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -721,6 +721,11 @@ struct async_poll { struct io_poll_iocb *double_poll; }; +struct io_task_work { + struct io_wq_work_node node; + task_work_func_t func; +}; + /* * NOTE! Each of the iocb union members has the file pointer * as the first entry in their struct definition. So you can @@ -779,7 +784,10 @@ struct io_kiocb { * 2. to track reqs with ->files (see io_op_def::file_table) */ struct list_head inflight_entry; - struct callback_head task_work; + union { + struct io_task_work io_task_work; + struct callback_head task_work; + }; /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ struct hlist_node hash_node; struct async_poll *apoll; @@ -2129,6 +2137,81 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) return __io_req_find_next(req); } +static bool __tctx_task_work(struct io_uring_task *tctx) +{ + struct io_wq_work_list list; + struct io_wq_work_node *node; + + if (wq_list_empty(&tctx->task_list)) + return false; + + spin_lock(&tctx->task_lock); + list = tctx->task_list; + INIT_WQ_LIST(&tctx->task_list); + spin_unlock(&tctx->task_lock); + + node = list.first; + while (node) { + struct io_wq_work_node *next = node->next; + struct io_kiocb *req; + + req = container_of(node, struct io_kiocb, io_task_work.node); + req->task_work.func(&req->task_work); + node = next; + } + + return list.first != NULL; +} + +static void tctx_task_work(struct callback_head *cb) +{ + struct io_uring_task *tctx = container_of(cb, struct io_uring_task, task_work); + + while (__tctx_task_work(tctx)) + cond_resched(); + + clear_bit(0, &tctx->task_state); +} + +static int io_task_work_add(struct task_struct *tsk, struct io_kiocb *req, + enum task_work_notify_mode notify) +{ + struct io_uring_task *tctx = tsk->io_uring; + struct io_wq_work_node *node, *prev; + int ret; + + WARN_ON_ONCE(!tctx); + + spin_lock(&tctx->task_lock); + wq_list_add_tail(&req->io_task_work.node, &tctx->task_list); + spin_unlock(&tctx->task_lock); + + /* task_work already pending, we're done */ + if (test_bit(0, &tctx->task_state) || + test_and_set_bit(0, &tctx->task_state)) + return 0; + + if (!task_work_add(tsk, &tctx->task_work, notify)) + return 0; + + /* + * Slow path - we failed, find and delete work. if the work is not + * in the list, it got run and we're fine. + */ + ret = 0; + spin_lock(&tctx->task_lock); + wq_list_for_each(node, prev, &tctx->task_list) { + if (&req->io_task_work.node == node) { + wq_list_del(&tctx->task_list, node, prev); + ret = 1; + break; + } + } + spin_unlock(&tctx->task_lock); + clear_bit(0, &tctx->task_state); + return ret; +} + static int io_req_task_work_add(struct io_kiocb *req) { struct task_struct *tsk = req->task; @@ -2149,7 +2232,7 @@ static int io_req_task_work_add(struct io_kiocb *req) if (!(ctx->flags & IORING_SETUP_SQPOLL)) notify = TWA_SIGNAL; - ret = task_work_add(tsk, &req->task_work, notify); + ret = io_task_work_add(tsk, req, notify); if (!ret) wake_up_process(tsk); @@ -2157,7 +2240,7 @@ static int io_req_task_work_add(struct io_kiocb *req) } static void io_req_task_work_add_fallback(struct io_kiocb *req, - void (*cb)(struct callback_head *)) + task_work_func_t cb) { struct task_struct *tsk = io_wq_get_task(req->ctx->io_wq); @@ -2216,7 +2299,7 @@ static void io_req_task_queue(struct io_kiocb *req) { int ret; - init_task_work(&req->task_work, io_req_task_submit); + req->task_work.func = io_req_task_submit; percpu_ref_get(&req->ctx->refs); ret = io_req_task_work_add(req); @@ -2347,7 +2430,7 @@ static void io_free_req_deferred(struct io_kiocb *req) { int ret; - init_task_work(&req->task_work, io_put_req_deferred_cb); + req->task_work.func = io_put_req_deferred_cb; ret = io_req_task_work_add(req); if (unlikely(ret)) io_req_task_work_add_fallback(req, io_put_req_deferred_cb); @@ -3392,7 +3475,7 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, req->rw.kiocb.ki_flags &= ~IOCB_WAITQ; list_del_init(&wait->entry); - init_task_work(&req->task_work, io_req_task_submit); + req->task_work.func = io_req_task_submit; percpu_ref_get(&req->ctx->refs); /* submit ref gets dropped, acquire a new one */ @@ -5083,7 +5166,7 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll, list_del_init(&poll->wait.entry); req->result = mask; - init_task_work(&req->task_work, func); + req->task_work.func = func; percpu_ref_get(&req->ctx->refs); /* @@ -8086,6 +8169,10 @@ static int io_uring_alloc_task_context(struct task_struct *task) io_init_identity(&tctx->__identity); tctx->identity = &tctx->__identity; task->io_uring = tctx; + spin_lock_init(&tctx->task_lock); + INIT_WQ_LIST(&tctx->task_list); + tctx->task_state = 0; + init_task_work(&tctx->task_work, tctx_task_work); return 0; } diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index 35b2d845704d..2eb6d19de336 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -22,6 +22,15 @@ struct io_identity { refcount_t count; }; +struct io_wq_work_node { + struct io_wq_work_node *next; +}; + +struct io_wq_work_list { + struct io_wq_work_node *first; + struct io_wq_work_node *last; +}; + struct io_uring_task { /* submission side */ struct xarray xa; @@ -32,6 +41,11 @@ struct io_uring_task { struct io_identity *identity; atomic_t in_idle; bool sqpoll; + + spinlock_t task_lock; + struct io_wq_work_list task_list; + unsigned long task_state; + struct callback_head task_work; }; #if defined(CONFIG_IO_URING) From 65453d1efbd20f3825beba2a9c93ffb2ec729ece Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 10 Feb 2021 00:03:21 +0000 Subject: [PATCH 074/103] io_uring: enable req cache for task_work items task_work is run without utilizing the req alloc cache, so any deferred items don't get to take advantage of either the alloc or free side of it. With task_work now being wrapped by io_uring, we can use the ctx completion state to both use the req cache and the completion flush batching. With this, the only request type that cannot take advantage of the req cache is IRQ driven IO for regular files / block devices. Anything else, including IOPOLL polled IO to those same tyes, will take advantage of it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index bfc8fcd93504..fe8921a728b0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1051,6 +1051,8 @@ static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, const struct iovec *fast_iov, struct iov_iter *iter, bool force); static void io_req_task_queue(struct io_kiocb *req); +static void io_submit_flush_completions(struct io_comp_state *cs, + struct io_ring_ctx *ctx); static struct kmem_cache *req_cachep; @@ -2139,6 +2141,7 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) static bool __tctx_task_work(struct io_uring_task *tctx) { + struct io_ring_ctx *ctx = NULL; struct io_wq_work_list list; struct io_wq_work_node *node; @@ -2153,11 +2156,28 @@ static bool __tctx_task_work(struct io_uring_task *tctx) node = list.first; while (node) { struct io_wq_work_node *next = node->next; + struct io_ring_ctx *this_ctx; struct io_kiocb *req; req = container_of(node, struct io_kiocb, io_task_work.node); + this_ctx = req->ctx; req->task_work.func(&req->task_work); node = next; + + if (!ctx) { + ctx = this_ctx; + } else if (ctx != this_ctx) { + mutex_lock(&ctx->uring_lock); + io_submit_flush_completions(&ctx->submit_state.comp, ctx); + mutex_unlock(&ctx->uring_lock); + ctx = this_ctx; + } + } + + if (ctx && ctx->submit_state.comp.nr) { + mutex_lock(&ctx->uring_lock); + io_submit_flush_completions(&ctx->submit_state.comp, ctx); + mutex_unlock(&ctx->uring_lock); } return list.first != NULL; @@ -2280,7 +2300,7 @@ static void __io_req_task_submit(struct io_kiocb *req) if (!ctx->sqo_dead && !__io_sq_thread_acquire_mm(ctx) && !__io_sq_thread_acquire_files(ctx)) - __io_queue_sqe(req, NULL); + __io_queue_sqe(req, &ctx->submit_state.comp); else __io_req_task_cancel(req, -EFAULT); mutex_unlock(&ctx->uring_lock); From c5eef2b9449ba267f53bfa7cf63d2bc93acbee32 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:22 +0000 Subject: [PATCH 075/103] io_uring: take comp_state from ctx __io_queue_sqe() is always called with a non-NULL comp_state, which is taken directly from context. Don't pass it around but infer from ctx. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fe8921a728b0..bff5bc4a2b6e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1042,7 +1042,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, static void __io_clean_op(struct io_kiocb *req); static struct file *io_file_get(struct io_submit_state *state, struct io_kiocb *req, int fd, bool fixed); -static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs); +static void __io_queue_sqe(struct io_kiocb *req); static void io_rsrc_put_work(struct work_struct *work); static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, @@ -2300,7 +2300,7 @@ static void __io_req_task_submit(struct io_kiocb *req) if (!ctx->sqo_dead && !__io_sq_thread_acquire_mm(ctx) && !__io_sq_thread_acquire_files(ctx)) - __io_queue_sqe(req, &ctx->submit_state.comp); + __io_queue_sqe(req); else __io_req_task_cancel(req, -EFAULT); mutex_unlock(&ctx->uring_lock); @@ -6551,14 +6551,12 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) return nxt; } -static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) +static void __io_queue_sqe(struct io_kiocb *req) { struct io_kiocb *linked_timeout; const struct cred *old_creds = NULL; - int ret, issue_flags = IO_URING_F_NONBLOCK; + int ret; - if (cs) - issue_flags |= IO_URING_F_COMPLETE_DEFER; again: linked_timeout = io_prep_linked_timeout(req); @@ -6573,7 +6571,7 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) old_creds = override_creds(req->work.identity->creds); } - ret = io_issue_sqe(req, issue_flags); + ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); /* * We async punt it if the file wasn't marked NOWAIT, or if the file @@ -6593,9 +6591,12 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) } else if (likely(!ret)) { /* drop submission reference */ if (req->flags & REQ_F_COMPLETE_INLINE) { + struct io_ring_ctx *ctx = req->ctx; + struct io_comp_state *cs = &ctx->submit_state.comp; + cs->reqs[cs->nr++] = req; if (cs->nr == IO_COMPL_BATCH) - io_submit_flush_completions(cs, req->ctx); + io_submit_flush_completions(cs, ctx); req = NULL; } else { req = io_put_req_find_next(req); @@ -6621,8 +6622,7 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) revert_creds(old_creds); } -static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, - struct io_comp_state *cs) +static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe) { int ret; @@ -6647,18 +6647,17 @@ static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, if (unlikely(ret)) goto fail_req; } - __io_queue_sqe(req, cs); + __io_queue_sqe(req); } } -static inline void io_queue_link_head(struct io_kiocb *req, - struct io_comp_state *cs) +static inline void io_queue_link_head(struct io_kiocb *req) { if (unlikely(req->flags & REQ_F_FAIL_LINK)) { io_put_req(req); io_req_complete(req, -ECANCELED); } else - io_queue_sqe(req, NULL, cs); + io_queue_sqe(req, NULL); } struct io_submit_link { @@ -6667,7 +6666,7 @@ struct io_submit_link { }; static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, - struct io_submit_link *link, struct io_comp_state *cs) + struct io_submit_link *link) { struct io_ring_ctx *ctx = req->ctx; int ret; @@ -6705,7 +6704,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, /* last request of a link, enqueue the link */ if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) { - io_queue_link_head(head, cs); + io_queue_link_head(head); link->head = NULL; } } else { @@ -6720,7 +6719,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, link->head = req; link->last = req; } else { - io_queue_sqe(req, sqe, cs); + io_queue_sqe(req, sqe); } } @@ -6961,7 +6960,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, true, ctx->flags & IORING_SETUP_SQPOLL); - err = io_submit_sqe(req, sqe, &link, &ctx->submit_state.comp); + err = io_submit_sqe(req, sqe, &link); if (err) goto fail_req; } @@ -6976,7 +6975,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) put_task_struct_many(current, unused); } if (link.head) - io_queue_link_head(link.head, &ctx->submit_state.comp); + io_queue_link_head(link.head); io_submit_state_end(&ctx->submit_state, ctx); /* Commit SQ ring head once we've consumed and submitted all SQEs */ From e5d1bc0a91f16959aa279aa3ee9fdc246d4bb382 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:23 +0000 Subject: [PATCH 076/103] io_uring: defer flushing cached reqs Awhile there are requests in the allocation cache -- use them, only if those ended go for the stashed memory in comp.free_list. As list manipulation are generally heavy and are not good for caches, flush them all or as much as can in one go. Signed-off-by: Pavel Begunkov [axboe: return success/failure from io_flush_cached_reqs()] Signed-off-by: Jens Axboe --- fs/io_uring.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index bff5bc4a2b6e..4a28032ba35b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1952,25 +1952,35 @@ static inline void io_req_complete(struct io_kiocb *req, long res) __io_req_complete(req, 0, res, 0); } +static bool io_flush_cached_reqs(struct io_submit_state *state) +{ + struct io_kiocb *req = NULL; + + while (!list_empty(&state->comp.free_list)) { + req = list_first_entry(&state->comp.free_list, struct io_kiocb, + compl.list); + list_del(&req->compl.list); + state->reqs[state->free_reqs++] = req; + if (state->free_reqs == ARRAY_SIZE(state->reqs)) + break; + } + + return req != NULL; +} + static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) { struct io_submit_state *state = &ctx->submit_state; BUILD_BUG_ON(IO_REQ_ALLOC_BATCH > ARRAY_SIZE(state->reqs)); - if (!list_empty(&state->comp.free_list)) { - struct io_kiocb *req; - - req = list_first_entry(&state->comp.free_list, struct io_kiocb, - compl.list); - list_del(&req->compl.list); - return req; - } - if (!state->free_reqs) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; int ret; + if (io_flush_cached_reqs(state)) + goto got_req; + ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH, state->reqs); @@ -1986,7 +1996,7 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) } state->free_reqs = ret; } - +got_req: state->free_reqs--; return state->reqs[state->free_reqs]; } From ed670c3f90a67d9e16ab6d8893be6f072d79cd4c Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Fri, 5 Feb 2021 16:34:21 +0800 Subject: [PATCH 077/103] io_uring: fix possible deadlock in io_uring_poll Abaci reported follow issue: [ 30.615891] ====================================================== [ 30.616648] WARNING: possible circular locking dependency detected [ 30.617423] 5.11.0-rc3-next-20210115 #1 Not tainted [ 30.618035] ------------------------------------------------------ [ 30.618914] a.out/1128 is trying to acquire lock: [ 30.619520] ffff88810b063868 (&ep->mtx){+.+.}-{3:3}, at: __ep_eventpoll_poll+0x9f/0x220 [ 30.620505] [ 30.620505] but task is already holding lock: [ 30.621218] ffff88810e952be8 (&ctx->uring_lock){+.+.}-{3:3}, at: __x64_sys_io_uring_enter+0x3f0/0x5b0 [ 30.622349] [ 30.622349] which lock already depends on the new lock. [ 30.622349] [ 30.623289] [ 30.623289] the existing dependency chain (in reverse order) is: [ 30.624243] [ 30.624243] -> #1 (&ctx->uring_lock){+.+.}-{3:3}: [ 30.625263] lock_acquire+0x2c7/0x390 [ 30.625868] __mutex_lock+0xae/0x9f0 [ 30.626451] io_cqring_overflow_flush.part.95+0x6d/0x70 [ 30.627278] io_uring_poll+0xcb/0xd0 [ 30.627890] ep_item_poll.isra.14+0x4e/0x90 [ 30.628531] do_epoll_ctl+0xb7e/0x1120 [ 30.629122] __x64_sys_epoll_ctl+0x70/0xb0 [ 30.629770] do_syscall_64+0x2d/0x40 [ 30.630332] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [ 30.631187] [ 30.631187] -> #0 (&ep->mtx){+.+.}-{3:3}: [ 30.631985] check_prevs_add+0x226/0xb00 [ 30.632584] __lock_acquire+0x1237/0x13a0 [ 30.633207] lock_acquire+0x2c7/0x390 [ 30.633740] __mutex_lock+0xae/0x9f0 [ 30.634258] __ep_eventpoll_poll+0x9f/0x220 [ 30.634879] __io_arm_poll_handler+0xbf/0x220 [ 30.635462] io_issue_sqe+0xa6b/0x13e0 [ 30.635982] __io_queue_sqe+0x10b/0x550 [ 30.636648] io_queue_sqe+0x235/0x470 [ 30.637281] io_submit_sqes+0xcce/0xf10 [ 30.637839] __x64_sys_io_uring_enter+0x3fb/0x5b0 [ 30.638465] do_syscall_64+0x2d/0x40 [ 30.638999] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [ 30.639643] [ 30.639643] other info that might help us debug this: [ 30.639643] [ 30.640618] Possible unsafe locking scenario: [ 30.640618] [ 30.641402] CPU0 CPU1 [ 30.641938] ---- ---- [ 30.642664] lock(&ctx->uring_lock); [ 30.643425] lock(&ep->mtx); [ 30.644498] lock(&ctx->uring_lock); [ 30.645668] lock(&ep->mtx); [ 30.646321] [ 30.646321] *** DEADLOCK *** [ 30.646321] [ 30.647642] 1 lock held by a.out/1128: [ 30.648424] #0: ffff88810e952be8 (&ctx->uring_lock){+.+.}-{3:3}, at: __x64_sys_io_uring_enter+0x3f0/0x5b0 [ 30.649954] [ 30.649954] stack backtrace: [ 30.650592] CPU: 1 PID: 1128 Comm: a.out Not tainted 5.11.0-rc3-next-20210115 #1 [ 30.651554] Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011 [ 30.652290] Call Trace: [ 30.652688] dump_stack+0xac/0xe3 [ 30.653164] check_noncircular+0x11e/0x130 [ 30.653747] ? check_prevs_add+0x226/0xb00 [ 30.654303] check_prevs_add+0x226/0xb00 [ 30.654845] ? add_lock_to_list.constprop.49+0xac/0x1d0 [ 30.655564] __lock_acquire+0x1237/0x13a0 [ 30.656262] lock_acquire+0x2c7/0x390 [ 30.656788] ? __ep_eventpoll_poll+0x9f/0x220 [ 30.657379] ? __io_queue_proc.isra.88+0x180/0x180 [ 30.658014] __mutex_lock+0xae/0x9f0 [ 30.658524] ? __ep_eventpoll_poll+0x9f/0x220 [ 30.659112] ? mark_held_locks+0x5a/0x80 [ 30.659648] ? __ep_eventpoll_poll+0x9f/0x220 [ 30.660229] ? _raw_spin_unlock_irqrestore+0x2d/0x40 [ 30.660885] ? trace_hardirqs_on+0x46/0x110 [ 30.661471] ? __io_queue_proc.isra.88+0x180/0x180 [ 30.662102] ? __ep_eventpoll_poll+0x9f/0x220 [ 30.662696] __ep_eventpoll_poll+0x9f/0x220 [ 30.663273] ? __ep_eventpoll_poll+0x220/0x220 [ 30.663875] __io_arm_poll_handler+0xbf/0x220 [ 30.664463] io_issue_sqe+0xa6b/0x13e0 [ 30.664984] ? __lock_acquire+0x782/0x13a0 [ 30.665544] ? __io_queue_proc.isra.88+0x180/0x180 [ 30.666170] ? __io_queue_sqe+0x10b/0x550 [ 30.666725] __io_queue_sqe+0x10b/0x550 [ 30.667252] ? __fget_files+0x131/0x260 [ 30.667791] ? io_req_prep+0xd8/0x1090 [ 30.668316] ? io_queue_sqe+0x235/0x470 [ 30.668868] io_queue_sqe+0x235/0x470 [ 30.669398] io_submit_sqes+0xcce/0xf10 [ 30.669931] ? xa_load+0xe4/0x1c0 [ 30.670425] __x64_sys_io_uring_enter+0x3fb/0x5b0 [ 30.671051] ? lockdep_hardirqs_on_prepare+0xde/0x180 [ 30.671719] ? syscall_enter_from_user_mode+0x2b/0x80 [ 30.672380] do_syscall_64+0x2d/0x40 [ 30.672901] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [ 30.673503] RIP: 0033:0x7fd89c813239 [ 30.673962] Code: 01 00 48 81 c4 80 00 00 00 e9 f1 fe ff ff 0f 1f 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 3d 01 f0 ff ff 73 01 c3 48 8b 0d 27 ec 2c 00 f7 d8 64 89 01 48 [ 30.675920] RSP: 002b:00007ffc65a7c628 EFLAGS: 00000217 ORIG_RAX: 00000000000001aa [ 30.676791] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007fd89c813239 [ 30.677594] RDX: 0000000000000000 RSI: 0000000000000014 RDI: 0000000000000003 [ 30.678678] RBP: 00007ffc65a7c720 R08: 0000000000000000 R09: 0000000003000000 [ 30.679492] R10: 0000000000000000 R11: 0000000000000217 R12: 0000000000400ff0 [ 30.680282] R13: 00007ffc65a7c840 R14: 0000000000000000 R15: 0000000000000000 This might happen if we do epoll_wait on a uring fd while reading/writing the former epoll fd in a sqe in the former uring instance. So let's don't flush cqring overflow list, just do a simple check. Reported-by: Abaci Fixes: 6c503150ae33 ("io_uring: patch up IOPOLL overflow_flush sync") Signed-off-by: Hao Xu Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 4a28032ba35b..e73ca37c6a3b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8820,8 +8820,21 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) smp_rmb(); if (!io_sqring_full(ctx)) mask |= EPOLLOUT | EPOLLWRNORM; - io_cqring_overflow_flush(ctx, false, NULL, NULL); - if (io_cqring_events(ctx)) + + /* + * Don't flush cqring overflow list here, just do a simple check. + * Otherwise there could possible be ABBA deadlock: + * CPU0 CPU1 + * ---- ---- + * lock(&ctx->uring_lock); + * lock(&ep->mtx); + * lock(&ctx->uring_lock); + * lock(&ep->mtx); + * + * Users may get EPOLLIN meanwhile seeing nothing in cqring, this + * pushs them to do the flush. + */ + if (io_cqring_events(ctx) || test_bit(0, &ctx->cq_check_overflow)) mask |= EPOLLIN | EPOLLRDNORM; return mask; From c7dae4ba46c9d7d56430b800907b708711995414 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 9 Feb 2021 19:53:37 -0700 Subject: [PATCH 078/103] io_uring: enable req cache for IRQ driven IO This is the last class of requests that cannot utilize the req alloc cache. Add a per-ctx req cache that is protected by the completion_lock, and refill our submit side cache when it gets over our batch count. Signed-off-by: Jens Axboe --- fs/io_uring.c | 71 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index e73ca37c6a3b..2c7ff0b1b086 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -272,7 +272,11 @@ struct io_sq_data { struct io_comp_state { struct io_kiocb *reqs[IO_COMPL_BATCH]; unsigned int nr; + unsigned int locked_free_nr; + /* inline/task_work completion list, under ->uring_lock */ struct list_head free_list; + /* IRQ completion list, under ->completion_lock */ + struct list_head locked_free_list; }; struct io_submit_state { @@ -1033,6 +1037,9 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res); static void io_put_req(struct io_kiocb *req); static void io_put_req_deferred(struct io_kiocb *req, int nr); static void io_double_put_req(struct io_kiocb *req); +static void io_dismantle_req(struct io_kiocb *req); +static void io_put_task(struct task_struct *task, int nr); +static void io_queue_next(struct io_kiocb *req); static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req); static void __io_queue_linked_timeout(struct io_kiocb *req); static void io_queue_linked_timeout(struct io_kiocb *req); @@ -1353,6 +1360,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); init_llist_head(&ctx->rsrc_put_llist); INIT_LIST_HEAD(&ctx->submit_state.comp.free_list); + INIT_LIST_HEAD(&ctx->submit_state.comp.locked_free_list); return ctx; err: kfree(ctx->cancel_hash); @@ -1908,8 +1916,8 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) __io_cqring_fill_event(req, res, 0); } -static void io_req_complete_post(struct io_kiocb *req, long res, - unsigned int cflags) +static inline void io_req_complete_post(struct io_kiocb *req, long res, + unsigned int cflags) { struct io_ring_ctx *ctx = req->ctx; unsigned long flags; @@ -1917,16 +1925,26 @@ static void io_req_complete_post(struct io_kiocb *req, long res, spin_lock_irqsave(&ctx->completion_lock, flags); __io_cqring_fill_event(req, res, cflags); io_commit_cqring(ctx); + /* + * If we're the last reference to this request, add to our locked + * free_list cache. + */ + if (refcount_dec_and_test(&req->refs)) { + struct io_comp_state *cs = &ctx->submit_state.comp; + + io_dismantle_req(req); + io_put_task(req->task, 1); + list_add(&req->compl.list, &cs->locked_free_list); + cs->locked_free_nr++; + } else + req = NULL; spin_unlock_irqrestore(&ctx->completion_lock, flags); io_cqring_ev_posted(ctx); -} - -static inline void io_req_complete_nostate(struct io_kiocb *req, long res, - unsigned int cflags) -{ - io_req_complete_post(req, res, cflags); - io_put_req(req); + if (req) { + io_queue_next(req); + percpu_ref_put(&ctx->refs); + } } static void io_req_complete_state(struct io_kiocb *req, long res, @@ -1944,7 +1962,7 @@ static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags, if (issue_flags & IO_URING_F_COMPLETE_DEFER) io_req_complete_state(req, res, cflags); else - io_req_complete_nostate(req, res, cflags); + io_req_complete_post(req, res, cflags); } static inline void io_req_complete(struct io_kiocb *req, long res) @@ -1952,12 +1970,26 @@ static inline void io_req_complete(struct io_kiocb *req, long res) __io_req_complete(req, 0, res, 0); } -static bool io_flush_cached_reqs(struct io_submit_state *state) +static bool io_flush_cached_reqs(struct io_ring_ctx *ctx) { + struct io_submit_state *state = &ctx->submit_state; + struct io_comp_state *cs = &state->comp; struct io_kiocb *req = NULL; - while (!list_empty(&state->comp.free_list)) { - req = list_first_entry(&state->comp.free_list, struct io_kiocb, + /* + * If we have more than a batch's worth of requests in our IRQ side + * locked cache, grab the lock and move them over to our submission + * side cache. + */ + if (READ_ONCE(cs->locked_free_nr) > IO_COMPL_BATCH) { + spin_lock_irq(&ctx->completion_lock); + list_splice_init(&cs->locked_free_list, &cs->free_list); + cs->locked_free_nr = 0; + spin_unlock_irq(&ctx->completion_lock); + } + + while (!list_empty(&cs->free_list)) { + req = list_first_entry(&cs->free_list, struct io_kiocb, compl.list); list_del(&req->compl.list); state->reqs[state->free_reqs++] = req; @@ -1978,7 +2010,7 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; int ret; - if (io_flush_cached_reqs(state)) + if (io_flush_cached_reqs(ctx)) goto got_req; ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH, @@ -8748,14 +8780,12 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) idr_destroy(&ctx->io_buffer_idr); } -static void io_req_cache_free(struct io_ring_ctx *ctx) +static void io_req_cache_free(struct list_head *list) { - struct io_comp_state *cs = &ctx->submit_state.comp; - - while (!list_empty(&cs->free_list)) { + while (!list_empty(list)) { struct io_kiocb *req; - req = list_first_entry(&cs->free_list, struct io_kiocb, compl.list); + req = list_first_entry(list, struct io_kiocb, compl.list); list_del(&req->compl.list); kmem_cache_free(req_cachep, req); } @@ -8803,7 +8833,8 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) free_uid(ctx->user); put_cred(ctx->creds); kfree(ctx->cancel_hash); - io_req_cache_free(ctx); + io_req_cache_free(&ctx->submit_state.comp.free_list); + io_req_cache_free(&ctx->submit_state.comp.locked_free_list); kfree(ctx); } From 91f245d5d5de0802428a478802ec051f7de2f5d6 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 9 Feb 2021 13:48:50 -0700 Subject: [PATCH 079/103] io_uring: enable kmemcg account for io_uring requests This puts io_uring under the memory cgroups accounting and limits for requests. Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 2c7ff0b1b086..bffed6aa5722 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -10350,7 +10350,8 @@ static int __init io_uring_init(void) BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int)); - req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC); + req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC | + SLAB_ACCOUNT); return 0; }; __initcall(io_uring_init); From 26bfa89e25f42d2b26fe951bbcf04bb13937fbba Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 9 Feb 2021 20:14:12 -0700 Subject: [PATCH 080/103] io_uring: place ring SQ/CQ arrays under memcg memory limits Instead of imposing rlimit memlock limits for the rings themselves, ensure that we account them properly under memcg with __GFP_ACCOUNT. We retain rlimit memlock for registered buffers, this is just for the ring arrays themselves. Signed-off-by: Jens Axboe --- fs/io_uring.c | 85 ++++++--------------------------------------------- 1 file changed, 10 insertions(+), 75 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index bffed6aa5722..7a1e4ecf5f94 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1017,11 +1017,6 @@ static const struct io_op_def io_op_defs[] = { }, }; -enum io_mem_account { - ACCT_LOCKED, - ACCT_PINNED, -}; - static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct task_struct *task, struct files_struct *files); @@ -8355,25 +8350,16 @@ static inline int __io_account_mem(struct user_struct *user, return 0; } -static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages, - enum io_mem_account acct) +static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages) { if (ctx->limit_mem) __io_unaccount_mem(ctx->user, nr_pages); - if (ctx->mm_account) { - if (acct == ACCT_LOCKED) { - mmap_write_lock(ctx->mm_account); - ctx->mm_account->locked_vm -= nr_pages; - mmap_write_unlock(ctx->mm_account); - }else if (acct == ACCT_PINNED) { - atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm); - } - } + if (ctx->mm_account) + atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm); } -static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages, - enum io_mem_account acct) +static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages) { int ret; @@ -8383,15 +8369,8 @@ static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages, return ret; } - if (ctx->mm_account) { - if (acct == ACCT_LOCKED) { - mmap_write_lock(ctx->mm_account); - ctx->mm_account->locked_vm += nr_pages; - mmap_write_unlock(ctx->mm_account); - } else if (acct == ACCT_PINNED) { - atomic64_add(nr_pages, &ctx->mm_account->pinned_vm); - } - } + if (ctx->mm_account) + atomic64_add(nr_pages, &ctx->mm_account->pinned_vm); return 0; } @@ -8411,7 +8390,7 @@ static void io_mem_free(void *ptr) static void *io_mem_alloc(size_t size) { gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP | - __GFP_NORETRY; + __GFP_NORETRY | __GFP_ACCOUNT; return (void *) __get_free_pages(gfp_flags, get_order(size)); } @@ -8445,18 +8424,6 @@ static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries, return off; } -static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries) -{ - size_t pages; - - pages = (size_t)1 << get_order( - rings_size(sq_entries, cq_entries, NULL)); - pages += (size_t)1 << get_order( - array_size(sizeof(struct io_uring_sqe), sq_entries)); - - return pages; -} - static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx) { int i, j; @@ -8471,7 +8438,7 @@ static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx) unpin_user_page(imu->bvec[j].bv_page); if (imu->acct_pages) - io_unaccount_mem(ctx, imu->acct_pages, ACCT_PINNED); + io_unaccount_mem(ctx, imu->acct_pages); kvfree(imu->bvec); imu->nr_bvecs = 0; } @@ -8569,7 +8536,7 @@ static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages, if (!imu->acct_pages) return 0; - ret = io_account_mem(ctx, imu->acct_pages, ACCT_PINNED); + ret = io_account_mem(ctx, imu->acct_pages); if (ret) imu->acct_pages = 0; return ret; @@ -8949,14 +8916,6 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) /* if we failed setting up the ctx, we might not have any rings */ io_iopoll_try_reap_events(ctx); - /* - * Do this upfront, so we won't have a grace period where the ring - * is closed but resources aren't reaped yet. This can cause - * spurious failure in setting up a new ring. - */ - io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries), - ACCT_LOCKED); - INIT_WORK(&ctx->exit_work, io_ring_exit_work); /* * Use system_unbound_wq to avoid spawning tons of event kworkers @@ -9780,7 +9739,6 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, struct user_struct *user = NULL; struct io_ring_ctx *ctx; struct file *file; - bool limit_mem; int ret; if (!entries) @@ -9821,26 +9779,14 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, } user = get_uid(current_user()); - limit_mem = !capable(CAP_IPC_LOCK); - - if (limit_mem) { - ret = __io_account_mem(user, - ring_pages(p->sq_entries, p->cq_entries)); - if (ret) { - free_uid(user); - return ret; - } - } ctx = io_ring_ctx_alloc(p); if (!ctx) { - if (limit_mem) - __io_unaccount_mem(user, ring_pages(p->sq_entries, - p->cq_entries)); free_uid(user); return -ENOMEM; } ctx->compat = in_compat_syscall(); + ctx->limit_mem = !capable(CAP_IPC_LOCK); ctx->user = user; ctx->creds = get_current_cred(); #ifdef CONFIG_AUDIT @@ -9876,17 +9822,6 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, goto err; } #endif - - /* - * Account memory _before_ installing the file descriptor. Once - * the descriptor is installed, it can get closed at any time. Also - * do this before hitting the general error path, as ring freeing - * will un-account as well. - */ - io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries), - ACCT_LOCKED); - ctx->limit_mem = limit_mem; - ret = io_allocate_scq_urings(ctx, p); if (ret) goto err; From 34343786ecc5ff493ca4d1f873b4386759ba52ee Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 11:45:42 +0000 Subject: [PATCH 081/103] io_uring: unpark SQPOLL thread for cancelation We park SQPOLL task before going into io_uring_cancel_files(), so the task won't run task_works including those that might be important for the cancellation passes. In this case it's io_poll_remove_one(), which frees requests via io_put_req_deferred(). Unpark it for while waiting, it's ok as we disable submissions beforehand, so no new requests will be generated. INFO: task syz-executor893:8493 blocked for more than 143 seconds. Call Trace: context_switch kernel/sched/core.c:4327 [inline] __schedule+0x90c/0x21a0 kernel/sched/core.c:5078 schedule+0xcf/0x270 kernel/sched/core.c:5157 io_uring_cancel_files fs/io_uring.c:8912 [inline] io_uring_cancel_task_requests+0xe70/0x11a0 fs/io_uring.c:8979 __io_uring_files_cancel+0x110/0x1b0 fs/io_uring.c:9067 io_uring_files_cancel include/linux/io_uring.h:51 [inline] do_exit+0x2fe/0x2ae0 kernel/exit.c:780 do_group_exit+0x125/0x310 kernel/exit.c:922 __do_sys_exit_group kernel/exit.c:933 [inline] __se_sys_exit_group kernel/exit.c:931 [inline] __x64_sys_exit_group+0x3a/0x50 kernel/exit.c:931 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xa9 Cc: stable@vger.kernel.org # 5.5+ Reported-by: syzbot+695b03d82fa8e4901b06@syzkaller.appspotmail.com Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index 7a1e4ecf5f94..9ed79509f389 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9047,11 +9047,16 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, break; io_uring_try_cancel_requests(ctx, task, files); + + if (ctx->sq_data) + io_sq_thread_unpark(ctx->sq_data); prepare_to_wait(&task->io_uring->wait, &wait, TASK_UNINTERRUPTIBLE); if (inflight == io_uring_count_inflight(ctx, task, files)) schedule(); finish_wait(&task->io_uring->wait, &wait); + if (ctx->sq_data) + io_sq_thread_park(ctx->sq_data); } } From 4a245479c2312e6b51862c21af134d4191ab9cf7 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 10 Feb 2021 20:00:07 +0000 Subject: [PATCH 082/103] io_uring: remove redundant initialization of variable ret The variable ret is being initialized with a value that is never read and it is being updated later with a new value. The initialization is redundant and can be removed. Addresses-Coverity: ("Unused value") Fixes: b63534c41e20 ("io_uring: re-issue block requests that failed because of resources") Signed-off-by: Colin Ian King Reviewed-by: Chaitanya Kulkarni Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 9ed79509f389..f730af32c17a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2796,7 +2796,7 @@ static void io_complete_rw_common(struct kiocb *kiocb, long res, static bool io_resubmit_prep(struct io_kiocb *req) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; - int rw, ret = -ECANCELED; + int rw, ret; struct iov_iter iter; /* already prepared */ From e68a3ff8c342b655f01f74a577c15605eec9aa12 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 11 Feb 2021 07:45:08 -0700 Subject: [PATCH 083/103] io_uring: assign file_slot prior to calling io_sqe_file_register() We use the assigned slot in io_sqe_file_register(), and a previous patch moved the assignment to after we have called it. This isn't super pretty, and will get cleaned up in the future. For now, fix the regression by restoring the previous assignment/clear of the file_slot. Fixes: ea64ec02b31d ("io_uring: deduplicate file table slot calculation") Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index f730af32c17a..cd9c4c05f6f5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8112,12 +8112,13 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, err = -EBADF; break; } + *file_slot = file; err = io_sqe_file_register(ctx, file, i); if (err) { + *file_slot = NULL; fput(file); break; } - *file_slot = file; } } From 3c1a2ead915c1bcb7b1f9e902469ea0ee1f7857f Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 11 Feb 2021 10:48:03 -0700 Subject: [PATCH 084/103] io_uring: move submit side state closer in the ring We recently added the submit side req cache, but it was placed at the end of the struct. Move it near the other submission state for better memory placement, and reshuffle a few other members at the same time. Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index cd9c4c05f6f5..8be7a24aa10e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -346,6 +346,13 @@ struct io_ring_ctx { struct io_uring_sqe *sq_sqes; } ____cacheline_aligned_in_smp; + struct { + struct mutex uring_lock; + wait_queue_head_t wait; + } ____cacheline_aligned_in_smp; + + struct io_submit_state submit_state; + struct io_rings *rings; /* IO offload */ @@ -413,11 +420,6 @@ struct io_ring_ctx { struct eventfd_ctx *cq_ev_fd; } ____cacheline_aligned_in_smp; - struct { - struct mutex uring_lock; - wait_queue_head_t wait; - } ____cacheline_aligned_in_smp; - struct { spinlock_t completion_lock; @@ -441,9 +443,10 @@ struct io_ring_ctx { struct list_head rsrc_ref_list; spinlock_t rsrc_ref_lock; - struct work_struct exit_work; struct io_restriction restrictions; - struct io_submit_state submit_state; + + /* Keep this last, we don't need it for the fast path */ + struct work_struct exit_work; }; /* From 6e833d538b3123767393c987d11c40b7728b3f79 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 11 Feb 2021 18:28:20 +0000 Subject: [PATCH 085/103] io_uring: clean up io_req_free_batch_finish() io_req_free_batch_finish() is final and does not permit struct req_batch to be reused without re-init. To be more consistent don't clear ->task there. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 8be7a24aa10e..fe06ca43e832 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2397,10 +2397,8 @@ static inline void io_init_req_batch(struct req_batch *rb) static void io_req_free_batch_finish(struct io_ring_ctx *ctx, struct req_batch *rb) { - if (rb->task) { + if (rb->task) io_put_task(rb->task, rb->task_refs); - rb->task = NULL; - } if (rb->ctx_refs) percpu_ref_put_many(&ctx->refs, rb->ctx_refs); } From f161340d9e85b9038031b497b32383e50ff00ca1 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 11 Feb 2021 18:28:21 +0000 Subject: [PATCH 086/103] io_uring: simplify iopoll reissuing Don't stash -EAGAIN'ed iopoll requests into a list to reissue it later, do it eagerly. It removes overhead on keeping and checking that list, and allows in case of failure for these requests to be completed through normal iopoll completion path. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fe06ca43e832..18e449b5d632 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1029,8 +1029,7 @@ static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( static void init_fixed_file_ref_node(struct io_ring_ctx *ctx, struct fixed_rsrc_ref_node *ref_node); -static void __io_complete_rw(struct io_kiocb *req, long res, long res2, - unsigned int issue_flags); +static bool io_rw_reissue(struct io_kiocb *req, long res); static void io_cqring_fill_event(struct io_kiocb *req, long res); static void io_put_req(struct io_kiocb *req); static void io_put_req_deferred(struct io_kiocb *req, int nr); @@ -2558,17 +2557,6 @@ static inline bool io_run_task_work(void) return false; } -static void io_iopoll_queue(struct list_head *again) -{ - struct io_kiocb *req; - - do { - req = list_first_entry(again, struct io_kiocb, inflight_entry); - list_del(&req->inflight_entry); - __io_complete_rw(req, -EAGAIN, 0, 0); - } while (!list_empty(again)); -} - /* * Find and free completed poll iocbs */ @@ -2577,7 +2565,6 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, { struct req_batch rb; struct io_kiocb *req; - LIST_HEAD(again); /* order with ->result store in io_complete_rw_iopoll() */ smp_rmb(); @@ -2587,14 +2574,14 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, int cflags = 0; req = list_first_entry(done, struct io_kiocb, inflight_entry); - if (READ_ONCE(req->result) == -EAGAIN) { - req->result = 0; - req->iopoll_completed = 0; - list_move_tail(&req->inflight_entry, &again); - continue; - } list_del(&req->inflight_entry); + if (READ_ONCE(req->result) == -EAGAIN) { + req->iopoll_completed = 0; + if (io_rw_reissue(req, -EAGAIN)) + continue; + } + if (req->flags & REQ_F_BUFFER_SELECTED) cflags = io_put_rw_kbuf(req); @@ -2608,9 +2595,6 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, io_commit_cqring(ctx); io_cqring_ev_posted_iopoll(ctx); io_req_free_batch_finish(ctx, &rb); - - if (!list_empty(&again)) - io_iopoll_queue(&again); } static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, From 23faba36ce287e4af9018dea51893a1067701508 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 11 Feb 2021 18:28:22 +0000 Subject: [PATCH 087/103] io_uring: move res check out of io_rw_reissue() We pass return code into io_rw_reissue() only to be able to check if it's -EAGAIN. That's not the cleanest approach and may prevent inlining of the non-EAGAIN fast path, so do it at call sites. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 18e449b5d632..c873ec113bcc 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1029,7 +1029,7 @@ static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( static void init_fixed_file_ref_node(struct io_ring_ctx *ctx, struct fixed_rsrc_ref_node *ref_node); -static bool io_rw_reissue(struct io_kiocb *req, long res); +static bool io_rw_reissue(struct io_kiocb *req); static void io_cqring_fill_event(struct io_kiocb *req, long res); static void io_put_req(struct io_kiocb *req); static void io_put_req_deferred(struct io_kiocb *req, int nr); @@ -2578,7 +2578,7 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, if (READ_ONCE(req->result) == -EAGAIN) { req->iopoll_completed = 0; - if (io_rw_reissue(req, -EAGAIN)) + if (io_rw_reissue(req)) continue; } @@ -2812,15 +2812,12 @@ static bool io_resubmit_prep(struct io_kiocb *req) } #endif -static bool io_rw_reissue(struct io_kiocb *req, long res) +static bool io_rw_reissue(struct io_kiocb *req) { #ifdef CONFIG_BLOCK - umode_t mode; + umode_t mode = file_inode(req->file)->i_mode; int ret; - if (res != -EAGAIN && res != -EOPNOTSUPP) - return false; - mode = file_inode(req->file)->i_mode; if (!S_ISBLK(mode) && !S_ISREG(mode)) return false; if ((req->flags & REQ_F_NOWAIT) || io_wq_current_is_worker()) @@ -2843,8 +2840,10 @@ static bool io_rw_reissue(struct io_kiocb *req, long res) static void __io_complete_rw(struct io_kiocb *req, long res, long res2, unsigned int issue_flags) { - if (!io_rw_reissue(req, res)) - io_complete_rw_common(&req->rw.kiocb, res, issue_flags); + if ((res == -EAGAIN || res == -EOPNOTSUPP) && io_rw_reissue(req)) + return; + + io_complete_rw_common(&req->rw.kiocb, res, issue_flags); } static void io_complete_rw(struct kiocb *kiocb, long res, long res2) From 2f8e45f16c57360dd4d8b1310c2952a29a8fa890 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 11 Feb 2021 18:28:23 +0000 Subject: [PATCH 088/103] io_uring: inline io_complete_rw_common() __io_complete_rw() casts request to kiocb for it to be immediately container_of()'ed by io_complete_rw_common(). And the last function's name doesn't do a great job of illuminating its purposes, so just inline it in its only user. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index c873ec113bcc..7b1979624320 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2761,22 +2761,6 @@ static void kiocb_end_write(struct io_kiocb *req) file_end_write(req->file); } -static void io_complete_rw_common(struct kiocb *kiocb, long res, - unsigned int issue_flags) -{ - struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); - int cflags = 0; - - if (kiocb->ki_flags & IOCB_WRITE) - kiocb_end_write(req); - - if (res != req->result) - req_set_fail_links(req); - if (req->flags & REQ_F_BUFFER_SELECTED) - cflags = io_put_rw_kbuf(req); - __io_req_complete(req, issue_flags, res, cflags); -} - #ifdef CONFIG_BLOCK static bool io_resubmit_prep(struct io_kiocb *req) { @@ -2840,10 +2824,18 @@ static bool io_rw_reissue(struct io_kiocb *req) static void __io_complete_rw(struct io_kiocb *req, long res, long res2, unsigned int issue_flags) { + int cflags = 0; + if ((res == -EAGAIN || res == -EOPNOTSUPP) && io_rw_reissue(req)) return; + if (res != req->result) + req_set_fail_links(req); - io_complete_rw_common(&req->rw.kiocb, res, issue_flags); + if (req->rw.kiocb.ki_flags & IOCB_WRITE) + kiocb_end_write(req); + if (req->flags & REQ_F_BUFFER_SELECTED) + cflags = io_put_rw_kbuf(req); + __io_req_complete(req, issue_flags, res, cflags); } static void io_complete_rw(struct kiocb *kiocb, long res, long res2) From bd75904590de1c2bbdff55180cef209b13bd50fa Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 03:23:50 +0000 Subject: [PATCH 089/103] io_uring: take compl state from submit state Completion and submission states are now coupled together, it's weird to get one from argument and another from ctx, do it consistently for io_req_free_batch(). It's also faster as we already have @state cached in registers. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 7b1979624320..8c2613bf54d3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2417,13 +2417,10 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, rb->ctx_refs++; io_dismantle_req(req); - if (state->free_reqs != ARRAY_SIZE(state->reqs)) { + if (state->free_reqs != ARRAY_SIZE(state->reqs)) state->reqs[state->free_reqs++] = req; - } else { - struct io_comp_state *cs = &req->ctx->submit_state.comp; - - list_add(&req->compl.list, &cs->free_list); - } + else + list_add(&req->compl.list, &state->comp.free_list); } static void io_submit_flush_completions(struct io_comp_state *cs, From d3d7298d05cb026305b0f5033acc9c9c4f281e14 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 03:23:51 +0000 Subject: [PATCH 090/103] io_uring: optimise out unlikely link queue __io_queue_sqe() tries to issue as much requests of a link as it can, and uses io_put_req_find_next() to extract a next one, targeting inline completed requests. As now __io_queue_sqe() is always used together with struct io_comp_state, it leaves next propagation only a small window and only for async reqs, that doesn't justify its existence. Remove it, make __io_queue_sqe() to issue only a head request. It simplifies the code and will allow other optimisations. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 42 ++++++++++-------------------------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 8c2613bf54d3..26d1080217e5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6563,26 +6563,20 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) static void __io_queue_sqe(struct io_kiocb *req) { - struct io_kiocb *linked_timeout; + struct io_kiocb *linked_timeout = io_prep_linked_timeout(req); const struct cred *old_creds = NULL; int ret; -again: - linked_timeout = io_prep_linked_timeout(req); - if ((req->flags & REQ_F_WORK_INITIALIZED) && (req->work.flags & IO_WQ_WORK_CREDS) && - req->work.identity->creds != current_cred()) { - if (old_creds) - revert_creds(old_creds); - if (old_creds == req->work.identity->creds) - old_creds = NULL; /* restored original creds */ - else - old_creds = override_creds(req->work.identity->creds); - } + req->work.identity->creds != current_cred()) + old_creds = override_creds(req->work.identity->creds); ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); + if (old_creds) + revert_creds(old_creds); + /* * We async punt it if the file wasn't marked NOWAIT, or if the file * doesn't support non-blocking read/write attempts @@ -6595,9 +6589,6 @@ static void __io_queue_sqe(struct io_kiocb *req) */ io_queue_async_work(req); } - - if (linked_timeout) - io_queue_linked_timeout(linked_timeout); } else if (likely(!ret)) { /* drop submission reference */ if (req->flags & REQ_F_COMPLETE_INLINE) { @@ -6605,31 +6596,18 @@ static void __io_queue_sqe(struct io_kiocb *req) struct io_comp_state *cs = &ctx->submit_state.comp; cs->reqs[cs->nr++] = req; - if (cs->nr == IO_COMPL_BATCH) + if (cs->nr == ARRAY_SIZE(cs->reqs)) io_submit_flush_completions(cs, ctx); - req = NULL; } else { - req = io_put_req_find_next(req); - } - - if (linked_timeout) - io_queue_linked_timeout(linked_timeout); - - if (req) { - if (!(req->flags & REQ_F_FORCE_ASYNC)) - goto again; - io_queue_async_work(req); + io_put_req(req); } } else { - /* un-prep timeout, so it'll be killed as any other linked */ - req->flags &= ~REQ_F_LINK_TIMEOUT; req_set_fail_links(req); io_put_req(req); io_req_complete(req, ret); } - - if (old_creds) - revert_creds(old_creds); + if (linked_timeout) + io_queue_linked_timeout(linked_timeout); } static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe) From 4e32635834a30b8aa9583d3899a8ecc6416023fb Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 03:23:52 +0000 Subject: [PATCH 091/103] io_uring: optimise SQPOLL mm/files grabbing There are two reasons for this. First is to optimise io_sq_thread_acquire_mm_files() for non-SQPOLL case, which currently do too many checks and function calls in the hot path, e.g. in io_init_req(). The second is to not grab mm/files when there are not needed. As __io_queue_sqe() issues only one request now, we can reuse io_sq_thread_acquire_mm_files() instead of unconditional acquire mm/files. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 26d1080217e5..813d1ccd7a69 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1145,9 +1145,6 @@ static void io_sq_thread_drop_mm_files(void) static int __io_sq_thread_acquire_files(struct io_ring_ctx *ctx) { - if (current->flags & PF_EXITING) - return -EFAULT; - if (!current->files) { struct files_struct *files; struct nsproxy *nsproxy; @@ -1175,15 +1172,9 @@ static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx) { struct mm_struct *mm; - if (current->flags & PF_EXITING) - return -EFAULT; if (current->mm) return 0; - /* Should never happen */ - if (unlikely(!(ctx->flags & IORING_SETUP_SQPOLL))) - return -EFAULT; - task_lock(ctx->sqo_task); mm = ctx->sqo_task->mm; if (unlikely(!mm || !mmget_not_zero(mm))) @@ -1198,8 +1189,8 @@ static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx) return -EFAULT; } -static int io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, - struct io_kiocb *req) +static int __io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, + struct io_kiocb *req) { const struct io_op_def *def = &io_op_defs[req->opcode]; int ret; @@ -1219,6 +1210,16 @@ static int io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, return 0; } +static inline int io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, + struct io_kiocb *req) +{ + if (unlikely(current->flags & PF_EXITING)) + return -EFAULT; + if (!(ctx->flags & IORING_SETUP_SQPOLL)) + return 0; + return __io_sq_thread_acquire_mm_files(ctx, req); +} + static void io_sq_thread_associate_blkcg(struct io_ring_ctx *ctx, struct cgroup_subsys_state **cur_css) @@ -2336,9 +2337,7 @@ static void __io_req_task_submit(struct io_kiocb *req) struct io_ring_ctx *ctx = req->ctx; mutex_lock(&ctx->uring_lock); - if (!ctx->sqo_dead && - !__io_sq_thread_acquire_mm(ctx) && - !__io_sq_thread_acquire_files(ctx)) + if (!ctx->sqo_dead && !io_sq_thread_acquire_mm_files(ctx, req)) __io_queue_sqe(req); else __io_req_task_cancel(req, -EFAULT); From 921b9054e0c4c443c479c21800f6c4c8b43fa1b0 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 03:23:53 +0000 Subject: [PATCH 092/103] io_uring: don't duplicate io_req_task_queue() Don't hand code io_req_task_queue() inside of io_async_buf_func(), just call it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 813d1ccd7a69..5c0b1a7dba80 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3494,7 +3494,6 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, struct wait_page_queue *wpq; struct io_kiocb *req = wait->private; struct wait_page_key *key = arg; - int ret; wpq = container_of(wait, struct wait_page_queue, wait); @@ -3504,14 +3503,9 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, req->rw.kiocb.ki_flags &= ~IOCB_WAITQ; list_del_init(&wait->entry); - req->task_work.func = io_req_task_submit; - percpu_ref_get(&req->ctx->refs); - /* submit ref gets dropped, acquire a new one */ refcount_inc(&req->refs); - ret = io_req_task_work_add(req); - if (unlikely(ret)) - io_req_task_work_add_fallback(req, io_req_task_cancel); + io_req_task_queue(req); return 1; } From 04fc6c802dfacba800f5a5d00bea0ebfcc60f840 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 03:23:54 +0000 Subject: [PATCH 093/103] io_uring: save ctx put/get for task_work submit Do a little trick in io_ring_ctx_free() briefly taking uring_lock, that will wait for everyone currently holding it, so we can skip pinning ctx with ctx->refs for __io_req_task_submit(), which is executed and loses its refs/reqs while holding the lock. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 5c0b1a7dba80..87f2f8e660e8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2336,6 +2336,7 @@ static void __io_req_task_submit(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; + /* ctx stays valid until unlock, even if we drop all ours ctx->refs */ mutex_lock(&ctx->uring_lock); if (!ctx->sqo_dead && !io_sq_thread_acquire_mm_files(ctx, req)) __io_queue_sqe(req); @@ -2347,10 +2348,8 @@ static void __io_req_task_submit(struct io_kiocb *req) static void io_req_task_submit(struct callback_head *cb) { struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); - struct io_ring_ctx *ctx = req->ctx; __io_req_task_submit(req); - percpu_ref_put(&ctx->refs); } static void io_req_task_queue(struct io_kiocb *req) @@ -2358,11 +2357,11 @@ static void io_req_task_queue(struct io_kiocb *req) int ret; req->task_work.func = io_req_task_submit; - percpu_ref_get(&req->ctx->refs); - ret = io_req_task_work_add(req); - if (unlikely(ret)) + if (unlikely(ret)) { + percpu_ref_get(&req->ctx->refs); io_req_task_work_add_fallback(req, io_req_task_cancel); + } } static inline void io_queue_next(struct io_kiocb *req) @@ -8707,6 +8706,14 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) { struct io_submit_state *submit_state = &ctx->submit_state; + /* + * Some may use context even when all refs and requests have been put, + * and they are free to do so while still holding uring_lock, see + * __io_req_task_submit(). Wait for them to finish. + */ + mutex_lock(&ctx->uring_lock); + mutex_unlock(&ctx->uring_lock); + io_finish_async(ctx); io_sqe_buffers_unregister(ctx); From 4fccfcbb733794634d4e873e7973c1847beca5bf Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 11:55:17 +0000 Subject: [PATCH 094/103] io_uring: don't split out consume out of SQE get Remove io_consume_sqe() and inline it back into io_get_sqe(). It requires req dealloc on error, but in exchange we get cleaner io_submit_sqes() and better locality for cached_sq_head. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 87f2f8e660e8..9c58be0579f3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6762,7 +6762,7 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) * 2) allows the kernel side to track the head on its own, even * though the application is the one updating it. */ - head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]); + head = READ_ONCE(sq_array[ctx->cached_sq_head++ & ctx->sq_mask]); if (likely(head < ctx->sq_entries)) return &ctx->sq_sqes[head]; @@ -6772,11 +6772,6 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) return NULL; } -static inline void io_consume_sqe(struct io_ring_ctx *ctx) -{ - ctx->cached_sq_head++; -} - /* * Check SQE restrictions (opcode and flags). * @@ -6915,18 +6910,17 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) struct io_kiocb *req; int err; - sqe = io_get_sqe(ctx); - if (unlikely(!sqe)) { - io_consume_sqe(ctx); - break; - } req = io_alloc_req(ctx); if (unlikely(!req)) { if (!submitted) submitted = -EAGAIN; break; } - io_consume_sqe(ctx); + sqe = io_get_sqe(ctx); + if (unlikely(!sqe)) { + kmem_cache_free(req_cachep, req); + break; + } /* will complete beyond this point, count as submitted */ submitted++; From dc0eced5d92052a84d58df03a3bc6382f64fecfa Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 18:41:15 +0000 Subject: [PATCH 095/103] io_uring: don't check PF_EXITING from syscall io_sq_thread_acquire_mm_files() can find a PF_EXITING task only when it's called from task_work context. Don't check it in all other cases, that are when we're in io_uring_enter(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 9c58be0579f3..66bbb0dc50af 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1213,8 +1213,6 @@ static int __io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, static inline int io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, struct io_kiocb *req) { - if (unlikely(current->flags & PF_EXITING)) - return -EFAULT; if (!(ctx->flags & IORING_SETUP_SQPOLL)) return 0; return __io_sq_thread_acquire_mm_files(ctx, req); @@ -2338,7 +2336,8 @@ static void __io_req_task_submit(struct io_kiocb *req) /* ctx stays valid until unlock, even if we drop all ours ctx->refs */ mutex_lock(&ctx->uring_lock); - if (!ctx->sqo_dead && !io_sq_thread_acquire_mm_files(ctx, req)) + if (!ctx->sqo_dead && !(current->flags & PF_EXITING) && + !io_sq_thread_acquire_mm_files(ctx, req)) __io_queue_sqe(req); else __io_req_task_cancel(req, -EFAULT); From cdbff98223330cdb6c57ead1533ce066dddd61b7 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 18:41:16 +0000 Subject: [PATCH 096/103] io_uring: clean io_req_find_next() fast check Indirectly io_req_find_next() is called for every request, optimise the check by testing flags as it was long before -- __io_req_find_next() tolerates false-positives well (i.e. link==NULL), and those should be really rare. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 66bbb0dc50af..776531f6e18b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2172,7 +2172,7 @@ static struct io_kiocb *__io_req_find_next(struct io_kiocb *req) static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) { - if (likely(!(req->link) && !(req->flags & REQ_F_LINK_TIMEOUT))) + if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK)))) return NULL; return __io_req_find_next(req); } From 5be9ad1e4287e1742fd8d253267c86446441bdaf Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 18:41:17 +0000 Subject: [PATCH 097/103] io_uring: optimise io_init_req() flags setting Invalid req->flags are tolerated by free/put well, avoid this dancing needlessly presetting it to zero, and then not even resetting but modifying it, i.e. "|=". Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 776531f6e18b..2e8cb739c835 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6806,14 +6806,15 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, { struct io_submit_state *state; unsigned int sqe_flags; - int id, ret; + int id, ret = 0; req->opcode = READ_ONCE(sqe->opcode); + /* same numerical values with corresponding REQ_F_*, safe to copy */ + req->flags = sqe_flags = READ_ONCE(sqe->flags); req->user_data = READ_ONCE(sqe->user_data); req->async_data = NULL; req->file = NULL; req->ctx = ctx; - req->flags = 0; req->link = NULL; req->fixed_rsrc_refs = NULL; /* one is dropped after submission, the other at completion */ @@ -6821,17 +6822,16 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->task = current; req->result = 0; + /* enforce forwards compatibility on users */ + if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) + return -EINVAL; + if (unlikely(req->opcode >= IORING_OP_LAST)) return -EINVAL; if (unlikely(io_sq_thread_acquire_mm_files(ctx, req))) return -EFAULT; - sqe_flags = READ_ONCE(sqe->flags); - /* enforce forwards compatibility on users */ - if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) - return -EINVAL; - if (unlikely(!io_check_restriction(ctx, req, sqe_flags))) return -EACCES; @@ -6854,8 +6854,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->work.flags |= IO_WQ_WORK_CREDS; } - /* same numerical values with corresponding REQ_F_*, safe to copy */ - req->flags |= sqe_flags; state = &ctx->submit_state; /* @@ -6868,7 +6866,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, state->plug_started = true; } - ret = 0; if (io_op_defs[req->opcode].needs_file) { bool fixed = req->flags & REQ_F_FIXED_FILE; From e06aa2e94f0532d04bad7713eb7c6a32ab9ba674 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 12 Feb 2021 14:02:54 -0700 Subject: [PATCH 098/103] io-wq: clear out worker ->fs and ->files By default, kernel threads have init_fs and init_files assigned. In the past, this has triggered security problems, as commands that don't ask for (and hence don't get assigned) fs/files from the originating task can then attempt path resolution etc with access to parts of the system they should not be able to. Rather than add checks in the fs code for misuse, just set these to NULL. If we do attempt to use them, then the resulting code will oops rather than provide access to something that it should not permit. Suggested-by: Linus Torvalds Signed-off-by: Jens Axboe --- fs/io-wq.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index 63ef195b1acb..c36bbcd823ce 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -64,9 +64,7 @@ struct io_worker { #endif const struct cred *cur_creds; const struct cred *saved_creds; - struct files_struct *restore_files; struct nsproxy *restore_nsproxy; - struct fs_struct *restore_fs; }; #if BITS_PER_LONG == 64 @@ -156,19 +154,19 @@ static bool __io_worker_unuse(struct io_wqe *wqe, struct io_worker *worker) worker->cur_creds = worker->saved_creds = NULL; } - if (current->files != worker->restore_files) { + if (current->files) { __acquire(&wqe->lock); raw_spin_unlock_irq(&wqe->lock); dropped_lock = true; task_lock(current); - current->files = worker->restore_files; + current->files = NULL; current->nsproxy = worker->restore_nsproxy; task_unlock(current); } - if (current->fs != worker->restore_fs) - current->fs = worker->restore_fs; + if (current->fs) + current->fs = NULL; /* * If we have an active mm, we need to drop the wq lock before unusing @@ -329,11 +327,11 @@ static void io_worker_start(struct io_wqe *wqe, struct io_worker *worker) allow_kernel_signal(SIGINT); current->flags |= PF_IO_WORKER; + current->fs = NULL; + current->files = NULL; worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING); - worker->restore_files = current->files; worker->restore_nsproxy = current->nsproxy; - worker->restore_fs = current->fs; io_wqe_inc_running(wqe, worker); } From 68e68ee6e359318c40891f614612616d219066d0 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 13 Feb 2021 09:00:02 -0700 Subject: [PATCH 099/103] io_uring: allow task match to be passed to io_req_cache_free() No changes in this patch, just allows a caller to pass in a targeted task that we must match for freeing requests in the cache. Signed-off-by: Jens Axboe --- fs/io_uring.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 2e8cb739c835..9cd7b03a6f34 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8681,12 +8681,13 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) idr_destroy(&ctx->io_buffer_idr); } -static void io_req_cache_free(struct list_head *list) +static void io_req_cache_free(struct list_head *list, struct task_struct *tsk) { - while (!list_empty(list)) { - struct io_kiocb *req; + struct io_kiocb *req, *nxt; - req = list_first_entry(list, struct io_kiocb, compl.list); + list_for_each_entry_safe(req, nxt, list, compl.list) { + if (tsk && req->task != tsk) + continue; list_del(&req->compl.list); kmem_cache_free(req_cachep, req); } @@ -8742,8 +8743,8 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) free_uid(ctx->user); put_cred(ctx->creds); kfree(ctx->cancel_hash); - io_req_cache_free(&ctx->submit_state.comp.free_list); - io_req_cache_free(&ctx->submit_state.comp.locked_free_list); + io_req_cache_free(&ctx->submit_state.comp.free_list, NULL); + io_req_cache_free(&ctx->submit_state.comp.locked_free_list, NULL); kfree(ctx); } From 9a4fdbd8ee0d8aca0cb5692446e5ca583b230cd7 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 13 Feb 2021 09:09:44 -0700 Subject: [PATCH 100/103] io_uring: add helper to free all request caches We have three different ones, put it in a helper for easy calling. This is in preparation for doing it outside of ring freeing as well. With that in mind, also ensure that we do the proper locking for safe calling from a context where the ring it still live. Signed-off-by: Jens Axboe --- fs/io_uring.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 9cd7b03a6f34..1895fc132252 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8693,10 +8693,27 @@ static void io_req_cache_free(struct list_head *list, struct task_struct *tsk) } } -static void io_ring_ctx_free(struct io_ring_ctx *ctx) +static void io_req_caches_free(struct io_ring_ctx *ctx, struct task_struct *tsk) { struct io_submit_state *submit_state = &ctx->submit_state; + mutex_lock(&ctx->uring_lock); + + if (submit_state->free_reqs) + kmem_cache_free_bulk(req_cachep, submit_state->free_reqs, + submit_state->reqs); + + io_req_cache_free(&submit_state->comp.free_list, NULL); + + spin_lock_irq(&ctx->completion_lock); + io_req_cache_free(&submit_state->comp.locked_free_list, NULL); + spin_unlock_irq(&ctx->completion_lock); + + mutex_unlock(&ctx->uring_lock); +} + +static void io_ring_ctx_free(struct io_ring_ctx *ctx) +{ /* * Some may use context even when all refs and requests have been put, * and they are free to do so while still holding uring_lock, see @@ -8715,10 +8732,6 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) ctx->mm_account = NULL; } - if (submit_state->free_reqs) - kmem_cache_free_bulk(req_cachep, submit_state->free_reqs, - submit_state->reqs); - #ifdef CONFIG_BLK_CGROUP if (ctx->sqo_blkcg_css) css_put(ctx->sqo_blkcg_css); @@ -8742,9 +8755,8 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) percpu_ref_exit(&ctx->refs); free_uid(ctx->user); put_cred(ctx->creds); + io_req_caches_free(ctx, NULL); kfree(ctx->cancel_hash); - io_req_cache_free(&ctx->submit_state.comp.free_list, NULL); - io_req_cache_free(&ctx->submit_state.comp.locked_free_list, NULL); kfree(ctx); } From 41be53e94fb04cc69fdf2f524c2a05d8069e047b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 13 Feb 2021 09:11:04 -0700 Subject: [PATCH 101/103] io_uring: kill cached requests from exiting task closing the ring Be nice and prune these upfront, in case the ring is being shared and one of the tasks is going away. This is a bit more important now that we account the allocations. Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 1895fc132252..a9d094f7060f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9232,8 +9232,10 @@ static int io_uring_flush(struct file *file, void *data) struct io_uring_task *tctx = current->io_uring; struct io_ring_ctx *ctx = file->private_data; - if (fatal_signal_pending(current) || (current->flags & PF_EXITING)) + if (fatal_signal_pending(current) || (current->flags & PF_EXITING)) { io_uring_cancel_task_requests(ctx, NULL); + io_req_caches_free(ctx, current); + } if (!tctx) return 0; From 0d4370cfe36b7f1719123b621a4ec4d9c7a25f89 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 14 Feb 2021 13:21:43 -0700 Subject: [PATCH 102/103] proc: don't allow async path resolution of /proc/thread-self components If this is attempted by an io-wq kthread, then return -EOPNOTSUPP as we don't currently support that. Once we can get task_pid_ptr() doing the right thing, then this can go away again. Use PF_IO_WORKER for this to speciically target the io_uring workers. Modify the /proc/self/ check to use PF_IO_WORKER as well. Cc: stable@vger.kernel.org Fixes: 8d4c3e76e3be ("proc: don't allow async path resolution of /proc/self components") Reported-by: Eric W. Biederman Signed-off-by: Jens Axboe --- fs/proc/self.c | 2 +- fs/proc/thread_self.c | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/fs/proc/self.c b/fs/proc/self.c index cc71ce3466dc..a4012154e109 100644 --- a/fs/proc/self.c +++ b/fs/proc/self.c @@ -20,7 +20,7 @@ static const char *proc_self_get_link(struct dentry *dentry, * Not currently supported. Once we can inherit all of struct pid, * we can allow this. */ - if (current->flags & PF_KTHREAD) + if (current->flags & PF_IO_WORKER) return ERR_PTR(-EOPNOTSUPP); if (!tgid) diff --git a/fs/proc/thread_self.c b/fs/proc/thread_self.c index a553273fbd41..d56681d86d28 100644 --- a/fs/proc/thread_self.c +++ b/fs/proc/thread_self.c @@ -17,6 +17,13 @@ static const char *proc_thread_self_get_link(struct dentry *dentry, pid_t pid = task_pid_nr_ns(current, ns); char *name; + /* + * Not currently supported. Once we can inherit all of struct pid, + * we can allow this. + */ + if (current->flags & PF_IO_WORKER) + return ERR_PTR(-EOPNOTSUPP); + if (!pid) return ERR_PTR(-ENOENT); name = kmalloc(10 + 6 + 10 + 1, dentry ? GFP_KERNEL : GFP_ATOMIC); From 0b81e80c813f92520667c872d499a2dba8377be6 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 16 Feb 2021 10:33:53 -0700 Subject: [PATCH 103/103] io_uring: tctx->task_lock should be IRQ safe We add task_work from any context, hence we need to ensure that we can tolerate it being from IRQ context as well. Fixes: 7cbf1722d5fc ("io_uring: provide FIFO ordering for task_work") Signed-off-by: Jens Axboe --- fs/io_uring.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index a9d094f7060f..58dd10481106 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2186,10 +2186,10 @@ static bool __tctx_task_work(struct io_uring_task *tctx) if (wq_list_empty(&tctx->task_list)) return false; - spin_lock(&tctx->task_lock); + spin_lock_irq(&tctx->task_lock); list = tctx->task_list; INIT_WQ_LIST(&tctx->task_list); - spin_unlock(&tctx->task_lock); + spin_unlock_irq(&tctx->task_lock); node = list.first; while (node) { @@ -2236,13 +2236,14 @@ static int io_task_work_add(struct task_struct *tsk, struct io_kiocb *req, { struct io_uring_task *tctx = tsk->io_uring; struct io_wq_work_node *node, *prev; + unsigned long flags; int ret; WARN_ON_ONCE(!tctx); - spin_lock(&tctx->task_lock); + spin_lock_irqsave(&tctx->task_lock, flags); wq_list_add_tail(&req->io_task_work.node, &tctx->task_list); - spin_unlock(&tctx->task_lock); + spin_unlock_irqrestore(&tctx->task_lock, flags); /* task_work already pending, we're done */ if (test_bit(0, &tctx->task_state) || @@ -2257,7 +2258,7 @@ static int io_task_work_add(struct task_struct *tsk, struct io_kiocb *req, * in the list, it got run and we're fine. */ ret = 0; - spin_lock(&tctx->task_lock); + spin_lock_irqsave(&tctx->task_lock, flags); wq_list_for_each(node, prev, &tctx->task_list) { if (&req->io_task_work.node == node) { wq_list_del(&tctx->task_list, node, prev); @@ -2265,7 +2266,7 @@ static int io_task_work_add(struct task_struct *tsk, struct io_kiocb *req, break; } } - spin_unlock(&tctx->task_lock); + spin_unlock_irqrestore(&tctx->task_lock, flags); clear_bit(0, &tctx->task_state); return ret; }