diff options
| -rw-r--r-- | io_uring/io_uring.c | 13 | ||||
| -rw-r--r-- | io_uring/io_uring.h | 1 | ||||
| -rw-r--r-- | io_uring/rw.c | 40 |
3 files changed, 29 insertions, 25 deletions
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 64845634d89f..2675cffbd9a4 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -527,19 +527,6 @@ static void io_queue_iowq(struct io_kiocb *req) io_queue_linked_timeout(link); } -static void io_tw_requeue_iowq(struct io_kiocb *req, struct io_tw_state *ts) -{ - req->flags &= ~REQ_F_REISSUE; - io_queue_iowq(req); -} - -void io_tw_queue_iowq(struct io_kiocb *req) -{ - req->flags |= REQ_F_REISSUE | REQ_F_BL_NO_RECYCLE; - req->io_task_work.func = io_tw_requeue_iowq; - io_req_task_work_add(req); -} - static __cold void io_queue_deferred(struct io_ring_ctx *ctx) { while (!list_empty(&ctx->defer_list)) { diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index b83a719c5443..624ca9076a50 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -75,7 +75,6 @@ struct file *io_file_get_fixed(struct io_kiocb *req, int fd, void __io_req_task_work_add(struct io_kiocb *req, unsigned flags); bool io_alloc_async_data(struct io_kiocb *req); void io_req_task_queue(struct io_kiocb *req); -void io_tw_queue_iowq(struct io_kiocb *req); void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts); void io_req_task_queue_fail(struct io_kiocb *req, int ret); void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts); diff --git a/io_uring/rw.c b/io_uring/rw.c index 4fed829fe97c..a6bf2ea8db91 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -396,9 +396,16 @@ static inline loff_t *io_kiocb_update_pos(struct io_kiocb *req) return NULL; } +#ifdef CONFIG_BLOCK +static void io_resubmit_prep(struct io_kiocb *req) +{ + struct io_async_rw *io = req->async_data; + + iov_iter_restore(&io->iter, &io->iter_state); +} + static bool io_rw_should_reissue(struct io_kiocb *req) { -#ifdef CONFIG_BLOCK umode_t mode = file_inode(req->file)->i_mode; struct io_ring_ctx *ctx = req->ctx; @@ -414,11 +421,23 @@ static bool io_rw_should_reissue(struct io_kiocb *req) */ if (percpu_ref_is_dying(&ctx->refs)) return false; + /* + * Play it safe and assume not safe to re-import and reissue if we're + * not in the original thread group (or in task context). + */ + if (!same_thread_group(req->task, current) || !in_task()) + return false; return true; +} #else +static void io_resubmit_prep(struct io_kiocb *req) +{ +} +static bool io_rw_should_reissue(struct io_kiocb *req) +{ return false; -#endif } +#endif static void io_req_end_write(struct io_kiocb *req) { @@ -455,7 +474,7 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res) * current cycle. */ io_req_io_end(req); - io_tw_queue_iowq(req); + req->flags |= REQ_F_REISSUE | REQ_F_BL_NO_RECYCLE; return true; } req_set_fail(req); @@ -521,7 +540,7 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res) io_req_end_write(req); if (unlikely(res != req->cqe.res)) { if (res == -EAGAIN && io_rw_should_reissue(req)) { - io_tw_queue_iowq(req); + req->flags |= REQ_F_REISSUE | REQ_F_BL_NO_RECYCLE; return; } req->cqe.res = res; @@ -583,10 +602,8 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, } if (req->flags & REQ_F_REISSUE) { - struct io_async_rw *io = req->async_data; - req->flags &= ~REQ_F_REISSUE; - iov_iter_restore(&io->iter, &io->iter_state); + io_resubmit_prep(req); return -EAGAIN; } return IOU_ISSUE_SKIP_COMPLETE; @@ -839,8 +856,7 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags) ret = io_iter_do_read(rw, &io->iter); if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { - if (req->flags & REQ_F_REISSUE) - return IOU_ISSUE_SKIP_COMPLETE; + req->flags &= ~REQ_F_REISSUE; /* If we can poll, just do that. */ if (io_file_can_poll(req)) return -EAGAIN; @@ -1035,8 +1051,10 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags) else ret2 = -EINVAL; - if (req->flags & REQ_F_REISSUE) - return IOU_ISSUE_SKIP_COMPLETE; + if (req->flags & REQ_F_REISSUE) { + req->flags &= ~REQ_F_REISSUE; + ret2 = -EAGAIN; + } /* * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just |