diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2023-04-04 13:39:49 +0100 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2023-04-04 09:30:39 -0600 |
| commit | ef8ae64ffa9578c12e44de42604004c2cc3e9c27 (patch) | |
| tree | a9aeabbb02273d7ca7ce7c38eebd96665b100156 /io_uring/rsrc.h | |
| parent | 03adabe81abb20221079b48343783b4327bd1186 (diff) | |
io_uring/rsrc: protect node refs with uring_lock
Currently, for nodes we have an atomic counter and some cached
(non-atomic) refs protected by uring_lock. Let's put all ref
manipulations under uring_lock and get rid of the atomic part.
It's free as in all cases we care about we already hold the lock.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/25b142feed7d831008257d90c8b17c0115d4fc15.1680576071.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'io_uring/rsrc.h')
| -rw-r--r-- | io_uring/rsrc.h | 29 |
1 files changed, 5 insertions, 24 deletions
diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 8164777279ba..a96103095f0f 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -37,13 +37,12 @@ struct io_rsrc_data { }; struct io_rsrc_node { - refcount_t refs; struct list_head node; struct list_head rsrc_list; struct io_rsrc_data *rsrc_data; struct llist_node llist; + int refs; bool done; - int cached_refs; }; struct io_mapped_ubuf { @@ -57,10 +56,8 @@ struct io_mapped_ubuf { void io_rsrc_put_tw(struct callback_head *cb); void io_rsrc_node_ref_zero(struct io_rsrc_node *node); void io_rsrc_put_work(struct work_struct *work); -void io_rsrc_refs_refill(struct io_ring_ctx *ctx, struct io_rsrc_node *node); void io_wait_rsrc_data(struct io_rsrc_data *data); void io_rsrc_node_destroy(struct io_rsrc_node *ref_node); -void io_rsrc_refs_drop(struct io_ring_ctx *ctx); int io_rsrc_node_switch_start(struct io_ring_ctx *ctx); int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, struct io_rsrc_node *node, void *rsrc); @@ -109,38 +106,22 @@ int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, unsigned int size, unsigned int type); -static inline void io_rsrc_put_node(struct io_rsrc_node *node, int nr) -{ - if (refcount_sub_and_test(nr, &node->refs)) - io_rsrc_node_ref_zero(node); -} - static inline void io_put_rsrc_node(struct io_rsrc_node *node) { - if (node) - io_rsrc_put_node(node, 1); + if (node && !--node->refs) + io_rsrc_node_ref_zero(node); } static inline void io_req_put_rsrc_locked(struct io_kiocb *req, struct io_ring_ctx *ctx) - __must_hold(&ctx->uring_lock) { - struct io_rsrc_node *node = req->rsrc_node; - - if (node) { - if (node == ctx->rsrc_node) - node->cached_refs++; - else - io_rsrc_put_node(node, 1); - } + io_put_rsrc_node(req->rsrc_node); } static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node) { - node->cached_refs--; - if (unlikely(node->cached_refs < 0)) - io_rsrc_refs_refill(ctx, node); + node->refs++; } static inline void io_req_set_rsrc_node(struct io_kiocb *req, |