Skip to content

Commit 360cd42

Browse files
isilenceaxboe
authored andcommitted
io_uring: optimise io_req_local_work_add
Chains of memory accesses are never good for performance. The req->task->io_uring->in_cancel in io_req_local_work_add() is there so that when a task is exiting via io_uring_try_cancel_requests() and starts waiting for completions, it gets woken up by every new task_work item queued. Do a little trick by announcing waiting in io_uring_try_cancel_requests(), making io_req_local_work_add() wake us up. We also need to check for deferred tw items after prepare_to_wait(TASK_INTERRUPTIBLE); Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Link: https://lore.kernel.org/r/fb11597e9bbcb365901824f8c5c2cf0d6ee100d0.1680782017.git.asml.silence@gmail.com Signed-off-by: Jens Axboe <axboe@kernel.dk>
1 parent c66ae3e commit 360cd42

File tree

1 file changed

+16
-8
lines changed

1 file changed

+16
-8
lines changed

io_uring/io_uring.c

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1335,10 +1335,6 @@ static void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
13351335
&req->io_task_work.node));
13361336

13371337
if (!first) {
1338-
if (unlikely(atomic_read(&req->task->io_uring->in_cancel))) {
1339-
io_move_task_work_from_local(ctx);
1340-
return;
1341-
}
13421338
if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
13431339
atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
13441340
if (ctx->has_evfd)
@@ -3205,6 +3201,12 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
32053201
enum io_wq_cancel cret;
32063202
bool ret = false;
32073203

3204+
/* set it so io_req_local_work_add() would wake us up */
3205+
if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
3206+
atomic_set(&ctx->cq_wait_nr, 1);
3207+
smp_mb();
3208+
}
3209+
32083210
/* failed during ring init, it couldn't have issued any requests */
32093211
if (!ctx->rings)
32103212
return false;
@@ -3259,6 +3261,8 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
32593261
{
32603262
struct io_uring_task *tctx = current->io_uring;
32613263
struct io_ring_ctx *ctx;
3264+
struct io_tctx_node *node;
3265+
unsigned long index;
32623266
s64 inflight;
32633267
DEFINE_WAIT(wait);
32643268

@@ -3280,9 +3284,6 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
32803284
break;
32813285

32823286
if (!sqd) {
3283-
struct io_tctx_node *node;
3284-
unsigned long index;
3285-
32863287
xa_for_each(&tctx->xa, index, node) {
32873288
/* sqpoll task will cancel all its requests */
32883289
if (node->ctx->sq_data)
@@ -3305,14 +3306,21 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
33053306
prepare_to_wait(&tctx->wait, &wait, TASK_INTERRUPTIBLE);
33063307
io_run_task_work();
33073308
io_uring_drop_tctx_refs(current);
3308-
3309+
xa_for_each(&tctx->xa, index, node) {
3310+
if (!llist_empty(&node->ctx->work_llist)) {
3311+
WARN_ON_ONCE(node->ctx->submitter_task &&
3312+
node->ctx->submitter_task != current);
3313+
goto end_wait;
3314+
}
3315+
}
33093316
/*
33103317
* If we've seen completions, retry without waiting. This
33113318
* avoids a race where a completion comes in before we did
33123319
* prepare_to_wait().
33133320
*/
33143321
if (inflight == tctx_inflight(tctx, !cancel_all))
33153322
schedule();
3323+
end_wait:
33163324
finish_wait(&tctx->wait, &wait);
33173325
} while (1);
33183326

0 commit comments

Comments
 (0)