Skip to content

Commit ed9f311

Browse files
keithbuschaxboe
keithbusch
authored and
axboe
committed
io_uring: cache nodes and mapped buffers
Frequent alloc/free cycles on these is pretty costly. Use an io cache to more efficiently reuse these buffers. Signed-off-by: Keith Busch <kbusch@kernel.org> Link: https://lore.kernel.org/r/20250227223916.143006-7-kbusch@meta.com [axboe: fix imu leak] Signed-off-by: Jens Axboe <axboe@kernel.dk>
1 parent 1f6540e commit ed9f311

File tree

5 files changed

+65
-16
lines changed

5 files changed

+65
-16
lines changed

‎include/linux/io_uring_types.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,8 @@ struct io_ring_ctx {
292292

293293
struct io_file_table file_table;
294294
struct io_rsrc_data buf_table;
295+
struct io_alloc_cache node_cache;
296+
struct io_alloc_cache imu_cache;
295297

296298
struct io_submit_state submit_state;
297299

‎io_uring/filetable.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
6868
if (slot_index >= ctx->file_table.data.nr)
6969
return -EINVAL;
7070

71-
node = io_rsrc_node_alloc(IORING_RSRC_FILE);
71+
node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE);
7272
if (!node)
7373
return -ENOMEM;
7474

‎io_uring/io_uring.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,7 @@ static void io_free_alloc_caches(struct io_ring_ctx *ctx)
291291
io_alloc_cache_free(&ctx->uring_cache, kfree);
292292
io_alloc_cache_free(&ctx->msg_cache, kfree);
293293
io_futex_cache_free(ctx);
294+
io_rsrc_cache_free(ctx);
294295
}
295296

296297
static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
@@ -338,6 +339,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
338339
ret |= io_alloc_cache_init(&ctx->msg_cache, IO_ALLOC_CACHE_MAX,
339340
sizeof(struct io_kiocb), 0);
340341
ret |= io_futex_cache_init(ctx);
342+
ret |= io_rsrc_cache_init(ctx);
341343
if (ret)
342344
goto free_ref;
343345
init_completion(&ctx->ref_comp);

‎io_uring/rsrc.c‎

Lines changed: 57 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
3333
#define IORING_MAX_FIXED_FILES (1U << 20)
3434
#define IORING_MAX_REG_BUFFERS (1U << 14)
3535

36+
#define IO_CACHED_BVECS_SEGS 32
37+
3638
int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
3739
{
3840
unsigned long page_limit, cur_pages, new_pages;
@@ -111,6 +113,22 @@ static void io_release_ubuf(void *priv)
111113
unpin_user_page(imu->bvec[i].bv_page);
112114
}
113115

116+
static struct io_mapped_ubuf *io_alloc_imu(struct io_ring_ctx *ctx,
117+
int nr_bvecs)
118+
{
119+
if (nr_bvecs <= IO_CACHED_BVECS_SEGS)
120+
return io_cache_alloc(&ctx->imu_cache, GFP_KERNEL);
121+
return kvmalloc(struct_size_t(struct io_mapped_ubuf, bvec, nr_bvecs),
122+
GFP_KERNEL);
123+
}
124+
125+
static void io_free_imu(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu)
126+
{
127+
if (imu->nr_bvecs > IO_CACHED_BVECS_SEGS ||
128+
!io_alloc_cache_put(&ctx->imu_cache, imu))
129+
kvfree(imu);
130+
}
131+
114132
static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu)
115133
{
116134
if (!refcount_dec_and_test(&imu->refs))
@@ -119,22 +137,45 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu)
119137
if (imu->acct_pages)
120138
io_unaccount_mem(ctx, imu->acct_pages);
121139
imu->release(imu->priv);
122-
kvfree(imu);
140+
io_free_imu(ctx, imu);
123141
}
124142

125-
struct io_rsrc_node *io_rsrc_node_alloc(int type)
143+
struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type)
126144
{
127145
struct io_rsrc_node *node;
128146

129-
node = kzalloc(sizeof(*node), GFP_KERNEL);
147+
node = io_cache_alloc(&ctx->node_cache, GFP_KERNEL);
130148
if (node) {
131149
node->type = type;
132150
node->refs = 1;
151+
node->tag = 0;
152+
node->file_ptr = 0;
133153
}
134154
return node;
135155
}
136156

137-
__cold void io_rsrc_data_free(struct io_ring_ctx *ctx, struct io_rsrc_data *data)
157+
bool io_rsrc_cache_init(struct io_ring_ctx *ctx)
158+
{
159+
const int imu_cache_size = struct_size_t(struct io_mapped_ubuf, bvec,
160+
IO_CACHED_BVECS_SEGS);
161+
const int node_size = sizeof(struct io_rsrc_node);
162+
bool ret;
163+
164+
ret = io_alloc_cache_init(&ctx->node_cache, IO_ALLOC_CACHE_MAX,
165+
node_size, 0);
166+
ret |= io_alloc_cache_init(&ctx->imu_cache, IO_ALLOC_CACHE_MAX,
167+
imu_cache_size, 0);
168+
return ret;
169+
}
170+
171+
void io_rsrc_cache_free(struct io_ring_ctx *ctx)
172+
{
173+
io_alloc_cache_free(&ctx->node_cache, kfree);
174+
io_alloc_cache_free(&ctx->imu_cache, kfree);
175+
}
176+
177+
__cold void io_rsrc_data_free(struct io_ring_ctx *ctx,
178+
struct io_rsrc_data *data)
138179
{
139180
if (!data->nr)
140181
return;
@@ -207,7 +248,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
207248
err = -EBADF;
208249
break;
209250
}
210-
node = io_rsrc_node_alloc(IORING_RSRC_FILE);
251+
node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE);
211252
if (!node) {
212253
err = -ENOMEM;
213254
fput(file);
@@ -465,7 +506,8 @@ void io_free_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
465506
break;
466507
}
467508

468-
kfree(node);
509+
if (!io_alloc_cache_put(&ctx->node_cache, node))
510+
kvfree(node);
469511
}
470512

471513
int io_sqe_files_unregister(struct io_ring_ctx *ctx)
@@ -527,7 +569,7 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
527569
goto fail;
528570
}
529571
ret = -ENOMEM;
530-
node = io_rsrc_node_alloc(IORING_RSRC_FILE);
572+
node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE);
531573
if (!node) {
532574
fput(file);
533575
goto fail;
@@ -732,7 +774,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
732774
if (!iov->iov_base)
733775
return NULL;
734776

735-
node = io_rsrc_node_alloc(IORING_RSRC_BUFFER);
777+
node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
736778
if (!node)
737779
return ERR_PTR(-ENOMEM);
738780
node->buf = NULL;
@@ -752,10 +794,11 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
752794
coalesced = io_coalesce_buffer(&pages, &nr_pages, &data);
753795
}
754796

755-
imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
797+
imu = io_alloc_imu(ctx, nr_pages);
756798
if (!imu)
757799
goto done;
758800

801+
imu->nr_bvecs = nr_pages;
759802
ret = io_buffer_account_pin(ctx, pages, nr_pages, imu, last_hpage);
760803
if (ret) {
761804
unpin_user_pages(pages, nr_pages);
@@ -766,7 +809,6 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
766809
/* store original address for later verification */
767810
imu->ubuf = (unsigned long) iov->iov_base;
768811
imu->len = iov->iov_len;
769-
imu->nr_bvecs = nr_pages;
770812
imu->folio_shift = PAGE_SHIFT;
771813
imu->release = io_release_ubuf;
772814
imu->priv = imu;
@@ -789,7 +831,8 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
789831
}
790832
done:
791833
if (ret) {
792-
kvfree(imu);
834+
if (imu)
835+
io_free_imu(ctx, imu);
793836
if (node)
794837
io_put_rsrc_node(ctx, node);
795838
node = ERR_PTR(ret);
@@ -893,14 +936,14 @@ int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,
893936
goto unlock;
894937
}
895938

896-
node = io_rsrc_node_alloc(IORING_RSRC_BUFFER);
939+
node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
897940
if (!node) {
898941
ret = -ENOMEM;
899942
goto unlock;
900943
}
901944

902945
nr_bvecs = blk_rq_nr_phys_segments(rq);
903-
imu = kvmalloc(struct_size(imu, bvec, nr_bvecs), GFP_KERNEL);
946+
imu = io_alloc_imu(ctx, nr_bvecs);
904947
if (!imu) {
905948
kfree(node);
906949
ret = -ENOMEM;
@@ -1137,7 +1180,7 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx
11371180
if (!src_node) {
11381181
dst_node = NULL;
11391182
} else {
1140-
dst_node = io_rsrc_node_alloc(IORING_RSRC_BUFFER);
1183+
dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
11411184
if (!dst_node) {
11421185
ret = -ENOMEM;
11431186
goto out_free;

‎io_uring/rsrc.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,9 @@ struct io_imu_folio_data {
4848
unsigned int nr_folios;
4949
};
5050

51-
struct io_rsrc_node *io_rsrc_node_alloc(int type);
51+
bool io_rsrc_cache_init(struct io_ring_ctx *ctx);
52+
void io_rsrc_cache_free(struct io_ring_ctx *ctx);
53+
struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type);
5254
void io_free_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node);
5355
void io_rsrc_data_free(struct io_ring_ctx *ctx, struct io_rsrc_data *data);
5456
int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr);

0 commit comments

Comments
 (0)