Skip to content

Commit

Permalink
io_uring/rsrc: merge nodes and io_rsrc_put
Browse files Browse the repository at this point in the history 
struct io_rsrc_node carries a number of resources represented by struct
io_rsrc_put. That was handy before for sync overhead ammortisation, but
all complexity is gone and nodes are simple and lightweight. Let's
allocate a separate node for each resource.

Nodes and io_rsrc_put and not much different in size, and former are
cached, so node allocation should work better. That also removes some
overhead for nested iteration in io_rsrc_node_ref_zero() /
__io_rsrc_put_work().

Another reason for the patch is that it greatly reduces complexity
by moving io_rsrc_node_switch[_start]() inside io_queue_rsrc_removal(),
so users don't have to care about it.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/c7d3a45b30cc14cd93700a710dd112edc703db98.1681822823.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
  • Loading branch information
@isilence @axboe
isilence authored and axboe committed Apr 19, 2023
1 parent 63fea89 commit c376644
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 97 deletions.
9 changes: 0 additions & 9 deletions io_uring/filetable.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,18 +80,13 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
if (file_slot->file_ptr) {
struct file *old_file;

ret = io_rsrc_node_switch_start(ctx);
if (ret)
return ret;

old_file = (struct file *)(file_slot->file_ptr & FFS_MASK);
ret = io_queue_rsrc_removal(ctx->file_data, slot_index, old_file);
if (ret)
return ret;

file_slot->file_ptr = 0;
io_file_bitmap_clear(&ctx->file_table, slot_index);
io_rsrc_node_switch(ctx, ctx->file_data);
}

ret = io_scm_file_account(ctx, file);
Expand Down Expand Up @@ -152,9 +147,6 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
return -ENXIO;
if (offset >= ctx->nr_user_files)
return -EINVAL;
ret = io_rsrc_node_switch_start(ctx);
if (ret)
return ret;

offset = array_index_nospec(offset, ctx->nr_user_files);
file_slot = io_fixed_file_slot(&ctx->file_table, offset);
Expand All @@ -168,7 +160,6 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)

file_slot->file_ptr = 0;
io_file_bitmap_clear(&ctx->file_table, offset);
io_rsrc_node_switch(ctx, ctx->file_data);
return 0;
}

Expand Down
91 changes: 23 additions & 68 deletions io_uring/rsrc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,17 +153,10 @@ static void io_rsrc_put_work_one(struct io_rsrc_data *rsrc_data,
static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
{
struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
struct io_rsrc_put *prsrc, *tmp;

if (ref_node->inline_items)
if (likely(ref_node->inline_items))
io_rsrc_put_work_one(rsrc_data, &ref_node->item);

list_for_each_entry_safe(prsrc, tmp, &ref_node->item_list, list) {
list_del(&prsrc->list);
io_rsrc_put_work_one(rsrc_data, prsrc);
kfree(prsrc);
}

io_rsrc_node_destroy(rsrc_data->ctx, ref_node);
}

Expand Down Expand Up @@ -206,53 +199,29 @@ struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
}

ref_node->rsrc_data = NULL;
ref_node->refs = 1;
INIT_LIST_HEAD(&ref_node->node);
INIT_LIST_HEAD(&ref_node->item_list);
ref_node->inline_items = 0;
ref_node->refs = 1;
return ref_node;
}

void io_rsrc_node_switch(struct io_ring_ctx *ctx,
struct io_rsrc_data *data_to_kill)
__must_hold(&ctx->uring_lock)
{
struct io_rsrc_node *node = ctx->rsrc_node;
struct io_rsrc_node *backup = io_rsrc_node_alloc(ctx);

if (WARN_ON_ONCE(!backup))
return;

node->rsrc_data = data_to_kill;
list_add_tail(&node->node, &ctx->rsrc_ref_list);
/* put master ref */
io_put_rsrc_node(ctx, node);
ctx->rsrc_node = backup;
}

int __io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
{
struct io_rsrc_node *node = kzalloc(sizeof(*node), GFP_KERNEL);

if (!node)
return -ENOMEM;
io_alloc_cache_put(&ctx->rsrc_node_cache, &node->cache);
return 0;
}

__cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
struct io_ring_ctx *ctx)
{
struct io_rsrc_node *backup;
DEFINE_WAIT(we);
int ret;

/* As we may drop ->uring_lock, other task may have started quiesce */
/* As We may drop ->uring_lock, other task may have started quiesce */
if (data->quiesce)
return -ENXIO;
ret = io_rsrc_node_switch_start(ctx);
if (ret)
return ret;
io_rsrc_node_switch(ctx, data);

backup = io_rsrc_node_alloc(ctx);
if (!backup)
return -ENOMEM;
ctx->rsrc_node->rsrc_data = data;
list_add_tail(&ctx->rsrc_node->node, &ctx->rsrc_ref_list);
io_put_rsrc_node(ctx, ctx->rsrc_node);
ctx->rsrc_node = backup;

if (list_empty(&ctx->rsrc_ref_list))
return 0;
Expand Down Expand Up @@ -382,7 +351,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
struct file *file;
int fd, i, err = 0;
unsigned int done;
bool needs_switch = false;

if (!ctx->file_data)
return -ENXIO;
Expand Down Expand Up @@ -414,7 +382,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
break;
file_slot->file_ptr = 0;
io_file_bitmap_clear(&ctx->file_table, i);
needs_switch = true;
}
if (fd != -1) {
file = fget(fd);
Expand Down Expand Up @@ -445,9 +412,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
io_file_bitmap_set(&ctx->file_table, i);
}
}

if (needs_switch)
io_rsrc_node_switch(ctx, data);
return done ? done : err;
}

Expand All @@ -458,7 +422,6 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
u64 __user *tags = u64_to_user_ptr(up->tags);
struct iovec iov, __user *iovs = u64_to_user_ptr(up->data);
struct page *last_hpage = NULL;
bool needs_switch = false;
__u32 done;
int i, err;

Expand Down Expand Up @@ -498,15 +461,11 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
break;
}
ctx->user_bufs[i] = ctx->dummy_ubuf;
needs_switch = true;
}

ctx->user_bufs[i] = imu;
*io_get_tag_slot(ctx->buf_data, i) = tag;
}

if (needs_switch)
io_rsrc_node_switch(ctx, ctx->buf_data);
return done ? done : err;
}

Expand All @@ -515,15 +474,11 @@ static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
unsigned nr_args)
{
__u32 tmp;
int err;

lockdep_assert_held(&ctx->uring_lock);

if (check_add_overflow(up->offset, nr_args, &tmp))
return -EOVERFLOW;
err = io_rsrc_node_switch_start(ctx);
if (err)
return err;

switch (type) {
case IORING_RSRC_FILE:
Expand Down Expand Up @@ -685,21 +640,21 @@ int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc)
struct io_ring_ctx *ctx = data->ctx;
struct io_rsrc_node *node = ctx->rsrc_node;
u64 *tag_slot = io_get_tag_slot(data, idx);
struct io_rsrc_put *prsrc;

if (!node->inline_items) {
prsrc = &node->item;
node->inline_items++;
} else {
prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
if (!prsrc)
return -ENOMEM;
list_add(&prsrc->list, &node->item_list);
ctx->rsrc_node = io_rsrc_node_alloc(ctx);
if (unlikely(!ctx->rsrc_node)) {
ctx->rsrc_node = node;
return -ENOMEM;
}

prsrc->tag = *tag_slot;
node->item.rsrc = rsrc;
node->item.tag = *tag_slot;
node->inline_items = 1;
*tag_slot = 0;
prsrc->rsrc = rsrc;

node->rsrc_data = data;
list_add_tail(&node->node, &ctx->rsrc_ref_list);
io_put_rsrc_node(ctx, node);
return 0;
}

Expand Down
22 changes: 2 additions & 20 deletions io_uring/rsrc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ enum {
};

struct io_rsrc_put {
struct list_head list;
u64 tag;
union {
void *rsrc;
Expand All @@ -43,17 +42,10 @@ struct io_rsrc_node {
struct io_cache_entry cache;
struct io_rsrc_data *rsrc_data;
};
struct list_head node;
int refs;

/*
* Keeps a list of struct io_rsrc_put to be completed. Each entry
* represents one rsrc (e.g. file or buffer), but all of them should've
* came from the same table and so are of the same type.
*/
struct list_head item_list;
struct io_rsrc_put item;
int inline_items;
struct list_head node;
struct io_rsrc_put item;
};

struct io_mapped_ubuf {
Expand All @@ -68,11 +60,8 @@ void io_rsrc_put_tw(struct callback_head *cb);
void io_rsrc_node_ref_zero(struct io_rsrc_node *node);
void io_rsrc_put_work(struct work_struct *work);
void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node);
int __io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc);
void io_rsrc_node_switch(struct io_ring_ctx *ctx,
struct io_rsrc_data *data_to_kill);

int io_import_fixed(int ddir, struct iov_iter *iter,
struct io_mapped_ubuf *imu,
Expand Down Expand Up @@ -109,13 +98,6 @@ static inline int io_scm_file_account(struct io_ring_ctx *ctx,
return __io_scm_file_account(ctx, file);
}

static inline int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
{
if (unlikely(io_alloc_cache_empty(&ctx->rsrc_node_cache)))
return __io_rsrc_node_switch_start(ctx);
return 0;
}

int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
unsigned nr_args);
int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
Expand Down

0 comments on commit c376644

Please sign in to comment.