Skip to content

Commit

Permalink
block/rnbd-clt: Dynamically allocate sglist for rnbd_iu
Browse files Browse the repository at this point in the history 
The BMAX_SEGMENT static array for scatterlist is embedded in
rnbd_iu structure to avoid memory allocation in hot IO path.
In many cases, we do need only several sg entries because many IOs
have only several segments.

This patch change rnbd_iu to check the number of segments in the request
and allocate sglist dynamically.

For io path, use sg_alloc_table_chained to allocate sg list faster.

First it makes two sg entries after pdu of request.
The sg_alloc_table_chained uses the pre-allocated sg entries
if the number of segments of the request is less than two.
So it reduces the number of memory allocation.

Signed-off-by: Gioh Kim <gi-oh.kim@cloud.ionos.com>
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
  • Loading branch information
@axboe
Gioh Kim authored and axboe committed Dec 16, 2020
1 parent 512c781 commit 5a1328d
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 27 deletions.
60 changes: 34 additions & 26 deletions drivers/block/rnbd/rnbd-clt.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,7 @@ static void rnbd_softirq_done_fn(struct request *rq)
struct rnbd_iu *iu;

iu = blk_mq_rq_to_pdu(rq);
sg_free_table_chained(&iu->sgt, RNBD_INLINE_SG_CNT);
rnbd_put_permit(sess, iu->permit);
blk_mq_end_request(rq, errno_to_blk_status(iu->errno));
}
Expand Down Expand Up @@ -477,7 +478,7 @@ static int send_msg_close(struct rnbd_clt_dev *dev, u32 device_id, bool wait)
iu->buf = NULL;
iu->dev = dev;

sg_mark_end(&iu->sglist[0]);
sg_alloc_table(&iu->sgt, 1, GFP_KERNEL);

msg.hdr.type = cpu_to_le16(RNBD_MSG_CLOSE);
msg.device_id = cpu_to_le32(device_id);
Expand All @@ -492,6 +493,7 @@ static int send_msg_close(struct rnbd_clt_dev *dev, u32 device_id, bool wait)
err = errno;
}

sg_free_table(&iu->sgt);
rnbd_put_iu(sess, iu);
return err;
}
Expand Down Expand Up @@ -564,15 +566,16 @@ static int send_msg_open(struct rnbd_clt_dev *dev, bool wait)
iu->buf = rsp;
iu->dev = dev;

sg_init_one(iu->sglist, rsp, sizeof(*rsp));
sg_alloc_table(&iu->sgt, 1, GFP_KERNEL);
sg_init_one(iu->sgt.sgl, rsp, sizeof(*rsp));

msg.hdr.type = cpu_to_le16(RNBD_MSG_OPEN);
msg.access_mode = dev->access_mode;
strlcpy(msg.dev_name, dev->pathname, sizeof(msg.dev_name));

WARN_ON(!rnbd_clt_get_dev(dev));
err = send_usr_msg(sess->rtrs, READ, iu,
&vec, sizeof(*rsp), iu->sglist, 1,
&vec, sizeof(*rsp), iu->sgt.sgl, 1,
msg_open_conf, &errno, wait);
if (err) {
rnbd_clt_put_dev(dev);
Expand All @@ -582,6 +585,7 @@ static int send_msg_open(struct rnbd_clt_dev *dev, bool wait)
err = errno;
}

sg_free_table(&iu->sgt);
rnbd_put_iu(sess, iu);
return err;
}
Expand Down Expand Up @@ -610,7 +614,8 @@ static int send_msg_sess_info(struct rnbd_clt_session *sess, bool wait)
iu->buf = rsp;
iu->sess = sess;

sg_init_one(iu->sglist, rsp, sizeof(*rsp));
sg_alloc_table(&iu->sgt, 1, GFP_KERNEL);
sg_init_one(iu->sgt.sgl, rsp, sizeof(*rsp));

msg.hdr.type = cpu_to_le16(RNBD_MSG_SESS_INFO);
msg.ver = RNBD_PROTO_VER_MAJOR;
Expand All @@ -626,7 +631,7 @@ static int send_msg_sess_info(struct rnbd_clt_session *sess, bool wait)
goto put_iu;
}
err = send_usr_msg(sess->rtrs, READ, iu,
&vec, sizeof(*rsp), iu->sglist, 1,
&vec, sizeof(*rsp), iu->sgt.sgl, 1,
msg_sess_info_conf, &errno, wait);
if (err) {
rnbd_clt_put_sess(sess);
Expand All @@ -636,7 +641,7 @@ static int send_msg_sess_info(struct rnbd_clt_session *sess, bool wait)
} else {
err = errno;
}

sg_free_table(&iu->sgt);
rnbd_put_iu(sess, iu);
return err;
}
Expand Down Expand Up @@ -1016,11 +1021,10 @@ static int rnbd_client_xfer_request(struct rnbd_clt_dev *dev,
* See queue limits.
*/
if (req_op(rq) != REQ_OP_DISCARD)
sg_cnt = blk_rq_map_sg(dev->queue, rq, iu->sglist);
sg_cnt = blk_rq_map_sg(dev->queue, rq, iu->sgt.sgl);

if (sg_cnt == 0)
/* Do not forget to mark the end */
sg_mark_end(&iu->sglist[0]);
sg_mark_end(&iu->sgt.sgl[0]);

msg.hdr.type = cpu_to_le16(RNBD_MSG_IO);
msg.device_id = cpu_to_le32(dev->device_id);
Expand All @@ -1029,13 +1033,13 @@ static int rnbd_client_xfer_request(struct rnbd_clt_dev *dev,
.iov_base = &msg,
.iov_len = sizeof(msg)
};
size = rnbd_clt_get_sg_size(iu->sglist, sg_cnt);
size = rnbd_clt_get_sg_size(iu->sgt.sgl, sg_cnt);
req_ops = (struct rtrs_clt_req_ops) {
.priv = iu,
.conf_fn = msg_io_conf,
};
err = rtrs_clt_request(rq_data_dir(rq), &req_ops, rtrs, permit,
&vec, 1, size, iu->sglist, sg_cnt);
&vec, 1, size, iu->sgt.sgl, sg_cnt);
if (unlikely(err)) {
rnbd_clt_err_rl(dev, "RTRS failed to transfer IO, err: %d\n",
err);
Expand Down Expand Up @@ -1122,6 +1126,7 @@ static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx,
struct rnbd_clt_dev *dev = rq->rq_disk->private_data;
struct rnbd_iu *iu = blk_mq_rq_to_pdu(rq);
int err;
blk_status_t ret = BLK_STS_IOERR;

if (unlikely(dev->dev_state != DEV_STATE_MAPPED))
return BLK_STS_IOERR;
Expand All @@ -1133,32 +1138,35 @@ static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx,
return BLK_STS_RESOURCE;
}

iu->sgt.sgl = iu->first_sgl;
err = sg_alloc_table_chained(&iu->sgt,
/* Even-if the request has no segment,
* sglist must have one entry at least */
blk_rq_nr_phys_segments(rq) ? : 1,
iu->sgt.sgl,
RNBD_INLINE_SG_CNT);
if (err) {
rnbd_clt_err_rl(dev, "sg_alloc_table_chained ret=%d\n", err);
rnbd_clt_dev_kick_mq_queue(dev, hctx, 10/*ms*/);
rnbd_put_permit(dev->sess, iu->permit);
return BLK_STS_RESOURCE;
}

blk_mq_start_request(rq);
err = rnbd_client_xfer_request(dev, rq, iu);
if (likely(err == 0))
return BLK_STS_OK;
if (unlikely(err == -EAGAIN || err == -ENOMEM)) {
rnbd_clt_dev_kick_mq_queue(dev, hctx, 10/*ms*/);
rnbd_put_permit(dev->sess, iu->permit);
return BLK_STS_RESOURCE;
ret = BLK_STS_RESOURCE;
}

sg_free_table_chained(&iu->sgt, RNBD_INLINE_SG_CNT);
rnbd_put_permit(dev->sess, iu->permit);
return BLK_STS_IOERR;
}

static int rnbd_init_request(struct blk_mq_tag_set *set, struct request *rq,
unsigned int hctx_idx, unsigned int numa_node)
{
struct rnbd_iu *iu = blk_mq_rq_to_pdu(rq);

sg_init_table(iu->sglist, BMAX_SEGMENTS);
return 0;
return ret;
}

static struct blk_mq_ops rnbd_mq_ops = {
.queue_rq = rnbd_queue_rq,
.init_request = rnbd_init_request,
.complete = rnbd_softirq_done_fn,
};

Expand All @@ -1172,7 +1180,7 @@ static int setup_mq_tags(struct rnbd_clt_session *sess)
tag_set->numa_node = NUMA_NO_NODE;
tag_set->flags = BLK_MQ_F_SHOULD_MERGE |
BLK_MQ_F_TAG_QUEUE_SHARED;
tag_set->cmd_size = sizeof(struct rnbd_iu);
tag_set->cmd_size = sizeof(struct rnbd_iu) + RNBD_RDMA_SGL_SIZE;
tag_set->nr_hw_queues = num_online_cpus();

return blk_mq_alloc_tag_set(tag_set);
Expand Down
10 changes: 9 additions & 1 deletion drivers/block/rnbd/rnbd-clt.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,13 @@ struct rnbd_iu_comp {
int errno;
};

#ifdef CONFIG_ARCH_NO_SG_CHAIN
#define RNBD_INLINE_SG_CNT 0
#else
#define RNBD_INLINE_SG_CNT 2
#endif
#define RNBD_RDMA_SGL_SIZE (sizeof(struct scatterlist) * RNBD_INLINE_SG_CNT)

struct rnbd_iu {
union {
struct request *rq; /* for block io */
Expand All @@ -56,11 +63,12 @@ struct rnbd_iu {
/* use to send msg associated with a sess */
struct rnbd_clt_session *sess;
};
struct scatterlist sglist[BMAX_SEGMENTS];
struct sg_table sgt;
struct work_struct work;
int errno;
struct rnbd_iu_comp comp;
atomic_t refcount;
struct scatterlist first_sgl[]; /* must be the last one */
};

struct rnbd_cpu_qlist {
Expand Down

0 comments on commit 5a1328d

Please sign in to comment.