Skip to content

Commit 6ec429d

Browse files
Junxian Huangrleon
Junxian Huang
authored and
rleon
committed
RDMA/hns: Support userspace configuring congestion control algorithm with QP granularity
Currently, congestion control algorithm is statically configured in FW, and all QPs use the same algorithm(except UD which has a fixed configuration of DCQCN). This is not flexible enough. Support userspace configuring congestion control algorithm with QP granularity while creating QPs. If the algorithm is not specified in userspace, use the default one. Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com> Link: https://lore.kernel.org/r/20240301104845.1141083-1-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky <leon@kernel.org>
1 parent 7a7b7f5 commit 6ec429d

File tree

6 files changed

+85
-12
lines changed

6 files changed

+85
-12
lines changed

‎drivers/infiniband/hw/hns/hns_roce_device.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -845,7 +845,8 @@ struct hns_roce_caps {
845845
u16 default_aeq_period;
846846
u16 default_aeq_arm_st;
847847
u16 default_ceq_arm_st;
848-
enum hns_roce_cong_type cong_type;
848+
u8 cong_cap;
849+
enum hns_roce_cong_type default_cong_type;
849850
};
850851

851852
enum hns_roce_device_state {

‎drivers/infiniband/hw/hns/hns_roce_hw_v2.c‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2209,11 +2209,12 @@ static int hns_roce_query_caps(struct hns_roce_dev *hr_dev)
22092209
caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth);
22102210

22112211
caps->num_srqs = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_SRQS);
2212-
caps->cong_type = hr_reg_read(resp_d, PF_CAPS_D_CONG_TYPE);
2212+
caps->cong_cap = hr_reg_read(resp_d, PF_CAPS_D_CONG_CAP);
22132213
caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth);
22142214
caps->ceqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_CEQ_DEPTH);
22152215
caps->num_comp_vectors = hr_reg_read(resp_d, PF_CAPS_D_NUM_CEQS);
22162216
caps->aeqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_AEQ_DEPTH);
2217+
caps->default_cong_type = hr_reg_read(resp_d, PF_CAPS_D_DEFAULT_ALG);
22172218
caps->reserved_pds = hr_reg_read(resp_d, PF_CAPS_D_RSV_PDS);
22182219
caps->num_uars = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_UARS);
22192220
caps->reserved_qps = hr_reg_read(resp_d, PF_CAPS_D_RSV_QPS);
@@ -4737,14 +4738,8 @@ enum {
47374738
static int check_cong_type(struct ib_qp *ibqp,
47384739
struct hns_roce_congestion_algorithm *cong_alg)
47394740
{
4740-
struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
47414741
struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
47424742

4743-
if (ibqp->qp_type == IB_QPT_UD || ibqp->qp_type == IB_QPT_GSI)
4744-
hr_qp->cong_type = CONG_TYPE_DCQCN;
4745-
else
4746-
hr_qp->cong_type = hr_dev->caps.cong_type;
4747-
47484743
/* different congestion types match different configurations */
47494744
switch (hr_qp->cong_type) {
47504745
case CONG_TYPE_DCQCN:
@@ -4772,9 +4767,6 @@ static int check_cong_type(struct ib_qp *ibqp,
47724767
cong_alg->wnd_mode_sel = WND_LIMIT;
47734768
break;
47744769
default:
4775-
ibdev_warn(&hr_dev->ib_dev,
4776-
"invalid type(%u) for congestion selection.\n",
4777-
hr_qp->cong_type);
47784770
hr_qp->cong_type = CONG_TYPE_DCQCN;
47794771
cong_alg->alg_sel = CONG_DCQCN;
47804772
cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL;

‎drivers/infiniband/hw/hns/hns_roce_hw_v2.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1214,12 +1214,13 @@ struct hns_roce_query_pf_caps_d {
12141214
#define PF_CAPS_D_RQWQE_HOP_NUM PF_CAPS_D_FIELD_LOC(21, 20)
12151215
#define PF_CAPS_D_EX_SGE_HOP_NUM PF_CAPS_D_FIELD_LOC(23, 22)
12161216
#define PF_CAPS_D_SQWQE_HOP_NUM PF_CAPS_D_FIELD_LOC(25, 24)
1217-
#define PF_CAPS_D_CONG_TYPE PF_CAPS_D_FIELD_LOC(29, 26)
1217+
#define PF_CAPS_D_CONG_CAP PF_CAPS_D_FIELD_LOC(29, 26)
12181218
#define PF_CAPS_D_CEQ_DEPTH PF_CAPS_D_FIELD_LOC(85, 64)
12191219
#define PF_CAPS_D_NUM_CEQS PF_CAPS_D_FIELD_LOC(95, 86)
12201220
#define PF_CAPS_D_AEQ_DEPTH PF_CAPS_D_FIELD_LOC(117, 96)
12211221
#define PF_CAPS_D_AEQ_ARM_ST PF_CAPS_D_FIELD_LOC(119, 118)
12221222
#define PF_CAPS_D_CEQ_ARM_ST PF_CAPS_D_FIELD_LOC(121, 120)
1223+
#define PF_CAPS_D_DEFAULT_ALG PF_CAPS_D_FIELD_LOC(127, 122)
12231224
#define PF_CAPS_D_RSV_PDS PF_CAPS_D_FIELD_LOC(147, 128)
12241225
#define PF_CAPS_D_NUM_UARS PF_CAPS_D_FIELD_LOC(155, 148)
12251226
#define PF_CAPS_D_RSV_QPS PF_CAPS_D_FIELD_LOC(179, 160)

‎drivers/infiniband/hw/hns/hns_roce_main.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,9 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,
394394
resp.config |= HNS_ROCE_RSP_CQE_INLINE_FLAGS;
395395
}
396396

397+
if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
398+
resp.congest_type = hr_dev->caps.cong_cap;
399+
397400
ret = hns_roce_uar_alloc(hr_dev, &context->uar);
398401
if (ret)
399402
goto error_out;

‎drivers/infiniband/hw/hns/hns_roce_qp.c‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1004,6 +1004,60 @@ static void free_kernel_wrid(struct hns_roce_qp *hr_qp)
10041004
kfree(hr_qp->sq.wrid);
10051005
}
10061006

1007+
static void default_congest_type(struct hns_roce_dev *hr_dev,
1008+
struct hns_roce_qp *hr_qp)
1009+
{
1010+
if (hr_qp->ibqp.qp_type == IB_QPT_UD ||
1011+
hr_qp->ibqp.qp_type == IB_QPT_GSI)
1012+
hr_qp->cong_type = CONG_TYPE_DCQCN;
1013+
else
1014+
hr_qp->cong_type = hr_dev->caps.default_cong_type;
1015+
}
1016+
1017+
static int set_congest_type(struct hns_roce_qp *hr_qp,
1018+
struct hns_roce_ib_create_qp *ucmd)
1019+
{
1020+
struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device);
1021+
1022+
switch (ucmd->cong_type_flags) {
1023+
case HNS_ROCE_CREATE_QP_FLAGS_DCQCN:
1024+
hr_qp->cong_type = CONG_TYPE_DCQCN;
1025+
break;
1026+
case HNS_ROCE_CREATE_QP_FLAGS_LDCP:
1027+
hr_qp->cong_type = CONG_TYPE_LDCP;
1028+
break;
1029+
case HNS_ROCE_CREATE_QP_FLAGS_HC3:
1030+
hr_qp->cong_type = CONG_TYPE_HC3;
1031+
break;
1032+
case HNS_ROCE_CREATE_QP_FLAGS_DIP:
1033+
hr_qp->cong_type = CONG_TYPE_DIP;
1034+
break;
1035+
default:
1036+
return -EINVAL;
1037+
}
1038+
1039+
if (!test_bit(hr_qp->cong_type, (unsigned long *)&hr_dev->caps.cong_cap))
1040+
return -EOPNOTSUPP;
1041+
1042+
if (hr_qp->ibqp.qp_type == IB_QPT_UD &&
1043+
hr_qp->cong_type != CONG_TYPE_DCQCN)
1044+
return -EOPNOTSUPP;
1045+
1046+
return 0;
1047+
}
1048+
1049+
static int set_congest_param(struct hns_roce_dev *hr_dev,
1050+
struct hns_roce_qp *hr_qp,
1051+
struct hns_roce_ib_create_qp *ucmd)
1052+
{
1053+
if (ucmd->comp_mask & HNS_ROCE_CREATE_QP_MASK_CONGEST_TYPE)
1054+
return set_congest_type(hr_qp, ucmd);
1055+
1056+
default_congest_type(hr_dev, hr_qp);
1057+
1058+
return 0;
1059+
}
1060+
10071061
static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
10081062
struct ib_qp_init_attr *init_attr,
10091063
struct ib_udata *udata,
@@ -1043,6 +1097,10 @@ static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
10431097
ibdev_err(ibdev,
10441098
"failed to set user SQ size, ret = %d.\n",
10451099
ret);
1100+
1101+
ret = set_congest_param(hr_dev, hr_qp, ucmd);
1102+
if (ret)
1103+
return ret;
10461104
} else {
10471105
if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
10481106
hr_qp->config = HNS_ROCE_EXSGE_FLAGS;
@@ -1051,6 +1109,8 @@ static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
10511109
ibdev_err(ibdev,
10521110
"failed to set kernel SQ size, ret = %d.\n",
10531111
ret);
1112+
1113+
default_congest_type(hr_dev, hr_qp);
10541114
}
10551115

10561116
return ret;

‎include/uapi/rdma/hns-abi.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,17 @@ struct hns_roce_ib_create_srq_resp {
7373
__u32 cap_flags; /* Use enum hns_roce_srq_cap_flags */
7474
};
7575

76+
enum hns_roce_congest_type_flags {
77+
HNS_ROCE_CREATE_QP_FLAGS_DCQCN,
78+
HNS_ROCE_CREATE_QP_FLAGS_LDCP,
79+
HNS_ROCE_CREATE_QP_FLAGS_HC3,
80+
HNS_ROCE_CREATE_QP_FLAGS_DIP,
81+
};
82+
83+
enum hns_roce_create_qp_comp_mask {
84+
HNS_ROCE_CREATE_QP_MASK_CONGEST_TYPE = 1 << 0,
85+
};
86+
7687
struct hns_roce_ib_create_qp {
7788
__aligned_u64 buf_addr;
7889
__aligned_u64 db_addr;
@@ -81,6 +92,9 @@ struct hns_roce_ib_create_qp {
8192
__u8 sq_no_prefetch;
8293
__u8 reserved[5];
8394
__aligned_u64 sdb_addr;
95+
__aligned_u64 comp_mask; /* Use enum hns_roce_create_qp_comp_mask */
96+
__aligned_u64 create_flags;
97+
__aligned_u64 cong_type_flags;
8498
};
8599

86100
enum hns_roce_qp_cap_flags {
@@ -114,6 +128,8 @@ struct hns_roce_ib_alloc_ucontext_resp {
114128
__u32 reserved;
115129
__u32 config;
116130
__u32 max_inline_data;
131+
__u8 congest_type;
132+
__u8 reserved0[7];
117133
};
118134

119135
struct hns_roce_ib_alloc_ucontext {

0 commit comments

Comments
 (0)