Skip to content

Commit cb3c7fd

Browse files
gilr8davem330
gilr8
authored and
davem330
committed
net/mlx5e: Support adaptive RX coalescing
Striving for high message rate and low interrupt rate. Usage: ethtool -C <interface> adaptive-rx on/off Signed-off-by: Gil Rockah <gilr@mellanox.com> Signed-off-by: Achiad Shochat <achiad@mellanox.com> Signed-off-by: Saeed Mahameed <saeedm@mellanox.com> CC: Arnd Bergmann <arnd@arndb.de> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent 9908aa2 commit cb3c7fd

File tree

6 files changed

+416
-8
lines changed

6 files changed

+416
-8
lines changed

‎drivers/net/ethernet/mellanox/mlx5/core/Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
77

88
mlx5_core-$(CONFIG_MLX5_CORE_EN) += wq.o eswitch.o \
99
en_main.o en_fs.o en_ethtool.o en_tx.o en_rx.o \
10-
en_txrx.o en_clock.o vxlan.o en_tc.o en_arfs.o
10+
en_rx_am.o en_txrx.o en_clock.o vxlan.o en_tc.o \
11+
en_arfs.o
1112

1213
mlx5_core-$(CONFIG_MLX5_CORE_EN_DCB) += en_dcbnl.o

‎drivers/net/ethernet/mellanox/mlx5/core/en.h‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,7 @@ struct mlx5e_params {
195195
#ifdef CONFIG_MLX5_CORE_EN_DCB
196196
struct ieee_ets ets;
197197
#endif
198+
bool rx_am_enabled;
198199
};
199200

200201
struct mlx5e_tstamp {
@@ -213,13 +214,15 @@ struct mlx5e_tstamp {
213214
enum {
214215
MLX5E_RQ_STATE_POST_WQES_ENABLE,
215216
MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS,
217+
MLX5E_RQ_STATE_AM,
216218
};
217219

218220
struct mlx5e_cq {
219221
/* data path - accessed per cqe */
220222
struct mlx5_cqwq wq;
221223

222224
/* data path - accessed per napi poll */
225+
u16 event_ctr;
223226
struct napi_struct *napi;
224227
struct mlx5_core_cq mcq;
225228
struct mlx5e_channel *channel;
@@ -247,6 +250,30 @@ struct mlx5e_dma_info {
247250
dma_addr_t addr;
248251
};
249252

253+
struct mlx5e_rx_am_stats {
254+
int ppms; /* packets per msec */
255+
int epms; /* events per msec */
256+
};
257+
258+
struct mlx5e_rx_am_sample {
259+
ktime_t time;
260+
unsigned int pkt_ctr;
261+
u16 event_ctr;
262+
};
263+
264+
struct mlx5e_rx_am { /* Adaptive Moderation */
265+
u8 state;
266+
struct mlx5e_rx_am_stats prev_stats;
267+
struct mlx5e_rx_am_sample start_sample;
268+
struct work_struct work;
269+
u8 profile_ix;
270+
u8 mode;
271+
u8 tune_state;
272+
u8 steps_right;
273+
u8 steps_left;
274+
u8 tired;
275+
};
276+
250277
struct mlx5e_rq {
251278
/* data path */
252279
struct mlx5_wq_ll wq;
@@ -267,6 +294,8 @@ struct mlx5e_rq {
267294
unsigned long state;
268295
int ix;
269296

297+
struct mlx5e_rx_am am; /* Adaptive Moderation */
298+
270299
/* control */
271300
struct mlx5_wq_ctrl wq_ctrl;
272301
u8 wq_type;
@@ -637,6 +666,10 @@ void mlx5e_free_rx_fragmented_mpwqe(struct mlx5e_rq *rq,
637666
struct mlx5e_mpw_info *wi);
638667
struct mlx5_cqe64 *mlx5e_get_cqe(struct mlx5e_cq *cq);
639668

669+
void mlx5e_rx_am(struct mlx5e_rq *rq);
670+
void mlx5e_rx_am_work(struct work_struct *work);
671+
struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode);
672+
640673
void mlx5e_update_stats(struct mlx5e_priv *priv);
641674

642675
int mlx5e_create_flow_steering(struct mlx5e_priv *priv);

‎drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -528,6 +528,7 @@ static int mlx5e_get_coalesce(struct net_device *netdev,
528528
coal->rx_max_coalesced_frames = priv->params.rx_cq_moderation.pkts;
529529
coal->tx_coalesce_usecs = priv->params.tx_cq_moderation.usec;
530530
coal->tx_max_coalesced_frames = priv->params.tx_cq_moderation.pkts;
531+
coal->use_adaptive_rx_coalesce = priv->params.rx_am_enabled;
531532

532533
return 0;
533534
}
@@ -538,6 +539,10 @@ static int mlx5e_set_coalesce(struct net_device *netdev,
538539
struct mlx5e_priv *priv = netdev_priv(netdev);
539540
struct mlx5_core_dev *mdev = priv->mdev;
540541
struct mlx5e_channel *c;
542+
bool restart =
543+
!!coal->use_adaptive_rx_coalesce != priv->params.rx_am_enabled;
544+
bool was_opened;
545+
int err = 0;
541546
int tc;
542547
int i;
543548

@@ -546,12 +551,18 @@ static int mlx5e_set_coalesce(struct net_device *netdev,
546551

547552
mutex_lock(&priv->state_lock);
548553

554+
was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
555+
if (was_opened && restart) {
556+
mlx5e_close_locked(netdev);
557+
priv->params.rx_am_enabled = !!coal->use_adaptive_rx_coalesce;
558+
}
559+
549560
priv->params.tx_cq_moderation.usec = coal->tx_coalesce_usecs;
550561
priv->params.tx_cq_moderation.pkts = coal->tx_max_coalesced_frames;
551562
priv->params.rx_cq_moderation.usec = coal->rx_coalesce_usecs;
552563
priv->params.rx_cq_moderation.pkts = coal->rx_max_coalesced_frames;
553564

554-
if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
565+
if (!was_opened || restart)
555566
goto out;
556567

557568
for (i = 0; i < priv->params.num_channels; ++i) {
@@ -570,8 +581,11 @@ static int mlx5e_set_coalesce(struct net_device *netdev,
570581
}
571582

572583
out:
584+
if (was_opened && restart)
585+
err = mlx5e_open_locked(netdev);
586+
573587
mutex_unlock(&priv->state_lock);
574-
return 0;
588+
return err;
575589
}
576590

577591
static u32 ptys2ethtool_supported_link(u32 eth_proto_cap)

‎drivers/net/ethernet/mellanox/mlx5/core/en_main.c‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,9 @@
4040
#include "vxlan.h"
4141

4242
struct mlx5e_rq_param {
43-
u32 rqc[MLX5_ST_SZ_DW(rqc)];
44-
struct mlx5_wq_param wq;
43+
u32 rqc[MLX5_ST_SZ_DW(rqc)];
44+
struct mlx5_wq_param wq;
45+
bool am_enabled;
4546
};
4647

4748
struct mlx5e_sq_param {
@@ -337,6 +338,9 @@ static int mlx5e_create_rq(struct mlx5e_channel *c,
337338
wqe->data.byte_count = cpu_to_be32(byte_count);
338339
}
339340

341+
INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
342+
rq->am.mode = priv->params.rx_cq_period_mode;
343+
340344
rq->wq_type = priv->params.rq_wq_type;
341345
rq->pdev = c->pdev;
342346
rq->netdev = c->netdev;
@@ -509,6 +513,9 @@ static int mlx5e_open_rq(struct mlx5e_channel *c,
509513
if (err)
510514
goto err_disable_rq;
511515

516+
if (param->am_enabled)
517+
set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
518+
512519
set_bit(MLX5E_RQ_STATE_POST_WQES_ENABLE, &rq->state);
513520

514521
sq->ico_wqe_info[pi].opcode = MLX5_OPCODE_NOP;
@@ -537,6 +544,8 @@ static void mlx5e_close_rq(struct mlx5e_rq *rq)
537544
/* avoid destroying rq before mlx5e_poll_rx_cq() is done with it */
538545
napi_synchronize(&rq->channel->napi);
539546

547+
cancel_work_sync(&rq->am.work);
548+
540549
mlx5e_disable_rq(rq);
541550
mlx5e_destroy_rq(rq);
542551
}
@@ -1112,6 +1121,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
11121121
{
11131122
struct mlx5e_cq_moder icosq_cq_moder = {0, 0};
11141123
struct net_device *netdev = priv->netdev;
1124+
struct mlx5e_cq_moder rx_cq_profile;
11151125
int cpu = mlx5e_get_cpu(priv, ix);
11161126
struct mlx5e_channel *c;
11171127
struct mlx5e_sq *sq;
@@ -1130,6 +1140,11 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
11301140
c->mkey_be = cpu_to_be32(priv->mkey.key);
11311141
c->num_tc = priv->params.num_tc;
11321142

1143+
if (priv->params.rx_am_enabled)
1144+
rx_cq_profile = mlx5e_am_get_def_profile(priv->params.rx_cq_period_mode);
1145+
else
1146+
rx_cq_profile = priv->params.rx_cq_moderation;
1147+
11331148
mlx5e_build_channeltc_to_txq_map(priv, ix);
11341149

11351150
netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
@@ -1143,7 +1158,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
11431158
goto err_close_icosq_cq;
11441159

11451160
err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq,
1146-
priv->params.rx_cq_moderation);
1161+
rx_cq_profile);
11471162
if (err)
11481163
goto err_close_tx_cqs;
11491164

@@ -1243,6 +1258,8 @@ static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
12431258

12441259
param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
12451260
param->wq.linear = 1;
1261+
1262+
param->am_enabled = priv->params.rx_am_enabled;
12461263
}
12471264

12481265
static void mlx5e_build_drop_rq_param(struct mlx5e_rq_param *param)
@@ -2883,6 +2900,9 @@ static void mlx5e_build_netdev_priv(struct mlx5_core_dev *mdev,
28832900
struct mlx5e_priv *priv = netdev_priv(netdev);
28842901
u32 link_speed = 0;
28852902
u32 pci_bw = 0;
2903+
u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
2904+
MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
2905+
MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
28862906

28872907
priv->params.log_sq_size =
28882908
MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
@@ -2929,8 +2949,8 @@ static void mlx5e_build_netdev_priv(struct mlx5_core_dev *mdev,
29292949
priv->params.min_rx_wqes = mlx5_min_rx_wqes(priv->params.rq_wq_type,
29302950
BIT(priv->params.log_rq_size));
29312951

2932-
mlx5e_set_rx_cq_mode_params(&priv->params,
2933-
MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2952+
priv->params.rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
2953+
mlx5e_set_rx_cq_mode_params(&priv->params, cq_period_mode);
29342954

29352955
priv->params.tx_cq_moderation.usec =
29362956
MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;

0 commit comments

Comments
 (0)