diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index a3700c57b073..c190061763fd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -409,14 +409,17 @@ struct mlx5e_xdp_info_fifo { struct mlx5e_xdp_wqe_info { u8 num_wqebbs; - u8 num_ds; + u8 num_pkts; }; struct mlx5e_xdp_mpwqe { /* Current MPWQE session */ struct mlx5e_tx_wqe *wqe; u8 ds_count; + u8 pkt_count; u8 max_ds_count; + u8 complete; + u8 inline_on; }; struct mlx5e_xdpsq; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index a9075b526ab9..c3d4efbf60ba 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -113,7 +113,9 @@ static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq) mlx5e_xdpsq_fetch_wqe(sq, &session->wqe); prefetchw(session->wqe->data); - session->ds_count = MLX5E_XDP_TX_EMPTY_DS_COUNT; + session->ds_count = MLX5E_XDP_TX_EMPTY_DS_COUNT; + session->pkt_count = 0; + session->complete = 0; pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); @@ -132,6 +134,9 @@ static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq) MLX5E_XDP_MPW_MAX_WQEBBS); session->max_ds_count = MLX5_SEND_WQEBB_NUM_DS * wqebbs; + + mlx5e_xdp_update_inline_state(sq); + stats->mpwqe++; } @@ -149,7 +154,7 @@ static void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq) cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_count); wi->num_wqebbs = DIV_ROUND_UP(ds_count, MLX5_SEND_WQEBB_NUM_DS); - wi->num_ds = ds_count - MLX5E_XDP_TX_EMPTY_DS_COUNT; + wi->num_pkts = session->pkt_count; sq->pc += wi->num_wqebbs; @@ -164,11 +169,9 @@ static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; struct mlx5e_xdpsq_stats *stats = sq->stats; - dma_addr_t dma_addr = xdpi->dma_addr; struct xdp_frame *xdpf = xdpi->xdpf; - unsigned int dma_len = xdpf->len; - if (unlikely(sq->hw_mtu < dma_len)) { + if (unlikely(sq->hw_mtu < xdpf->len)) { stats->err++; return false; } @@ -185,9 +188,10 @@ static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, mlx5e_xdp_mpwqe_session_start(sq); } - mlx5e_xdp_mpwqe_add_dseg(sq, dma_addr, dma_len); + mlx5e_xdp_mpwqe_add_dseg(sq, xdpi, stats); - if (unlikely(session->ds_count == session->max_ds_count)) + if (unlikely(session->complete || + session->ds_count == session->max_ds_count)) mlx5e_xdp_mpwqe_complete(sq); mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi); @@ -301,7 +305,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) sqcc += wi->num_wqebbs; - for (j = 0; j < wi->num_ds; j++) { + for (j = 0; j < wi->num_pkts; j++) { struct mlx5e_xdp_info xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo); @@ -342,7 +346,7 @@ void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq) sq->cc += wi->num_wqebbs; - for (i = 0; i < wi->num_ds; i++) { + for (i = 0; i < wi->num_pkts; i++) { struct mlx5e_xdp_info xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h index ee27a7c8cd87..858e7a2a13ca 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h @@ -75,16 +75,68 @@ static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq) } } +/* Enable inline WQEs to shift some load from a congested HCA (HW) to + * a less congested cpu (SW). + */ +static inline void mlx5e_xdp_update_inline_state(struct mlx5e_xdpsq *sq) +{ + u16 outstanding = sq->xdpi_fifo_pc - sq->xdpi_fifo_cc; + struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; + +#define MLX5E_XDP_INLINE_WATERMARK_LOW 10 +#define MLX5E_XDP_INLINE_WATERMARK_HIGH 128 + + if (session->inline_on) { + if (outstanding <= MLX5E_XDP_INLINE_WATERMARK_LOW) + session->inline_on = 0; + return; + } + + /* inline is false */ + if (outstanding >= MLX5E_XDP_INLINE_WATERMARK_HIGH) + session->inline_on = 1; +} + static inline void -mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq, dma_addr_t dma_addr, u16 dma_len) +mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_info *xdpi, + struct mlx5e_xdpsq_stats *stats) { struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; + dma_addr_t dma_addr = xdpi->dma_addr; + struct xdp_frame *xdpf = xdpi->xdpf; struct mlx5_wqe_data_seg *dseg = - (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count++; + (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count; + u16 dma_len = xdpf->len; + session->pkt_count++; + +#define MLX5E_XDP_INLINE_WQE_SZ_THRSD (256 - sizeof(struct mlx5_wqe_inline_seg)) + + if (session->inline_on && dma_len <= MLX5E_XDP_INLINE_WQE_SZ_THRSD) { + struct mlx5_wqe_inline_seg *inline_dseg = + (struct mlx5_wqe_inline_seg *)dseg; + u16 ds_len = sizeof(*inline_dseg) + dma_len; + u16 ds_cnt = DIV_ROUND_UP(ds_len, MLX5_SEND_WQE_DS); + + if (unlikely(session->ds_count + ds_cnt > session->max_ds_count)) { + /* Not enough space for inline wqe, send with memory pointer */ + session->complete = true; + goto no_inline; + } + + inline_dseg->byte_count = cpu_to_be32(dma_len | MLX5_INLINE_SEG); + memcpy(inline_dseg->data, xdpf->data, dma_len); + + session->ds_count += ds_cnt; + stats->inlnw++; + return; + } + +no_inline: dseg->addr = cpu_to_be64(dma_addr); dseg->byte_count = cpu_to_be32(dma_len); dseg->lkey = sq->mkey_be; + session->ds_count++; } static inline void mlx5e_xdpsq_fetch_wqe(struct mlx5e_xdpsq *sq, @@ -111,5 +163,4 @@ mlx5e_xdpi_fifo_pop(struct mlx5e_xdp_info_fifo *fifo) { return fifo->xi[(*fifo->cc)++ & fifo->mask]; } - #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 7ab195ac7299..23df6f486c6a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -1532,7 +1532,7 @@ static int mlx5e_open_xdpsq(struct mlx5e_channel *c, dseg->lkey = sq->mkey_be; wi->num_wqebbs = 1; - wi->num_ds = 1; + wi->num_pkts = 1; } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 80ee48dcc0a3..ca0ff3b3fbd1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -66,6 +66,7 @@ static const struct counter_desc sw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_redirect) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_xmit) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_mpwqe) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_inlnw) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_full) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_err) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_cqe) }, @@ -81,6 +82,7 @@ static const struct counter_desc sw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_cqe_err) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_xmit) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_mpwqe) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_inlnw) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_full) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_err) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_cqes) }, @@ -163,6 +165,7 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) s->rx_xdp_redirect += rq_stats->xdp_redirect; s->rx_xdp_tx_xmit += xdpsq_stats->xmit; s->rx_xdp_tx_mpwqe += xdpsq_stats->mpwqe; + s->rx_xdp_tx_inlnw += xdpsq_stats->inlnw; s->rx_xdp_tx_full += xdpsq_stats->full; s->rx_xdp_tx_err += xdpsq_stats->err; s->rx_xdp_tx_cqe += xdpsq_stats->cqes; @@ -188,7 +191,8 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) s->ch_eq_rearm += ch_stats->eq_rearm; /* xdp redirect */ s->tx_xdp_xmit += xdpsq_red_stats->xmit; - s->tx_xdp_mpwqe += xdpsq_red_stats->mpwqe; + s->tx_xdp_mpwqe += xdpsq_red_stats->mpwqe; + s->tx_xdp_inlnw += xdpsq_red_stats->inlnw; s->tx_xdp_full += xdpsq_red_stats->full; s->tx_xdp_err += xdpsq_red_stats->err; s->tx_xdp_cqes += xdpsq_red_stats->cqes; @@ -1250,6 +1254,7 @@ static const struct counter_desc sq_stats_desc[] = { static const struct counter_desc rq_xdpsq_stats_desc[] = { { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) }, { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, mpwqe) }, + { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, inlnw) }, { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) }, { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) }, { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) }, @@ -1258,6 +1263,7 @@ static const struct counter_desc rq_xdpsq_stats_desc[] = { static const struct counter_desc xdpsq_stats_desc[] = { { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) }, { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, mpwqe) }, + { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, inlnw) }, { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) }, { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) }, { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) }, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index 1f05ffa086b1..ac3c7c2a0964 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -78,6 +78,7 @@ struct mlx5e_sw_stats { u64 rx_xdp_redirect; u64 rx_xdp_tx_xmit; u64 rx_xdp_tx_mpwqe; + u64 rx_xdp_tx_inlnw; u64 rx_xdp_tx_full; u64 rx_xdp_tx_err; u64 rx_xdp_tx_cqe; @@ -93,6 +94,7 @@ struct mlx5e_sw_stats { u64 tx_cqe_err; u64 tx_xdp_xmit; u64 tx_xdp_mpwqe; + u64 tx_xdp_inlnw; u64 tx_xdp_full; u64 tx_xdp_err; u64 tx_xdp_cqes; @@ -244,6 +246,7 @@ struct mlx5e_sq_stats { struct mlx5e_xdpsq_stats { u64 xmit; u64 mpwqe; + u64 inlnw; u64 full; u64 err; /* dirtied @completion */ diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h index 0343c81d4c5f..3ba4edbd17a6 100644 --- a/include/linux/mlx5/qp.h +++ b/include/linux/mlx5/qp.h @@ -395,6 +395,7 @@ struct mlx5_wqe_signature_seg { struct mlx5_wqe_inline_seg { __be32 byte_count; + __be32 data[0]; }; enum mlx5_sig_type {