openkylin
/
linux
mirror of https://gitee.com/openkylin/linux.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

mlx5-updates-2017-09-03 

This series from Tariq includes micro data path optimization for mlx5e
 netdevice driver.
 
 Mainly Tariq introduces the following changes to NAPI and RX handling
 path of the driver:
  - RX ring structure reorganizing
  - Trivial code refactoring and optimization
  - NAPI busy-poll for when fast UMR is in progress
  - Non-atomic state operations in NAPI context
  - Remove unnecessary fields from fast path structures
  - page-cache micro optimization
  - Rely on NAPI to avoid missing an IRQ for RX/TX shared NAPI contexts
  - Stop NAPI when irq changes affinity
  - Distribute RSS table among all RX rings
 
 Thanks,
 Saeed.
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v1
 
 iQEcBAABAgAGBQJZq3r/AAoJEEg/ir3gV/o+ZtwIAK6LcyUxNSa+Q8a7+57EIQgj
 xwA76EG1x5MskZ30QRJpZG6VP6C2WEOtG0/WDi6yfEzZh5J0+clqVv1cHVIJIFhC
 vB+0FCl8GIlTE/VMpRFqFTZapz6/BWCWNQEW3a1raHb026cpeRzq7c+g1x4lKXx5
 RN0QhOd/G+yUz6A+xt6GCRlHsIkvFpigL90rhfQqcvg/T8QepxZ1trJiytpu2J51
 OhEtnl9mIapgj0Z9nQMMKV+BnLSaJxlJ2j5xGWa5x8zuySrGv/P26TYDsAvYp8pV
 +f9OgISoQr7d2mWyus3IYSi31F3AkbGE01K3vBQTXpyd8pdEqv5bzXXWJtQASeQ=
 =dmvv
 -----END PGP SIGNATURE-----

Merge tag 'mlx5-updates-2017-09-03' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux

Saeed Mahameed says:

====================
mlx5-updates-2017-09-03

This series from Tariq includes micro data path optimization for mlx5e
netdevice driver.

Mainly Tariq introduces the following changes to NAPI and RX handling
path of the driver:
 - RX ring structure reorganizing
 - Trivial code refactoring and optimization
 - NAPI busy-poll for when fast UMR is in progress
 - Non-atomic state operations in NAPI context
 - Remove unnecessary fields from fast path structures
 - page-cache micro optimization
 - Rely on NAPI to avoid missing an IRQ for RX/TX shared NAPI contexts
 - Stop NAPI when irq changes affinity
 - Distribute RSS table among all RX rings
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller 2017-09-03 21:17:07 -07:00
parent ccfdf21bbb d4b6c48800
commit 18a4ded9d1
8 changed files with 218 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
drivers/net/ethernet/mellanox/mlx5/core/en.h
View File

@ -291,10 +291,11 @@ struct mlx5e_tstamp {
enum { enum {
MLX5E_RQ_STATE_ENABLED, MLX5E_RQ_STATE_ENABLED,
MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS,
MLX5E_RQ_STATE_AM, MLX5E_RQ_STATE_AM,
}; };
#define MLX5E_TEST_BIT(state, nr) (state & BIT(nr))
struct mlx5e_cq { struct mlx5e_cq {
/* data path - accessed per cqe */ /* data path - accessed per cqe */
struct mlx5_cqwq wq; struct mlx5_cqwq wq;
@ -342,7 +343,6 @@ enum {
struct mlx5e_sq_wqe_info { struct mlx5e_sq_wqe_info {
u8 opcode; u8 opcode;
u8 num_wqebbs;
}; };
struct mlx5e_txqsq { struct mlx5e_txqsq {
@ -418,13 +418,8 @@ struct mlx5e_xdpsq {
struct mlx5e_icosq { struct mlx5e_icosq {
/* data path */ /* data path */
/* dirtied @completion */
u16 cc;
/* dirtied @xmit */ /* dirtied @xmit */
u16 pc ____cacheline_aligned_in_smp; u16 pc ____cacheline_aligned_in_smp;
u32 dma_fifo_pc;
u16 prev_cc;
struct mlx5e_cq cq; struct mlx5e_cq cq;
@ -438,7 +433,6 @@ struct mlx5e_icosq {
void __iomem *uar_map; void __iomem *uar_map;
u32 sqn; u32 sqn;
u16 edge; u16 edge;
struct device *pdev;
__be32 mkey_be; __be32 mkey_be;
unsigned long state; unsigned long state;
@ -507,7 +501,7 @@ struct mlx5e_rx_am { /* Adaptive Moderation */
*/ */
#define MLX5E_CACHE_UNIT (MLX5_MPWRQ_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \ #define MLX5E_CACHE_UNIT (MLX5_MPWRQ_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \
MLX5_MPWRQ_PAGES_PER_WQE : NAPI_POLL_WEIGHT) MLX5_MPWRQ_PAGES_PER_WQE : NAPI_POLL_WEIGHT)
#define MLX5E_CACHE_SIZE (2 * roundup_pow_of_two(MLX5E_CACHE_UNIT)) #define MLX5E_CACHE_SIZE (4 * roundup_pow_of_two(MLX5E_CACHE_UNIT))
struct mlx5e_page_cache { struct mlx5e_page_cache {
u32 head; u32 head;
u32 tail; u32 tail;
@ -516,7 +510,7 @@ struct mlx5e_page_cache {
struct mlx5e_rq; struct mlx5e_rq;
typedef void (*mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq*, struct mlx5_cqe64*); typedef void (*mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq*, struct mlx5_cqe64*);
typedef int (*mlx5e_fp_alloc_wqe)(struct mlx5e_rq*, struct mlx5e_rx_wqe*, u16); typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq);
typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16); typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16);
struct mlx5e_rq { struct mlx5e_rq {
@ -527,21 +521,26 @@ struct mlx5e_rq {
struct { struct {
struct mlx5e_wqe_frag_info *frag_info; struct mlx5e_wqe_frag_info *frag_info;
u32 frag_sz; /* max possible skb frag_sz */ u32 frag_sz; /* max possible skb frag_sz */
bool page_reuse; union {
bool xdp_xmit; bool page_reuse;
bool xdp_xmit;
};
} wqe; } wqe;
struct { struct {
struct mlx5e_mpw_info *info; struct mlx5e_mpw_info *info;
void *mtt_no_align; void *mtt_no_align;
u16 num_strides;
u8 log_stride_sz;
bool umr_in_progress;
} mpwqe; } mpwqe;
}; };
struct { struct {
u16 headroom;
u8 page_order; u8 page_order;
u32 wqe_sz; /* wqe data buffer size */
u8 map_dir; /* dma map direction */ u8 map_dir; /* dma map direction */
} buff; } buff;
__be32 mkey_be;
struct mlx5e_channel *channel;
struct device *pdev; struct device *pdev;
struct net_device *netdev; struct net_device *netdev;
struct mlx5e_tstamp *tstamp; struct mlx5e_tstamp *tstamp;
@ -550,12 +549,11 @@ struct mlx5e_rq {
struct mlx5e_page_cache page_cache; struct mlx5e_page_cache page_cache;
mlx5e_fp_handle_rx_cqe handle_rx_cqe; mlx5e_fp_handle_rx_cqe handle_rx_cqe;
mlx5e_fp_alloc_wqe alloc_wqe; mlx5e_fp_post_rx_wqes post_wqes;
mlx5e_fp_dealloc_wqe dealloc_wqe; mlx5e_fp_dealloc_wqe dealloc_wqe;
unsigned long state; unsigned long state;
int ix; int ix;
u16 rx_headroom;
struct mlx5e_rx_am am; /* Adaptive Moderation */ struct mlx5e_rx_am am; /* Adaptive Moderation */
@ -565,19 +563,13 @@ struct mlx5e_rq {
/* control */ /* control */
struct mlx5_wq_ctrl wq_ctrl; struct mlx5_wq_ctrl wq_ctrl;
__be32 mkey_be;
u8 wq_type; u8 wq_type;
u32 mpwqe_stride_sz;
u32 mpwqe_num_strides;
u32 rqn; u32 rqn;
struct mlx5e_channel *channel;
struct mlx5_core_dev *mdev; struct mlx5_core_dev *mdev;
struct mlx5_core_mkey umr_mkey; struct mlx5_core_mkey umr_mkey;
} ____cacheline_aligned_in_smp; } ____cacheline_aligned_in_smp;
enum channel_flags {
MLX5E_CHANNEL_NAPI_SCHED = 1,
};
struct mlx5e_channel { struct mlx5e_channel {
/* data path */ /* data path */
struct mlx5e_rq rq; struct mlx5e_rq rq;
@ -589,7 +581,9 @@ struct mlx5e_channel {
struct net_device *netdev; struct net_device *netdev;
__be32 mkey_be; __be32 mkey_be;
u8 num_tc; u8 num_tc;
unsigned long flags;
/* data path - accessed per napi poll */
struct irq_desc *irq_desc;
/* control */ /* control */
struct mlx5e_priv *priv; struct mlx5e_priv *priv;
@ -850,11 +844,9 @@ void mlx5e_page_release(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info,
void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe); void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe); void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq); bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq);
int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix); bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq);
int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix);
void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix); void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix); void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq);
void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi); void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi);
void mlx5e_rx_am(struct mlx5e_rq *rq); void mlx5e_rx_am(struct mlx5e_rq *rq);
@ -933,8 +925,7 @@ void mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
void mlx5e_activate_priv_channels(struct mlx5e_priv *priv); void mlx5e_activate_priv_channels(struct mlx5e_priv *priv);
void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv); void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv);
void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev, void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len,
u32 *indirection_rqt, int len,
int num_channels); int num_channels);
int mlx5e_get_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed); int mlx5e_get_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed);

3
drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
View File

@ -663,8 +663,7 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv,
new_channels.params = priv->channels.params; new_channels.params = priv->channels.params;
new_channels.params.num_channels = count; new_channels.params.num_channels = count;
if (!netif_is_rxfh_configured(priv->netdev)) if (!netif_is_rxfh_configured(priv->netdev))
mlx5e_build_default_indir_rqt(priv->mdev, mlx5e_build_default_indir_rqt(new_channels.params.indirection_rqt,
new_channels.params.indirection_rqt,
MLX5E_INDIR_RQT_SIZE, count); MLX5E_INDIR_RQT_SIZE, count);
if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {

55
drivers/net/ethernet/mellanox/mlx5/core/en_main.c
View File

@ -208,6 +208,7 @@ static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
s->rx_cache_full += rq_stats->cache_full; s->rx_cache_full += rq_stats->cache_full;
s->rx_cache_empty += rq_stats->cache_empty; s->rx_cache_empty += rq_stats->cache_empty;
s->rx_cache_busy += rq_stats->cache_busy; s->rx_cache_busy += rq_stats->cache_busy;
s->rx_cache_waive += rq_stats->cache_waive;
for (j = 0; j < priv->channels.params.num_tc; j++) { for (j = 0; j < priv->channels.params.num_tc; j++) {
sq_stats = &c->sq[j].stats; sq_stats = &c->sq[j].stats;
@ -593,12 +594,12 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
} }
rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
rq->rx_headroom = params->rq_headroom; rq->buff.headroom = params->rq_headroom;
switch (rq->wq_type) { switch (rq->wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
rq->alloc_wqe = mlx5e_alloc_rx_mpwqe; rq->post_wqes = mlx5e_post_rx_mpwqes;
rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe; rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe_mpwqe; rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe_mpwqe;
@ -615,11 +616,10 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
goto err_rq_wq_destroy; goto err_rq_wq_destroy;
} }
rq->mpwqe_stride_sz = BIT(params->mpwqe_log_stride_sz); rq->mpwqe.log_stride_sz = params->mpwqe_log_stride_sz;
rq->mpwqe_num_strides = BIT(params->mpwqe_log_num_strides); rq->mpwqe.num_strides = BIT(params->mpwqe_log_num_strides);
rq->buff.wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides; byte_count = rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;
byte_count = rq->buff.wqe_sz;
err = mlx5e_create_rq_umr_mkey(mdev, rq); err = mlx5e_create_rq_umr_mkey(mdev, rq);
if (err) if (err)
@ -638,7 +638,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
err = -ENOMEM; err = -ENOMEM;
goto err_rq_wq_destroy; goto err_rq_wq_destroy;
} }
rq->alloc_wqe = mlx5e_alloc_rx_wqe; rq->post_wqes = mlx5e_post_rx_wqes;
rq->dealloc_wqe = mlx5e_dealloc_rx_wqe; rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
#ifdef CONFIG_MLX5_EN_IPSEC #ifdef CONFIG_MLX5_EN_IPSEC
@ -654,18 +654,17 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
goto err_rq_wq_destroy; goto err_rq_wq_destroy;
} }
rq->buff.wqe_sz = params->lro_en ? byte_count = params->lro_en ?
params->lro_wqe_sz : params->lro_wqe_sz :
MLX5E_SW2HW_MTU(c->priv, c->netdev->mtu); MLX5E_SW2HW_MTU(c->priv, c->netdev->mtu);
#ifdef CONFIG_MLX5_EN_IPSEC #ifdef CONFIG_MLX5_EN_IPSEC
if (MLX5_IPSEC_DEV(mdev)) if (MLX5_IPSEC_DEV(mdev))
rq->buff.wqe_sz += MLX5E_METADATA_ETHER_LEN; byte_count += MLX5E_METADATA_ETHER_LEN;
#endif #endif
rq->wqe.page_reuse = !params->xdp_prog && !params->lro_en; rq->wqe.page_reuse = !params->xdp_prog && !params->lro_en;
byte_count = rq->buff.wqe_sz;
/* calc the required page order */ /* calc the required page order */
rq->wqe.frag_sz = MLX5_SKB_FRAG_SZ(rq->rx_headroom + byte_count); rq->wqe.frag_sz = MLX5_SKB_FRAG_SZ(rq->buff.headroom + byte_count);
npages = DIV_ROUND_UP(rq->wqe.frag_sz, PAGE_SIZE); npages = DIV_ROUND_UP(rq->wqe.frag_sz, PAGE_SIZE);
rq->buff.page_order = order_base_2(npages); rq->buff.page_order = order_base_2(npages);
@ -676,6 +675,12 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
for (i = 0; i < wq_sz; i++) { for (i = 0; i < wq_sz; i++) {
struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i); struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
u64 dma_offset = (u64)mlx5e_get_wqe_mtt_offset(rq, i) << PAGE_SHIFT;
wqe->data.addr = cpu_to_be64(dma_offset);
}
wqe->data.byte_count = cpu_to_be32(byte_count); wqe->data.byte_count = cpu_to_be32(byte_count);
wqe->data.lkey = rq->mkey_be; wqe->data.lkey = rq->mkey_be;
} }
@ -882,7 +887,8 @@ static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
u16 wqe_ix; u16 wqe_ix;
/* UMR WQE (if in progress) is always at wq->head */ /* UMR WQE (if in progress) is always at wq->head */
if (test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state)) if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
rq->mpwqe.umr_in_progress)
mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]); mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
while (!mlx5_wq_ll_is_empty(wq)) { while (!mlx5_wq_ll_is_empty(wq)) {
@ -925,7 +931,7 @@ static int mlx5e_open_rq(struct mlx5e_channel *c,
goto err_destroy_rq; goto err_destroy_rq;
if (params->rx_am_enabled) if (params->rx_am_enabled)
set_bit(MLX5E_RQ_STATE_AM, &c->rq.state); c->rq.state |= BIT(MLX5E_RQ_STATE_AM);
return 0; return 0;
@ -945,7 +951,6 @@ static void mlx5e_activate_rq(struct mlx5e_rq *rq)
set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP; sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
sq->db.ico_wqe[pi].num_wqebbs = 1;
nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc); nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);
mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &nopwqe->ctrl); mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
} }
@ -1047,7 +1052,6 @@ static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
struct mlx5_core_dev *mdev = c->mdev; struct mlx5_core_dev *mdev = c->mdev;
int err; int err;
sq->pdev = c->pdev;
sq->mkey_be = c->mkey_be; sq->mkey_be = c->mkey_be;
sq->channel = c; sq->channel = c;
sq->uar_map = mdev->mlx5e_res.bfreg.map; sq->uar_map = mdev->mlx5e_res.bfreg.map;
@ -1757,7 +1761,9 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
struct net_device *netdev = priv->netdev; struct net_device *netdev = priv->netdev;
int cpu = mlx5e_get_cpu(priv, ix); int cpu = mlx5e_get_cpu(priv, ix);
struct mlx5e_channel *c; struct mlx5e_channel *c;
unsigned int irq;
int err; int err;
int eqn;
c = kzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu)); c = kzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
if (!c) if (!c)
@ -1774,6 +1780,9 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
c->num_tc = params->num_tc; c->num_tc = params->num_tc;
c->xdp = !!params->xdp_prog; c->xdp = !!params->xdp_prog;
mlx5_vector2eqn(priv->mdev, ix, &eqn, &irq);
c->irq_desc = irq_to_desc(irq);
netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64); netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq); err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq);
@ -3693,7 +3702,6 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
set_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state); set_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
/* napi_schedule in case we have missed anything */ /* napi_schedule in case we have missed anything */
set_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags);
napi_schedule(&c->napi); napi_schedule(&c->napi);
if (old_prog) if (old_prog)
@ -3825,22 +3833,11 @@ u16 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/; 2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/;
} }
void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev, void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len,
u32 *indirection_rqt, int len,
int num_channels) int num_channels)
{ {
int node = mdev->priv.numa_node;
int node_num_of_cores;
int i; int i;
if (node == -1)
node = first_online_node;
node_num_of_cores = cpumask_weight(cpumask_of_node(node));
if (node_num_of_cores)
num_channels = min_t(int, num_channels, node_num_of_cores);
for (i = 0; i < len; i++) for (i = 0; i < len; i++)
indirection_rqt[i] = i % num_channels; indirection_rqt[i] = i % num_channels;
} }
@ -3979,7 +3976,7 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
/* RSS */ /* RSS */
params->rss_hfunc = ETH_RSS_HASH_XOR; params->rss_hfunc = ETH_RSS_HASH_XOR;
netdev_rss_key_fill(params->toeplitz_hash_key, sizeof(params->toeplitz_hash_key)); netdev_rss_key_fill(params->toeplitz_hash_key, sizeof(params->toeplitz_hash_key));
mlx5e_build_default_indir_rqt(mdev, params->indirection_rqt, mlx5e_build_default_indir_rqt(params->indirection_rqt,
MLX5E_INDIR_RQT_SIZE, max_channels); MLX5E_INDIR_RQT_SIZE, max_channels);
} }

227
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
View File

@ -163,7 +163,7 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
static inline bool mlx5e_page_is_reserved(struct page *page) static inline bool mlx5e_page_is_reserved(struct page *page)
{ {
return page_is_pfmemalloc(page) || page_to_nid(page) != numa_node_id(); return page_is_pfmemalloc(page) || page_to_nid(page) != numa_mem_id();
} }
static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq, static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
@ -177,8 +177,10 @@ static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
return false; return false;
} }
if (unlikely(page_is_pfmemalloc(dma_info->page))) if (unlikely(mlx5e_page_is_reserved(dma_info->page))) {
rq->stats.cache_waive++;
return false; return false;
}
cache->page_cache[cache->tail] = *dma_info; cache->page_cache[cache->tail] = *dma_info;
cache->tail = tail_next; cache->tail = tail_next;
@ -252,7 +254,7 @@ static inline bool mlx5e_page_reuse(struct mlx5e_rq *rq,
!mlx5e_page_is_reserved(wi->di.page); !mlx5e_page_is_reserved(wi->di.page);
} }
int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix) static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix)
{ {
struct mlx5e_wqe_frag_info *wi = &rq->wqe.frag_info[ix]; struct mlx5e_wqe_frag_info *wi = &rq->wqe.frag_info[ix];
@ -263,8 +265,7 @@ int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix)
wi->offset = 0; wi->offset = 0;
} }
wqe->data.addr = cpu_to_be64(wi->di.addr + wi->offset + wqe->data.addr = cpu_to_be64(wi->di.addr + wi->offset + rq->buff.headroom);
rq->rx_headroom);
return 0; return 0;
} }
@ -296,7 +297,7 @@ void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
static inline int mlx5e_mpwqe_strides_per_page(struct mlx5e_rq *rq) static inline int mlx5e_mpwqe_strides_per_page(struct mlx5e_rq *rq)
{ {
return rq->mpwqe_num_strides >> MLX5_MPWRQ_WQE_PAGE_ORDER; return rq->mpwqe.num_strides >> MLX5_MPWRQ_WQE_PAGE_ORDER;
} }
static inline void mlx5e_add_skb_frag_mpwqe(struct mlx5e_rq *rq, static inline void mlx5e_add_skb_frag_mpwqe(struct mlx5e_rq *rq,
@ -305,7 +306,7 @@ static inline void mlx5e_add_skb_frag_mpwqe(struct mlx5e_rq *rq,
u32 page_idx, u32 frag_offset, u32 page_idx, u32 frag_offset,
u32 len) u32 len)
{ {
unsigned int truesize = ALIGN(len, rq->mpwqe_stride_sz); unsigned int truesize = ALIGN(len, BIT(rq->mpwqe.log_stride_sz));
dma_sync_single_for_cpu(rq->pdev, dma_sync_single_for_cpu(rq->pdev,
wi->umr.dma_info[page_idx].addr + frag_offset, wi->umr.dma_info[page_idx].addr + frag_offset,
@ -358,7 +359,6 @@ static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
/* fill sq edge with nops to avoid wqe wrap around */ /* fill sq edge with nops to avoid wqe wrap around */
while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) { while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) {
sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP; sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
sq->db.ico_wqe[pi].num_wqebbs = 1;
mlx5e_post_nop(wq, sq->sqn, &sq->pc); mlx5e_post_nop(wq, sq->sqn, &sq->pc);
} }
@ -369,41 +369,35 @@ static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
MLX5_OPCODE_UMR); MLX5_OPCODE_UMR);
sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR; sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR;
sq->db.ico_wqe[pi].num_wqebbs = num_wqebbs;
sq->pc += num_wqebbs; sq->pc += num_wqebbs;
mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &wqe->ctrl); mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &wqe->ctrl);
} }
static int mlx5e_alloc_rx_umr_mpwqe(struct mlx5e_rq *rq, static int mlx5e_alloc_rx_umr_mpwqe(struct mlx5e_rq *rq,
struct mlx5e_rx_wqe *wqe,
u16 ix) u16 ix)
{ {
struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix]; struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
u64 dma_offset = (u64)mlx5e_get_wqe_mtt_offset(rq, ix) << PAGE_SHIFT;
int pg_strides = mlx5e_mpwqe_strides_per_page(rq); int pg_strides = mlx5e_mpwqe_strides_per_page(rq);
struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
int err; int err;
int i; int i;
for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) { for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) {
struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[i];
err = mlx5e_page_alloc_mapped(rq, dma_info); err = mlx5e_page_alloc_mapped(rq, dma_info);
if (unlikely(err)) if (unlikely(err))
goto err_unmap; goto err_unmap;
wi->umr.mtt[i] = cpu_to_be64(dma_info->addr | MLX5_EN_WR); wi->umr.mtt[i] = cpu_to_be64(dma_info->addr | MLX5_EN_WR);
page_ref_add(dma_info->page, pg_strides); page_ref_add(dma_info->page, pg_strides);
wi->skbs_frags[i] = 0;
} }
memset(wi->skbs_frags, 0, sizeof(*wi->skbs_frags) * MLX5_MPWRQ_PAGES_PER_WQE);
wi->consumed_strides = 0; wi->consumed_strides = 0;
wqe->data.addr = cpu_to_be64(dma_offset);
return 0; return 0;
err_unmap: err_unmap:
while (--i >= 0) { while (--i >= 0) {
struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[i]; dma_info--;
page_ref_sub(dma_info->page, pg_strides); page_ref_sub(dma_info->page, pg_strides);
mlx5e_page_release(rq, dma_info, true); mlx5e_page_release(rq, dma_info, true);
} }
@ -414,27 +408,21 @@ static int mlx5e_alloc_rx_umr_mpwqe(struct mlx5e_rq *rq,
void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi) void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi)
{ {
int pg_strides = mlx5e_mpwqe_strides_per_page(rq); int pg_strides = mlx5e_mpwqe_strides_per_page(rq);
struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
int i; int i;
for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) { for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) {
struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[i];
page_ref_sub(dma_info->page, pg_strides - wi->skbs_frags[i]); page_ref_sub(dma_info->page, pg_strides - wi->skbs_frags[i]);
mlx5e_page_release(rq, dma_info, true); mlx5e_page_release(rq, dma_info, true);
} }
} }
void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq) static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
{ {
struct mlx5_wq_ll *wq = &rq->wq; struct mlx5_wq_ll *wq = &rq->wq;
struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(wq, wq->head); struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(wq, wq->head);
clear_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state); rq->mpwqe.umr_in_progress = false;
if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state))) {
mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
return;
}
mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index)); mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index));
@ -444,16 +432,18 @@ void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
mlx5_wq_ll_update_db_record(wq); mlx5_wq_ll_update_db_record(wq);
} }
int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix) static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
{ {
int err; int err;
err = mlx5e_alloc_rx_umr_mpwqe(rq, wqe, ix); err = mlx5e_alloc_rx_umr_mpwqe(rq, ix);
if (unlikely(err)) if (unlikely(err)) {
rq->stats.buff_alloc_err++;
return err; return err;
set_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state); }
rq->mpwqe.umr_in_progress = true;
mlx5e_post_umr_wqe(rq, ix); mlx5e_post_umr_wqe(rq, ix);
return -EBUSY; return 0;
} }
void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
@ -463,94 +453,150 @@ void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
mlx5e_free_rx_mpwqe(rq, wi); mlx5e_free_rx_mpwqe(rq, wi);
} }
#define RQ_CANNOT_POST(rq) \
(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state) || \
test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
{ {
struct mlx5_wq_ll *wq = &rq->wq; struct mlx5_wq_ll *wq = &rq->wq;
int err;
if (unlikely(RQ_CANNOT_POST(rq))) if (unlikely(!MLX5E_TEST_BIT(rq->state, MLX5E_RQ_STATE_ENABLED)))
return false; return false;
while (!mlx5_wq_ll_is_full(wq)) { if (mlx5_wq_ll_is_full(wq))
struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(wq, wq->head); return false;
int err;
err = rq->alloc_wqe(rq, wqe, wq->head); do {
if (err == -EBUSY) struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(wq, wq->head);
return true;
err = mlx5e_alloc_rx_wqe(rq, wqe, wq->head);
if (unlikely(err)) { if (unlikely(err)) {
rq->stats.buff_alloc_err++; rq->stats.buff_alloc_err++;
break; break;
} }
mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index)); mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index));
} } while (!mlx5_wq_ll_is_full(wq));
/* ensure wqes are visible to device before updating doorbell record */ /* ensure wqes are visible to device before updating doorbell record */
dma_wmb(); dma_wmb();
mlx5_wq_ll_update_db_record(wq); mlx5_wq_ll_update_db_record(wq);
return !mlx5_wq_ll_is_full(wq); return !!err;
}
static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq *cq,
struct mlx5e_icosq *sq,
struct mlx5e_rq *rq,
struct mlx5_cqe64 *cqe)
{
struct mlx5_wq_cyc *wq = &sq->wq;
u16 ci = be16_to_cpu(cqe->wqe_counter) & wq->sz_m1;
struct mlx5e_sq_wqe_info *icowi = &sq->db.ico_wqe[ci];
mlx5_cqwq_pop(&cq->wq);
if (unlikely((cqe->op_own >> 4) != MLX5_CQE_REQ)) {
WARN_ONCE(true, "mlx5e: Bad OP in ICOSQ CQE: 0x%x\n",
cqe->op_own);
return;
}
if (likely(icowi->opcode == MLX5_OPCODE_UMR)) {
mlx5e_post_rx_mpwqe(rq);
return;
}
if (unlikely(icowi->opcode != MLX5_OPCODE_NOP))
WARN_ONCE(true,
"mlx5e: Bad OPCODE in ICOSQ WQE info: 0x%x\n",
icowi->opcode);
}
static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq)
{
struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq);
struct mlx5_cqe64 *cqe;
if (unlikely(!MLX5E_TEST_BIT(sq->state, MLX5E_SQ_STATE_ENABLED)))
return;
cqe = mlx5_cqwq_get_cqe(&cq->wq);
if (likely(!cqe))
return;
/* by design, there's only a single cqe */
mlx5e_poll_ico_single_cqe(cq, sq, rq, cqe);
mlx5_cqwq_update_db_record(&cq->wq);
}
bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
{
struct mlx5_wq_ll *wq = &rq->wq;
if (unlikely(!MLX5E_TEST_BIT(rq->state, MLX5E_RQ_STATE_ENABLED)))
return false;
mlx5e_poll_ico_cq(&rq->channel->icosq.cq, rq);
if (mlx5_wq_ll_is_full(wq))
return false;
if (!rq->mpwqe.umr_in_progress)
mlx5e_alloc_rx_mpwqe(rq, wq->head);
return true;
} }
static void mlx5e_lro_update_hdr(struct sk_buff *skb, struct mlx5_cqe64 *cqe, static void mlx5e_lro_update_hdr(struct sk_buff *skb, struct mlx5_cqe64 *cqe,
u32 cqe_bcnt) u32 cqe_bcnt)
{ {
struct ethhdr *eth = (struct ethhdr *)(skb->data); struct ethhdr *eth = (struct ethhdr *)(skb->data);
struct iphdr *ipv4;
struct ipv6hdr *ipv6;
struct tcphdr *tcp; struct tcphdr *tcp;
int network_depth = 0; int network_depth = 0;
__be16 proto; __be16 proto;
u16 tot_len; u16 tot_len;
void *ip_p;
u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe); u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe);
int tcp_ack = ((l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA) || u8 tcp_ack = (l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA) ||
(l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA)); (l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA);
skb->mac_len = ETH_HLEN; skb->mac_len = ETH_HLEN;
proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth); proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth);
ipv4 = (struct iphdr *)(skb->data + network_depth);
ipv6 = (struct ipv6hdr *)(skb->data + network_depth);
tot_len = cqe_bcnt - network_depth; tot_len = cqe_bcnt - network_depth;
ip_p = skb->data + network_depth;
if (proto == htons(ETH_P_IP)) { if (proto == htons(ETH_P_IP)) {
tcp = (struct tcphdr *)(skb->data + network_depth + struct iphdr *ipv4 = ip_p;
sizeof(struct iphdr));
ipv6 = NULL; tcp = ip_p + sizeof(struct iphdr);
skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
} else {
tcp = (struct tcphdr *)(skb->data + network_depth +
sizeof(struct ipv6hdr));
ipv4 = NULL;
skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
}
if (get_cqe_lro_tcppsh(cqe))
tcp->psh = 1;
if (tcp_ack) {
tcp->ack = 1;
tcp->ack_seq = cqe->lro_ack_seq_num;
tcp->window = cqe->lro_tcp_win;
}
if (ipv4) {
ipv4->ttl = cqe->lro_min_ttl; ipv4->ttl = cqe->lro_min_ttl;
ipv4->tot_len = cpu_to_be16(tot_len); ipv4->tot_len = cpu_to_be16(tot_len);
ipv4->check = 0; ipv4->check = 0;
ipv4->check = ip_fast_csum((unsigned char *)ipv4, ipv4->check = ip_fast_csum((unsigned char *)ipv4,
ipv4->ihl); ipv4->ihl);
} else { } else {
struct ipv6hdr *ipv6 = ip_p;
tcp = ip_p + sizeof(struct ipv6hdr);
skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
ipv6->hop_limit = cqe->lro_min_ttl; ipv6->hop_limit = cqe->lro_min_ttl;
ipv6->payload_len = cpu_to_be16(tot_len - ipv6->payload_len = cpu_to_be16(tot_len -
sizeof(struct ipv6hdr)); sizeof(struct ipv6hdr));
} }
tcp->psh = get_cqe_lro_tcppsh(cqe);
if (tcp_ack) {
tcp->ack = 1;
tcp->ack_seq = cqe->lro_ack_seq_num;
tcp->window = cqe->lro_tcp_win;
}
} }
static inline void mlx5e_skb_set_hash(struct mlx5_cqe64 *cqe, static inline void mlx5e_skb_set_hash(struct mlx5_cqe64 *cqe,
@ -776,9 +822,9 @@ struct sk_buff *skb_from_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt) struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
{ {
struct mlx5e_dma_info *di = &wi->di; struct mlx5e_dma_info *di = &wi->di;
u16 rx_headroom = rq->buff.headroom;
struct sk_buff *skb; struct sk_buff *skb;
void *va, *data; void *va, *data;
u16 rx_headroom = rq->rx_headroom;
bool consumed; bool consumed;
u32 frag_size; u32 frag_size;
@ -911,7 +957,7 @@ static inline void mlx5e_mpwqe_fill_rx_skb(struct mlx5e_rq *rq,
struct sk_buff *skb) struct sk_buff *skb)
{ {
u16 stride_ix = mpwrq_get_cqe_stride_index(cqe); u16 stride_ix = mpwrq_get_cqe_stride_index(cqe);
u32 wqe_offset = stride_ix * rq->mpwqe_stride_sz; u32 wqe_offset = stride_ix << rq->mpwqe.log_stride_sz;
u32 head_offset = wqe_offset & (PAGE_SIZE - 1); u32 head_offset = wqe_offset & (PAGE_SIZE - 1);
u32 page_idx = wqe_offset >> PAGE_SHIFT; u32 page_idx = wqe_offset >> PAGE_SHIFT;
u32 head_page_idx = page_idx; u32 head_page_idx = page_idx;
@ -979,7 +1025,7 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
napi_gro_receive(rq->cq.napi, skb); napi_gro_receive(rq->cq.napi, skb);
mpwrq_cqe_out: mpwrq_cqe_out:
if (likely(wi->consumed_strides < rq->mpwqe_num_strides)) if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
return; return;
mlx5e_free_rx_mpwqe(rq, wi); mlx5e_free_rx_mpwqe(rq, wi);
@ -989,21 +1035,23 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget) int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
{ {
struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq); struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq);
struct mlx5e_xdpsq *xdpsq = &rq->xdpsq; struct mlx5e_xdpsq *xdpsq;
struct mlx5_cqe64 *cqe;
int work_done = 0; int work_done = 0;
if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state))) if (unlikely(!MLX5E_TEST_BIT(rq->state, MLX5E_RQ_STATE_ENABLED)))
return 0; return 0;
if (cq->decmprs_left) if (cq->decmprs_left)
work_done += mlx5e_decompress_cqes_cont(rq, cq, 0, budget); work_done += mlx5e_decompress_cqes_cont(rq, cq, 0, budget);
for (; work_done < budget; work_done++) { cqe = mlx5_cqwq_get_cqe(&cq->wq);
struct mlx5_cqe64 *cqe = mlx5_cqwq_get_cqe(&cq->wq); if (!cqe)
return 0;
if (!cqe) xdpsq = &rq->xdpsq;
break;
do {
if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) { if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) {
work_done += work_done +=
mlx5e_decompress_cqes_start(rq, cq, mlx5e_decompress_cqes_start(rq, cq,
@ -1014,7 +1062,7 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
mlx5_cqwq_pop(&cq->wq); mlx5_cqwq_pop(&cq->wq);
rq->handle_rx_cqe(rq, cqe); rq->handle_rx_cqe(rq, cqe);
} } while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
if (xdpsq->db.doorbell) { if (xdpsq->db.doorbell) {
mlx5e_xmit_xdp_doorbell(xdpsq); mlx5e_xmit_xdp_doorbell(xdpsq);
@ -1032,13 +1080,18 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq) bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
{ {
struct mlx5e_xdpsq *sq; struct mlx5e_xdpsq *sq;
struct mlx5_cqe64 *cqe;
struct mlx5e_rq *rq; struct mlx5e_rq *rq;
u16 sqcc; u16 sqcc;
int i; int i;
sq = container_of(cq, struct mlx5e_xdpsq, cq); sq = container_of(cq, struct mlx5e_xdpsq, cq);
if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))) if (unlikely(!MLX5E_TEST_BIT(sq->state, MLX5E_SQ_STATE_ENABLED)))
return false;
cqe = mlx5_cqwq_get_cqe(&cq->wq);
if (!cqe)
return false; return false;
rq = container_of(sq, struct mlx5e_rq, xdpsq); rq = container_of(sq, struct mlx5e_rq, xdpsq);
@ -1048,15 +1101,11 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
*/ */
sqcc = sq->cc; sqcc = sq->cc;
for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) { i = 0;
struct mlx5_cqe64 *cqe; do {
u16 wqe_counter; u16 wqe_counter;
bool last_wqe; bool last_wqe;
cqe = mlx5_cqwq_get_cqe(&cq->wq);
if (!cqe)
break;
mlx5_cqwq_pop(&cq->wq); mlx5_cqwq_pop(&cq->wq);
wqe_counter = be16_to_cpu(cqe->wqe_counter); wqe_counter = be16_to_cpu(cqe->wqe_counter);
@ -1074,7 +1123,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
/* Recycle RX page */ /* Recycle RX page */
mlx5e_page_release(rq, di, true); mlx5e_page_release(rq, di, true);
} while (!last_wqe); } while (!last_wqe);
} } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
mlx5_cqwq_update_db_record(&cq->wq); mlx5_cqwq_update_db_record(&cq->wq);

4
drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
View File

@ -84,6 +84,7 @@ struct mlx5e_sw_stats {
u64 rx_cache_full; u64 rx_cache_full;
u64 rx_cache_empty; u64 rx_cache_empty;
u64 rx_cache_busy; u64 rx_cache_busy;
u64 rx_cache_waive;
/* Special handling counters */ /* Special handling counters */
u64 link_down_events_phy; u64 link_down_events_phy;
@ -123,6 +124,7 @@ static const struct counter_desc sw_stats_desc[] = {
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_full) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_full) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_busy) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_busy) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_waive) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, link_down_events_phy) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, link_down_events_phy) },
}; };
@ -354,6 +356,7 @@ struct mlx5e_rq_stats {
u64 cache_full; u64 cache_full;
u64 cache_empty; u64 cache_empty;
u64 cache_busy; u64 cache_busy;
u64 cache_waive;
}; };
static const struct counter_desc rq_stats_desc[] = { static const struct counter_desc rq_stats_desc[] = {
@ -377,6 +380,7 @@ static const struct counter_desc rq_stats_desc[] = {
{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_full) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_full) },
{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_empty) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_empty) },
{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_busy) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_busy) },
{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_waive) },
}; };
struct mlx5e_sq_stats { struct mlx5e_sq_stats {

18
drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
View File

@ -394,6 +394,7 @@ netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev)
bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget) bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
{ {
struct mlx5e_txqsq *sq; struct mlx5e_txqsq *sq;
struct mlx5_cqe64 *cqe;
u32 dma_fifo_cc; u32 dma_fifo_cc;
u32 nbytes; u32 nbytes;
u16 npkts; u16 npkts;
@ -402,7 +403,11 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
sq = container_of(cq, struct mlx5e_txqsq, cq); sq = container_of(cq, struct mlx5e_txqsq, cq);
if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))) if (unlikely(!MLX5E_TEST_BIT(sq->state, MLX5E_SQ_STATE_ENABLED)))
return false;
cqe = mlx5_cqwq_get_cqe(&cq->wq);
if (!cqe)
return false; return false;
npkts = 0; npkts = 0;
@ -416,15 +421,11 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
/* avoid dirtying sq cache line every cqe */ /* avoid dirtying sq cache line every cqe */
dma_fifo_cc = sq->dma_fifo_cc; dma_fifo_cc = sq->dma_fifo_cc;
for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) { i = 0;
struct mlx5_cqe64 *cqe; do {
u16 wqe_counter; u16 wqe_counter;
bool last_wqe; bool last_wqe;
cqe = mlx5_cqwq_get_cqe(&cq->wq);
if (!cqe)
break;
mlx5_cqwq_pop(&cq->wq); mlx5_cqwq_pop(&cq->wq);
wqe_counter = be16_to_cpu(cqe->wqe_counter); wqe_counter = be16_to_cpu(cqe->wqe_counter);
@ -467,7 +468,8 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
sqcc += wi->num_wqebbs; sqcc += wi->num_wqebbs;
napi_consume_skb(skb, napi_budget); napi_consume_skb(skb, napi_budget);
} while (!last_wqe); } while (!last_wqe);
}
} while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
mlx5_cqwq_update_db_record(&cq->wq); mlx5_cqwq_update_db_record(&cq->wq);

90
drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
View File

@ -30,66 +30,18 @@
* SOFTWARE. * SOFTWARE.
*/ */
#include <linux/irq.h>
#include "en.h" #include "en.h"
static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq *cq, static inline bool mlx5e_channel_no_affinity_change(struct mlx5e_channel *c)
struct mlx5e_icosq *sq,
struct mlx5_cqe64 *cqe,
u16 *sqcc)
{ {
struct mlx5_wq_cyc *wq = &sq->wq; int current_cpu = smp_processor_id();
u16 ci = be16_to_cpu(cqe->wqe_counter) & wq->sz_m1; const struct cpumask *aff;
struct mlx5e_sq_wqe_info *icowi = &sq->db.ico_wqe[ci]; struct irq_data *idata;
struct mlx5e_rq *rq = &sq->channel->rq;
prefetch(rq); idata = irq_desc_get_irq_data(c->irq_desc);
mlx5_cqwq_pop(&cq->wq); aff = irq_data_get_affinity_mask(idata);
*sqcc += icowi->num_wqebbs; return cpumask_test_cpu(current_cpu, aff);
if (unlikely((cqe->op_own >> 4) != MLX5_CQE_REQ)) {
WARN_ONCE(true, "mlx5e: Bad OP in ICOSQ CQE: 0x%x\n",
cqe->op_own);
return;
}
if (likely(icowi->opcode == MLX5_OPCODE_UMR)) {
mlx5e_post_rx_mpwqe(rq);
return;
}
if (unlikely(icowi->opcode != MLX5_OPCODE_NOP))
WARN_ONCE(true,
"mlx5e: Bad OPCODE in ICOSQ WQE info: 0x%x\n",
icowi->opcode);
}
static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq)
{
struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq);
struct mlx5_cqe64 *cqe;
u16 sqcc;
if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
return;
cqe = mlx5_cqwq_get_cqe(&cq->wq);
if (likely(!cqe))
return;
/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
* otherwise a cq overrun may occur
*/
sqcc = sq->cc;
/* by design, there's only a single cqe */
mlx5e_poll_ico_single_cqe(cq, sq, cqe, &sqcc);
mlx5_cqwq_update_db_record(&cq->wq);
/* ensure cq space is freed before enabling more cqes */
wmb();
sq->cc = sqcc;
} }
int mlx5e_napi_poll(struct napi_struct *napi, int budget) int mlx5e_napi_poll(struct napi_struct *napi, int budget)
@ -100,8 +52,6 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
int work_done; int work_done;
int i; int i;
clear_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags);
for (i = 0; i < c->num_tc; i++) for (i = 0; i < c->num_tc; i++)
busy |= mlx5e_poll_tx_cq(&c->sq[i].cq, budget); busy |= mlx5e_poll_tx_cq(&c->sq[i].cq, budget);
@ -111,25 +61,22 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
work_done = mlx5e_poll_rx_cq(&c->rq.cq, budget); work_done = mlx5e_poll_rx_cq(&c->rq.cq, budget);
busy |= work_done == budget; busy |= work_done == budget;
mlx5e_poll_ico_cq(&c->icosq.cq); busy |= c->rq.post_wqes(&c->rq);
busy |= mlx5e_post_rx_wqes(&c->rq); if (busy) {
if (likely(mlx5e_channel_no_affinity_change(c)))
if (busy) return budget;
return budget; if (work_done == budget)
work_done--;
napi_complete_done(napi, work_done);
/* avoid losing completion event during/after polling cqs */
if (test_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags)) {
napi_schedule(napi);
return work_done;
} }
if (unlikely(!napi_complete_done(napi, work_done)))
return work_done;
for (i = 0; i < c->num_tc; i++) for (i = 0; i < c->num_tc; i++)
mlx5e_cq_arm(&c->sq[i].cq); mlx5e_cq_arm(&c->sq[i].cq);
if (test_bit(MLX5E_RQ_STATE_AM, &c->rq.state)) if (MLX5E_TEST_BIT(c->rq.state, MLX5E_RQ_STATE_AM))
mlx5e_rx_am(&c->rq); mlx5e_rx_am(&c->rq);
mlx5e_cq_arm(&c->rq.cq); mlx5e_cq_arm(&c->rq.cq);
@ -143,7 +90,6 @@ void mlx5e_completion_event(struct mlx5_core_cq *mcq)
struct mlx5e_cq *cq = container_of(mcq, struct mlx5e_cq, mcq); struct mlx5e_cq *cq = container_of(mcq, struct mlx5e_cq, mcq);
cq->event_ctr++; cq->event_ctr++;
set_bit(MLX5E_CHANNEL_NAPI_SCHED, &cq->channel->flags);
napi_schedule(cq->napi); napi_schedule(cq->napi);
} }

2
include/linux/mlx5/device.h
View File

@ -709,7 +709,7 @@ static inline int mlx5_get_cqe_format(struct mlx5_cqe64 *cqe)
return (cqe->op_own >> 2) & 0x3; return (cqe->op_own >> 2) & 0x3;
} }
static inline int get_cqe_lro_tcppsh(struct mlx5_cqe64 *cqe) static inline u8 get_cqe_lro_tcppsh(struct mlx5_cqe64 *cqe)
{ {
return (cqe->lro_tcppsh_abort_dupack >> 6) & 1; return (cqe->lro_tcppsh_abort_dupack >> 6) & 1;
} }