openkylin
/
linux
mirror of https://gitee.com/openkylin/linux.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

mlx5-fixes-2019-10-18 

-----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCAAdFiEEGhZs6bAKwk/OTgTpSD+KveBX+j4FAl2qDnsACgkQSD+KveBX
 +j60KQgAo4bJcC3e0p38gZvt6GMt5Li0+BK1OnhIf/hW9DsEJPzqNtw4g7hqXCU+
 cH0YqB3q26Gq6Qg9Akuza/rY8Vfmpl1kZzfqg5/7REqC1jJBwBeIh8lOpwQh4QEh
 m//hxSq/r9M59J1SdQzUu8H7xj1+aD0xrJJ3LCL2uVwe1pk7ho8sOAwjCwvidJaJ
 fb56sBrkWjcL/k/Zxtic2e9Y3awfLWwN0QifPOWc/tehaMCwUpc04/o4yAyjyOy5
 jY5pioEYZ6BBlLHNVZO2/kjXPIQQQt9n6H+60uVud9DWDhGwQMI2b1/3LJha6q1t
 Hg2/wiHIcsoe5TZQoI5HiQNTIw6bVw==
 =jBeW
 -----END PGP SIGNATURE-----

Merge tag 'mlx5-fixes-2019-10-18' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux

Saeed Mahameed says:

====================
Mellanox, mlx5 kTLS fixes 18-10-2019

This series introduces kTLS related fixes to mlx5 driver from Tariq,
and two misc memory leak fixes form Navid Emamdoost.

Please pull and let me know if there is any problem.

I would appreciate it if you queue up kTLS fixes from the list below to
stable kernel v5.3 !

For -stable v4.13:
  nett/mlx5: prevent memory leak in mlx5_fpga_conn_create_cq
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller 2019-10-21 09:15:58 -07:00
parent 531e93d114 c7ed6d0183
commit 40c5b2bd24
11 changed files with 202 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
drivers/net/ethernet/mellanox/mlx5/core/en.h
View File

@ -345,7 +345,7 @@ struct mlx5e_tx_wqe_info {
u8 num_wqebbs;
u8 num_dma;
#ifdef CONFIG_MLX5_EN_TLS
skb_frag_t *resync_dump_frag;
struct page *resync_dump_frag_page;
#endif
};
@ -410,6 +410,7 @@ struct mlx5e_txqsq {
struct device *pdev;
__be32 mkey_be;
unsigned long state;
unsigned int hw_mtu;
struct hwtstamp_config *tstamp;
struct mlx5_clock *clock;

13
drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
View File

@ -15,15 +15,14 @@
#else
/* TLS offload requires additional stop_room for:
* - a resync SKB.
* kTLS offload requires additional stop_room for:
* - static params WQE,
* - progress params WQE, and
* - resync DUMP per frag.
* kTLS offload requires fixed additional stop_room for:
* - a static params WQE, and a progress params WQE.
* The additional MTU-depending room for the resync DUMP WQEs
* will be calculated and added in runtime.
*/
#define MLX5E_SQ_TLS_ROOM \
(MLX5_SEND_WQE_MAX_WQEBBS + \
MLX5E_KTLS_STATIC_WQEBBS + MLX5E_KTLS_PROGRESS_WQEBBS + \
MAX_SKB_FRAGS * MLX5E_KTLS_MAX_DUMP_WQEBBS)
MLX5E_KTLS_STATIC_WQEBBS + MLX5E_KTLS_PROGRESS_WQEBBS)
#endif
#define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start))
@ -92,7 +91,7 @@ mlx5e_fill_sq_frag_edge(struct mlx5e_txqsq *sq, struct mlx5_wq_cyc *wq,
/* fill sq frag edge with nops to avoid wqe wrapping two pages */
for (; wi < edge_wi; wi++) {
wi->skb = NULL;
memset(wi, 0, sizeof(*wi));
wi->num_wqebbs = 1;
mlx5e_post_nop(wq, sq->sqn, &sq->pc);
}

2
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
View File

@ -38,7 +38,7 @@ static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk,
return -ENOMEM;
tx_priv->expected_seq = start_offload_tcp_sn;
tx_priv->crypto_info = crypto_info;
tx_priv->crypto_info = *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
mlx5e_set_ktls_tx_priv_ctx(tls_ctx, tx_priv);
/* tc and underlay_qpn values are not in use for tls tis */

29
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
View File

@ -21,7 +21,14 @@
MLX5_ST_SZ_BYTES(tls_progress_params))
#define MLX5E_KTLS_PROGRESS_WQEBBS \
(DIV_ROUND_UP(MLX5E_KTLS_PROGRESS_WQE_SZ, MLX5_SEND_WQE_BB))
#define MLX5E_KTLS_MAX_DUMP_WQEBBS 2
struct mlx5e_dump_wqe {
struct mlx5_wqe_ctrl_seg ctrl;
struct mlx5_wqe_data_seg data;
};
#define MLX5E_KTLS_DUMP_WQEBBS \
(DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB))
enum {
MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD = 0,
@ -37,7 +44,7 @@ enum {
struct mlx5e_ktls_offload_context_tx {
struct tls_offload_context_tx *tx_ctx;
struct tls_crypto_info *crypto_info;
struct tls12_crypto_info_aes_gcm_128 crypto_info;
u32 expected_seq;
u32 tisn;
u32 key_id;
@ -86,14 +93,28 @@ struct sk_buff *mlx5e_ktls_handle_tx_skb(struct net_device *netdev,
struct mlx5e_tx_wqe **wqe, u16 *pi);
void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
struct mlx5e_tx_wqe_info *wi,
struct mlx5e_sq_dma *dma);
u32 *dma_fifo_cc);
static inline u8
mlx5e_ktls_dumps_num_wqebbs(struct mlx5e_txqsq *sq, unsigned int nfrags,
unsigned int sync_len)
{
/* Given the MTU and sync_len, calculates an upper bound for the
* number of WQEBBs needed for the TX resync DUMP WQEs of a record.
*/
return MLX5E_KTLS_DUMP_WQEBBS *
(nfrags + DIV_ROUND_UP(sync_len, sq->hw_mtu));
}
#else
static inline void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv)
{
}
static inline void
mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
struct mlx5e_tx_wqe_info *wi,
u32 *dma_fifo_cc) {}
#endif
#endif /* __MLX5E_TLS_H__ */

188
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c
View File

@ -24,17 +24,12 @@ enum {
static void
fill_static_params_ctx(void *ctx, struct mlx5e_ktls_offload_context_tx *priv_tx)
{
struct tls_crypto_info *crypto_info = priv_tx->crypto_info;
struct tls12_crypto_info_aes_gcm_128 *info;
struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info;
char *initial_rn, *gcm_iv;
u16 salt_sz, rec_seq_sz;
char *salt, *rec_seq;
u8 tls_version;
if (WARN_ON(crypto_info->cipher_type != TLS_CIPHER_AES_GCM_128))
return;
info = (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
EXTRACT_INFO_FIELDS;
gcm_iv = MLX5_ADDR_OF(tls_static_params, ctx, gcm_iv);
@ -108,16 +103,15 @@ build_progress_params(struct mlx5e_tx_wqe *wqe, u16 pc, u32 sqn,
}
static void tx_fill_wi(struct mlx5e_txqsq *sq,
u16 pi, u8 num_wqebbs,
skb_frag_t *resync_dump_frag,
u32 num_bytes)
u16 pi, u8 num_wqebbs, u32 num_bytes,
struct page *page)
{
struct mlx5e_tx_wqe_info *wi = &sq->db.wqe_info[pi];
wi->skb = NULL;
wi->num_wqebbs = num_wqebbs;
wi->resync_dump_frag = resync_dump_frag;
wi->num_bytes = num_bytes;
memset(wi, 0, sizeof(*wi));
wi->num_wqebbs = num_wqebbs;
wi->num_bytes = num_bytes;
wi->resync_dump_frag_page = page;
}
void mlx5e_ktls_tx_offload_set_pending(struct mlx5e_ktls_offload_context_tx *priv_tx)
@ -145,7 +139,7 @@ post_static_params(struct mlx5e_txqsq *sq,
umr_wqe = mlx5e_sq_fetch_wqe(sq, MLX5E_KTLS_STATIC_UMR_WQE_SZ, &pi);
build_static_params(umr_wqe, sq->pc, sq->sqn, priv_tx, fence);
tx_fill_wi(sq, pi, MLX5E_KTLS_STATIC_WQEBBS, NULL, 0);
tx_fill_wi(sq, pi, MLX5E_KTLS_STATIC_WQEBBS, 0, NULL);
sq->pc += MLX5E_KTLS_STATIC_WQEBBS;
}
@ -159,7 +153,7 @@ post_progress_params(struct mlx5e_txqsq *sq,
wqe = mlx5e_sq_fetch_wqe(sq, MLX5E_KTLS_PROGRESS_WQE_SZ, &pi);
build_progress_params(wqe, sq->pc, sq->sqn, priv_tx, fence);
tx_fill_wi(sq, pi, MLX5E_KTLS_PROGRESS_WQEBBS, NULL, 0);
tx_fill_wi(sq, pi, MLX5E_KTLS_PROGRESS_WQEBBS, 0, NULL);
sq->pc += MLX5E_KTLS_PROGRESS_WQEBBS;
}
@ -169,6 +163,14 @@ mlx5e_ktls_tx_post_param_wqes(struct mlx5e_txqsq *sq,
bool skip_static_post, bool fence_first_post)
{
bool progress_fence = skip_static_post || !fence_first_post;
struct mlx5_wq_cyc *wq = &sq->wq;
u16 contig_wqebbs_room, pi;
pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
contig_wqebbs_room = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
if (unlikely(contig_wqebbs_room <
MLX5E_KTLS_STATIC_WQEBBS + MLX5E_KTLS_PROGRESS_WQEBBS))
mlx5e_fill_sq_frag_edge(sq, wq, pi, contig_wqebbs_room);
if (!skip_static_post)
post_static_params(sq, priv_tx, fence_first_post);
@ -180,29 +182,36 @@ struct tx_sync_info {
u64 rcd_sn;
s32 sync_len;
int nr_frags;
skb_frag_t *frags[MAX_SKB_FRAGS];
skb_frag_t frags[MAX_SKB_FRAGS];
};
static bool tx_sync_info_get(struct mlx5e_ktls_offload_context_tx *priv_tx,
u32 tcp_seq, struct tx_sync_info *info)
enum mlx5e_ktls_sync_retval {
MLX5E_KTLS_SYNC_DONE,
MLX5E_KTLS_SYNC_FAIL,
MLX5E_KTLS_SYNC_SKIP_NO_DATA,
};
static enum mlx5e_ktls_sync_retval
tx_sync_info_get(struct mlx5e_ktls_offload_context_tx *priv_tx,
u32 tcp_seq, struct tx_sync_info *info)
{
struct tls_offload_context_tx *tx_ctx = priv_tx->tx_ctx;
enum mlx5e_ktls_sync_retval ret = MLX5E_KTLS_SYNC_DONE;
struct tls_record_info *record;
int remaining, i = 0;
unsigned long flags;
bool ret = true;
spin_lock_irqsave(&tx_ctx->lock, flags);
record = tls_get_record(tx_ctx, tcp_seq, &info->rcd_sn);
if (unlikely(!record)) {
ret = false;
ret = MLX5E_KTLS_SYNC_FAIL;
goto out;
}
if (unlikely(tcp_seq < tls_record_start_seq(record))) {
if (!tls_record_is_start_marker(record))
ret = false;
ret = tls_record_is_start_marker(record) ?
MLX5E_KTLS_SYNC_SKIP_NO_DATA : MLX5E_KTLS_SYNC_FAIL;
goto out;
}
@ -211,13 +220,13 @@ static bool tx_sync_info_get(struct mlx5e_ktls_offload_context_tx *priv_tx,
while (remaining > 0) {
skb_frag_t *frag = &record->frags[i];
__skb_frag_ref(frag);
get_page(skb_frag_page(frag));
remaining -= skb_frag_size(frag);
info->frags[i++] = frag;
info->frags[i++] = *frag;
}
/* reduce the part which will be sent with the original SKB */
if (remaining < 0)
skb_frag_size_add(info->frags[i - 1], remaining);
skb_frag_size_add(&info->frags[i - 1], remaining);
info->nr_frags = i;
out:
spin_unlock_irqrestore(&tx_ctx->lock, flags);
@ -229,17 +238,12 @@ tx_post_resync_params(struct mlx5e_txqsq *sq,
struct mlx5e_ktls_offload_context_tx *priv_tx,
u64 rcd_sn)
{
struct tls_crypto_info *crypto_info = priv_tx->crypto_info;
struct tls12_crypto_info_aes_gcm_128 *info;
struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info;
__be64 rn_be = cpu_to_be64(rcd_sn);
bool skip_static_post;
u16 rec_seq_sz;
char *rec_seq;
if (WARN_ON(crypto_info->cipher_type != TLS_CIPHER_AES_GCM_128))
return;
info = (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
rec_seq = info->rec_seq;
rec_seq_sz = sizeof(info->rec_seq);
@ -250,11 +254,6 @@ tx_post_resync_params(struct mlx5e_txqsq *sq,
mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, skip_static_post, true);
}
struct mlx5e_dump_wqe {
struct mlx5_wqe_ctrl_seg ctrl;
struct mlx5_wqe_data_seg data;
};
static int
tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool first)
{
@ -262,7 +261,6 @@ tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool fir
struct mlx5_wqe_data_seg *dseg;
struct mlx5e_dump_wqe *wqe;
dma_addr_t dma_addr = 0;
u8 num_wqebbs;
u16 ds_cnt;
int fsz;
u16 pi;
@ -270,7 +268,6 @@ tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool fir
wqe = mlx5e_sq_fetch_wqe(sq, sizeof(*wqe), &pi);
ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
cseg = &wqe->ctrl;
dseg = &wqe->data;
@ -291,24 +288,27 @@ tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool fir
dseg->byte_count = cpu_to_be32(fsz);
mlx5e_dma_push(sq, dma_addr, fsz, MLX5E_DMA_MAP_PAGE);
tx_fill_wi(sq, pi, num_wqebbs, frag, fsz);
sq->pc += num_wqebbs;
WARN(num_wqebbs > MLX5E_KTLS_MAX_DUMP_WQEBBS,
"unexpected DUMP num_wqebbs, %d > %d",
num_wqebbs, MLX5E_KTLS_MAX_DUMP_WQEBBS);
tx_fill_wi(sq, pi, MLX5E_KTLS_DUMP_WQEBBS, fsz, skb_frag_page(frag));
sq->pc += MLX5E_KTLS_DUMP_WQEBBS;
return 0;
}
void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
struct mlx5e_tx_wqe_info *wi,
struct mlx5e_sq_dma *dma)
u32 *dma_fifo_cc)
{
struct mlx5e_sq_stats *stats = sq->stats;
struct mlx5e_sq_stats *stats;
struct mlx5e_sq_dma *dma;
if (!wi->resync_dump_frag_page)
return;
dma = mlx5e_dma_get(sq, (*dma_fifo_cc)++);
stats = sq->stats;
mlx5e_tx_dma_unmap(sq->pdev, dma);
__skb_frag_unref(wi->resync_dump_frag);
put_page(wi->resync_dump_frag_page);
stats->tls_dump_packets++;
stats->tls_dump_bytes += wi->num_bytes;
}
@ -318,25 +318,31 @@ static void tx_post_fence_nop(struct mlx5e_txqsq *sq)
struct mlx5_wq_cyc *wq = &sq->wq;
u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
tx_fill_wi(sq, pi, 1, NULL, 0);
tx_fill_wi(sq, pi, 1, 0, NULL);
mlx5e_post_nop_fence(wq, sq->sqn, &sq->pc);
}
static struct sk_buff *
static enum mlx5e_ktls_sync_retval
mlx5e_ktls_tx_handle_ooo(struct mlx5e_ktls_offload_context_tx *priv_tx,
struct mlx5e_txqsq *sq,
struct sk_buff *skb,
int datalen,
u32 seq)
{
struct mlx5e_sq_stats *stats = sq->stats;
struct mlx5_wq_cyc *wq = &sq->wq;
enum mlx5e_ktls_sync_retval ret;
struct tx_sync_info info = {};
u16 contig_wqebbs_room, pi;
u8 num_wqebbs;
int i;
int i = 0;
if (!tx_sync_info_get(priv_tx, seq, &info)) {
ret = tx_sync_info_get(priv_tx, seq, &info);
if (unlikely(ret != MLX5E_KTLS_SYNC_DONE)) {
if (ret == MLX5E_KTLS_SYNC_SKIP_NO_DATA) {
stats->tls_skip_no_sync_data++;
return MLX5E_KTLS_SYNC_SKIP_NO_DATA;
}
/* We might get here if a retransmission reaches the driver
* after the relevant record is acked.
* It should be safe to drop the packet in this case
@ -346,13 +352,8 @@ mlx5e_ktls_tx_handle_ooo(struct mlx5e_ktls_offload_context_tx *priv_tx,
}
if (unlikely(info.sync_len < 0)) {
u32 payload;
int headln;
headln = skb_transport_offset(skb) + tcp_hdrlen(skb);
payload = skb->len - headln;
if (likely(payload <= -info.sync_len))
return skb;
if (likely(datalen <= -info.sync_len))
return MLX5E_KTLS_SYNC_DONE;
stats->tls_drop_bypass_req++;
goto err_out;
@ -360,30 +361,62 @@ mlx5e_ktls_tx_handle_ooo(struct mlx5e_ktls_offload_context_tx *priv_tx,
stats->tls_ooo++;
num_wqebbs = MLX5E_KTLS_STATIC_WQEBBS + MLX5E_KTLS_PROGRESS_WQEBBS +
(info.nr_frags ? info.nr_frags * MLX5E_KTLS_MAX_DUMP_WQEBBS : 1);
tx_post_resync_params(sq, priv_tx, info.rcd_sn);
/* If no dump WQE was sent, we need to have a fence NOP WQE before the
* actual data xmit.
*/
if (!info.nr_frags) {
tx_post_fence_nop(sq);
return MLX5E_KTLS_SYNC_DONE;
}
num_wqebbs = mlx5e_ktls_dumps_num_wqebbs(sq, info.nr_frags, info.sync_len);
pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
contig_wqebbs_room = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
if (unlikely(contig_wqebbs_room < num_wqebbs))
mlx5e_fill_sq_frag_edge(sq, wq, pi, contig_wqebbs_room);
tx_post_resync_params(sq, priv_tx, info.rcd_sn);
for (i = 0; i < info.nr_frags; i++)
if (tx_post_resync_dump(sq, info.frags[i], priv_tx->tisn, !i))
goto err_out;
for (; i < info.nr_frags; i++) {
unsigned int orig_fsz, frag_offset = 0, n = 0;
skb_frag_t *f = &info.frags[i];
/* If no dump WQE was sent, we need to have a fence NOP WQE before the
* actual data xmit.
*/
if (!info.nr_frags)
tx_post_fence_nop(sq);
orig_fsz = skb_frag_size(f);
return skb;
do {
bool fence = !(i || frag_offset);
unsigned int fsz;
n++;
fsz = min_t(unsigned int, sq->hw_mtu, orig_fsz - frag_offset);
skb_frag_size_set(f, fsz);
if (tx_post_resync_dump(sq, f, priv_tx->tisn, fence)) {
page_ref_add(skb_frag_page(f), n - 1);
goto err_out;
}
skb_frag_off_add(f, fsz);
frag_offset += fsz;
} while (frag_offset < orig_fsz);
page_ref_add(skb_frag_page(f), n - 1);
}
return MLX5E_KTLS_SYNC_DONE;
err_out:
dev_kfree_skb_any(skb);
return NULL;
for (; i < info.nr_frags; i++)
/* The put_page() here undoes the page ref obtained in tx_sync_info_get().
* Page refs obtained for the DUMP WQEs above (by page_ref_add) will be
* released only upon their completions (or in mlx5e_free_txqsq_descs,
* if channel closes).
*/
put_page(skb_frag_page(&info.frags[i]));
return MLX5E_KTLS_SYNC_FAIL;
}
struct sk_buff *mlx5e_ktls_handle_tx_skb(struct net_device *netdev,
@ -419,10 +452,15 @@ struct sk_buff *mlx5e_ktls_handle_tx_skb(struct net_device *netdev,
seq = ntohl(tcp_hdr(skb)->seq);
if (unlikely(priv_tx->expected_seq != seq)) {
skb = mlx5e_ktls_tx_handle_ooo(priv_tx, sq, skb, seq);
if (unlikely(!skb))
enum mlx5e_ktls_sync_retval ret =
mlx5e_ktls_tx_handle_ooo(priv_tx, sq, datalen, seq);
if (likely(ret == MLX5E_KTLS_SYNC_DONE))
*wqe = mlx5e_sq_fetch_wqe(sq, sizeof(**wqe), pi);
else if (ret == MLX5E_KTLS_SYNC_FAIL)
goto err_out;
else /* ret == MLX5E_KTLS_SYNC_SKIP_NO_DATA */
goto out;
*wqe = mlx5e_sq_fetch_wqe(sq, sizeof(**wqe), pi);
}
priv_tx->expected_seq = seq + datalen;

13
drivers/net/ethernet/mellanox/mlx5/core/en_main.c
View File

@ -1128,6 +1128,7 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
sq->txq_ix = txq_ix;
sq->uar_map = mdev->mlx5e_res.bfreg.map;
sq->min_inline_mode = params->tx_min_inline_mode;
sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
sq->stats = &c->priv->channel_stats[c->ix].sq[tc];
sq->stop_room = MLX5E_SQ_STOP_ROOM;
INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work);
@ -1135,10 +1136,14 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state);
if (MLX5_IPSEC_DEV(c->priv->mdev))
set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state);
#ifdef CONFIG_MLX5_EN_TLS
if (mlx5_accel_is_tls_device(c->priv->mdev)) {
set_bit(MLX5E_SQ_STATE_TLS, &sq->state);
sq->stop_room += MLX5E_SQ_TLS_ROOM;
sq->stop_room += MLX5E_SQ_TLS_ROOM +
mlx5e_ktls_dumps_num_wqebbs(sq, MAX_SKB_FRAGS,
TLS_MAX_PAYLOAD_SIZE);
}
#endif
param->wq.db_numa_node = cpu_to_node(c->cpu);
err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
@ -1349,9 +1354,13 @@ static void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
/* last doorbell out, godspeed .. */
if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) {
u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
struct mlx5e_tx_wqe_info *wi;
struct mlx5e_tx_wqe *nop;
sq->db.wqe_info[pi].skb = NULL;
wi = &sq->db.wqe_info[pi];
memset(wi, 0, sizeof(*wi));
wi->num_wqebbs = 1;
nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl);
}

20
drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
View File

@ -52,11 +52,12 @@ static const struct counter_desc sw_stats_desc[] = {
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_encrypted_bytes) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_ctx) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_ooo) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_resync_bytes) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_drop_no_sync_data) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_drop_bypass_req) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_dump_packets) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_dump_bytes) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_resync_bytes) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_skip_no_sync_data) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_drop_no_sync_data) },
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_drop_bypass_req) },
#endif
{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_lro_packets) },
@ -288,11 +289,12 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv)
s->tx_tls_encrypted_bytes += sq_stats->tls_encrypted_bytes;
s->tx_tls_ctx += sq_stats->tls_ctx;
s->tx_tls_ooo += sq_stats->tls_ooo;
s->tx_tls_resync_bytes += sq_stats->tls_resync_bytes;
s->tx_tls_drop_no_sync_data += sq_stats->tls_drop_no_sync_data;
s->tx_tls_drop_bypass_req += sq_stats->tls_drop_bypass_req;
s->tx_tls_dump_bytes += sq_stats->tls_dump_bytes;
s->tx_tls_dump_packets += sq_stats->tls_dump_packets;
s->tx_tls_resync_bytes += sq_stats->tls_resync_bytes;
s->tx_tls_skip_no_sync_data += sq_stats->tls_skip_no_sync_data;
s->tx_tls_drop_no_sync_data += sq_stats->tls_drop_no_sync_data;
s->tx_tls_drop_bypass_req += sq_stats->tls_drop_bypass_req;
#endif
s->tx_cqes += sq_stats->cqes;
}
@ -1472,10 +1474,12 @@ static const struct counter_desc sq_stats_desc[] = {
{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_encrypted_bytes) },
{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_ctx) },
{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_ooo) },
{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_drop_no_sync_data) },
{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_drop_bypass_req) },
{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_dump_packets) },
{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_dump_bytes) },
{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_resync_bytes) },
{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_skip_no_sync_data) },
{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_drop_no_sync_data) },
{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_drop_bypass_req) },
#endif
{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, csum_none) },
{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, stopped) },

14
drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
View File

@ -129,11 +129,12 @@ struct mlx5e_sw_stats {
u64 tx_tls_encrypted_bytes;
u64 tx_tls_ctx;
u64 tx_tls_ooo;
u64 tx_tls_resync_bytes;
u64 tx_tls_drop_no_sync_data;
u64 tx_tls_drop_bypass_req;
u64 tx_tls_dump_packets;
u64 tx_tls_dump_bytes;
u64 tx_tls_resync_bytes;
u64 tx_tls_skip_no_sync_data;
u64 tx_tls_drop_no_sync_data;
u64 tx_tls_drop_bypass_req;
#endif
u64 rx_xsk_packets;
@ -273,11 +274,12 @@ struct mlx5e_sq_stats {
u64 tls_encrypted_bytes;
u64 tls_ctx;
u64 tls_ooo;
u64 tls_resync_bytes;
u64 tls_drop_no_sync_data;
u64 tls_drop_bypass_req;
u64 tls_dump_packets;
u64 tls_dump_bytes;
u64 tls_resync_bytes;
u64 tls_skip_no_sync_data;
u64 tls_drop_no_sync_data;
u64 tls_drop_bypass_req;
#endif
/* less likely accessed in data path */
u64 csum_none;

35
drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
View File

@ -403,7 +403,10 @@ netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev)
static void mlx5e_dump_error_cqe(struct mlx5e_txqsq *sq,
struct mlx5_err_cqe *err_cqe)
{
u32 ci = mlx5_cqwq_get_ci(&sq->cq.wq);
struct mlx5_cqwq *wq = &sq->cq.wq;
u32 ci;
ci = mlx5_cqwq_ctr2ix(wq, wq->cc - 1);
netdev_err(sq->channel->netdev,
"Error cqe on cqn 0x%x, ci 0x%x, sqn 0x%x, opcode 0x%x, syndrome 0x%x, vendor syndrome 0x%x\n",
@ -479,14 +482,7 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
skb = wi->skb;
if (unlikely(!skb)) {
#ifdef CONFIG_MLX5_EN_TLS
if (wi->resync_dump_frag) {
struct mlx5e_sq_dma *dma =
mlx5e_dma_get(sq, dma_fifo_cc++);
mlx5e_ktls_tx_handle_resync_dump_comp(sq, wi, dma);
}
#endif
mlx5e_ktls_tx_handle_resync_dump_comp(sq, wi, &dma_fifo_cc);
sqcc += wi->num_wqebbs;
continue;
}
@ -542,29 +538,38 @@ void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq)
{
struct mlx5e_tx_wqe_info *wi;
struct sk_buff *skb;
u32 dma_fifo_cc;
u16 sqcc;
u16 ci;
int i;
while (sq->cc != sq->pc) {
ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc);
sqcc = sq->cc;
dma_fifo_cc = sq->dma_fifo_cc;
while (sqcc != sq->pc) {
ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
wi = &sq->db.wqe_info[ci];
skb = wi->skb;
if (!skb) { /* nop */
sq->cc++;
if (!skb) {
mlx5e_ktls_tx_handle_resync_dump_comp(sq, wi, &dma_fifo_cc);
sqcc += wi->num_wqebbs;
continue;
}
for (i = 0; i < wi->num_dma; i++) {
struct mlx5e_sq_dma *dma =
mlx5e_dma_get(sq, sq->dma_fifo_cc++);
mlx5e_dma_get(sq, dma_fifo_cc++);
mlx5e_tx_dma_unmap(sq->pdev, dma);
}
dev_kfree_skb_any(skb);
sq->cc += wi->num_wqebbs;
sqcc += wi->num_wqebbs;
}
sq->dma_fifo_cc = dma_fifo_cc;
sq->cc = sqcc;
}
#ifdef CONFIG_MLX5_CORE_IPOIB

4
drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
View File

@ -464,8 +464,10 @@ static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size)
}
err = mlx5_vector2eqn(mdev, smp_processor_id(), &eqn, &irqn);
if (err)
if (err) {
kvfree(in);
goto err_cqwq;
}
cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
MLX5_SET(cqc, cqc, log_cq_size, ilog2(cq_size));

2
drivers/net/ethernet/mellanox/mlx5/core/health.c
View File

@ -572,7 +572,7 @@ mlx5_fw_fatal_reporter_dump(struct devlink_health_reporter *reporter,
return -ENOMEM;
err = mlx5_crdump_collect(dev, cr_data);
if (err)
return err;
goto free_data;
if (priv_ctx) {
struct mlx5_fw_reporter_ctx *fw_reporter_ctx = priv_ctx;