2018-03-20 22:58:10 +08:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
/* Copyright (c) 2018, Intel Corporation. */
|
|
|
|
|
|
|
|
#ifndef _ICE_TXRX_H_
|
|
|
|
#define _ICE_TXRX_H_
|
|
|
|
|
|
|
|
#define ICE_DFLT_IRQ_WORK 256
|
2018-03-20 22:58:13 +08:00
|
|
|
#define ICE_RXBUF_2048 2048
|
|
|
|
#define ICE_MAX_CHAINED_RX_BUFS 5
|
2018-03-20 22:58:14 +08:00
|
|
|
#define ICE_MAX_BUF_TXD 8
|
|
|
|
#define ICE_MIN_TX_LEN 17
|
|
|
|
|
|
|
|
/* The size limit for a transmit buffer in a descriptor is (16K - 1).
|
|
|
|
* In order to align with the read requests we will align the value to
|
|
|
|
* the nearest 4K which represents our maximum read request size.
|
|
|
|
*/
|
|
|
|
#define ICE_MAX_READ_REQ_SIZE 4096
|
|
|
|
#define ICE_MAX_DATA_PER_TXD (16 * 1024 - 1)
|
|
|
|
#define ICE_MAX_DATA_PER_TXD_ALIGNED \
|
|
|
|
(~(ICE_MAX_READ_REQ_SIZE - 1) & ICE_MAX_DATA_PER_TXD)
|
|
|
|
|
|
|
|
#define ICE_RX_BUF_WRITE 16 /* Must be power of 2 */
|
2018-03-20 22:58:13 +08:00
|
|
|
#define ICE_MAX_TXQ_PER_TXQG 128
|
|
|
|
|
ice: Fix tx_timeout in PF driver
Prior to this commit the driver was running into tx_timeouts when a
queue was stressed enough. This was happening because the HW tail
and SW tail (NTU) were incorrectly out of sync. Consequently this was
causing the HW head to collide with the HW tail, which to the hardware
means that all descriptors posted for Tx have been processed.
Due to the Tx logic used in the driver SW tail and HW tail are allowed
to be out of sync. This is done as an optimization because it allows the
driver to write HW tail as infrequently as possible, while still
updating the SW tail index to keep track. However, there are situations
where this results in the tail never getting updated, resulting in Tx
timeouts.
Tx HW tail write condition:
if (netif_xmit_stopped(txring_txq(tx_ring) || !skb->xmit_more)
writel(sw_tail, tx_ring->tail);
An issue was found in the Tx logic that was causing the afore mentioned
condition for updating HW tail to never happen, causing tx_timeouts.
In ice_xmit_frame_ring we calculate how many descriptors we need for the
Tx transaction based on the skb the kernel hands us. This is then passed
into ice_maybe_stop_tx along with some extra padding to determine if we
have enough descriptors available for this transaction. If we don't then
we return -EBUSY to the stack, otherwise we move on and eventually
prepare the Tx descriptors accordingly in ice_tx_map and set
next_to_watch. In ice_tx_map we make another call to ice_maybe_stop_tx
with a value of MAX_SKB_FRAGS + 4. The key here is that this value is
possibly less than the value we sent in the first call to
ice_maybe_stop_tx in ice_xmit_frame_ring. Now, if the number of unused
descriptors is between MAX_SKB_FRAGS + 4 and the value used in the first
call to ice_maybe_stop_tx in ice_xmit_frame_ring then we do not update
the HW tail because of the "Tx HW tail write condition" above. This is
because in ice_maybe_stop_tx we return success from ice_maybe_stop_tx
instead of calling __ice_maybe_stop_tx and subsequently calling
netif_stop_subqueue, which sets the __QUEUE_STATE_DEV_XOFF bit. This
bit is then checked in the "Tx HW tail write condition" by calling
netif_xmit_stopped and subsequently updating HW tail if the
afore mentioned bit is set.
In ice_clean_tx_irq, if next_to_watch is not NULL, we end up cleaning
the descriptors that HW sets the DD bit on and we have the budget. The
HW head will eventually run into the HW tail in response to the
description in the paragraph above.
The next time through ice_xmit_frame_ring we make the initial call to
ice_maybe_stop_tx with another skb from the stack. This time we do not
have enough descriptors available and we return NETDEV_TX_BUSY to the
stack and end up setting next_to_watch to NULL.
This is where we are stuck. In ice_clean_tx_irq we never clean anything
because next_to_watch is always NULL and in ice_xmit_frame_ring we never
update HW tail because we already return NETDEV_TX_BUSY to the stack and
eventually we hit a tx_timeout.
This issue was fixed by making sure that the second call to
ice_maybe_stop_tx in ice_tx_map is passed a value that is >= the value
that was used on the initial call to ice_maybe_stop_tx in
ice_xmit_frame_ring. This was done by adding the following defines to
make the logic more clear and to reduce the chance of mucking this up
again:
ICE_CACHE_LINE_BYTES 64
ICE_DESCS_PER_CACHE_LINE (ICE_CACHE_LINE_BYTES / \
sizeof(struct ice_tx_desc))
ICE_DESCS_FOR_CTX_DESC 1
ICE_DESCS_FOR_SKB_DATA_PTR 1
The ICE_CACHE_LINE_BYTES being 64 is an assumption being made so we
don't have to figure this out on every pass through the Tx path. Instead
I added a sanity check in ice_probe to verify cache line size and print
a message if it's not 64 Bytes. This will make it easier to file issues
if they are seen when the cache line size is not 64 Bytes when reading
from the GLPCI_CNF2 register.
Signed-off-by: Brett Creeley <brett.creeley@intel.com>
Signed-off-by: Anirudh Venkataramanan <anirudh.venkataramanan@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
2018-10-27 01:40:58 +08:00
|
|
|
/* We are assuming that the cache line is always 64 Bytes here for ice.
|
|
|
|
* In order to make sure that is a correct assumption there is a check in probe
|
|
|
|
* to print a warning if the read from GLPCI_CNF2 tells us that the cache line
|
|
|
|
* size is 128 bytes. We do it this way because we do not want to read the
|
|
|
|
* GLPCI_CNF2 register or a variable containing the value on every pass through
|
|
|
|
* the Tx path.
|
|
|
|
*/
|
|
|
|
#define ICE_CACHE_LINE_BYTES 64
|
|
|
|
#define ICE_DESCS_PER_CACHE_LINE (ICE_CACHE_LINE_BYTES / \
|
|
|
|
sizeof(struct ice_tx_desc))
|
|
|
|
#define ICE_DESCS_FOR_CTX_DESC 1
|
|
|
|
#define ICE_DESCS_FOR_SKB_DATA_PTR 1
|
|
|
|
/* Tx descriptors needed, worst case */
|
|
|
|
#define DESC_NEEDED (MAX_SKB_FRAGS + ICE_DESCS_FOR_CTX_DESC + \
|
|
|
|
ICE_DESCS_PER_CACHE_LINE + ICE_DESCS_FOR_SKB_DATA_PTR)
|
2018-03-20 22:58:13 +08:00
|
|
|
#define ICE_DESC_UNUSED(R) \
|
|
|
|
((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->count) + \
|
|
|
|
(R)->next_to_clean - (R)->next_to_use - 1)
|
|
|
|
|
2018-03-20 22:58:15 +08:00
|
|
|
#define ICE_TX_FLAGS_TSO BIT(0)
|
|
|
|
#define ICE_TX_FLAGS_HW_VLAN BIT(1)
|
|
|
|
#define ICE_TX_FLAGS_SW_VLAN BIT(2)
|
|
|
|
#define ICE_TX_FLAGS_VLAN_M 0xffff0000
|
|
|
|
#define ICE_TX_FLAGS_VLAN_S 16
|
|
|
|
|
2019-02-14 02:51:07 +08:00
|
|
|
#define ICE_RX_DMA_ATTR \
|
|
|
|
(DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
|
|
|
|
|
2018-03-20 22:58:13 +08:00
|
|
|
struct ice_tx_buf {
|
|
|
|
struct ice_tx_desc *next_to_watch;
|
|
|
|
struct sk_buff *skb;
|
|
|
|
unsigned int bytecount;
|
|
|
|
unsigned short gso_segs;
|
|
|
|
u32 tx_flags;
|
|
|
|
DEFINE_DMA_UNMAP_ADDR(dma);
|
|
|
|
DEFINE_DMA_UNMAP_LEN(len);
|
|
|
|
};
|
|
|
|
|
2018-03-20 22:58:15 +08:00
|
|
|
struct ice_tx_offload_params {
|
|
|
|
u8 header_len;
|
|
|
|
u32 td_cmd;
|
|
|
|
u32 td_offset;
|
|
|
|
u32 td_l2tag1;
|
|
|
|
u16 cd_l2tag2;
|
|
|
|
u32 cd_tunnel_params;
|
|
|
|
u64 cd_qw1;
|
|
|
|
struct ice_ring *tx_ring;
|
|
|
|
};
|
|
|
|
|
2018-03-20 22:58:13 +08:00
|
|
|
struct ice_rx_buf {
|
|
|
|
struct sk_buff *skb;
|
|
|
|
dma_addr_t dma;
|
|
|
|
struct page *page;
|
|
|
|
unsigned int page_offset;
|
2019-02-14 02:51:04 +08:00
|
|
|
u16 pagecnt_bias;
|
2018-03-20 22:58:13 +08:00
|
|
|
};
|
2018-03-20 22:58:10 +08:00
|
|
|
|
2018-03-20 22:58:14 +08:00
|
|
|
struct ice_q_stats {
|
|
|
|
u64 pkts;
|
|
|
|
u64 bytes;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct ice_txq_stats {
|
|
|
|
u64 restart_q;
|
|
|
|
u64 tx_busy;
|
|
|
|
u64 tx_linearize;
|
2018-08-09 21:29:53 +08:00
|
|
|
int prev_pkt; /* negative if no pending Tx descriptors */
|
2018-03-20 22:58:14 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
struct ice_rxq_stats {
|
|
|
|
u64 non_eop_descs;
|
|
|
|
u64 alloc_page_failed;
|
|
|
|
u64 alloc_buf_failed;
|
|
|
|
u64 page_reuse_count;
|
|
|
|
};
|
|
|
|
|
2018-03-20 22:58:10 +08:00
|
|
|
/* this enum matches hardware bits and is meant to be used by DYN_CTLN
|
|
|
|
* registers and QINT registers or more generally anywhere in the manual
|
|
|
|
* mentioning ITR_INDX, ITR_NONE cannot be used as an index 'n' into any
|
|
|
|
* register but instead is a special value meaning "don't update" ITR0/1/2.
|
|
|
|
*/
|
|
|
|
enum ice_dyn_idx_t {
|
|
|
|
ICE_IDX_ITR0 = 0,
|
|
|
|
ICE_IDX_ITR1 = 1,
|
|
|
|
ICE_IDX_ITR2 = 2,
|
|
|
|
ICE_ITR_NONE = 3 /* ITR_NONE must not be used as an index */
|
|
|
|
};
|
|
|
|
|
2018-03-20 22:58:13 +08:00
|
|
|
/* Header split modes defined by DTYPE field of Rx RLAN context */
|
|
|
|
enum ice_rx_dtype {
|
|
|
|
ICE_RX_DTYPE_NO_SPLIT = 0,
|
|
|
|
ICE_RX_DTYPE_HEADER_SPLIT = 1,
|
|
|
|
ICE_RX_DTYPE_SPLIT_ALWAYS = 2,
|
|
|
|
};
|
|
|
|
|
2018-03-20 22:58:10 +08:00
|
|
|
/* indices into GLINT_ITR registers */
|
|
|
|
#define ICE_RX_ITR ICE_IDX_ITR0
|
2018-03-20 22:58:13 +08:00
|
|
|
#define ICE_TX_ITR ICE_IDX_ITR1
|
2018-12-20 02:03:29 +08:00
|
|
|
#define ICE_ITR_8K 124
|
2018-09-20 08:43:05 +08:00
|
|
|
#define ICE_ITR_20K 50
|
2018-12-20 02:03:30 +08:00
|
|
|
#define ICE_ITR_MAX 8160
|
2018-12-20 02:03:29 +08:00
|
|
|
#define ICE_DFLT_TX_ITR (ICE_ITR_20K | ICE_ITR_DYNAMIC)
|
|
|
|
#define ICE_DFLT_RX_ITR (ICE_ITR_20K | ICE_ITR_DYNAMIC)
|
|
|
|
#define ICE_ITR_DYNAMIC 0x8000 /* used as flag for itr_setting */
|
2018-12-20 02:03:30 +08:00
|
|
|
#define ITR_IS_DYNAMIC(setting) (!!((setting) & ICE_ITR_DYNAMIC))
|
2018-12-20 02:03:29 +08:00
|
|
|
#define ITR_TO_REG(setting) ((setting) & ~ICE_ITR_DYNAMIC)
|
|
|
|
#define ICE_ITR_GRAN_S 1 /* Assume ITR granularity is 2us */
|
2019-02-09 04:50:55 +08:00
|
|
|
#define ICE_ITR_GRAN_US BIT(ICE_ITR_GRAN_S)
|
2018-12-20 02:03:29 +08:00
|
|
|
#define ICE_ITR_MASK 0x1FFE /* ITR register value alignment mask */
|
|
|
|
#define ITR_REG_ALIGN(setting) __ALIGN_MASK(setting, ~ICE_ITR_MASK)
|
2018-03-20 22:58:10 +08:00
|
|
|
|
2019-02-20 07:04:01 +08:00
|
|
|
#define ICE_ITR_ADAPTIVE_MIN_INC 0x0002
|
|
|
|
#define ICE_ITR_ADAPTIVE_MIN_USECS 0x0002
|
|
|
|
#define ICE_ITR_ADAPTIVE_MAX_USECS 0x00FA
|
|
|
|
#define ICE_ITR_ADAPTIVE_LATENCY 0x8000
|
|
|
|
#define ICE_ITR_ADAPTIVE_BULK 0x0000
|
|
|
|
|
2018-09-20 08:23:19 +08:00
|
|
|
#define ICE_DFLT_INTRL 0
|
2018-03-20 22:58:10 +08:00
|
|
|
|
2018-03-20 22:58:13 +08:00
|
|
|
/* Legacy or Advanced Mode Queue */
|
|
|
|
#define ICE_TX_ADVANCED 0
|
|
|
|
#define ICE_TX_LEGACY 1
|
|
|
|
|
2018-03-20 22:58:11 +08:00
|
|
|
/* descriptor ring, associated with a VSI */
|
|
|
|
struct ice_ring {
|
|
|
|
struct ice_ring *next; /* pointer to next ring in q_vector */
|
2018-03-20 22:58:13 +08:00
|
|
|
void *desc; /* Descriptor ring memory */
|
2018-03-20 22:58:11 +08:00
|
|
|
struct device *dev; /* Used for DMA mapping */
|
|
|
|
struct net_device *netdev; /* netdev ring maps to */
|
|
|
|
struct ice_vsi *vsi; /* Backreference to associated VSI */
|
|
|
|
struct ice_q_vector *q_vector; /* Backreference to associated vector */
|
2018-03-20 22:58:13 +08:00
|
|
|
u8 __iomem *tail;
|
|
|
|
union {
|
|
|
|
struct ice_tx_buf *tx_buf;
|
|
|
|
struct ice_rx_buf *rx_buf;
|
|
|
|
};
|
2018-03-20 22:58:11 +08:00
|
|
|
u16 q_index; /* Queue number of ring */
|
2018-03-20 22:58:13 +08:00
|
|
|
u32 txq_teid; /* Added Tx queue TEID */
|
|
|
|
|
2018-03-20 22:58:11 +08:00
|
|
|
u16 count; /* Number of descriptors */
|
|
|
|
u16 reg_idx; /* HW register index of the ring */
|
2018-03-20 22:58:13 +08:00
|
|
|
|
|
|
|
/* used in interrupt processing */
|
|
|
|
u16 next_to_use;
|
|
|
|
u16 next_to_clean;
|
|
|
|
|
2018-08-09 21:29:02 +08:00
|
|
|
u8 ring_active; /* is ring online or not */
|
2018-03-20 22:58:14 +08:00
|
|
|
|
|
|
|
/* stats structs */
|
|
|
|
struct ice_q_stats stats;
|
|
|
|
struct u64_stats_sync syncp;
|
|
|
|
union {
|
|
|
|
struct ice_txq_stats tx_stats;
|
|
|
|
struct ice_rxq_stats rx_stats;
|
|
|
|
};
|
|
|
|
|
2018-03-20 22:58:13 +08:00
|
|
|
unsigned int size; /* length of descriptor ring in bytes */
|
|
|
|
dma_addr_t dma; /* physical address of ring */
|
2018-03-20 22:58:11 +08:00
|
|
|
struct rcu_head rcu; /* to avoid race on free */
|
2018-03-20 22:58:13 +08:00
|
|
|
u16 next_to_alloc;
|
2018-03-20 22:58:11 +08:00
|
|
|
} ____cacheline_internodealigned_in_smp;
|
|
|
|
|
2018-03-20 22:58:13 +08:00
|
|
|
enum ice_latency_range {
|
|
|
|
ICE_LOWEST_LATENCY = 0,
|
|
|
|
ICE_LOW_LATENCY = 1,
|
|
|
|
ICE_BULK_LATENCY = 2,
|
|
|
|
ICE_ULTRA_LATENCY = 3,
|
|
|
|
};
|
|
|
|
|
2018-03-20 22:58:11 +08:00
|
|
|
struct ice_ring_container {
|
2018-12-20 02:03:29 +08:00
|
|
|
/* head of linked-list of rings */
|
2018-03-20 22:58:11 +08:00
|
|
|
struct ice_ring *ring;
|
2018-12-20 02:03:29 +08:00
|
|
|
unsigned long next_update; /* jiffies value of next queue update */
|
2018-03-20 22:58:11 +08:00
|
|
|
unsigned int total_bytes; /* total bytes processed this int */
|
|
|
|
unsigned int total_pkts; /* total packets processed this int */
|
2018-03-20 22:58:13 +08:00
|
|
|
enum ice_latency_range latency_range;
|
2018-12-20 02:03:29 +08:00
|
|
|
int itr_idx; /* index in the interrupt vector */
|
|
|
|
u16 target_itr; /* value in usecs divided by the hw->itr_gran */
|
|
|
|
u16 current_itr; /* value in usecs divided by the hw->itr_gran */
|
|
|
|
/* high bit set means dynamic ITR, rest is used to store user
|
|
|
|
* readable ITR value in usecs and must be converted before programming
|
|
|
|
* to a register.
|
|
|
|
*/
|
|
|
|
u16 itr_setting;
|
2018-03-20 22:58:11 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
/* iterator for handling rings in ring container */
|
|
|
|
#define ice_for_each_ring(pos, head) \
|
|
|
|
for (pos = (head).ring; pos; pos = pos->next)
|
|
|
|
|
2018-03-20 22:58:13 +08:00
|
|
|
bool ice_alloc_rx_bufs(struct ice_ring *rxr, u16 cleaned_count);
|
2018-03-20 22:58:14 +08:00
|
|
|
netdev_tx_t ice_start_xmit(struct sk_buff *skb, struct net_device *netdev);
|
2018-03-20 22:58:13 +08:00
|
|
|
void ice_clean_tx_ring(struct ice_ring *tx_ring);
|
|
|
|
void ice_clean_rx_ring(struct ice_ring *rx_ring);
|
|
|
|
int ice_setup_tx_ring(struct ice_ring *tx_ring);
|
|
|
|
int ice_setup_rx_ring(struct ice_ring *rx_ring);
|
|
|
|
void ice_free_tx_ring(struct ice_ring *tx_ring);
|
|
|
|
void ice_free_rx_ring(struct ice_ring *rx_ring);
|
2018-03-20 22:58:14 +08:00
|
|
|
int ice_napi_poll(struct napi_struct *napi, int budget);
|
|
|
|
|
2018-03-20 22:58:10 +08:00
|
|
|
#endif /* _ICE_TXRX_H_ */
|