From 15d2ee42a3087089e73ad52fd8c1b37ab496b87c Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Mon, 26 Feb 2018 22:47:06 +0100 Subject: [PATCH 1/4] net: stmmac: ensure that the MSS desc is the last desc to set the own bit A dma_wmb() is used to guarantee the ordering, with respect to other writes, to cache coherent DMA memory. There is a dma_wmb() in prepare_tx_desc()/prepare_tso_tx_desc() which ensures that TDES0/1/2 is written before TDES3 (which contains the own bit), for First Desc. However, in the rare case that MSS changes, there will be a MSS context descriptor in front of the regular DMA descriptors: <- DMA Next Descriptor Thus, for this special case, we need a dma_wmb() after prepare_tso_tx_desc()/before writing the own bit to the MSS desc, so that we flush the write to TDES3 for First Desc, in order to ensure that the MSS descriptor is the last descriptor to set the own bit. Signed-off-by: Niklas Cassel Signed-off-by: David S. Miller --- drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index c8d86d77e03d..3b5e7b06e796 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -2983,8 +2983,15 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len)); /* If context desc is used to change MSS */ - if (mss_desc) + if (mss_desc) { + /* Make sure that first descriptor has been completely + * written, including its own bit. This is because MSS is + * actually before first descriptor, so we need to make + * sure that MSS's own bit is the last thing written. + */ + dma_wmb(); priv->hw->desc->set_tx_owner(mss_desc); + } /* The own bit must be the latest setting done when prepare the * descriptor and then barrier is needed to make sure that From 95eb930a40a0973f9b984d87a4986bb81f897ede Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Mon, 26 Feb 2018 22:47:07 +0100 Subject: [PATCH 2/4] net: stmmac: use correct barrier between coherent memory and MMIO The last memory barrier in stmmac_xmit()/stmmac_tso_xmit() is placed between a coherent memory write and a MMIO write: The own bit is written in First Desc (TSO: MSS desc or First Desc). The DMA engine is started by a write to the tx desc tail pointer/ enable dma transmission register, i.e. a MMIO write. This barrier cannot be a simple dma_wmb(), since a dma_wmb() is only used to guarantee the ordering, with respect to other writes, to cache coherent DMA memory. To guarantee that the cache coherent memory writes have completed before we attempt to write to the cache incoherent MMIO region, we need to use the more heavyweight barrier wmb(). Signed-off-by: Niklas Cassel Signed-off-by: David S. Miller --- drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 3b5e7b06e796..6dd04f237b2a 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -2997,7 +2997,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) * descriptor and then barrier is needed to make sure that * all is coherent before granting the DMA engine. */ - dma_wmb(); + wmb(); if (netif_msg_pktdata(priv)) { pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", @@ -3221,7 +3221,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) * descriptor and then barrier is needed to make sure that * all is coherent before granting the DMA engine. */ - dma_wmb(); + wmb(); } netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); From a6b25da5e7ba212af5826a662e6a035a79bffabd Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Mon, 26 Feb 2018 22:47:08 +0100 Subject: [PATCH 3/4] net: stmmac: ensure that the device has released ownership before reading data According to Documentation/memory-barriers.txt, we need to use a dma_rmb() after reading the status/own bit, to ensure that all descriptor fields are read after reading the own bit. This way, we ensure that the DMA engine is done with the DMA descriptor before we read the other descriptor fields, e.g. reading the tx hardware timestamp (if PTP is enabled). Signed-off-by: Niklas Cassel Signed-off-by: David S. Miller --- drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 6dd04f237b2a..a9856a8bf8ad 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -1844,6 +1844,11 @@ static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue) if (unlikely(status & tx_dma_own)) break; + /* Make sure descriptor fields are read after reading + * the own bit. + */ + dma_rmb(); + /* Just consider the last segment and ...*/ if (likely(!(status & tx_not_ls))) { /* ... verify the status error condition */ From 1e88f6e01bc094c9fc7021cc1944d14a63257703 Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Mon, 26 Feb 2018 22:47:09 +0100 Subject: [PATCH 4/4] net: stmmac: make dwmac4_release_tx_desc() clear all descriptor fields Make dwmac4_release_tx_desc() clear all descriptor fields, not just TDES2 and TDES3. I'm suspecting that TDES0 and TDES1 wasn't cleared because the DMA engine uses them to store the tx hardware timestamp (if PTP is enabled). However, stmmac_tx_clean() calls stmmac_get_tx_hwtstamp(), which reads and saves the timestamp, before it calls release_tx_desc(), so this is not an issue. stmmac_xmit() and stmmac_tso_xmit() both always overwrite TDES0, however, stmmac_tso_xmit() sometimes sets TDES1, and since neither stmmac_xmit() nor stmmac_tso_xmit() explicitly clears TDES1, both functions might reuse a DMA descriptor with old TDES1 data. I haven't observed any misbehavior even though TDES1 sometimes point to an old skb, however, explicitly clearing both TDES0 and TDES1 in dwmac4_release_tx_desc() minimizes the chances of undefined behavior. Signed-off-by: Niklas Cassel Signed-off-by: David S. Miller --- drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c index c728ffa095de..2a6521d33e43 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c @@ -389,6 +389,8 @@ static void dwmac4_rd_prepare_tso_tx_desc(struct dma_desc *p, int is_fs, static void dwmac4_release_tx_desc(struct dma_desc *p, int mode) { + p->des0 = 0; + p->des1 = 0; p->des2 = 0; p->des3 = 0; }