Merge branch 'enetc-Add-adaptive-interrupt-coalescing'

Claudiu Manoil says:

====================
enetc: Add adaptive interrupt coalescing

Apart from some related cleanup patches, this set
introduces in a straightforward way the support needed
to enable and configure interrupt coalescing for ENETC.

Patch 5 introduces the support needed for configuring the
interrupt coalescing parameters and for switching between
moderated (int. coalescing) and per-packet interrupt modes.
When interrupt coalescing is enabled the Rx/Tx time
thresholds are configurable, packet thresholds are fixed.
To make this work reliably, patch 5 uses the traffic
pause procedure introduced in patch 2.

Patch 6 adds DIM (Dynamic Interrupt Moderation) to implement
adaptive coalescing based on time thresholds, for the Rx 'channel'.
On the Tx side a default optimal value is used instead, optimized for
TCP traffic over 1G and 2.5G links.  This default 'optimal' value can
be overridden anytime via 'ethtool -C tx-usecs'.

netperf -t TCP_MAERTS measurements show a significant CPU load
reduction correlated w/ reduced interrupt rates. For the
measurement results refer to the comments in patch 6.

v2: Replaced Tx DIM with predefined optimal value, giving
better results. This was also suggested by Jakub (cc).
Switched order of patches 4 and 5, for better grouping.

v3: minor cleanup/improvements
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller 2020-07-21 15:38:30 -07:00
commit ccbc6dacac
5 changed files with 258 additions and 45 deletions

View File

@ -4,6 +4,7 @@ config FSL_ENETC
depends on PCI && PCI_MSI
select FSL_ENETC_MDIO
select PHYLIB
select DIMLIB
help
This driver supports NXP ENETC gigabit ethernet controller PCIe
physical function (PF) devices, managing ENETC Ports at a privileged
@ -15,6 +16,7 @@ config FSL_ENETC_VF
tristate "ENETC VF driver"
depends on PCI && PCI_MSI
select PHYLIB
select DIMLIB
help
This driver supports NXP ENETC gigabit ethernet controller PCIe
virtual function (VF) devices enabled by the ENETC PF driver.

View File

@ -265,6 +265,7 @@ static irqreturn_t enetc_msix(int irq, void *data)
/* disable interrupts */
enetc_wr_reg(v->rbier, 0);
enetc_wr_reg(v->ricr1, v->rx_ictt);
for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 0);
@ -278,6 +279,34 @@ static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget);
static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
struct napi_struct *napi, int work_limit);
static void enetc_rx_dim_work(struct work_struct *w)
{
struct dim *dim = container_of(w, struct dim, work);
struct dim_cq_moder moder =
net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
struct enetc_int_vector *v =
container_of(dim, struct enetc_int_vector, rx_dim);
v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
dim->state = DIM_START_MEASURE;
}
static void enetc_rx_net_dim(struct enetc_int_vector *v)
{
struct dim_sample dim_sample;
v->comp_cnt++;
if (!v->rx_napi_work)
return;
dim_update_sample(v->comp_cnt,
v->rx_ring.stats.packets,
v->rx_ring.stats.bytes,
&dim_sample);
net_dim(&v->rx_dim, dim_sample);
}
static int enetc_poll(struct napi_struct *napi, int budget)
{
struct enetc_int_vector
@ -293,12 +322,19 @@ static int enetc_poll(struct napi_struct *napi, int budget)
work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget);
if (work_done == budget)
complete = false;
if (work_done)
v->rx_napi_work = true;
if (!complete)
return budget;
napi_complete_done(napi, work_done);
if (likely(v->rx_dim_en))
enetc_rx_net_dim(v);
v->rx_napi_work = false;
/* enable interrupts */
enetc_wr_reg(v->rbier, ENETC_RBIER_RXTIE);
@ -1064,8 +1100,8 @@ void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
struct enetc_si *si = priv->si;
int cpus = num_online_cpus();
priv->tx_bd_count = ENETC_BDR_DEFAULT_SIZE;
priv->rx_bd_count = ENETC_BDR_DEFAULT_SIZE;
priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
/* Enable all available TX rings in order to configure as many
* priorities as possible, when needed.
@ -1074,6 +1110,8 @@ void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
priv->num_tx_rings = si->num_tx_rings;
priv->bdr_int_num = cpus;
priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
priv->tx_ictt = ENETC_TXIC_TIMETHR;
/* SI specific */
si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE;
@ -1140,7 +1178,7 @@ static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
/* enable Tx ints by setting pkt thr to 1 */
enetc_txbdr_wr(hw, idx, ENETC_TBICIR0, ENETC_TBICIR0_ICEN | 0x1);
enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
tbmr = ENETC_TBMR_EN;
if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
@ -1174,7 +1212,7 @@ static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
/* enable Rx ints by setting pkt thr to 1 */
enetc_rxbdr_wr(hw, idx, ENETC_RBICIR0, ENETC_RBICIR0_ICEN | 0x1);
enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
rbmr = ENETC_RBMR_EN;
@ -1264,9 +1302,11 @@ static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
dev_err(priv->dev, "request_irq() failed!\n");
goto irq_err;
}
disable_irq(irq);
v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
@ -1306,23 +1346,42 @@ static void enetc_free_irqs(struct enetc_ndev_priv *priv)
}
}
static void enetc_enable_interrupts(struct enetc_ndev_priv *priv)
static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
{
struct enetc_hw *hw = &priv->si->hw;
u32 icpt, ictt;
int i;
/* enable Tx & Rx event indication */
for (i = 0; i < priv->num_rx_rings; i++) {
enetc_rxbdr_wr(&priv->si->hw, i,
ENETC_RBIER, ENETC_RBIER_RXTIE);
if (priv->ic_mode &
(ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
/* init to non-0 minimum, will be adjusted later */
ictt = 0x1;
} else {
icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
ictt = 0;
}
for (i = 0; i < priv->num_rx_rings; i++) {
enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
}
if (priv->ic_mode & ENETC_IC_TX_MANUAL)
icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
else
icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
for (i = 0; i < priv->num_tx_rings; i++) {
enetc_txbdr_wr(&priv->si->hw, i,
ENETC_TBIER, ENETC_TBIER_TXTIE);
enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
}
}
static void enetc_disable_interrupts(struct enetc_ndev_priv *priv)
static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
{
int i;
@ -1369,10 +1428,33 @@ static int enetc_phy_connect(struct net_device *ndev)
return 0;
}
void enetc_start(struct net_device *ndev)
{
struct enetc_ndev_priv *priv = netdev_priv(ndev);
int i;
enetc_setup_interrupts(priv);
for (i = 0; i < priv->bdr_int_num; i++) {
int irq = pci_irq_vector(priv->si->pdev,
ENETC_BDR_INT_BASE_IDX + i);
napi_enable(&priv->int_vector[i]->napi);
enable_irq(irq);
}
if (ndev->phydev)
phy_start(ndev->phydev);
else
netif_carrier_on(ndev);
netif_tx_start_all_queues(ndev);
}
int enetc_open(struct net_device *ndev)
{
struct enetc_ndev_priv *priv = netdev_priv(ndev);
int i, err;
int err;
err = enetc_setup_irqs(priv);
if (err)
@ -1390,8 +1472,6 @@ int enetc_open(struct net_device *ndev)
if (err)
goto err_alloc_rx;
enetc_setup_bdrs(priv);
err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
if (err)
goto err_set_queues;
@ -1400,17 +1480,8 @@ int enetc_open(struct net_device *ndev)
if (err)
goto err_set_queues;
for (i = 0; i < priv->bdr_int_num; i++)
napi_enable(&priv->int_vector[i]->napi);
enetc_enable_interrupts(priv);
if (ndev->phydev)
phy_start(ndev->phydev);
else
netif_carrier_on(ndev);
netif_tx_start_all_queues(ndev);
enetc_setup_bdrs(priv);
enetc_start(ndev);
return 0;
@ -1427,28 +1498,39 @@ int enetc_open(struct net_device *ndev)
return err;
}
int enetc_close(struct net_device *ndev)
void enetc_stop(struct net_device *ndev)
{
struct enetc_ndev_priv *priv = netdev_priv(ndev);
int i;
netif_tx_stop_all_queues(ndev);
if (ndev->phydev) {
phy_stop(ndev->phydev);
phy_disconnect(ndev->phydev);
} else {
netif_carrier_off(ndev);
}
for (i = 0; i < priv->bdr_int_num; i++) {
int irq = pci_irq_vector(priv->si->pdev,
ENETC_BDR_INT_BASE_IDX + i);
disable_irq(irq);
napi_synchronize(&priv->int_vector[i]->napi);
napi_disable(&priv->int_vector[i]->napi);
}
enetc_disable_interrupts(priv);
if (ndev->phydev)
phy_stop(ndev->phydev);
else
netif_carrier_off(ndev);
enetc_clear_interrupts(priv);
}
int enetc_close(struct net_device *ndev)
{
struct enetc_ndev_priv *priv = netdev_priv(ndev);
enetc_stop(ndev);
enetc_clear_bdrs(priv);
if (ndev->phydev)
phy_disconnect(ndev->phydev);
enetc_free_rxtx_rings(priv);
enetc_free_rx_resources(priv);
enetc_free_tx_resources(priv);
@ -1742,6 +1824,12 @@ int enetc_alloc_msix(struct enetc_ndev_priv *priv)
priv->int_vector[i] = v;
/* init defaults for adaptive IC */
if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
v->rx_ictt = 0x1;
v->rx_dim_en = true;
}
INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
netif_napi_add(priv->ndev, &v->napi, enetc_poll,
NAPI_POLL_WEIGHT);
v->count_tx_rings = v_tx_rings;
@ -1777,6 +1865,7 @@ int enetc_alloc_msix(struct enetc_ndev_priv *priv)
fail:
while (i--) {
netif_napi_del(&priv->int_vector[i]->napi);
cancel_work_sync(&priv->int_vector[i]->rx_dim.work);
kfree(priv->int_vector[i]);
}
@ -1793,6 +1882,7 @@ void enetc_free_msix(struct enetc_ndev_priv *priv)
struct enetc_int_vector *v = priv->int_vector[i];
netif_napi_del(&v->napi);
cancel_work_sync(&v->rx_dim.work);
}
for (i = 0; i < priv->num_rx_rings; i++)

View File

@ -10,6 +10,7 @@
#include <linux/ethtool.h>
#include <linux/if_vlan.h>
#include <linux/phy.h>
#include <linux/dim.h>
#include "enetc_hw.h"
@ -44,8 +45,9 @@ struct enetc_ring_stats {
unsigned int rx_alloc_errs;
};
#define ENETC_BDR_DEFAULT_SIZE 1024
#define ENETC_DEFAULT_TX_WORK 256
#define ENETC_RX_RING_DEFAULT_SIZE 512
#define ENETC_TX_RING_DEFAULT_SIZE 256
#define ENETC_DEFAULT_TX_WORK (ENETC_TX_RING_DEFAULT_SIZE / 2)
struct enetc_bdr {
struct device *dev; /* for DMA mapping */
@ -189,14 +191,19 @@ static inline bool enetc_si_is_pf(struct enetc_si *si)
struct enetc_int_vector {
void __iomem *rbier;
void __iomem *tbier_base;
void __iomem *ricr1;
unsigned long tx_rings_map;
int count_tx_rings;
struct napi_struct napi;
u32 rx_ictt;
u16 comp_cnt;
bool rx_dim_en, rx_napi_work;
struct napi_struct napi ____cacheline_aligned_in_smp;
struct dim rx_dim ____cacheline_aligned_in_smp;
char name[ENETC_INT_NAME_MAX];
struct enetc_bdr rx_ring ____cacheline_aligned_in_smp;
struct enetc_bdr rx_ring;
struct enetc_bdr tx_ring[];
};
} ____cacheline_aligned_in_smp;
struct enetc_cls_rule {
struct ethtool_rx_flow_spec fs;
@ -220,6 +227,21 @@ enum enetc_active_offloads {
ENETC_F_QCI = BIT(3),
};
/* interrupt coalescing modes */
enum enetc_ic_mode {
/* one interrupt per frame */
ENETC_IC_NONE = 0,
/* activated when int coalescing time is set to a non-0 value */
ENETC_IC_RX_MANUAL = BIT(0),
ENETC_IC_TX_MANUAL = BIT(1),
/* use dynamic interrupt moderation */
ENETC_IC_RX_ADAPTIVE = BIT(2),
};
#define ENETC_RXIC_PKTTHR min_t(u32, 256, ENETC_RX_RING_DEFAULT_SIZE / 2)
#define ENETC_TXIC_PKTTHR min_t(u32, 128, ENETC_TX_RING_DEFAULT_SIZE / 2)
#define ENETC_TXIC_TIMETHR enetc_usecs_to_cycles(600)
struct enetc_ndev_priv {
struct net_device *ndev;
struct device *dev; /* dma-mapping device */
@ -244,6 +266,8 @@ struct enetc_ndev_priv {
struct device_node *phy_node;
phy_interface_t if_mode;
int ic_mode;
u32 tx_ictt;
};
/* Messaging */
@ -273,6 +297,8 @@ void enetc_free_si_resources(struct enetc_ndev_priv *priv);
int enetc_open(struct net_device *ndev);
int enetc_close(struct net_device *ndev);
void enetc_start(struct net_device *ndev);
void enetc_stop(struct net_device *ndev);
netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev);
struct net_device_stats *enetc_get_stats(struct net_device *ndev);
int enetc_set_features(struct net_device *ndev,

View File

@ -14,12 +14,14 @@ static const u32 enetc_si_regs[] = {
static const u32 enetc_txbdr_regs[] = {
ENETC_TBMR, ENETC_TBSR, ENETC_TBBAR0, ENETC_TBBAR1,
ENETC_TBPIR, ENETC_TBCIR, ENETC_TBLENR, ENETC_TBIER
ENETC_TBPIR, ENETC_TBCIR, ENETC_TBLENR, ENETC_TBIER, ENETC_TBICR0,
ENETC_TBICR1
};
static const u32 enetc_rxbdr_regs[] = {
ENETC_RBMR, ENETC_RBSR, ENETC_RBBSR, ENETC_RBCIR, ENETC_RBBAR0,
ENETC_RBBAR1, ENETC_RBPIR, ENETC_RBLENR, ENETC_RBICIR0, ENETC_RBIER
ENETC_RBBAR1, ENETC_RBPIR, ENETC_RBLENR, ENETC_RBIER, ENETC_RBICR0,
ENETC_RBICR1
};
static const u32 enetc_port_regs[] = {
@ -561,6 +563,74 @@ static void enetc_get_ringparam(struct net_device *ndev,
}
}
static int enetc_get_coalesce(struct net_device *ndev,
struct ethtool_coalesce *ic)
{
struct enetc_ndev_priv *priv = netdev_priv(ndev);
struct enetc_int_vector *v = priv->int_vector[0];
ic->tx_coalesce_usecs = enetc_cycles_to_usecs(priv->tx_ictt);
ic->rx_coalesce_usecs = enetc_cycles_to_usecs(v->rx_ictt);
ic->tx_max_coalesced_frames = ENETC_TXIC_PKTTHR;
ic->rx_max_coalesced_frames = ENETC_RXIC_PKTTHR;
ic->use_adaptive_rx_coalesce = priv->ic_mode & ENETC_IC_RX_ADAPTIVE;
return 0;
}
static int enetc_set_coalesce(struct net_device *ndev,
struct ethtool_coalesce *ic)
{
struct enetc_ndev_priv *priv = netdev_priv(ndev);
u32 rx_ictt, tx_ictt;
int i, ic_mode;
bool changed;
tx_ictt = enetc_usecs_to_cycles(ic->tx_coalesce_usecs);
rx_ictt = enetc_usecs_to_cycles(ic->rx_coalesce_usecs);
if (ic->rx_max_coalesced_frames != ENETC_RXIC_PKTTHR)
return -EOPNOTSUPP;
if (ic->tx_max_coalesced_frames != ENETC_TXIC_PKTTHR)
return -EOPNOTSUPP;
ic_mode = ENETC_IC_NONE;
if (ic->use_adaptive_rx_coalesce) {
ic_mode |= ENETC_IC_RX_ADAPTIVE;
rx_ictt = 0x1;
} else {
ic_mode |= rx_ictt ? ENETC_IC_RX_MANUAL : 0;
}
ic_mode |= tx_ictt ? ENETC_IC_TX_MANUAL : 0;
/* commit the settings */
changed = (ic_mode != priv->ic_mode) || (priv->tx_ictt != tx_ictt);
priv->ic_mode = ic_mode;
priv->tx_ictt = tx_ictt;
for (i = 0; i < priv->bdr_int_num; i++) {
struct enetc_int_vector *v = priv->int_vector[i];
v->rx_ictt = rx_ictt;
v->rx_dim_en = !!(ic_mode & ENETC_IC_RX_ADAPTIVE);
}
if (netif_running(ndev) && changed) {
/* reconfigure the operation mode of h/w interrupts,
* traffic needs to be paused in the process
*/
enetc_stop(ndev);
enetc_start(ndev);
}
return 0;
}
static int enetc_get_ts_info(struct net_device *ndev,
struct ethtool_ts_info *info)
{
@ -617,6 +687,9 @@ static int enetc_set_wol(struct net_device *dev,
}
static const struct ethtool_ops enetc_pf_ethtool_ops = {
.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
ETHTOOL_COALESCE_MAX_FRAMES |
ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
.get_regs_len = enetc_get_reglen,
.get_regs = enetc_get_regs,
.get_sset_count = enetc_get_sset_count,
@ -629,6 +702,8 @@ static const struct ethtool_ops enetc_pf_ethtool_ops = {
.get_rxfh = enetc_get_rxfh,
.set_rxfh = enetc_set_rxfh,
.get_ringparam = enetc_get_ringparam,
.get_coalesce = enetc_get_coalesce,
.set_coalesce = enetc_set_coalesce,
.get_link_ksettings = phy_ethtool_get_link_ksettings,
.set_link_ksettings = phy_ethtool_set_link_ksettings,
.get_link = ethtool_op_get_link,
@ -638,6 +713,9 @@ static const struct ethtool_ops enetc_pf_ethtool_ops = {
};
static const struct ethtool_ops enetc_vf_ethtool_ops = {
.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
ETHTOOL_COALESCE_MAX_FRAMES |
ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
.get_regs_len = enetc_get_reglen,
.get_regs = enetc_get_regs,
.get_sset_count = enetc_get_sset_count,
@ -649,6 +727,8 @@ static const struct ethtool_ops enetc_vf_ethtool_ops = {
.get_rxfh = enetc_get_rxfh,
.set_rxfh = enetc_set_rxfh,
.get_ringparam = enetc_get_ringparam,
.get_coalesce = enetc_get_coalesce,
.set_coalesce = enetc_set_coalesce,
.get_link = ethtool_op_get_link,
.get_ts_info = enetc_get_ts_info,
};

View File

@ -121,8 +121,11 @@ enum enetc_bdr_type {TX, RX};
#define ENETC_RBIER 0xa0
#define ENETC_RBIER_RXTIE BIT(0)
#define ENETC_RBIDR 0xa4
#define ENETC_RBICIR0 0xa8
#define ENETC_RBICIR0_ICEN BIT(31)
#define ENETC_RBICR0 0xa8
#define ENETC_RBICR0_ICEN BIT(31)
#define ENETC_RBICR0_ICPT_MASK 0x1ff
#define ENETC_RBICR0_SET_ICPT(n) ((n) & ENETC_RBICR0_ICPT_MASK)
#define ENETC_RBICR1 0xac
/* TX BDR reg offsets */
#define ENETC_TBMR 0
@ -141,8 +144,11 @@ enum enetc_bdr_type {TX, RX};
#define ENETC_TBIER 0xa0
#define ENETC_TBIER_TXTIE BIT(0)
#define ENETC_TBIDR 0xa4
#define ENETC_TBICIR0 0xa8
#define ENETC_TBICIR0_ICEN BIT(31)
#define ENETC_TBICR0 0xa8
#define ENETC_TBICR0_ICEN BIT(31)
#define ENETC_TBICR0_ICPT_MASK 0xf
#define ENETC_TBICR0_SET_ICPT(n) ((ilog2(n) + 1) & ENETC_TBICR0_ICPT_MASK)
#define ENETC_TBICR1 0xac
#define ENETC_RTBLENR_LEN(n) ((n) & ~0x7)
@ -787,6 +793,15 @@ struct enetc_cbd {
};
#define ENETC_CLK 400000000ULL
static inline u32 enetc_cycles_to_usecs(u32 cycles)
{
return (u32)div_u64(cycles * 1000000ULL, ENETC_CLK);
}
static inline u32 enetc_usecs_to_cycles(u32 usecs)
{
return (u32)div_u64(usecs * ENETC_CLK, 1000000ULL);
}
/* port time gating control register */
#define ENETC_QBV_PTGCR_OFFSET 0x11a00