openkylin
/
linux
mirror of https://gitee.com/openkylin/linux.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

NTB: NTB-RP support 

Add support for Non-Transparent Bridge connected to a PCI-E Root Port on
the remote system (also known as NTB-RP mode).  This allows for a NTB
enabled system to be connected to a non-NTB enabled system/slot.

Modifications to the registers and BARs/MWs on the Secondary side by the
remote system are reflected into registers on the Primary side for the
local system.  Similarly, modifications of registers and BARs/MWs on
Primary side by the local system are reflected into registers on the
Secondary side for the Remote System.  This allows communication between
the 2 sides via these registers and BARs/MWs.

Note: there is not a fix for the Xeon Errata (that was already worked
around in NTB-B2B mode) for NTB-RP mode.  Due to this limitation, NTB-RP
will not work on the Secondary side with the Xeon Errata workaround
enabled.  To get around this, disable the workaround via the
xeon_errata_workaround=0 modparm.  However, this can cause the hang
described in the errata.

Signed-off-by: Jon Mason <jon.mason@intel.com>
This commit is contained in:
Jon Mason 2013-07-15 16:43:54 -07:00
parent 497938890a
commit ed6c24eda9
2 changed files with 171 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
drivers/ntb/ntb_hw.c
View File

@ -69,7 +69,7 @@ module_param(xeon_errata_workaround, bool, 0644);
MODULE_PARM_DESC(xeon_errata_workaround, "Workaround for the Xeon Errata"); MODULE_PARM_DESC(xeon_errata_workaround, "Workaround for the Xeon Errata");
enum { enum {
NTB_CONN_CLASSIC = 0, NTB_CONN_TRANSPARENT = 0,
NTB_CONN_B2B, NTB_CONN_B2B,
NTB_CONN_RP, NTB_CONN_RP,
}; };
@ -509,7 +509,8 @@ static void ntb_link_event(struct ntb_device *ndev, int link_state)
ndev->link_status = NTB_LINK_UP; ndev->link_status = NTB_LINK_UP;
event = NTB_EVENT_HW_LINK_UP; event = NTB_EVENT_HW_LINK_UP;
if (ndev->hw_type == BWD_HW) if (ndev->hw_type == BWD_HW ||
ndev->conn_type == NTB_CONN_TRANSPARENT)
status = readw(ndev->reg_ofs.lnk_stat); status = readw(ndev->reg_ofs.lnk_stat);
else { else {
int rc = pci_read_config_word(ndev->pdev, int rc = pci_read_config_word(ndev->pdev,
@ -649,30 +650,21 @@ static int ntb_xeon_setup(struct ntb_device *ndev)
if (rc) if (rc)
return rc; return rc;
switch (val & SNB_PPD_CONN_TYPE) {
case NTB_CONN_B2B:
ndev->conn_type = NTB_CONN_B2B;
break;
case NTB_CONN_CLASSIC:
case NTB_CONN_RP:
default:
dev_err(&ndev->pdev->dev, "Only B2B supported at this time\n");
return -EINVAL;
}
if (val & SNB_PPD_DEV_TYPE) if (val & SNB_PPD_DEV_TYPE)
ndev->dev_type = NTB_DEV_USD; ndev->dev_type = NTB_DEV_USD;
else else
ndev->dev_type = NTB_DEV_DSD; ndev->dev_type = NTB_DEV_DSD;
switch (val & SNB_PPD_CONN_TYPE) {
case NTB_CONN_B2B:
dev_info(&ndev->pdev->dev, "Conn Type = B2B\n");
ndev->conn_type = NTB_CONN_B2B;
ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET; ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET; ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET; ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET; ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET; ndev->limits.max_spads = SNB_MAX_B2B_SPADS;
ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_LINK_STATUS_OFFSET;
ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;
/* There is a Xeon hardware errata related to writes to /* There is a Xeon hardware errata related to writes to
* SDOORBELL or B2BDOORBELL in conjunction with inbound access * SDOORBELL or B2BDOORBELL in conjunction with inbound access
@ -724,7 +716,7 @@ static int ntb_xeon_setup(struct ntb_device *ndev)
/* B2B_XLAT_OFFSET is a 64bit register, but can /* B2B_XLAT_OFFSET is a 64bit register, but can
* only take 32bit writes * only take 32bit writes
*/ */
writel(SNB_MBAR01_USD_ADDR & 0xffffffff, writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
ndev->reg_base + SNB_B2B_XLAT_OFFSETL); ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
writel(SNB_MBAR01_DSD_ADDR >> 32, writel(SNB_MBAR01_DSD_ADDR >> 32,
ndev->reg_base + SNB_B2B_XLAT_OFFSETU); ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
@ -748,7 +740,7 @@ static int ntb_xeon_setup(struct ntb_device *ndev)
/* B2B_XLAT_OFFSET is a 64bit register, but can /* B2B_XLAT_OFFSET is a 64bit register, but can
* only take 32bit writes * only take 32bit writes
*/ */
writel(SNB_MBAR01_USD_ADDR & 0xffffffff, writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
ndev->reg_base + SNB_B2B_XLAT_OFFSETL); ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
writel(SNB_MBAR01_USD_ADDR >> 32, writel(SNB_MBAR01_USD_ADDR >> 32,
ndev->reg_base + SNB_B2B_XLAT_OFFSETU); ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
@ -760,8 +752,72 @@ static int ntb_xeon_setup(struct ntb_device *ndev)
writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base + writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base +
SNB_SBAR4BASE_OFFSET); SNB_SBAR4BASE_OFFSET);
} }
break;
case NTB_CONN_RP:
dev_info(&ndev->pdev->dev, "Conn Type = RP\n");
ndev->conn_type = NTB_CONN_RP;
if (xeon_errata_workaround) {
dev_err(&ndev->pdev->dev,
"NTB-RP disabled due to hardware errata. To disregard this warning and potentially lock-up the system, add the parameter 'xeon_errata_workaround=0'.\n");
return -EINVAL;
}
/* Scratch pads need to have exclusive access from the primary
* or secondary side. Halve the num spads so that each side can
* have an equal amount.
*/
ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
/* Note: The SDOORBELL is the cause of the errata. You REALLY
* don't want to touch it.
*/
ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
/* Offset the start of the spads to correspond to whether it is
* primary or secondary
*/
ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET +
ndev->limits.max_spads * 4;
ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
ndev->limits.max_mw = SNB_MAX_MW;
break;
case NTB_CONN_TRANSPARENT:
dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n");
ndev->conn_type = NTB_CONN_TRANSPARENT;
/* Scratch pads need to have exclusive access from the primary
* or secondary side. Halve the num spads so that each side can
* have an equal amount.
*/
ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET;
ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET;
/* Offset the start of the spads to correspond to whether it is
* primary or secondary
*/
ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET +
ndev->limits.max_spads * 4;
ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET;
ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET;
ndev->limits.max_mw = SNB_MAX_MW;
break;
default:
/* Most likely caused by the remote NTB-RP device not being
* configured
*/
dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", val);
return -EINVAL;
}
ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET;
ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET;
ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;
ndev->limits.max_spads = SNB_MAX_B2B_SPADS;
ndev->limits.max_db_bits = SNB_MAX_DB_BITS; ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
ndev->limits.msix_cnt = SNB_MSIX_CNT; ndev->limits.msix_cnt = SNB_MSIX_CNT;
ndev->bits_per_vector = SNB_DB_BITS_PER_VEC; ndev->bits_per_vector = SNB_DB_BITS_PER_VEC;
@ -865,8 +921,10 @@ static int ntb_device_setup(struct ntb_device *ndev)
dev_info(&ndev->pdev->dev, "Device Type = %s\n", dev_info(&ndev->pdev->dev, "Device Type = %s\n",
ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP"); ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP");
if (ndev->conn_type == NTB_CONN_B2B)
/* Enable Bus Master and Memory Space on the secondary side */ /* Enable Bus Master and Memory Space on the secondary side */
writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, ndev->reg_ofs.spci_cmd); writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
ndev->reg_ofs.spci_cmd);
return 0; return 0;
} }
@ -1360,7 +1418,7 @@ static void ntb_pci_remove(struct pci_dev *pdev)
/* Bring NTB link down */ /* Bring NTB link down */
ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
ntb_cntl |= NTB_LINK_DISABLE; ntb_cntl |= NTB_CNTL_LINK_DISABLE;
writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
ntb_transport_free(ndev->ntb_transport); ntb_transport_free(ndev->ntb_transport);

5
drivers/ntb/ntb_regs.h
View File

@ -46,8 +46,6 @@
* Jon Mason <jon.mason@intel.com> * Jon Mason <jon.mason@intel.com>
*/ */
#define NTB_LINK_ENABLE 0x0000
#define NTB_LINK_DISABLE 0x0002
#define NTB_LINK_STATUS_ACTIVE 0x2000 #define NTB_LINK_STATUS_ACTIVE 0x2000
#define NTB_LINK_SPEED_MASK 0x000f #define NTB_LINK_SPEED_MASK 0x000f
#define NTB_LINK_WIDTH_MASK 0x03f0 #define NTB_LINK_WIDTH_MASK 0x03f0
@ -65,6 +63,7 @@
#define SNB_PCICMD_OFFSET 0x0504 #define SNB_PCICMD_OFFSET 0x0504
#define SNB_DEVCTRL_OFFSET 0x0598 #define SNB_DEVCTRL_OFFSET 0x0598
#define SNB_SLINK_STATUS_OFFSET 0x05A2
#define SNB_LINK_STATUS_OFFSET 0x01A2 #define SNB_LINK_STATUS_OFFSET 0x01A2
#define SNB_PBAR2LMT_OFFSET 0x0000 #define SNB_PBAR2LMT_OFFSET 0x0000
@ -147,6 +146,8 @@
#define BWD_LTSSMSTATEJMP_FORCEDETECT (1 << 2) #define BWD_LTSSMSTATEJMP_FORCEDETECT (1 << 2)
#define BWD_IBIST_ERR_OFLOW 0x7FFF7FFF #define BWD_IBIST_ERR_OFLOW 0x7FFF7FFF
#define NTB_CNTL_CFG_LOCK (1 << 0)
#define NTB_CNTL_LINK_DISABLE (1 << 1)
#define NTB_CNTL_BAR23_SNOOP (1 << 2) #define NTB_CNTL_BAR23_SNOOP (1 << 2)
#define NTB_CNTL_BAR45_SNOOP (1 << 6) #define NTB_CNTL_BAR45_SNOOP (1 << 6)
#define BWD_CNTL_LINK_DOWN (1 << 16) #define BWD_CNTL_LINK_DOWN (1 << 16)