From 60369a4f8d61bacd437adab0518581afb90bea24 Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Tue, 28 Apr 2020 15:36:40 +0000 Subject: [PATCH 1/4] x86/PCI: Drop unused xen_register_pirq() gsi_override parameter All callers of xen_register_pirq() pass -1 (no override) for the gsi_override parameter. Remove it and related code. Link: https://lore.kernel.org/r/20200428153640.76476-1-wei.liu@kernel.org Signed-off-by: Wei Liu Signed-off-by: Bjorn Helgaas Reviewed-by: Boris Ostrovsky --- arch/x86/pci/xen.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index 91220cc25854..e3f1ca316068 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -60,8 +60,7 @@ static int xen_pcifront_enable_irq(struct pci_dev *dev) } #ifdef CONFIG_ACPI -static int xen_register_pirq(u32 gsi, int gsi_override, int triggering, - bool set_pirq) +static int xen_register_pirq(u32 gsi, int triggering, bool set_pirq) { int rc, pirq = -1, irq = -1; struct physdev_map_pirq map_irq; @@ -94,9 +93,6 @@ static int xen_register_pirq(u32 gsi, int gsi_override, int triggering, name = "ioapic-level"; } - if (gsi_override >= 0) - gsi = gsi_override; - irq = xen_bind_pirq_gsi_to_irq(gsi, map_irq.pirq, shareable, name); if (irq < 0) goto out; @@ -112,12 +108,12 @@ static int acpi_register_gsi_xen_hvm(struct device *dev, u32 gsi, if (!xen_hvm_domain()) return -1; - return xen_register_pirq(gsi, -1 /* no GSI override */, trigger, + return xen_register_pirq(gsi, trigger, false /* no mapping of GSI to PIRQ */); } #ifdef CONFIG_XEN_DOM0 -static int xen_register_gsi(u32 gsi, int gsi_override, int triggering, int polarity) +static int xen_register_gsi(u32 gsi, int triggering, int polarity) { int rc, irq; struct physdev_setup_gsi setup_gsi; @@ -128,7 +124,7 @@ static int xen_register_gsi(u32 gsi, int gsi_override, int triggering, int polar printk(KERN_DEBUG "xen: registering gsi %u triggering %d polarity %d\n", gsi, triggering, polarity); - irq = xen_register_pirq(gsi, gsi_override, triggering, true); + irq = xen_register_pirq(gsi, triggering, true); setup_gsi.gsi = gsi; setup_gsi.triggering = (triggering == ACPI_EDGE_SENSITIVE ? 0 : 1); @@ -148,7 +144,7 @@ static int xen_register_gsi(u32 gsi, int gsi_override, int triggering, int polar static int acpi_register_gsi_xen(struct device *dev, u32 gsi, int trigger, int polarity) { - return xen_register_gsi(gsi, -1 /* no GSI override */, trigger, polarity); + return xen_register_gsi(gsi, trigger, polarity); } #endif #endif @@ -491,7 +487,7 @@ int __init pci_xen_initial_domain(void) if (acpi_get_override_irq(irq, &trigger, &polarity) == -1) continue; - xen_register_pirq(irq, -1 /* no GSI override */, + xen_register_pirq(irq, trigger ? ACPI_LEVEL_SENSITIVE : ACPI_EDGE_SENSITIVE, true /* Map GSI to PIRQ */); } From 0d14f06cd6657ba3446a5eb780672da487b068e7 Mon Sep 17 00:00:00 2001 From: Marcos Scriven Date: Wed, 20 May 2020 18:23:30 -0500 Subject: [PATCH 2/4] PCI: Avoid FLR for AMD Matisse HD Audio & USB 3.0 The AMD Matisse HD Audio & USB 3.0 devices advertise Function Level Reset support, but hang when an FLR is triggered. To reproduce the problem, attach the device to a VM, then detach and try to attach again. Rename the existing quirk_intel_no_flr(), which was not Intel-specific, to quirk_no_flr(), and apply it to prevent the use of FLR on these AMD devices. Link: https://lore.kernel.org/r/CAAri2DpkcuQZYbT6XsALhx2e6vRqPHwtbjHYeiH7MNp4zmt1RA@mail.gmail.com Signed-off-by: Marcos Scriven Signed-off-by: Bjorn Helgaas --- drivers/pci/quirks.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 28c9a2409c50..ff310f0cac22 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -5129,13 +5129,23 @@ static void quirk_intel_qat_vf_cap(struct pci_dev *pdev) } DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x443, quirk_intel_qat_vf_cap); -/* FLR may cause some 82579 devices to hang */ -static void quirk_intel_no_flr(struct pci_dev *dev) +/* + * FLR may cause the following to devices to hang: + * + * AMD Starship/Matisse HD Audio Controller 0x1487 + * AMD Matisse USB 3.0 Host Controller 0x149c + * Intel 82579LM Gigabit Ethernet Controller 0x1502 + * Intel 82579V Gigabit Ethernet Controller 0x1503 + * + */ +static void quirk_no_flr(struct pci_dev *dev) { dev->dev_flags |= PCI_DEV_FLAGS_NO_FLR_RESET; } -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1502, quirk_intel_no_flr); -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1503, quirk_intel_no_flr); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x1487, quirk_no_flr); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x149c, quirk_no_flr); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1502, quirk_no_flr); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1503, quirk_no_flr); static void quirk_no_ext_tags(struct pci_dev *pdev) { From 5727043c73fdfe04597971b5f3f4850d879c1f4f Mon Sep 17 00:00:00 2001 From: Kevin Buettner Date: Sun, 24 May 2020 00:35:29 -0700 Subject: [PATCH 3/4] PCI: Avoid FLR for AMD Starship USB 3.0 The AMD Starship USB 3.0 host controller advertises Function Level Reset support, but it apparently doesn't work. Add a quirk to prevent use of FLR on this device. Without this quirk, when attempting to assign (pass through) an AMD Starship USB 3.0 host controller to a guest OS, the system becomes increasingly unresponsive over the course of several minutes, eventually requiring a hard reset. Shortly after attempting to start the guest, I see these messages: vfio-pci 0000:05:00.3: not ready 1023ms after FLR; waiting vfio-pci 0000:05:00.3: not ready 2047ms after FLR; waiting vfio-pci 0000:05:00.3: not ready 4095ms after FLR; waiting vfio-pci 0000:05:00.3: not ready 8191ms after FLR; waiting And then eventually: vfio-pci 0000:05:00.3: not ready 65535ms after FLR; giving up INFO: NMI handler (perf_event_nmi_handler) took too long to run: 0.000 msecs perf: interrupt took too long (642744 > 2500), lowering kernel.perf_event_max_sample_rate to 1000 INFO: NMI handler (perf_event_nmi_handler) took too long to run: 82.270 msecs INFO: NMI handler (perf_event_nmi_handler) took too long to run: 680.608 msecs INFO: NMI handler (perf_event_nmi_handler) took too long to run: 100.952 msecs ... watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [qemu-system-x86:7487] Tested on a Micro-Star International Co., Ltd. MS-7C59/Creator TRX40 motherboard with an AMD Ryzen Threadripper 3970X. Link: https://lore.kernel.org/r/20200524003529.598434ff@f31-4.lan Signed-off-by: Kevin Buettner Signed-off-by: Bjorn Helgaas --- drivers/pci/quirks.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index ff310f0cac22..15341eacc50d 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -5133,6 +5133,7 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x443, quirk_intel_qat_vf_cap); * FLR may cause the following to devices to hang: * * AMD Starship/Matisse HD Audio Controller 0x1487 + * AMD Starship USB 3.0 Host Controller 0x148c * AMD Matisse USB 3.0 Host Controller 0x149c * Intel 82579LM Gigabit Ethernet Controller 0x1502 * Intel 82579V Gigabit Ethernet Controller 0x1503 @@ -5143,6 +5144,7 @@ static void quirk_no_flr(struct pci_dev *dev) dev->dev_flags |= PCI_DEV_FLAGS_NO_FLR_RESET; } DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x1487, quirk_no_flr); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x148c, quirk_no_flr); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x149c, quirk_no_flr); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1502, quirk_no_flr); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1503, quirk_no_flr); From 3247bd10a4502a3075ce8e1c3c7d31ef76f193ce Mon Sep 17 00:00:00 2001 From: Ashok Raj Date: Thu, 28 May 2020 13:57:42 -0700 Subject: [PATCH 4/4] PCI: Add ACS quirk for Intel Root Complex Integrated Endpoints All Intel platforms guarantee that all root complex implementations must send transactions up to IOMMU for address translations. Hence for Intel RCiEP devices, we can assume some ACS-type isolation even without an ACS capability. From the Intel VT-d spec, r3.1, sec 3.16 ("Root-Complex Peer to Peer Considerations"): When DMA remapping is enabled, peer-to-peer requests through the Root-Complex must be handled as follows: - The input address in the request is translated (through first-level, second-level or nested translation) to a host physical address (HPA). The address decoding for peer addresses must be done only on the translated HPA. Hardware implementations are free to further limit peer-to-peer accesses to specific host physical address regions (or to completely disallow peer-forwarding of translated requests). - Since address translation changes the contents (address field) of the PCI Express Transaction Layer Packet (TLP), for PCI Express peer-to-peer requests with ECRC, the Root-Complex hardware must use the new ECRC (re-computed with the translated address) if it decides to forward the TLP as a peer request. - Root-ports, and multi-function root-complex integrated endpoints, may support additional peer-to-peer control features by supporting PCI Express Access Control Services (ACS) capability. Refer to ACS capability in PCI Express specifications for details. Since Linux didn't give special treatment to allow this exception, certain RCiEP MFD devices were grouped in a single IOMMU group. This doesn't permit a single device to be assigned to a guest for instance. In one vendor system: Device 14.x were grouped in a single IOMMU group. /sys/kernel/iommu_groups/5/devices/0000:00:14.0 /sys/kernel/iommu_groups/5/devices/0000:00:14.2 /sys/kernel/iommu_groups/5/devices/0000:00:14.3 After this patch: /sys/kernel/iommu_groups/5/devices/0000:00:14.0 /sys/kernel/iommu_groups/5/devices/0000:00:14.2 /sys/kernel/iommu_groups/6/devices/0000:00:14.3 <<< new group 14.0 and 14.2 are integrated devices, but legacy end points, whereas 14.3 was a PCIe-compliant RCiEP. 00:14.3 Network controller: Intel Corporation Device 9df0 (rev 30) Capabilities: [40] Express (v2) Root Complex Integrated Endpoint, MSI 00 This permits assigning this device to a guest VM. [bhelgaas: drop "Fixes" tag since this doesn't fix a bug in that commit] Link: https://lore.kernel.org/r/1590699462-7131-1-git-send-email-ashok.raj@intel.com Tested-by: Darrel Goeddel Signed-off-by: Ashok Raj Signed-off-by: Bjorn Helgaas Reviewed-by: Alex Williamson Cc: stable@vger.kernel.org Cc: Lu Baolu Cc: Mark Scott , Cc: Romil Sharma --- drivers/pci/quirks.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 15341eacc50d..5cd2a3bd115a 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -4682,6 +4682,20 @@ static int pci_quirk_mf_endpoint_acs(struct pci_dev *dev, u16 acs_flags) PCI_ACS_CR | PCI_ACS_UF | PCI_ACS_DT); } +static int pci_quirk_rciep_acs(struct pci_dev *dev, u16 acs_flags) +{ + /* + * Intel RCiEP's are required to allow p2p only on translated + * addresses. Refer to Intel VT-d specification, r3.1, sec 3.16, + * "Root-Complex Peer to Peer Considerations". + */ + if (pci_pcie_type(dev) != PCI_EXP_TYPE_RC_END) + return -ENOTTY; + + return pci_acs_ctrl_enabled(acs_flags, + PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF); +} + static int pci_quirk_brcm_acs(struct pci_dev *dev, u16 acs_flags) { /* @@ -4764,6 +4778,7 @@ static const struct pci_dev_acs_enabled { /* I219 */ { PCI_VENDOR_ID_INTEL, 0x15b7, pci_quirk_mf_endpoint_acs }, { PCI_VENDOR_ID_INTEL, 0x15b8, pci_quirk_mf_endpoint_acs }, + { PCI_VENDOR_ID_INTEL, PCI_ANY_ID, pci_quirk_rciep_acs }, /* QCOM QDF2xxx root ports */ { PCI_VENDOR_ID_QCOM, 0x0400, pci_quirk_qcom_rp_acs }, { PCI_VENDOR_ID_QCOM, 0x0401, pci_quirk_qcom_rp_acs },