From 37d7f818a4629ebbb94c2c092c533a437ba7f95c Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 20 Jul 2017 19:33:53 -0400 Subject: [PATCH 1/8] PCI: vmd: Reserve IRQ pre-vector for better affinity The driver has a special purpose for the VMD device's first IRQ, so this one shouldn't be considered for IRQ affinity. Signed-off-by: Keith Busch Signed-off-by: Bjorn Helgaas --- drivers/pci/host/vmd.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/pci/host/vmd.c b/drivers/pci/host/vmd.c index 6088c3083194..539da102f3d7 100644 --- a/drivers/pci/host/vmd.c +++ b/drivers/pci/host/vmd.c @@ -671,6 +671,14 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id) struct vmd_dev *vmd; int i, err; + /* + * The first vector is reserved for special use, so start affinity at + * the second vector + */ + struct irq_affinity affd = { + .pre_vectors = 1, + }; + if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20)) return -ENOMEM; @@ -696,8 +704,8 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id) if (vmd->msix_count < 0) return -ENODEV; - vmd->msix_count = pci_alloc_irq_vectors(dev, 1, vmd->msix_count, - PCI_IRQ_MSIX | PCI_IRQ_AFFINITY); + vmd->msix_count = pci_alloc_irq_vectors_affinity(dev, 1, vmd->msix_count, + PCI_IRQ_MSIX | PCI_IRQ_AFFINITY, &affd); if (vmd->msix_count < 0) return vmd->msix_count; From f2586c678cb29f40a37c9c88a22b13d35484ffdc Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 20 Jul 2017 19:33:54 -0400 Subject: [PATCH 2/8] PCI: vmd: Assign vector zero to all bridges We don't want slower IRQ handlers impacting faster devices that happen to be assigned the same VMD interrupt vector. The driver was trying to separate such devices by checking if MSI-X wasn't used, but really we just don't want endpoint devices to share with bridges. Most bridges may use MSI currently, so that criteria happened to work, but newer ones may use MSI-X, so this patch explicitly checks the device type when choosing a vector. Signed-off-by: Keith Busch Signed-off-by: Bjorn Helgaas --- drivers/pci/host/vmd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pci/host/vmd.c b/drivers/pci/host/vmd.c index 539da102f3d7..7e967a8784b2 100644 --- a/drivers/pci/host/vmd.c +++ b/drivers/pci/host/vmd.c @@ -183,7 +183,7 @@ static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *d int i, best = 1; unsigned long flags; - if (!desc->msi_attrib.is_msix || vmd->msix_count == 1) + if (pci_is_bridge(msi_desc_to_pci_dev(desc)) || vmd->msix_count == 1) return &vmd->irqs[0]; raw_spin_lock_irqsave(&list_lock, flags); From e2b1820bd5d0962d6f271b0d47c3a0e38647df2f Mon Sep 17 00:00:00 2001 From: Scott Bauer Date: Fri, 11 Aug 2017 14:54:32 -0600 Subject: [PATCH 3/8] PCI: vmd: Free up IRQs on suspend path Free up the IRQs we request on the suspend path and reallocate them on the resume path. Fixes this error: CPU 111 disable failed: CPU has 9 vectors assigned and there are only 0 available. Error taking CPU111 down: -34 Non-boot CPUs are not disabled Enabling non-boot CPUs ... Signed-off-by: Scott Bauer Signed-off-by: Bjorn Helgaas Acked-by: Keith Busch --- drivers/pci/host/vmd.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/pci/host/vmd.c b/drivers/pci/host/vmd.c index 7e967a8784b2..4fe1756af010 100644 --- a/drivers/pci/host/vmd.c +++ b/drivers/pci/host/vmd.c @@ -763,6 +763,11 @@ static void vmd_remove(struct pci_dev *dev) static int vmd_suspend(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); + struct vmd_dev *vmd = pci_get_drvdata(pdev); + int i; + + for (i = 0; i < vmd->msix_count; i++) + devm_free_irq(dev, pci_irq_vector(pdev, i), &vmd->irqs[i]); pci_save_state(pdev); return 0; @@ -771,6 +776,16 @@ static int vmd_suspend(struct device *dev) static int vmd_resume(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); + struct vmd_dev *vmd = pci_get_drvdata(pdev); + int err, i; + + for (i = 0; i < vmd->msix_count; i++) { + err = devm_request_irq(dev, pci_irq_vector(pdev, i), + vmd_irq, IRQF_NO_THREAD, + "vmd", &vmd->irqs[i]); + if (err) + return err; + } pci_restore_state(pdev); return 0; From 46a6561b29cb42c06ebf193ccec3c5b666c0fc3f Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Wed, 30 Aug 2017 12:15:04 -0400 Subject: [PATCH 4/8] PCI: vmd: Remove IRQ affinity so we can allocate more IRQs VMD hardware has to share its vectors among child devices in its PCI domain so we should allocate as many as possible rather than just ones that can be affinitized. pci_alloc_irq_vectors_affinity() limits the number of affinitized IRQs to the number of present CPUs (see irq_calc_affinity_vectors()). But we'd prefer to have more vectors, even if they aren't distributed across the CPUs, so use pci_alloc_irq_vectors() instead. Reported-by: Brad Goodman Signed-off-by: Keith Busch [bhelgaas: add irq_calc_affinity_vectors() reference to changelog] Signed-off-by: Bjorn Helgaas --- drivers/pci/host/vmd.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/drivers/pci/host/vmd.c b/drivers/pci/host/vmd.c index 4fe1756af010..509893bc3e63 100644 --- a/drivers/pci/host/vmd.c +++ b/drivers/pci/host/vmd.c @@ -671,14 +671,6 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id) struct vmd_dev *vmd; int i, err; - /* - * The first vector is reserved for special use, so start affinity at - * the second vector - */ - struct irq_affinity affd = { - .pre_vectors = 1, - }; - if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20)) return -ENOMEM; @@ -704,8 +696,8 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id) if (vmd->msix_count < 0) return -ENODEV; - vmd->msix_count = pci_alloc_irq_vectors_affinity(dev, 1, vmd->msix_count, - PCI_IRQ_MSIX | PCI_IRQ_AFFINITY, &affd); + vmd->msix_count = pci_alloc_irq_vectors(dev, 1, vmd->msix_count, + PCI_IRQ_MSIX); if (vmd->msix_count < 0) return vmd->msix_count; From 7674d05dd2dcf3904b3ea6beb222e020879626f1 Mon Sep 17 00:00:00 2001 From: Jon Derrick Date: Thu, 17 Aug 2017 12:10:11 -0600 Subject: [PATCH 5/8] MAINTAINERS: Add Jonathan Derrick as VMD maintainer Add Jonathan Derrick as VMD maintainer. Signed-off-by: Jon Derrick Signed-off-by: Bjorn Helgaas Acked-by: Keith Busch --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index f66488dfdbc9..3ec39df93f93 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10090,6 +10090,7 @@ F: drivers/pci/dwc/*imx6* PCI DRIVER FOR INTEL VOLUME MANAGEMENT DEVICE (VMD) M: Keith Busch +M: Jonathan Derrick L: linux-pci@vger.kernel.org S: Supported F: drivers/pci/host/vmd.c From f1b0e54e16b3db7882b4158ab4a26b87841753fd Mon Sep 17 00:00:00 2001 From: Jon Derrick Date: Thu, 17 Aug 2017 12:10:12 -0600 Subject: [PATCH 6/8] x86/PCI: Move VMD quirk to x86 fixups VMD currently only exists for Intel x86 products, so move the VMD quirk to arch/x86. Signed-off-by: Jon Derrick Signed-off-by: Bjorn Helgaas --- arch/x86/pci/fixup.c | 17 +++++++++++++++++ drivers/pci/quirks.c | 17 ----------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/arch/x86/pci/fixup.c b/arch/x86/pci/fixup.c index 11e407489db0..4c2e318845ba 100644 --- a/arch/x86/pci/fixup.c +++ b/arch/x86/pci/fixup.c @@ -618,3 +618,20 @@ static void quirk_apple_mbp_poweroff(struct pci_dev *pdev) dev_info(dev, "can't work around MacBook Pro poweroff issue\n"); } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x8c10, quirk_apple_mbp_poweroff); + +/* + * VMD-enabled root ports will change the source ID for all messages + * to the VMD device. Rather than doing device matching with the source + * ID, the AER driver should traverse the child device tree, reading + * AER registers to find the faulting device. + */ +static void quirk_no_aersid(struct pci_dev *pdev) +{ + /* VMD Domain */ + if (pdev->bus->sysdata && pci_domain_nr(pdev->bus) >= 0x10000) + pdev->bus->bus_flags |= PCI_BUS_FLAGS_NO_AERSID; +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2030, quirk_no_aersid); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2031, quirk_no_aersid); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2032, quirk_no_aersid); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2033, quirk_no_aersid); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 6967c6b4cf6b..0b0a2ae9a853 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -4657,23 +4657,6 @@ static void quirk_intel_qat_vf_cap(struct pci_dev *pdev) } DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x443, quirk_intel_qat_vf_cap); -/* - * VMD-enabled root ports will change the source ID for all messages - * to the VMD device. Rather than doing device matching with the source - * ID, the AER driver should traverse the child device tree, reading - * AER registers to find the faulting device. - */ -static void quirk_no_aersid(struct pci_dev *pdev) -{ - /* VMD Domain */ - if (pdev->bus->sysdata && pci_domain_nr(pdev->bus) >= 0x10000) - pdev->bus->bus_flags |= PCI_BUS_FLAGS_NO_AERSID; -} -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2030, quirk_no_aersid); -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2031, quirk_no_aersid); -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2032, quirk_no_aersid); -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2033, quirk_no_aersid); - /* FLR may cause some 82579 devices to hang. */ static void quirk_intel_no_flr(struct pci_dev *dev) { From c37f23d44e866d079d30399cb358dd19617562a1 Mon Sep 17 00:00:00 2001 From: Jon Derrick Date: Thu, 17 Aug 2017 12:10:13 -0600 Subject: [PATCH 7/8] x86/PCI: Use is_vmd() rather than relying on the domain number Use the is_vmd() predicate to identify devices below a VMD host rather than relying on the domain number. Signed-off-by: Jon Derrick Signed-off-by: Bjorn Helgaas --- arch/x86/pci/fixup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/pci/fixup.c b/arch/x86/pci/fixup.c index 4c2e318845ba..f2228b150faa 100644 --- a/arch/x86/pci/fixup.c +++ b/arch/x86/pci/fixup.c @@ -628,7 +628,7 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x8c10, quirk_apple_mbp_poweroff); static void quirk_no_aersid(struct pci_dev *pdev) { /* VMD Domain */ - if (pdev->bus->sysdata && pci_domain_nr(pdev->bus) >= 0x10000) + if (is_vmd(pdev->bus)) pdev->bus->bus_flags |= PCI_BUS_FLAGS_NO_AERSID; } DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2030, quirk_no_aersid); From 5823e330b58c52d74bcf4588626863994a53f6d9 Mon Sep 17 00:00:00 2001 From: Jon Derrick Date: Wed, 30 Aug 2017 15:05:59 -0600 Subject: [PATCH 8/8] iommu/vt-d: Prevent VMD child devices from being remapping targets VMD child devices must use the VMD endpoint's ID as the requester. Because of this, there needs to be a way to link the parent VMD endpoint's IOMMU group and associated mappings to the VMD child devices such that attaching and detaching child devices modify the endpoint's mappings, while preventing early detaching on a singular device removal or unbinding. The reassignment of individual VMD child devices devices to VMs is outside the scope of VMD, but may be implemented in the future. For now it is best to prevent any such attempts. Prevent VMD child devices from returning an IOMMU, which prevents it from exposing an iommu_group sysfs directory and allowing subsequent binding by userspace-access drivers such as VFIO. Signed-off-by: Jon Derrick Signed-off-by: Bjorn Helgaas --- drivers/iommu/intel-iommu.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 687f18f65cea..2800a6ee8cc9 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -901,6 +901,13 @@ static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devf struct pci_dev *pf_pdev; pdev = to_pci_dev(dev); + +#ifdef CONFIG_X86 + /* VMD child devices currently cannot be handled individually */ + if (is_vmd(pdev->bus)) + return NULL; +#endif + /* VFs aren't listed in scope tables; we need to look up * the PF instead to find the IOMMU. */ pf_pdev = pci_physfn(pdev);