From b913efe78a7ce1b2e64af7e5dc3a03748b997c61 Mon Sep 17 00:00:00 2001 From: Sricharan R Date: Mon, 10 Apr 2017 16:51:04 +0530 Subject: arm64: dma-mapping: Remove the notifier trick to handle early setting of dma_ops With arch_setup_dma_ops now being called late during device's probe after the device's iommu is probed, the notifier trick required to handle the early setup of dma_ops before the iommu group gets created is not required. So removing the notifier's here. Tested-by: Marek Szyprowski Tested-by: Hanjun Guo Acked-by: Will Deacon Signed-off-by: Sricharan R [rm: clean up even more] Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- arch/arm64/mm/dma-mapping.c | 142 ++++++-------------------------------------- 1 file changed, 18 insertions(+), 124 deletions(-) (limited to 'arch') diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 81cdb2e844ed..b46575954032 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -813,34 +813,26 @@ static const struct dma_map_ops iommu_dma_ops = { .mapping_error = iommu_dma_mapping_error, }; -/* - * TODO: Right now __iommu_setup_dma_ops() gets called too early to do - * everything it needs to - the device is only partially created and the - * IOMMU driver hasn't seen it yet, so it can't have a group. Thus we - * need this delayed attachment dance. Once IOMMU probe ordering is sorted - * to move the arch_setup_dma_ops() call later, all the notifier bits below - * become unnecessary, and will go away. - */ -struct iommu_dma_notifier_data { - struct list_head list; - struct device *dev; - const struct iommu_ops *ops; - u64 dma_base; - u64 size; -}; -static LIST_HEAD(iommu_dma_masters); -static DEFINE_MUTEX(iommu_dma_notifier_lock); +static int __init __iommu_dma_init(void) +{ + return iommu_dma_init(); +} +arch_initcall(__iommu_dma_init); -static bool do_iommu_attach(struct device *dev, const struct iommu_ops *ops, - u64 dma_base, u64 size) +static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, + const struct iommu_ops *ops) { - struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct iommu_domain *domain; + + if (!ops) + return; /* - * If the IOMMU driver has the DMA domain support that we require, - * then the IOMMU core will have already configured a group for this - * device, and allocated the default domain for that group. + * The IOMMU core code allocates the default DMA domain, which the + * underlying IOMMU driver needs to support via the dma-iommu layer. */ + domain = iommu_get_domain_for_dev(dev); + if (!domain) goto out_err; @@ -851,109 +843,11 @@ static bool do_iommu_attach(struct device *dev, const struct iommu_ops *ops, dev->dma_ops = &iommu_dma_ops; } - return true; + return; + out_err: - pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n", + pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n", dev_name(dev)); - return false; -} - -static void queue_iommu_attach(struct device *dev, const struct iommu_ops *ops, - u64 dma_base, u64 size) -{ - struct iommu_dma_notifier_data *iommudata; - - iommudata = kzalloc(sizeof(*iommudata), GFP_KERNEL); - if (!iommudata) - return; - - iommudata->dev = dev; - iommudata->ops = ops; - iommudata->dma_base = dma_base; - iommudata->size = size; - - mutex_lock(&iommu_dma_notifier_lock); - list_add(&iommudata->list, &iommu_dma_masters); - mutex_unlock(&iommu_dma_notifier_lock); -} - -static int __iommu_attach_notifier(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct iommu_dma_notifier_data *master, *tmp; - - if (action != BUS_NOTIFY_BIND_DRIVER) - return 0; - - mutex_lock(&iommu_dma_notifier_lock); - list_for_each_entry_safe(master, tmp, &iommu_dma_masters, list) { - if (data == master->dev && do_iommu_attach(master->dev, - master->ops, master->dma_base, master->size)) { - list_del(&master->list); - kfree(master); - break; - } - } - mutex_unlock(&iommu_dma_notifier_lock); - return 0; -} - -static int __init register_iommu_dma_ops_notifier(struct bus_type *bus) -{ - struct notifier_block *nb = kzalloc(sizeof(*nb), GFP_KERNEL); - int ret; - - if (!nb) - return -ENOMEM; - - nb->notifier_call = __iommu_attach_notifier; - - ret = bus_register_notifier(bus, nb); - if (ret) { - pr_warn("Failed to register DMA domain notifier; IOMMU DMA ops unavailable on bus '%s'\n", - bus->name); - kfree(nb); - } - return ret; -} - -static int __init __iommu_dma_init(void) -{ - int ret; - - ret = iommu_dma_init(); - if (!ret) - ret = register_iommu_dma_ops_notifier(&platform_bus_type); - if (!ret) - ret = register_iommu_dma_ops_notifier(&amba_bustype); -#ifdef CONFIG_PCI - if (!ret) - ret = register_iommu_dma_ops_notifier(&pci_bus_type); -#endif - return ret; -} -arch_initcall(__iommu_dma_init); - -static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - const struct iommu_ops *ops) -{ - struct iommu_group *group; - - if (!ops) - return; - /* - * TODO: As a concession to the future, we're ready to handle being - * called both early and late (i.e. after bus_add_device). Once all - * the platform bus code is reworked to call us late and the notifier - * junk above goes away, move the body of do_iommu_attach here. - */ - group = iommu_group_get(dev); - if (group) { - do_iommu_attach(dev, ops, dma_base, size); - iommu_group_put(group); - } else { - queue_iommu_attach(dev, ops, dma_base, size); - } } void arch_teardown_dma_ops(struct device *dev) -- cgit v1.2.3-58-ga151 From bfd20f1cc85010d2f2d77e544da05cd8c149ba9b Mon Sep 17 00:00:00 2001 From: Shaohua Li Date: Wed, 26 Apr 2017 09:18:35 -0700 Subject: x86, iommu/vt-d: Add an option to disable Intel IOMMU force on IOMMU harms performance signficantly when we run very fast networking workloads. It's 40GB networking doing XDP test. Software overhead is almost unaware, but it's the IOTLB miss (based on our analysis) which kills the performance. We observed the same performance issue even with software passthrough (identity mapping), only the hardware passthrough survives. The pps with iommu (with software passthrough) is only about ~30% of that without it. This is a limitation in hardware based on our observation, so we'd like to disable the IOMMU force on, but we do want to use TBOOT and we can sacrifice the DMA security bought by IOMMU. I must admit I know nothing about TBOOT, but TBOOT guys (cc-ed) think not eabling IOMMU is totally ok. So introduce a new boot option to disable the force on. It's kind of silly we need to run into intel_iommu_init even without force on, but we need to disable TBOOT PMR registers. For system without the boot option, nothing is changed. Signed-off-by: Shaohua Li Signed-off-by: Joerg Roedel --- Documentation/admin-guide/kernel-parameters.txt | 9 +++++++++ arch/x86/kernel/tboot.c | 3 +++ drivers/iommu/intel-iommu.c | 18 ++++++++++++++++++ include/linux/dma_remapping.h | 1 + 4 files changed, 31 insertions(+) (limited to 'arch') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 2ba45caabada..17135bfade6a 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1578,6 +1578,15 @@ extended tables themselves, and also PASID support. With this option set, extended tables will not be used even on hardware which claims to support them. + tboot_noforce [Default Off] + Do not force the Intel IOMMU enabled under tboot. + By default, tboot will force Intel IOMMU on, which + could harm performance of some high-throughput + devices like 40GBit network cards, even if identity + mapping is enabled. + Note that using this option lowers the security + provided by tboot because it makes the system + vulnerable to DMA attacks. intel_idle.max_cstate= [KNL,HW,ACPI,X86] 0 disables intel_idle and fall back on acpi_idle. diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c index b868fa1b812b..edbdfe6ab60a 100644 --- a/arch/x86/kernel/tboot.c +++ b/arch/x86/kernel/tboot.c @@ -510,6 +510,9 @@ int tboot_force_iommu(void) if (!tboot_enabled()) return 0; + if (!intel_iommu_tboot_noforce) + return 1; + if (no_iommu || swiotlb || dmar_disabled) pr_warning("Forcing Intel-IOMMU to enabled\n"); diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 5f08ba13972b..b0ced1c13713 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -183,6 +183,7 @@ static int rwbf_quirk; * (used when kernel is launched w/ TXT) */ static int force_on = 0; +int intel_iommu_tboot_noforce; /* * 0: Present @@ -607,6 +608,10 @@ static int __init intel_iommu_setup(char *str) "Intel-IOMMU: enable pre-production PASID support\n"); intel_iommu_pasid28 = 1; iommu_identity_mapping |= IDENTMAP_GFX; + } else if (!strncmp(str, "tboot_noforce", 13)) { + printk(KERN_INFO + "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n"); + intel_iommu_tboot_noforce = 1; } str += strcspn(str, ","); @@ -4850,6 +4855,19 @@ int __init intel_iommu_init(void) } if (no_iommu || dmar_disabled) { + /* + * We exit the function here to ensure IOMMU's remapping and + * mempool aren't setup, which means that the IOMMU's PMRs + * won't be disabled via the call to init_dmars(). So disable + * it explicitly here. The PMRs were setup by tboot prior to + * calling SENTER, but the kernel is expected to reset/tear + * down the PMRs. + */ + if (intel_iommu_tboot_noforce) { + for_each_iommu(iommu, drhd) + iommu_disable_protect_mem_regions(iommu); + } + /* * Make sure the IOMMUs are switched off, even when we * boot into a kexec kernel and the previous kernel left diff --git a/include/linux/dma_remapping.h b/include/linux/dma_remapping.h index 187c10299722..90884072fa73 100644 --- a/include/linux/dma_remapping.h +++ b/include/linux/dma_remapping.h @@ -39,6 +39,7 @@ extern int iommu_calculate_agaw(struct intel_iommu *iommu); extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu); extern int dmar_disabled; extern int intel_iommu_enabled; +extern int intel_iommu_tboot_noforce; #else static inline int iommu_calculate_agaw(struct intel_iommu *iommu) { -- cgit v1.2.3-58-ga151 From 26b37b946a5c2658dbc37dd5d6df40aaa9685d70 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Fri, 15 May 2015 02:00:02 +0300 Subject: arm: dma-mapping: Don't override dma_ops in arch_setup_dma_ops() The arch_setup_dma_ops() function is in charge of setting dma_ops with a call to set_dma_ops(). set_dma_ops() is also called from - highbank and mvebu bus notifiers - dmabounce (to be replaced with swiotlb) - arm_iommu_attach_device (arm_iommu_attach_device is itself called from IOMMU and bus master device drivers) To allow the arch_setup_dma_ops() call to be moved from device add time to device probe time we must ensure that dma_ops already setup by any of the above callers will not be overriden. Aftering replacing dmabounce with swiotlb, converting IOMMU drivers to of_xlate and taking care of highbank and mvebu, the workaround should be removed. Signed-off-by: Sricharan R Signed-off-by: Laurent Pinchart Tested-by: Ralph Sennhauser Signed-off-by: Joerg Roedel --- arch/arm/mm/dma-mapping.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'arch') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 63eabb06f9f1..ff131928be50 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -2390,6 +2390,15 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, const struct dma_map_ops *dma_ops; dev->archdata.dma_coherent = coherent; + + /* + * Don't override the dma_ops if they have already been set. Ideally + * this should be the only location where dma_ops are set, remove this + * check when all other callers of set_dma_ops will have disappeared. + */ + if (dev->dma_ops) + return; + if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu)) dma_ops = arm_get_iommu_dma_map_ops(coherent); else -- cgit v1.2.3-58-ga151 From 461a6946b1f93f6720577fb06aa78e8cbd9291c9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 26 Apr 2017 15:46:20 +0200 Subject: iommu: Remove pci.h include from trace/events/iommu.h The include file does not need any PCI specifics, so remove that include. Also fix the places that relied on it. Signed-off-by: Joerg Roedel --- arch/arm64/mm/dma-mapping.c | 1 + drivers/infiniband/hw/qedr/main.c | 1 + drivers/iommu/fsl_pamu.h | 1 + drivers/iommu/rockchip-iommu.c | 1 + drivers/iommu/tegra-smmu.c | 1 + include/linux/dma-iommu.h | 1 + include/trace/events/iommu.h | 1 - 7 files changed, 6 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 81cdb2e844ed..982f85b81624 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -28,6 +28,7 @@ #include #include #include +#include #include diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c index b9b47e5cc8b3..33033624cd9b 100644 --- a/drivers/infiniband/hw/qedr/main.c +++ b/drivers/infiniband/hw/qedr/main.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/iommu/fsl_pamu.h b/drivers/iommu/fsl_pamu.h index aab723f91f12..c3434f29c967 100644 --- a/drivers/iommu/fsl_pamu.h +++ b/drivers/iommu/fsl_pamu.h @@ -20,6 +20,7 @@ #define __FSL_PAMU_H #include +#include #include diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 9afcbf79f0b0..0ba303a184dd 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 9305964250ac..eeb19f560a05 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h index 5725c94b1f12..abd946569515 100644 --- a/include/linux/dma-iommu.h +++ b/include/linux/dma-iommu.h @@ -20,6 +20,7 @@ #include #ifdef CONFIG_IOMMU_DMA +#include #include #include diff --git a/include/trace/events/iommu.h b/include/trace/events/iommu.h index 2c7befb10f13..99254ed89212 100644 --- a/include/trace/events/iommu.h +++ b/include/trace/events/iommu.h @@ -11,7 +11,6 @@ #define _TRACE_IOMMU_H #include -#include struct device; -- cgit v1.2.3-58-ga151