powerpc/dma: Fallback to dma_ops when persistent memory present

So far we have been using huge DMA windows to map all the RAM available. The RAM is normally mapped to the VM address space contiguously, and there is always a reasonable upper limit for possible future hot plugged RAM which makes it easy to map all RAM via IOMMU. Now there is persistent memory ("ibm,pmemory" in the FDT) which (unlike normal RAM) can map anywhere in the VM space beyond the maximum RAM size and since it can be used for DMA, it requires extending the huge window up to MAX_PHYSMEM_BITS which requires hypervisor support for: 1. huge TCE tables; 2. multilevel TCE tables; 3. huge IOMMU pages. Certain hypervisors cannot do either so the only option left is restricting the huge DMA window to include only RAM and fallback to the default DMA window for persistent memory. This defines arch_dma_map_direct/etc to allow generic DMA code perform additional checks on whether direct DMA is still possible. This checks if the system has persistent memory. If it does not, the DMA bypass mode is selected, i.e. * dev->bus_dma_limit = 0 * dev->dma_ops_bypass = true <- this avoid calling dma_ops for mapping. If there is such memory, this creates identity mapping only for RAM and sets the dev->bus_dma_limit to let the generic code decide whether to call into the direct DMA or the indirect DMA ops. This should not change the existing behaviour when no persistent memory as dev->dma_ops_bypass is expected to be set.

Alexey Kardashevskiy committed 6 years ago

Commit bf6e2d51 parent 8d8d53c3 files · +111 −12Browse files

Unified · Split

3 files

arch/powerpc/Kconfig +1 −0
arch/powerpc/kernel/dma-iommu.c +69 −2
arch/powerpc/platforms/pseries/iommu.c +41 −10

arch/powerpc/Kconfig+1−0

@@ -159,6 +159,7 @@ config PPC · 158 unmodified lines
159	select DCACHE_WORD_ACCESS if PPC64 && CPU_LITTLE_ENDIAN	159	select DCACHE_WORD_ACCESS if PPC64 && CPU_LITTLE_ENDIAN
160	select DMA_OPS if PPC64	160	select DMA_OPS if PPC64
161	select DMA_OPS_BYPASS if PPC64	161	select DMA_OPS_BYPASS if PPC64
		162	select ARCH_HAS_DMA_MAP_DIRECT if PPC64 && PPC_PSERIES
162	select DYNAMIC_FTRACE if FUNCTION_TRACER	163	select DYNAMIC_FTRACE if FUNCTION_TRACER
163	select EDAC_ATOMIC_SCRUB	164	select EDAC_ATOMIC_SCRUB
164	select EDAC_SUPPORT	165	select EDAC_SUPPORT

arch/powerpc/kernel/dma-iommu.c+69−2

@@ -10,6 +10,63 @@ · 9 unmodified lines
10	#include <linux/pci.h>	10	#include <linux/pci.h>
11	#include <asm/iommu.h>	11	#include <asm/iommu.h>
12		12
		13	#ifdef CONFIG_ARCH_HAS_DMA_MAP_DIRECT
		14	#define can_map_direct(dev, addr) \
		15	((dev)->bus_dma_limit >= phys_to_dma((dev), (addr)))
		16
		17	bool arch_dma_map_page_direct(struct device *dev, phys_addr_t addr)
		18	{
		19	if (likely(!dev->bus_dma_limit))
		20	return false;
		21
		22	return can_map_direct(dev, addr);
		23	}
		24
		25	#define is_direct_handle(dev, h) ((h) >= (dev)->archdata.dma_offset)
		26
		27	bool arch_dma_unmap_page_direct(struct device *dev, dma_addr_t dma_handle)
		28	{
		29	if (likely(!dev->bus_dma_limit))
		30	return false;
		31
		32	return is_direct_handle(dev, dma_handle);
		33	}
		34
		35	bool arch_dma_map_sg_direct(struct device dev, struct scatterlist sg,
		36	int nents)
		37	{
		38	struct scatterlist *s;
		39	int i;
		40
		41	if (likely(!dev->bus_dma_limit))
		42	return false;
		43
		44	for_each_sg(sg, s, nents, i) {
		45	if (!can_map_direct(dev, sg_phys(s) + s->offset + s->length))
		46	return false;
		47	}
		48
		49	return true;
		50	}
		51
		52	bool arch_dma_unmap_sg_direct(struct device dev, struct scatterlist sg,
		53	int nents)
		54	{
		55	struct scatterlist *s;
		56	int i;
		57
		58	if (likely(!dev->bus_dma_limit))
		59	return false;
		60
		61	for_each_sg(sg, s, nents, i) {
		62	if (!is_direct_handle(dev, s->dma_address + s->length))
		63	return false;
		64	}
		65
		66	return true;
		67	}
		68	#endif /* CONFIG_ARCH_HAS_DMA_MAP_DIRECT */
		69
13	/*	70	/*
14	* Generic iommu implementation	71	* Generic iommu implementation
15	*/	72	*/
@@ -90,8 +147,18 @@ int dma_iommu_dma_supported(struct device *dev, u64 mask) · 74 unmodified lines
90	struct iommu_table *tbl = get_iommu_table_base(dev);	147	struct iommu_table *tbl = get_iommu_table_base(dev);
91		148
92	if (dev_is_pci(dev) && dma_iommu_bypass_supported(dev, mask)) {	149	if (dev_is_pci(dev) && dma_iommu_bypass_supported(dev, mask)) {
93	dev->dma_ops_bypass = true;	150	/*
94	dev_dbg(dev, "iommu: 64-bit OK, using fixed ops\n");	151	* dma_iommu_bypass_supported() sets dma_max when there is
		152	* 1:1 mapping but it is somehow limited.
		153	* ibm,pmemory is one example.
		154	*/
		155	dev->dma_ops_bypass = dev->bus_dma_limit == 0;
		156	if (!dev->dma_ops_bypass)
		157	dev_warn(dev,
		158	"iommu: 64-bit OK but direct DMA is limited by %llx\n",
		159	dev->bus_dma_limit);
		160	else
		161	dev_dbg(dev, "iommu: 64-bit OK, using fixed ops\n");
95	return 1;	162	return 1;
96	}	163	}
97		164

arch/powerpc/platforms/pseries/iommu.c+41−10

 			np, ret);
 }
-static u64 find_existing_ddw(struct device_node *pdn)
+static u64 find_existing_ddw(struct device_node *pdn, int *window_shift)
 {
 	struct direct_window *window;
 	const struct dynamic_dma_window_prop *direct64;
 		if (window->device == pdn) {
 			direct64 = window->prop;
 			dma_addr = be64_to_cpu(direct64->dma_base);
+			*window_shift = be32_to_cpu(direct64->window_shift);
 			break;
 		}
 	}
  */
 static u64 enable_ddw(struct pci_dev *dev, struct device_node *pdn)
 {
-	int len, ret;
+	int len = 0, ret;
+	int max_ram_len = order_base_2(ddw_memory_hotplug_max());
 	struct ddw_query_response query;
 	struct ddw_create_response create;
 	int page_shift;
-	u64 dma_addr, max_addr;
+	u64 dma_addr;
 	struct device_node *dn;
 	u32 ddw_avail[DDW_APPLICABLE_SIZE];
 	struct direct_window *window;
 	struct dynamic_dma_window_prop *ddwprop;
 	struct failed_ddw_pdn *fpdn;
 	bool default_win_removed = false;
+	bool pmem_present;
+	dn = of_find_node_by_type(NULL, "ibm,pmemory");
+	pmem_present = dn != NULL;
+	of_node_put(dn);
 	mutex_lock(&direct_window_init_mutex);
-	dma_addr = find_existing_ddw(pdn);
+	dma_addr = find_existing_ddw(pdn, &len);
 	if (dma_addr != 0)
 		goto out_unlock;
 	}
 	/* verify the window * number of ptes will map the partition */
 	/* check largest block * page size > max memory hotplug addr */
-	max_addr = ddw_memory_hotplug_max();
+	/*
-	if (query.largest_available_block < (max_addr >> page_shift)) {
+	 * The "ibm,pmemory" can appear anywhere in the address space.
-		dev_dbg(&dev->dev, "can't map partition max 0x%llx with %llu "
+	 * Assuming it is still backed by page structs, try MAX_PHYSMEM_BITS
-			  "%llu-sized pages\n", max_addr,  query.largest_available_block,
+	 * for the upper limit and fallback to max RAM otherwise but this
-			  1ULL << page_shift);
+	 * disables device::dma_ops_bypass.
+	 */
+	len = max_ram_len;
+	if (pmem_present) {
+		if (query.largest_available_block >=
+		    (1ULL << (MAX_PHYSMEM_BITS - page_shift)))
+			len = MAX_PHYSMEM_BITS - page_shift;
+		else
+			dev_info(&dev->dev, "Skipping ibm,pmemory");
+	}
+	if (query.largest_available_block < (1ULL << (len - page_shift))) {
+		dev_dbg(&dev->dev,
+			"can't map partition max 0x%llx with %llu %llu-sized pages\n",
+			1ULL << len,
+			query.largest_available_block,
+			1ULL << page_shift);
 		goto out_failed;
 	}
-	len = order_base_2(max_addr);
 	win64 = kzalloc(sizeof(struct property), GFP_KERNEL);
 	if (!win64) {
 		dev_info(&dev->dev,
 out_unlock:
 	mutex_unlock(&direct_window_init_mutex);
+	/*
+	 * If we have persistent memory and the window size is only as big
+	 * as RAM, then we failed to create a window to cover persistent
+	 * memory and need to set the DMA limit.
+	 */
+	if (pmem_present && dma_addr && (len == max_ram_len))
+		dev->dev.bus_dma_limit = dma_addr + (1ULL << len);
 	return dma_addr;
 }

powerpc/dma: Fallback to dma_ops when persistent memory present

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code