dma-pool: add additional coherent pools to map to gfp mask

The single atomic pool is allocated from the lowest zone possible since it is guaranteed to be applicable for any DMA allocation. Devices may allocate through the DMA API but not have a strict reliance on GFP_DMA memory. Since the atomic pool will be used for all non-blockable allocations, returning all memory from ZONE_DMA may unnecessarily deplete the zone. Provision for multiple atomic pools that will map to the optimal gfp mask of the device. When allocating non-blockable memory, determine the optimal gfp mask of the device and use the appropriate atomic pool. The coherent DMA mask will remain the same between allocation and free and, thus, memory will be freed to the same atomic pool it was allocated from. __dma_atomic_pool_init() will be changed to return struct gen_pool * later once dynamic expansion is added.

David Rientjes committed 6 years ago

Commit c84dc6e1 parent e860c295 files · +91 −54Browse files

Unified · Split

5 files

drivers/iommu/dma-iommu.c +3 −2
include/linux/dma-direct.h +2 −0
include/linux/dma-mapping.h +3 −3
kernel/dma/direct.c +6 −6
kernel/dma/pool.c +77 −43

drivers/iommu/dma-iommu.c+3−2

@@ -952,7 +952,7 @@ static void __iommu_dma_free(struct device dev, size_t size, void cpu_addr) · 951 unmodified lines
952	952
953	953	/* Non-coherent atomic allocation? Easy */
954	954	if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
955		− dma_free_from_pool(cpu_addr, alloc_size))
	955	+ dma_free_from_pool(dev, cpu_addr, alloc_size))
956	956	return;
957	957
958	958	if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) {
@@ -1035,7 +1035,8 @@ static void iommu_dma_alloc(struct device dev, size_t size, · 76 unmodified lines
1035	1035
1036	1036	if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
1037	1037	!gfpflags_allow_blocking(gfp) && !coherent)
1038		− cpu_addr = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp);
	1038	+ cpu_addr = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &page,
	1039	+ gfp);
1039	1040	else
1040	1041	cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
1041	1042	if (!cpu_addr)

include/linux/dma-direct.h+2−0

@@ -67,6 +67,8 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size, · 66 unmodified lines
67	67	}
68	68
69	69	u64 dma_direct_get_required_mask(struct device *dev);
	70	+gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
	71	+ u64 *phys_mask);
70	72	void dma_direct_alloc(struct device dev, size_t size, dma_addr_t *dma_handle,
71	73	gfp_t gfp, unsigned long attrs);
72	74	void dma_direct_free(struct device dev, size_t size, void cpu_addr,

include/linux/dma-mapping.h+3−3

@@ -630,9 +630,9 @@ void dma_common_pages_remap(struct page *pages, size_t size, · 629 unmodified lines
630	630	pgprot_t prot, const void *caller);
631	631	void dma_common_free_remap(void *cpu_addr, size_t size);
632	632
633		−bool dma_in_atomic_pool(void *start, size_t size);
634		−void dma_alloc_from_pool(size_t size, struct page *ret_page, gfp_t flags);
635		−bool dma_free_from_pool(void *start, size_t size);
	633	+void dma_alloc_from_pool(struct device dev, size_t size,
	634	+ struct page **ret_page, gfp_t flags);
	635	+bool dma_free_from_pool(struct device dev, void start, size_t size);
636	636
637	637	int
638	638	dma_common_get_sgtable(struct device dev, struct sg_table sgt, void *cpu_addr,

kernel/dma/direct.c+6−6

@@ -45,8 +45,8 @@ u64 dma_direct_get_required_mask(struct device *dev) · 44 unmodified lines
45	45	return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
46	46	}
47	47
48		−static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
49		− u64 *phys_limit)
	48	+gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
	49	+ u64 *phys_limit)
50	50	{
51	51	u64 dma_limit = min_not_zero(dma_mask, dev->bus_dma_limit);
52	52
@@ -89,8 +89,8 @@ struct page __dma_direct_alloc_pages(struct device dev, size_t size, · 36 unmodified lines
89	89
90	90	/* we always manually zero the memory once we are done: */
91	91	gfp &= ~__GFP_ZERO;
92		− gfp \|= __dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
93		− &phys_limit);
	92	+ gfp \|= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
	93	+ &phys_limit);
94	94	page = dma_alloc_contiguous(dev, alloc_size, gfp);
95	95	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
96	96	dma_free_contiguous(dev, page, alloc_size);
@@ -128,7 +128,7 @@ void dma_direct_alloc_pages(struct device dev, size_t size, · 31 unmodified lines
128	128	if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
129	129	dma_alloc_need_uncached(dev, attrs) &&
130	130	!gfpflags_allow_blocking(gfp)) {
131		− ret = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp);
	131	+ ret = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &page, gfp);
132	132	if (!ret)
133	133	return NULL;
134	134	goto done;
@@ -212,7 +212,7 @@ void dma_direct_free_pages(struct device dev, size_t size, void cpu_addr, · 77 unmodified lines
212	212	}
213	213
214	214	if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
215		− dma_free_from_pool(cpu_addr, PAGE_ALIGN(size)))
	215	+ dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))
216	216	return;
217	217
218	218	if (force_dma_unencrypted(dev))

kernel/dma/pool.c+77−43

@@ -10,7 +10,9 @@ · 9 unmodified lines
10	10	#include <linux/genalloc.h>
11	11	#include <linux/slab.h>
12	12
13		−static struct gen_pool *atomic_pool __ro_after_init;
	13	+static struct gen_pool *atomic_pool_dma __ro_after_init;
	14	+static struct gen_pool *atomic_pool_dma32 __ro_after_init;
	15	+static struct gen_pool *atomic_pool_kernel __ro_after_init;
14	16
15	17	#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
16	18	static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
@@ -22,89 +24,119 @@ static int __init early_coherent_pool(char *p) · 5 unmodified lines
22	24	}
23	25	early_param("coherent_pool", early_coherent_pool);
24	26
25		−static gfp_t dma_atomic_pool_gfp(void)
	27	+static int __init __dma_atomic_pool_init(struct gen_pool **pool,
	28	+ size_t pool_size, gfp_t gfp)
26	29	{
27		− if (IS_ENABLED(CONFIG_ZONE_DMA))
28		− return GFP_DMA;
29		− if (IS_ENABLED(CONFIG_ZONE_DMA32))
30		− return GFP_DMA32;
31		− return GFP_KERNEL;
32		−}
33		−
34		−static int __init dma_atomic_pool_init(void)
35		−{
36		− unsigned int pool_size_order = get_order(atomic_pool_size);
37		− unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
	30	+ const unsigned int order = get_order(pool_size);
	31	+ const unsigned long nr_pages = pool_size >> PAGE_SHIFT;
38	32	struct page *page;
39	33	void *addr;
40	34	int ret;
41	35
42	36	if (dev_get_cma_area(NULL))
43		− page = dma_alloc_from_contiguous(NULL, nr_pages,
44		− pool_size_order, false);
	37	+ page = dma_alloc_from_contiguous(NULL, nr_pages, order, false);
45	38	else
46		− page = alloc_pages(dma_atomic_pool_gfp(), pool_size_order);
	39	+ page = alloc_pages(gfp, order);
47	40	if (!page)
48	41	goto out;
49	42
50		− arch_dma_prep_coherent(page, atomic_pool_size);
	43	+ arch_dma_prep_coherent(page, pool_size);
51	44
52		− atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
53		− if (!atomic_pool)
	45	+ *pool = gen_pool_create(PAGE_SHIFT, -1);
	46	+ if (!*pool)
54	47	goto free_page;
55	48
56		− addr = dma_common_contiguous_remap(page, atomic_pool_size,
	49	+ addr = dma_common_contiguous_remap(page, pool_size,
57	50	pgprot_dmacoherent(PAGE_KERNEL),
58	51	__builtin_return_address(0));
59	52	if (!addr)
60	53	goto destroy_genpool;
61	54
62		− ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr,
63		− page_to_phys(page), atomic_pool_size, -1);
	55	+ ret = gen_pool_add_virt(*pool, (unsigned long)addr, page_to_phys(page),
	56	+ pool_size, -1);
64	57	if (ret)
65	58	goto remove_mapping;
66		− gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);
	59	+ gen_pool_set_algo(*pool, gen_pool_first_fit_order_align, NULL);
67	60
68		− pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
69		− atomic_pool_size / 1024);
	61	+ pr_info("DMA: preallocated %zu KiB %pGg pool for atomic allocations\n",
	62	+ pool_size >> 10, &gfp);
70	63	return 0;
71	64
72	65	remove_mapping:
73		− dma_common_free_remap(addr, atomic_pool_size);
	66	+ dma_common_free_remap(addr, pool_size);
74	67	destroy_genpool:
75		− gen_pool_destroy(atomic_pool);
76		− atomic_pool = NULL;
	68	+ gen_pool_destroy(*pool);
	69	+ *pool = NULL;
77	70	free_page:
78	71	if (!dma_release_from_contiguous(NULL, page, nr_pages))
79		− __free_pages(page, pool_size_order);
	72	+ __free_pages(page, order);
80	73	out:
81		− pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
82		− atomic_pool_size / 1024);
	74	+ pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\n",
	75	+ pool_size >> 10, &gfp);
83	76	return -ENOMEM;
84	77	}
	78	+
	79	+static int __init dma_atomic_pool_init(void)
	80	+{
	81	+ int ret = 0;
	82	+ int err;
	83	+
	84	+ ret = __dma_atomic_pool_init(&atomic_pool_kernel, atomic_pool_size,
	85	+ GFP_KERNEL);
	86	+ if (IS_ENABLED(CONFIG_ZONE_DMA)) {
	87	+ err = __dma_atomic_pool_init(&atomic_pool_dma,
	88	+ atomic_pool_size, GFP_DMA);
	89	+ if (!ret && err)
	90	+ ret = err;
	91	+ }
	92	+ if (IS_ENABLED(CONFIG_ZONE_DMA32)) {
	93	+ err = __dma_atomic_pool_init(&atomic_pool_dma32,
	94	+ atomic_pool_size, GFP_DMA32);
	95	+ if (!ret && err)
	96	+ ret = err;
	97	+ }
	98	+ return ret;
	99	+}
85	100	postcore_initcall(dma_atomic_pool_init);
86	101
87		−bool dma_in_atomic_pool(void *start, size_t size)
	102	+static inline struct gen_pool dev_to_pool(struct device dev)
88	103	{
89		− if (unlikely(!atomic_pool))
90		− return false;
	104	+ u64 phys_mask;
	105	+ gfp_t gfp;
	106	+
	107	+ gfp = dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
	108	+ &phys_mask);
	109	+ if (IS_ENABLED(CONFIG_ZONE_DMA) && gfp == GFP_DMA)
	110	+ return atomic_pool_dma;
	111	+ if (IS_ENABLED(CONFIG_ZONE_DMA32) && gfp == GFP_DMA32)
	112	+ return atomic_pool_dma32;
	113	+ return atomic_pool_kernel;
	114	+}
91	115
92		− return gen_pool_has_addr(atomic_pool, (unsigned long)start, size);
	116	+static bool dma_in_atomic_pool(struct device dev, void start, size_t size)
	117	+{
	118	+ struct gen_pool *pool = dev_to_pool(dev);
	119	+
	120	+ if (unlikely(!pool))
	121	+ return false;
	122	+ return gen_pool_has_addr(pool, (unsigned long)start, size);
93	123	}
94	124
95		−void dma_alloc_from_pool(size_t size, struct page *ret_page, gfp_t flags)
	125	+void dma_alloc_from_pool(struct device dev, size_t size,
	126	+ struct page **ret_page, gfp_t flags)
96	127	{
	128	+ struct gen_pool *pool = dev_to_pool(dev);
97	129	unsigned long val;
98	130	void *ptr = NULL;
99	131
100		− if (!atomic_pool) {
101		− WARN(1, "coherent pool not initialised!\n");
	132	+ if (!pool) {
	133	+ WARN(1, "%pGg atomic pool not initialised!\n", &flags);
102	134	return NULL;
103	135	}
104	136
105		− val = gen_pool_alloc(atomic_pool, size);
	137	+ val = gen_pool_alloc(pool, size);
106	138	if (val) {
107		− phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
	139	+ phys_addr_t phys = gen_pool_virt_to_phys(pool, val);
108	140
109	141	*ret_page = pfn_to_page(__phys_to_pfn(phys));
110	142	ptr = (void *)val;
@@ -114,10 +146,12 @@ void dma_alloc_from_pool(size_t size, struct page *ret_page, gfp_t flags) · 3 unmodified lines
114	146	return ptr;
115	147	}
116	148
117		−bool dma_free_from_pool(void *start, size_t size)
	149	+bool dma_free_from_pool(struct device dev, void start, size_t size)
118	150	{
119		− if (!dma_in_atomic_pool(start, size))
	151	+ struct gen_pool *pool = dev_to_pool(dev);
	152	+
	153	+ if (!dma_in_atomic_pool(dev, start, size))
120	154	return false;
121		− gen_pool_free(atomic_pool, (unsigned long)start, size);
	155	+ gen_pool_free(pool, (unsigned long)start, size);
122	156	return true;
123	157	}