History

dma-pool: add additional coherent pools to map to gfp mask

The single atomic pool is allocated from the lowest zone possible since it is guaranteed to be applicable for any DMA allocation. Devices may allocate through the DMA API but not have a strict reliance on GFP_DMA memory. Since the atomic pool will be used for all non-blockable allocations, returning all memory from ZONE_DMA may unnecessarily deplete the zone. Provision for multiple atomic pools that will map to the optimal gfp mask of the device. When allocating non-blockable memory, determine the optimal gfp mask of the device and use the appropriate atomic pool. The coherent DMA mask will remain the same between allocation and free and, thus, memory will be freed to the same atomic pool it was allocated from. __dma_atomic_pool_init() will be changed to return struct gen_pool * later once dynamic expansion is added.

David Rientjes committed 6 years ago

Commit c84dc6e1 parent e860c295 files · +91 −54Browse files

Unified · Split

5 files
  1. drivers/iommu/dma-iommu.c +3 −2
  2. include/linux/dma-direct.h +2 −0
  3. include/linux/dma-mapping.h +3 −3
  4. kernel/dma/direct.c +6 −6
  5. kernel/dma/pool.c +77 −43
drivers/iommu/dma-iommu.c+3−2
@@ -952,7 +952,7 @@ static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr) · 951 unmodified lines
952 952
953 /* Non-coherent atomic allocation? Easy */953 /* Non-coherent atomic allocation? Easy */
954 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&954 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
955 dma_free_from_pool(cpu_addr, alloc_size))955 dma_free_from_pool(dev, cpu_addr, alloc_size))
956 return;956 return;
957 957
958 if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) {958 if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) {
@@ -1035,7 +1035,8 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, · 76 unmodified lines
1035 1035
1036 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&1036 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
1037 !gfpflags_allow_blocking(gfp) && !coherent)1037 !gfpflags_allow_blocking(gfp) && !coherent)
1038 cpu_addr = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp);1038 cpu_addr = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &page,
1039 gfp);
1039 else1040 else
1040 cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);1041 cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
1041 if (!cpu_addr)1042 if (!cpu_addr)
include/linux/dma-direct.h+2−0
@@ -67,6 +67,8 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size, · 66 unmodified lines
67}67}
68 68
69u64 dma_direct_get_required_mask(struct device *dev);69u64 dma_direct_get_required_mask(struct device *dev);
70gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
71 u64 *phys_mask);
70void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,72void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
71 gfp_t gfp, unsigned long attrs);73 gfp_t gfp, unsigned long attrs);
72void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,74void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
include/linux/dma-mapping.h+3−3
@@ -630,9 +630,9 @@ void *dma_common_pages_remap(struct page **pages, size_t size, · 629 unmodified lines
630 pgprot_t prot, const void *caller);630 pgprot_t prot, const void *caller);
631void dma_common_free_remap(void *cpu_addr, size_t size);631void dma_common_free_remap(void *cpu_addr, size_t size);
632 632
633bool dma_in_atomic_pool(void *start, size_t size);633void *dma_alloc_from_pool(struct device *dev, size_t size,
634void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags);634 struct page **ret_page, gfp_t flags);
635bool dma_free_from_pool(void *start, size_t size);635bool dma_free_from_pool(struct device *dev, void *start, size_t size);
636 636
637int637int
638dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr,638dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr,
kernel/dma/direct.c+6−6
@@ -45,8 +45,8 @@ u64 dma_direct_get_required_mask(struct device *dev) · 44 unmodified lines
45 return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;45 return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
46}46}
47 47
48static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,48gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
49 u64 *phys_limit)49 u64 *phys_limit)
50{50{
51 u64 dma_limit = min_not_zero(dma_mask, dev->bus_dma_limit);51 u64 dma_limit = min_not_zero(dma_mask, dev->bus_dma_limit);
52 52
@@ -89,8 +89,8 @@ struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, · 36 unmodified lines
89 89
90 /* we always manually zero the memory once we are done: */90 /* we always manually zero the memory once we are done: */
91 gfp &= ~__GFP_ZERO;91 gfp &= ~__GFP_ZERO;
92 gfp |= __dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,92 gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
93 &phys_limit);93 &phys_limit);
94 page = dma_alloc_contiguous(dev, alloc_size, gfp);94 page = dma_alloc_contiguous(dev, alloc_size, gfp);
95 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {95 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
96 dma_free_contiguous(dev, page, alloc_size);96 dma_free_contiguous(dev, page, alloc_size);
@@ -128,7 +128,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, · 31 unmodified lines
128 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&128 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
129 dma_alloc_need_uncached(dev, attrs) &&129 dma_alloc_need_uncached(dev, attrs) &&
130 !gfpflags_allow_blocking(gfp)) {130 !gfpflags_allow_blocking(gfp)) {
131 ret = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp);131 ret = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &page, gfp);
132 if (!ret)132 if (!ret)
133 return NULL;133 return NULL;
134 goto done;134 goto done;
@@ -212,7 +212,7 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, · 77 unmodified lines
212 }212 }
213 213
214 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&214 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
215 dma_free_from_pool(cpu_addr, PAGE_ALIGN(size)))215 dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))
216 return;216 return;
217 217
218 if (force_dma_unencrypted(dev))218 if (force_dma_unencrypted(dev))
kernel/dma/pool.c+77−43
@@ -10,7 +10,9 @@ · 9 unmodified lines
10#include <linux/genalloc.h>10#include <linux/genalloc.h>
11#include <linux/slab.h>11#include <linux/slab.h>
12 12
13static struct gen_pool *atomic_pool __ro_after_init;13static struct gen_pool *atomic_pool_dma __ro_after_init;
14static struct gen_pool *atomic_pool_dma32 __ro_after_init;
15static struct gen_pool *atomic_pool_kernel __ro_after_init;
14 16
15#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K17#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
16static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;18static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
@@ -22,89 +24,119 @@ static int __init early_coherent_pool(char *p) · 5 unmodified lines
22}24}
23early_param("coherent_pool", early_coherent_pool);25early_param("coherent_pool", early_coherent_pool);
24 26
25static gfp_t dma_atomic_pool_gfp(void)27static int __init __dma_atomic_pool_init(struct gen_pool **pool,
28 size_t pool_size, gfp_t gfp)
26{29{
27 if (IS_ENABLED(CONFIG_ZONE_DMA))30 const unsigned int order = get_order(pool_size);
28 return GFP_DMA;31 const unsigned long nr_pages = pool_size >> PAGE_SHIFT;
29 if (IS_ENABLED(CONFIG_ZONE_DMA32))
30 return GFP_DMA32;
31 return GFP_KERNEL;
32}
33
34static int __init dma_atomic_pool_init(void)
35{
36 unsigned int pool_size_order = get_order(atomic_pool_size);
37 unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
38 struct page *page;32 struct page *page;
39 void *addr;33 void *addr;
40 int ret;34 int ret;
41 35
42 if (dev_get_cma_area(NULL))36 if (dev_get_cma_area(NULL))
43 page = dma_alloc_from_contiguous(NULL, nr_pages,37 page = dma_alloc_from_contiguous(NULL, nr_pages, order, false);
44 pool_size_order, false);
45 else38 else
46 page = alloc_pages(dma_atomic_pool_gfp(), pool_size_order);39 page = alloc_pages(gfp, order);
47 if (!page)40 if (!page)
48 goto out;41 goto out;
49 42
50 arch_dma_prep_coherent(page, atomic_pool_size);43 arch_dma_prep_coherent(page, pool_size);
51 44
52 atomic_pool = gen_pool_create(PAGE_SHIFT, -1);45 *pool = gen_pool_create(PAGE_SHIFT, -1);
53 if (!atomic_pool)46 if (!*pool)
54 goto free_page;47 goto free_page;
55 48
56 addr = dma_common_contiguous_remap(page, atomic_pool_size,49 addr = dma_common_contiguous_remap(page, pool_size,
57 pgprot_dmacoherent(PAGE_KERNEL),50 pgprot_dmacoherent(PAGE_KERNEL),
58 __builtin_return_address(0));51 __builtin_return_address(0));
59 if (!addr)52 if (!addr)
60 goto destroy_genpool;53 goto destroy_genpool;
61 54
62 ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr,55 ret = gen_pool_add_virt(*pool, (unsigned long)addr, page_to_phys(page),
63 page_to_phys(page), atomic_pool_size, -1);56 pool_size, -1);
64 if (ret)57 if (ret)
65 goto remove_mapping;58 goto remove_mapping;
66 gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);59 gen_pool_set_algo(*pool, gen_pool_first_fit_order_align, NULL);
67 60
68 pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",61 pr_info("DMA: preallocated %zu KiB %pGg pool for atomic allocations\n",
69 atomic_pool_size / 1024);62 pool_size >> 10, &gfp);
70 return 0;63 return 0;
71 64
72remove_mapping:65remove_mapping:
73 dma_common_free_remap(addr, atomic_pool_size);66 dma_common_free_remap(addr, pool_size);
74destroy_genpool:67destroy_genpool:
75 gen_pool_destroy(atomic_pool);68 gen_pool_destroy(*pool);
76 atomic_pool = NULL;69 *pool = NULL;
77free_page:70free_page:
78 if (!dma_release_from_contiguous(NULL, page, nr_pages))71 if (!dma_release_from_contiguous(NULL, page, nr_pages))
79 __free_pages(page, pool_size_order);72 __free_pages(page, order);
80out:73out:
81 pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",74 pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\n",
82 atomic_pool_size / 1024);75 pool_size >> 10, &gfp);
83 return -ENOMEM;76 return -ENOMEM;
84}77}
78
79static int __init dma_atomic_pool_init(void)
80{
81 int ret = 0;
82 int err;
83
84 ret = __dma_atomic_pool_init(&atomic_pool_kernel, atomic_pool_size,
85 GFP_KERNEL);
86 if (IS_ENABLED(CONFIG_ZONE_DMA)) {
87 err = __dma_atomic_pool_init(&atomic_pool_dma,
88 atomic_pool_size, GFP_DMA);
89 if (!ret && err)
90 ret = err;
91 }
92 if (IS_ENABLED(CONFIG_ZONE_DMA32)) {
93 err = __dma_atomic_pool_init(&atomic_pool_dma32,
94 atomic_pool_size, GFP_DMA32);
95 if (!ret && err)
96 ret = err;
97 }
98 return ret;
99}
85postcore_initcall(dma_atomic_pool_init);100postcore_initcall(dma_atomic_pool_init);
86 101
87bool dma_in_atomic_pool(void *start, size_t size)102static inline struct gen_pool *dev_to_pool(struct device *dev)
88{103{
89 if (unlikely(!atomic_pool))104 u64 phys_mask;
90 return false;105 gfp_t gfp;
106
107 gfp = dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
108 &phys_mask);
109 if (IS_ENABLED(CONFIG_ZONE_DMA) && gfp == GFP_DMA)
110 return atomic_pool_dma;
111 if (IS_ENABLED(CONFIG_ZONE_DMA32) && gfp == GFP_DMA32)
112 return atomic_pool_dma32;
113 return atomic_pool_kernel;
114}
91 115
92 return gen_pool_has_addr(atomic_pool, (unsigned long)start, size);116static bool dma_in_atomic_pool(struct device *dev, void *start, size_t size)
117{
118 struct gen_pool *pool = dev_to_pool(dev);
119
120 if (unlikely(!pool))
121 return false;
122 return gen_pool_has_addr(pool, (unsigned long)start, size);
93}123}
94 124
95void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags)125void *dma_alloc_from_pool(struct device *dev, size_t size,
126 struct page **ret_page, gfp_t flags)
96{127{
128 struct gen_pool *pool = dev_to_pool(dev);
97 unsigned long val;129 unsigned long val;
98 void *ptr = NULL;130 void *ptr = NULL;
99 131
100 if (!atomic_pool) {132 if (!pool) {
101 WARN(1, "coherent pool not initialised!\n");133 WARN(1, "%pGg atomic pool not initialised!\n", &flags);
102 return NULL;134 return NULL;
103 }135 }
104 136
105 val = gen_pool_alloc(atomic_pool, size);137 val = gen_pool_alloc(pool, size);
106 if (val) {138 if (val) {
107 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);139 phys_addr_t phys = gen_pool_virt_to_phys(pool, val);
108 140
109 *ret_page = pfn_to_page(__phys_to_pfn(phys));141 *ret_page = pfn_to_page(__phys_to_pfn(phys));
110 ptr = (void *)val;142 ptr = (void *)val;
@@ -114,10 +146,12 @@ void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags) · 3 unmodified lines
114 return ptr;146 return ptr;
115}147}
116 148
117bool dma_free_from_pool(void *start, size_t size)149bool dma_free_from_pool(struct device *dev, void *start, size_t size)
118{150{
119 if (!dma_in_atomic_pool(start, size))151 struct gen_pool *pool = dev_to_pool(dev);
152
153 if (!dma_in_atomic_pool(dev, start, size))
120 return false;154 return false;
121 gen_pool_free(atomic_pool, (unsigned long)start, size);155 gen_pool_free(pool, (unsigned long)start, size);
122 return true;156 return true;
123}157}