History

dma-pool: add additional coherent pools to map to gfp mask

The single atomic pool is allocated from the lowest zone possible since it is guaranteed to be applicable for any DMA allocation. Devices may allocate through the DMA API but not have a strict reliance on GFP_DMA memory. Since the atomic pool will be used for all non-blockable allocations, returning all memory from ZONE_DMA may unnecessarily deplete the zone. Provision for multiple atomic pools that will map to the optimal gfp mask of the device. When allocating non-blockable memory, determine the optimal gfp mask of the device and use the appropriate atomic pool. The coherent DMA mask will remain the same between allocation and free and, thus, memory will be freed to the same atomic pool it was allocated from. __dma_atomic_pool_init() will be changed to return struct gen_pool * later once dynamic expansion is added.

David Rientjes committed 6 years ago

Commit c84dc6e1 parent e860c295 files · +91 −54Browse files

Showing the changes to kernel (2 of 5 files). Show all

Unified · Split

2 files
  1. kernel/dma/direct.c +6 −6
  2. kernel/dma/pool.c +77 −43
kernel/dma/direct.c+6−6
@@ -45,8 +45,8 @@ u64 dma_direct_get_required_mask(struct device *dev) · 44 unmodified lines
45 return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;45 return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
46}46}
47 47
48static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,48gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
49 u64 *phys_limit)49 u64 *phys_limit)
50{50{
51 u64 dma_limit = min_not_zero(dma_mask, dev->bus_dma_limit);51 u64 dma_limit = min_not_zero(dma_mask, dev->bus_dma_limit);
52 52
@@ -89,8 +89,8 @@ struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, · 36 unmodified lines
89 89
90 /* we always manually zero the memory once we are done: */90 /* we always manually zero the memory once we are done: */
91 gfp &= ~__GFP_ZERO;91 gfp &= ~__GFP_ZERO;
92 gfp |= __dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,92 gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
93 &phys_limit);93 &phys_limit);
94 page = dma_alloc_contiguous(dev, alloc_size, gfp);94 page = dma_alloc_contiguous(dev, alloc_size, gfp);
95 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {95 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
96 dma_free_contiguous(dev, page, alloc_size);96 dma_free_contiguous(dev, page, alloc_size);
@@ -128,7 +128,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, · 31 unmodified lines
128 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&128 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
129 dma_alloc_need_uncached(dev, attrs) &&129 dma_alloc_need_uncached(dev, attrs) &&
130 !gfpflags_allow_blocking(gfp)) {130 !gfpflags_allow_blocking(gfp)) {
131 ret = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp);131 ret = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &page, gfp);
132 if (!ret)132 if (!ret)
133 return NULL;133 return NULL;
134 goto done;134 goto done;
@@ -212,7 +212,7 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, · 77 unmodified lines
212 }212 }
213 213
214 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&214 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
215 dma_free_from_pool(cpu_addr, PAGE_ALIGN(size)))215 dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))
216 return;216 return;
217 217
218 if (force_dma_unencrypted(dev))218 if (force_dma_unencrypted(dev))
kernel/dma/pool.c+77−43
@@ -10,7 +10,9 @@ · 9 unmodified lines
10#include <linux/genalloc.h>10#include <linux/genalloc.h>
11#include <linux/slab.h>11#include <linux/slab.h>
12 12
13static struct gen_pool *atomic_pool __ro_after_init;13static struct gen_pool *atomic_pool_dma __ro_after_init;
14static struct gen_pool *atomic_pool_dma32 __ro_after_init;
15static struct gen_pool *atomic_pool_kernel __ro_after_init;
14 16
15#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K17#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
16static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;18static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
@@ -22,89 +24,119 @@ static int __init early_coherent_pool(char *p) · 5 unmodified lines
22}24}
23early_param("coherent_pool", early_coherent_pool);25early_param("coherent_pool", early_coherent_pool);
24 26
25static gfp_t dma_atomic_pool_gfp(void)27static int __init __dma_atomic_pool_init(struct gen_pool **pool,
28 size_t pool_size, gfp_t gfp)
26{29{
27 if (IS_ENABLED(CONFIG_ZONE_DMA))30 const unsigned int order = get_order(pool_size);
28 return GFP_DMA;31 const unsigned long nr_pages = pool_size >> PAGE_SHIFT;
29 if (IS_ENABLED(CONFIG_ZONE_DMA32))
30 return GFP_DMA32;
31 return GFP_KERNEL;
32}
33
34static int __init dma_atomic_pool_init(void)
35{
36 unsigned int pool_size_order = get_order(atomic_pool_size);
37 unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
38 struct page *page;32 struct page *page;
39 void *addr;33 void *addr;
40 int ret;34 int ret;
41 35
42 if (dev_get_cma_area(NULL))36 if (dev_get_cma_area(NULL))
43 page = dma_alloc_from_contiguous(NULL, nr_pages,37 page = dma_alloc_from_contiguous(NULL, nr_pages, order, false);
44 pool_size_order, false);
45 else38 else
46 page = alloc_pages(dma_atomic_pool_gfp(), pool_size_order);39 page = alloc_pages(gfp, order);
47 if (!page)40 if (!page)
48 goto out;41 goto out;
49 42
50 arch_dma_prep_coherent(page, atomic_pool_size);43 arch_dma_prep_coherent(page, pool_size);
51 44
52 atomic_pool = gen_pool_create(PAGE_SHIFT, -1);45 *pool = gen_pool_create(PAGE_SHIFT, -1);
53 if (!atomic_pool)46 if (!*pool)
54 goto free_page;47 goto free_page;
55 48
56 addr = dma_common_contiguous_remap(page, atomic_pool_size,49 addr = dma_common_contiguous_remap(page, pool_size,
57 pgprot_dmacoherent(PAGE_KERNEL),50 pgprot_dmacoherent(PAGE_KERNEL),
58 __builtin_return_address(0));51 __builtin_return_address(0));
59 if (!addr)52 if (!addr)
60 goto destroy_genpool;53 goto destroy_genpool;
61 54
62 ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr,55 ret = gen_pool_add_virt(*pool, (unsigned long)addr, page_to_phys(page),
63 page_to_phys(page), atomic_pool_size, -1);56 pool_size, -1);
64 if (ret)57 if (ret)
65 goto remove_mapping;58 goto remove_mapping;
66 gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);59 gen_pool_set_algo(*pool, gen_pool_first_fit_order_align, NULL);
67 60
68 pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",61 pr_info("DMA: preallocated %zu KiB %pGg pool for atomic allocations\n",
69 atomic_pool_size / 1024);62 pool_size >> 10, &gfp);
70 return 0;63 return 0;
71 64
72remove_mapping:65remove_mapping:
73 dma_common_free_remap(addr, atomic_pool_size);66 dma_common_free_remap(addr, pool_size);
74destroy_genpool:67destroy_genpool:
75 gen_pool_destroy(atomic_pool);68 gen_pool_destroy(*pool);
76 atomic_pool = NULL;69 *pool = NULL;
77free_page:70free_page:
78 if (!dma_release_from_contiguous(NULL, page, nr_pages))71 if (!dma_release_from_contiguous(NULL, page, nr_pages))
79 __free_pages(page, pool_size_order);72 __free_pages(page, order);
80out:73out:
81 pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",74 pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\n",
82 atomic_pool_size / 1024);75 pool_size >> 10, &gfp);
83 return -ENOMEM;76 return -ENOMEM;
84}77}
78
79static int __init dma_atomic_pool_init(void)
80{
81 int ret = 0;
82 int err;
83
84 ret = __dma_atomic_pool_init(&atomic_pool_kernel, atomic_pool_size,
85 GFP_KERNEL);
86 if (IS_ENABLED(CONFIG_ZONE_DMA)) {
87 err = __dma_atomic_pool_init(&atomic_pool_dma,
88 atomic_pool_size, GFP_DMA);
89 if (!ret && err)
90 ret = err;
91 }
92 if (IS_ENABLED(CONFIG_ZONE_DMA32)) {
93 err = __dma_atomic_pool_init(&atomic_pool_dma32,
94 atomic_pool_size, GFP_DMA32);
95 if (!ret && err)
96 ret = err;
97 }
98 return ret;
99}
85postcore_initcall(dma_atomic_pool_init);100postcore_initcall(dma_atomic_pool_init);
86 101
87bool dma_in_atomic_pool(void *start, size_t size)102static inline struct gen_pool *dev_to_pool(struct device *dev)
88{103{
89 if (unlikely(!atomic_pool))104 u64 phys_mask;
90 return false;105 gfp_t gfp;
106
107 gfp = dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
108 &phys_mask);
109 if (IS_ENABLED(CONFIG_ZONE_DMA) && gfp == GFP_DMA)
110 return atomic_pool_dma;
111 if (IS_ENABLED(CONFIG_ZONE_DMA32) && gfp == GFP_DMA32)
112 return atomic_pool_dma32;
113 return atomic_pool_kernel;
114}
91 115
92 return gen_pool_has_addr(atomic_pool, (unsigned long)start, size);116static bool dma_in_atomic_pool(struct device *dev, void *start, size_t size)
117{
118 struct gen_pool *pool = dev_to_pool(dev);
119
120 if (unlikely(!pool))
121 return false;
122 return gen_pool_has_addr(pool, (unsigned long)start, size);
93}123}
94 124
95void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags)125void *dma_alloc_from_pool(struct device *dev, size_t size,
126 struct page **ret_page, gfp_t flags)
96{127{
128 struct gen_pool *pool = dev_to_pool(dev);
97 unsigned long val;129 unsigned long val;
98 void *ptr = NULL;130 void *ptr = NULL;
99 131
100 if (!atomic_pool) {132 if (!pool) {
101 WARN(1, "coherent pool not initialised!\n");133 WARN(1, "%pGg atomic pool not initialised!\n", &flags);
102 return NULL;134 return NULL;
103 }135 }
104 136
105 val = gen_pool_alloc(atomic_pool, size);137 val = gen_pool_alloc(pool, size);
106 if (val) {138 if (val) {
107 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);139 phys_addr_t phys = gen_pool_virt_to_phys(pool, val);
108 140
109 *ret_page = pfn_to_page(__phys_to_pfn(phys));141 *ret_page = pfn_to_page(__phys_to_pfn(phys));
110 ptr = (void *)val;142 ptr = (void *)val;
@@ -114,10 +146,12 @@ void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags) · 3 unmodified lines
114 return ptr;146 return ptr;
115}147}
116 148
117bool dma_free_from_pool(void *start, size_t size)149bool dma_free_from_pool(struct device *dev, void *start, size_t size)
118{150{
119 if (!dma_in_atomic_pool(start, size))151 struct gen_pool *pool = dev_to_pool(dev);
152
153 if (!dma_in_atomic_pool(dev, start, size))
120 return false;154 return false;
121 gen_pool_free(atomic_pool, (unsigned long)start, size);155 gen_pool_free(pool, (unsigned long)start, size);
122 return true;156 return true;
123}157}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t