There is no guarantee to CMA's placement, so allocating a zone specific atomic pool from CMA might return memory from a completely different memory zone. To get around this double check CMA's placement before allocating from it.
Unified · Split
| @@ -3,7 +3,9 @@ · 2 unmodified lines | ||
| 3 | 3 | * Copyright (C) 2012 ARM Ltd. |
| 4 | 4 | * Copyright (C) 2020 Google LLC |
| 5 | 5 | */ |
| 6 | +#include <linux/cma.h> | |
| 6 | 7 | #include <linux/debugfs.h> |
| 8 | +#include <linux/dma-contiguous.h> | |
| 7 | 9 | #include <linux/dma-direct.h> |
| 8 | 10 | #include <linux/dma-noncoherent.h> |
| 9 | 11 | #include <linux/init.h> |
| @@ -55,6 +57,29 @@ static void dma_atomic_pool_size_add(gfp_t gfp, size_t size) · 45 unmodified lines | ||
| 55 | 57 | pool_size_kernel += size; |
| 56 | 58 | } |
| 57 | 59 | |
| 60 | +static bool cma_in_zone(gfp_t gfp) | |
| 61 | +{ | |
| 62 | + unsigned long size; | |
| 63 | + phys_addr_t end; | |
| 64 | + struct cma *cma; | |
| 65 | + | |
| 66 | + cma = dev_get_cma_area(NULL); | |
| 67 | + if (!cma) | |
| 68 | + return false; | |
| 69 | + | |
| 70 | + size = cma_get_size(cma); | |
| 71 | + if (!size) | |
| 72 | + return false; | |
| 73 | + | |
| 74 | + /* CMA can't cross zone boundaries, see cma_activate_area() */ | |
| 75 | + end = cma_get_base(cma) + size - 1; | |
| 76 | + if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp & GFP_DMA)) | |
| 77 | + return end <= DMA_BIT_MASK(zone_dma_bits); | |
| 78 | + if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp & GFP_DMA32)) | |
| 79 | + return end <= DMA_BIT_MASK(32); | |
| 80 | + return true; | |
| 81 | +} | |
| 82 | + | |
| 58 | 83 | static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size, |
| 59 | 84 | gfp_t gfp) |
| 60 | 85 | { |
| @@ -68,7 +93,11 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size, · 7 unmodified lines | ||
| 68 | 93 | |
| 69 | 94 | do { |
| 70 | 95 | pool_size = 1 << (PAGE_SHIFT + order); |
| 71 | − page = alloc_pages(gfp, order); | |
| 96 | + if (cma_in_zone(gfp)) | |
| 97 | + page = dma_alloc_from_contiguous(NULL, 1 << order, | |
| 98 | + order, false); | |
| 99 | + if (!page) | |
| 100 | + page = alloc_pages(gfp, order); | |
| 72 | 101 | } while (!page && order-- > 0); |
| 73 | 102 | if (!page) |
| 74 | 103 | goto out; |