History

mm: hugetlb: optionally allocate gigantic hugepages using cma

Commit 944d9fec8d7a ("hugetlb: add support for gigantic page allocation at runtime") has added the run-time allocation of gigantic pages. However it actually works only at early stages of the system loading, when the majority of memory is free. After some time the memory gets fragmented by non-movable pages, so the chances to find a contiguous 1GB block are getting close to zero. Even dropping caches manually doesn't help a lot. At large scale rebooting servers in order to allocate gigantic hugepages is quite expensive and complex. At the same time keeping some constant percentage of memory in reserved hugepages even if the workload isn't using it is a big waste: not all workloads can benefit from using 1 GB pages. The following solution can solve the problem: 1) On boot time a dedicated cma area* is reserved. The size is passed as a kernel argument. 2) Run-time allocations of gigantic hugepages are performed using the cma allocator and the dedicated cma area In this case gigantic hugepages can be allocated successfully with a high probability, however the memory isn't completely wasted if nobody is using 1GB hugepages: it can be used for pagecache, anon memory, THPs, etc. * On a multi-node machine a per-node cma area is allocated on each node. Following gigantic hugetlb allocation are using the first available numa node if the mask isn't specified by a user. Usage: 1) configure the kernel to allocate a cma area for hugetlb allocations: pass hugetlb_cma=10G as a kernel argument 2) allocate hugetlb pages as usual, e.g. echo 10 > /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages If the option isn't enabled or the allocation of the cma area failed, the current behavior of the system is preserved. x86 and arm-64 are covered by this patch, other architectures can be trivially added later. The patch contains clean-ups and fixes proposed and implemented by Aslan Bakirov and Randy Dunlap. It also contains ideas and suggestions proposed by Rik van Riel, Michal Hocko and Mike Kravetz. Thanks!

Roman Gushchin committed 6 years ago

Commit cf11e851 parent 8676af15 files · +139 −0Browse files

Unified · Split

5 files
  1. Documentation/admin-guide/kernel-parameters.txt +8 −0
  2. arch/arm64/mm/init.c +6 −0
  3. arch/x86/kernel/setup.c +4 −0
  4. include/linux/hugetlb.h +12 −0
  5. mm/hugetlb.c +109 −0
Documentation/admin-guide/kernel-parameters.txt+8−0
@@ -1475,6 +1475,14 @@ · 1474 unmodified lines
1475 hpet_mmap= [X86, HPET_MMAP] Allow userspace to mmap HPET1475 hpet_mmap= [X86, HPET_MMAP] Allow userspace to mmap HPET
1476 registers. Default set by CONFIG_HPET_MMAP_DEFAULT.1476 registers. Default set by CONFIG_HPET_MMAP_DEFAULT.
1477 1477
1478 hugetlb_cma= [HW] The size of a cma area used for allocation
1479 of gigantic hugepages.
1480 Format: nn[KMGTPE]
1481
1482 Reserve a cma area of given size and allocate gigantic
1483 hugepages using the cma allocator. If enabled, the
1484 boot-time allocation of gigantic hugepages is skipped.
1485
1478 hugepages= [HW,X86-32,IA-64] HugeTLB pages to allocate at boot.1486 hugepages= [HW,X86-32,IA-64] HugeTLB pages to allocate at boot.
1479 hugepagesz= [HW,IA-64,PPC,X86-64] The size of the HugeTLB pages.1487 hugepagesz= [HW,IA-64,PPC,X86-64] The size of the HugeTLB pages.
1480 On x86-64 and powerpc, this option can be specified1488 On x86-64 and powerpc, this option can be specified
arch/arm64/mm/init.c+6−0
@@ -29,6 +29,7 @@ · 28 unmodified lines
29#include <linux/mm.h>29#include <linux/mm.h>
30#include <linux/kexec.h>30#include <linux/kexec.h>
31#include <linux/crash_dump.h>31#include <linux/crash_dump.h>
32#include <linux/hugetlb.h>
32 33
33#include <asm/boot.h>34#include <asm/boot.h>
34#include <asm/fixmap.h>35#include <asm/fixmap.h>
@@ -457,6 +458,11 @@ void __init arm64_memblock_init(void) · 422 unmodified lines
457 high_memory = __va(memblock_end_of_DRAM() - 1) + 1;458 high_memory = __va(memblock_end_of_DRAM() - 1) + 1;
458 459
459 dma_contiguous_reserve(arm64_dma32_phys_limit);460 dma_contiguous_reserve(arm64_dma32_phys_limit);
461
462#ifdef CONFIG_ARM64_4K_PAGES
463 hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT);
464#endif
465
460}466}
461 467
462void __init bootmem_init(void)468void __init bootmem_init(void)
arch/x86/kernel/setup.c+4−0
@@ -16,6 +16,7 @@ · 15 unmodified lines
16#include <linux/pci.h>16#include <linux/pci.h>
17#include <linux/root_dev.h>17#include <linux/root_dev.h>
18#include <linux/sfi.h>18#include <linux/sfi.h>
19#include <linux/hugetlb.h>
19#include <linux/tboot.h>20#include <linux/tboot.h>
20#include <linux/usb/xhci-dbgp.h>21#include <linux/usb/xhci-dbgp.h>
21 22
@@ -1157,6 +1158,9 @@ void __init setup_arch(char **cmdline_p) · 1135 unmodified lines
1157 initmem_init();1158 initmem_init();
1158 dma_contiguous_reserve(max_pfn_mapped << PAGE_SHIFT);1159 dma_contiguous_reserve(max_pfn_mapped << PAGE_SHIFT);
1159 1160
1161 if (boot_cpu_has(X86_FEATURE_GBPAGES))
1162 hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT);
1163
1160 /*1164 /*
1161 * Reserve memory for crash kernel after SRAT is parsed so that it1165 * Reserve memory for crash kernel after SRAT is parsed so that it
1162 * won't consume hotpluggable memory.1166 * won't consume hotpluggable memory.
include/linux/hugetlb.h+12−0
@@ -895,4 +895,16 @@ static inline spinlock_t *huge_pte_lock(struct hstate *h, · 894 unmodified lines
895 return ptl;895 return ptl;
896}896}
897 897
898#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_CMA)
899extern void __init hugetlb_cma_reserve(int order);
900extern void __init hugetlb_cma_check(void);
901#else
902static inline __init void hugetlb_cma_reserve(int order)
903{
904}
905static inline __init void hugetlb_cma_check(void)
906{
907}
908#endif
909
898#endif /* _LINUX_HUGETLB_H */910#endif /* _LINUX_HUGETLB_H */
mm/hugetlb.c+109−0
@@ -28,6 +28,7 @@ · 27 unmodified lines
28#include <linux/jhash.h>28#include <linux/jhash.h>
29#include <linux/numa.h>29#include <linux/numa.h>
30#include <linux/llist.h>30#include <linux/llist.h>
31#include <linux/cma.h>
31 32
32#include <asm/page.h>33#include <asm/page.h>
33#include <asm/pgtable.h>34#include <asm/pgtable.h>
@@ -44,6 +45,9 @@ · 10 unmodified lines
44int hugetlb_max_hstate __read_mostly;45int hugetlb_max_hstate __read_mostly;
45unsigned int default_hstate_idx;46unsigned int default_hstate_idx;
46struct hstate hstates[HUGE_MAX_HSTATE];47struct hstate hstates[HUGE_MAX_HSTATE];
48
49static struct cma *hugetlb_cma[MAX_NUMNODES];
50
47/*51/*
48 * Minimum page order among possible hugepage sizes, set to a proper value52 * Minimum page order among possible hugepage sizes, set to a proper value
49 * at boot time.53 * at boot time.
@@ -1228,6 +1232,14 @@ static void destroy_compound_gigantic_page(struct page *page, · 1178 unmodified lines
1228 1232
1229static void free_gigantic_page(struct page *page, unsigned int order)1233static void free_gigantic_page(struct page *page, unsigned int order)
1230{1234{
1235 /*
1236 * If the page isn't allocated using the cma allocator,
1237 * cma_release() returns false.
1238 */
1239 if (IS_ENABLED(CONFIG_CMA) &&
1240 cma_release(hugetlb_cma[page_to_nid(page)], page, 1 << order))
1241 return;
1242
1231 free_contig_range(page_to_pfn(page), 1 << order);1243 free_contig_range(page_to_pfn(page), 1 << order);
1232}1244}
1233 1245
@@ -1237,6 +1249,21 @@ static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, · 3 unmodified lines
1237{1249{
1238 unsigned long nr_pages = 1UL << huge_page_order(h);1250 unsigned long nr_pages = 1UL << huge_page_order(h);
1239 1251
1252 if (IS_ENABLED(CONFIG_CMA)) {
1253 struct page *page;
1254 int node;
1255
1256 for_each_node_mask(node, *nodemask) {
1257 if (!hugetlb_cma[node])
1258 continue;
1259
1260 page = cma_alloc(hugetlb_cma[node], nr_pages,
1261 huge_page_order(h), true);
1262 if (page)
1263 return page;
1264 }
1265 }
1266
1240 return alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask);1267 return alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask);
1241}1268}
1242 1269
@@ -1281,8 +1308,14 @@ static void update_and_free_page(struct hstate *h, struct page *page) · 38 unmodified lines
1281 set_compound_page_dtor(page, NULL_COMPOUND_DTOR);1308 set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
1282 set_page_refcounted(page);1309 set_page_refcounted(page);
1283 if (hstate_is_gigantic(h)) {1310 if (hstate_is_gigantic(h)) {
1311 /*
1312 * Temporarily drop the hugetlb_lock, because
1313 * we might block in free_gigantic_page().
1314 */
1315 spin_unlock(&hugetlb_lock);
1284 destroy_compound_gigantic_page(page, huge_page_order(h));1316 destroy_compound_gigantic_page(page, huge_page_order(h));
1285 free_gigantic_page(page, huge_page_order(h));1317 free_gigantic_page(page, huge_page_order(h));
1318 spin_lock(&hugetlb_lock);
1286 } else {1319 } else {
1287 __free_pages(page, huge_page_order(h));1320 __free_pages(page, huge_page_order(h));
1288 }1321 }
@@ -2539,6 +2572,10 @@ static void __init hugetlb_hstate_alloc_pages(struct hstate *h) · 1250 unmodified lines
2539 2572
2540 for (i = 0; i < h->max_huge_pages; ++i) {2573 for (i = 0; i < h->max_huge_pages; ++i) {
2541 if (hstate_is_gigantic(h)) {2574 if (hstate_is_gigantic(h)) {
2575 if (IS_ENABLED(CONFIG_CMA) && hugetlb_cma[0]) {
2576 pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n");
2577 break;
2578 }
2542 if (!alloc_bootmem_huge_page(h))2579 if (!alloc_bootmem_huge_page(h))
2543 break;2580 break;
2544 } else if (!alloc_pool_huge_page(h,2581 } else if (!alloc_pool_huge_page(h,
@@ -3194,6 +3231,7 @@ static int __init hugetlb_init(void) · 649 unmodified lines
3194 default_hstate.max_huge_pages = default_hstate_max_huge_pages;3231 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
3195 }3232 }
3196 3233
3234 hugetlb_cma_check();
3197 hugetlb_init_hstates();3235 hugetlb_init_hstates();
3198 gather_bootmem_prealloc();3236 gather_bootmem_prealloc();
3199 report_hugepages();3237 report_hugepages();
@@ -5506,3 +5544,74 @@ void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason) · 2306 unmodified lines
5506 spin_unlock(&hugetlb_lock);5544 spin_unlock(&hugetlb_lock);
5507 }5545 }
5508}5546}
5547
5548#ifdef CONFIG_CMA
5549static unsigned long hugetlb_cma_size __initdata;
5550static bool cma_reserve_called __initdata;
5551
5552static int __init cmdline_parse_hugetlb_cma(char *p)
5553{
5554 hugetlb_cma_size = memparse(p, &p);
5555 return 0;
5556}
5557
5558early_param("hugetlb_cma", cmdline_parse_hugetlb_cma);
5559
5560void __init hugetlb_cma_reserve(int order)
5561{
5562 unsigned long size, reserved, per_node;
5563 int nid;
5564
5565 cma_reserve_called = true;
5566
5567 if (!hugetlb_cma_size)
5568 return;
5569
5570 if (hugetlb_cma_size < (PAGE_SIZE << order)) {
5571 pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n",
5572 (PAGE_SIZE << order) / SZ_1M);
5573 return;
5574 }
5575
5576 /*
5577 * If 3 GB area is requested on a machine with 4 numa nodes,
5578 * let's allocate 1 GB on first three nodes and ignore the last one.
5579 */
5580 per_node = DIV_ROUND_UP(hugetlb_cma_size, nr_online_nodes);
5581 pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
5582 hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
5583
5584 reserved = 0;
5585 for_each_node_state(nid, N_ONLINE) {
5586 int res;
5587
5588 size = min(per_node, hugetlb_cma_size - reserved);
5589 size = round_up(size, PAGE_SIZE << order);
5590
5591 res = cma_declare_contiguous_nid(0, size, 0, PAGE_SIZE << order,
5592 0, false, "hugetlb",
5593 &hugetlb_cma[nid], nid);
5594 if (res) {
5595 pr_warn("hugetlb_cma: reservation failed: err %d, node %d",
5596 res, nid);
5597 continue;
5598 }
5599
5600 reserved += size;
5601 pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",
5602 size / SZ_1M, nid);
5603
5604 if (reserved >= hugetlb_cma_size)
5605 break;
5606 }
5607}
5608
5609void __init hugetlb_cma_check(void)
5610{
5611 if (!hugetlb_cma_size || cma_reserve_called)
5612 return;
5613
5614 pr_warn("hugetlb_cma: the option isn't supported by current arch\n");
5615}
5616
5617#endif /* CONFIG_CMA */