History

mm: memmap defer init doesn't work as expected

VMware observed a performance regression during memmap init on their platform, and bisected to commit 73a6e474cb376 ("mm: memmap_init: iterate over memblock regions rather that check each PFN") causing it. Before the commit: [0.033176] Normal zone: 1445888 pages used for memmap [0.033176] Normal zone: 89391104 pages, LIFO batch:63 [0.035851] ACPI: PM-Timer IO Port: 0x448 With commit [0.026874] Normal zone: 1445888 pages used for memmap [0.026875] Normal zone: 89391104 pages, LIFO batch:63 [2.028450] ACPI: PM-Timer IO Port: 0x448 The root cause is the current memmap defer init doesn't work as expected. Before, memmap_init_zone() was used to do memmap init of one whole zone, to initialize all low zones of one numa node, but defer memmap init of the last zone in that numa node. However, since commit 73a6e474cb376, function memmap_init() is adapted to iterater over memblock regions inside one zone, then call memmap_init_zone() to do memmap init for each region. E.g, on VMware's system, the memory layout is as below, there are two memory regions in node 2. The current code will mistakenly initialize the whole 1st region [mem 0xab00000000-0xfcffffffff], then do memmap defer to iniatialize only one memmory section on the 2nd region [mem 0x10000000000-0x1033fffffff]. In fact, we only expect to see that there's only one memory section's memmap initialized. That's why more time is costed at the time. [ 0.008842] ACPI: SRAT: Node 0 PXM 0 [mem 0x00000000-0x0009ffff] [ 0.008842] ACPI: SRAT: Node 0 PXM 0 [mem 0x00100000-0xbfffffff] [ 0.008843] ACPI: SRAT: Node 0 PXM 0 [mem 0x100000000-0x55ffffffff] [ 0.008844] ACPI: SRAT: Node 1 PXM 1 [mem 0x5600000000-0xaaffffffff] [ 0.008844] ACPI: SRAT: Node 2 PXM 2 [mem 0xab00000000-0xfcffffffff] [ 0.008845] ACPI: SRAT: Node 2 PXM 2 [mem 0x10000000000-0x1033fffffff] Now, let's add a parameter 'zone_end_pfn' to memmap_init_zone() to pass down the real zone end pfn so that defer_init() can use it to judge whether defer need be taken in zone wide.

Baoquan He committed 5 years ago

Commit dc2da7b1 parent 6d87d0e4 files · +11 −8Browse files

Unified · Split

4 files
  1. arch/ia64/mm/init.c +2 −2
  2. include/linux/mm.h +3 −2
  3. mm/memory_hotplug.c +1 −1
  4. mm/page_alloc.c +5 −3
arch/ia64/mm/init.c+2−2
@@ -536,7 +536,7 @@ virtual_memmap_init(u64 start, u64 end, void *arg) · 535 unmodified lines
536 536
537 if (map_start < map_end)537 if (map_start < map_end)
538 memmap_init_zone((unsigned long)(map_end - map_start),538 memmap_init_zone((unsigned long)(map_end - map_start),
539 args->nid, args->zone, page_to_pfn(map_start),539 args->nid, args->zone, page_to_pfn(map_start), page_to_pfn(map_end),
540 MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);540 MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);
541 return 0;541 return 0;
542}542}
@@ -546,7 +546,7 @@ memmap_init (unsigned long size, int nid, unsigned long zone, · 3 unmodified lines
546 unsigned long start_pfn)546 unsigned long start_pfn)
547{547{
548 if (!vmem_map) {548 if (!vmem_map) {
549 memmap_init_zone(size, nid, zone, start_pfn,549 memmap_init_zone(size, nid, zone, start_pfn, start_pfn + size,
550 MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);550 MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);
551 } else {551 } else {
552 struct page *start;552 struct page *start;
include/linux/mm.h+3−2
@@ -2439,8 +2439,9 @@ extern int __meminit early_pfn_to_nid(unsigned long pfn); · 2438 unmodified lines
2439#endif2439#endif
2440 2440
2441extern void set_dma_reserve(unsigned long new_dma_reserve);2441extern void set_dma_reserve(unsigned long new_dma_reserve);
2442extern void memmap_init_zone(unsigned long, int, unsigned long, unsigned long,2442extern void memmap_init_zone(unsigned long, int, unsigned long,
2443 enum meminit_context, struct vmem_altmap *, int migratetype);2443 unsigned long, unsigned long, enum meminit_context,
2444 struct vmem_altmap *, int migratetype);
2444extern void setup_per_zone_wmarks(void);2445extern void setup_per_zone_wmarks(void);
2445extern int __meminit init_per_zone_wmark_min(void);2446extern int __meminit init_per_zone_wmark_min(void);
2446extern void mem_init(void);2447extern void mem_init(void);
mm/memory_hotplug.c+1−1
@@ -713,7 +713,7 @@ void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, · 712 unmodified lines
713 * expects the zone spans the pfn range. All the pages in the range713 * expects the zone spans the pfn range. All the pages in the range
714 * are reserved so nobody should be touching them so we should be safe714 * are reserved so nobody should be touching them so we should be safe
715 */715 */
716 memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn,716 memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn, 0,
717 MEMINIT_HOTPLUG, altmap, migratetype);717 MEMINIT_HOTPLUG, altmap, migratetype);
718 718
719 set_zone_contiguous(zone);719 set_zone_contiguous(zone);
mm/page_alloc.c+5−3
@@ -423,6 +423,8 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn) · 422 unmodified lines
423 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))423 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
424 return false;424 return false;
425 425
426 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX)
427 return true;
426 /*428 /*
427 * We start only with one section of pages, more pages are added as429 * We start only with one section of pages, more pages are added as
428 * needed until the rest of deferred pages are initialized.430 * needed until the rest of deferred pages are initialized.
@@ -6116,7 +6118,7 @@ overlap_memmap_init(unsigned long zone, unsigned long *pfn) · 5687 unmodified lines
6116 * zone stats (e.g., nr_isolate_pageblock) are touched.6118 * zone stats (e.g., nr_isolate_pageblock) are touched.
6117 */6119 */
6118void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,6120void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
6119 unsigned long start_pfn,6121 unsigned long start_pfn, unsigned long zone_end_pfn,
6120 enum meminit_context context,6122 enum meminit_context context,
6121 struct vmem_altmap *altmap, int migratetype)6123 struct vmem_altmap *altmap, int migratetype)
6122{6124{
@@ -6152,7 +6154,7 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, · 29 unmodified lines
6152 if (context == MEMINIT_EARLY) {6154 if (context == MEMINIT_EARLY) {
6153 if (overlap_memmap_init(zone, &pfn))6155 if (overlap_memmap_init(zone, &pfn))
6154 continue;6156 continue;
6155 if (defer_init(nid, pfn, end_pfn))6157 if (defer_init(nid, pfn, zone_end_pfn))
6156 break;6158 break;
6157 }6159 }
6158 6160
@@ -6266,7 +6268,7 @@ void __meminit __weak memmap_init(unsigned long size, int nid, · 107 unmodified lines
6266 6268
6267 if (end_pfn > start_pfn) {6269 if (end_pfn > start_pfn) {
6268 size = end_pfn - start_pfn;6270 size = end_pfn - start_pfn;
6269 memmap_init_zone(size, nid, zone, start_pfn,6271 memmap_init_zone(size, nid, zone, start_pfn, range_end_pfn,
6270 MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);6272 MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);
6271 }6273 }
6272 }6274 }