/* * linux/mm/memory_hotplug.c * * Copyright (C) */ #include <linux/stddef.h>#include <linux/mm.h>#include <linux/sched/signal.h>#include <linux/swap.h>#include <linux/interrupt.h>#include <linux/pagemap.h>#include <linux/compiler.h>#include <linux/export.h>#include <linux/pagevec.h>#include <linux/writeback.h>#include <linux/slab.h>#include <linux/sysctl.h>#include <linux/cpu.h>#include <linux/memory.h>#include <linux/memremap.h>#include <linux/memory_hotplug.h>#include <linux/highmem.h>#include <linux/vmalloc.h>#include <linux/ioport.h>#include <linux/delay.h>#include <linux/migrate.h>#include <linux/page-isolation.h>#include <linux/pfn.h>#include <linux/suspend.h>#include <linux/mm_inline.h>#include <linux/firmware-map.h>#include <linux/stop_machine.h>#include <linux/hugetlb.h>#include <linux/memblock.h>#include <linux/bootmem.h>#include <linux/compaction.h> #include <asm/tlbflush.h> #include "internal.h" /* * online_page_callback contains pointer to current page onlining function. * Initially it is generic_online_page(). If it is required it could be * changed by calling set_online_page_callback() for callback registration * and restore_online_page_callback() for generic callback restore. */ static void generic_online_page(struct page *page); static online_page_callback_t online_page_callback = generic_online_page;static DEFINE_MUTEX(online_page_callback_lock); /* The same as the cpu_hotplug lock, but for memory hotplug. */static struct { struct task_struct *active_writer; struct mutex lock; /* Synchronizes accesses to refcount, */ /* * Also blocks the new readers during * an ongoing mem hotplug operation. */ int refcount; #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map;#endif} mem_hotplug = { .active_writer = NULL, .lock = __MUTEX_INITIALIZER(mem_hotplug.lock), .refcount = 0,#ifdef CONFIG_DEBUG_LOCK_ALLOC .dep_map = {.name = "mem_hotplug.lock" },#endif}; /* Lockdep annotations for get/put_online_mems() and mem_hotplug_begin/end() */#define memhp_lock_acquire_read() lock_map_acquire_read(&mem_hotplug.dep_map)#define memhp_lock_acquire() lock_map_acquire(&mem_hotplug.dep_map)#define memhp_lock_release() lock_map_release(&mem_hotplug.dep_map) #ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINEbool memhp_auto_online;#elsebool memhp_auto_online = true;#endifEXPORT_SYMBOL_GPL(memhp_auto_online); static int __init setup_memhp_default_state(char *str){ if (!strcmp(str, "online")) memhp_auto_online = true; else if (!strcmp(str, "offline")) memhp_auto_online = false; return 1;}__setup("memhp_default_state=", setup_memhp_default_state); void get_online_mems(void){ might_sleep(); if (mem_hotplug.active_writer == current) return; memhp_lock_acquire_read(); mutex_lock(&mem_hotplug.lock); mem_hotplug.refcount++; mutex_unlock(&mem_hotplug.lock); } void put_online_mems(void){ if (mem_hotplug.active_writer == current) return; mutex_lock(&mem_hotplug.lock); if (WARN_ON(!mem_hotplug.refcount)) mem_hotplug.refcount++; /* try to fix things up */ if (!--mem_hotplug.refcount && unlikely(mem_hotplug.active_writer)) wake_up_process(mem_hotplug.active_writer); mutex_unlock(&mem_hotplug.lock); memhp_lock_release(); } /* Serializes write accesses to mem_hotplug.active_writer. */static DEFINE_MUTEX(memory_add_remove_lock); void mem_hotplug_begin(void){ mutex_lock(&memory_add_remove_lock); mem_hotplug.active_writer = current; memhp_lock_acquire(); for (;;) { mutex_lock(&mem_hotplug.lock); if (likely(!mem_hotplug.refcount)) break; __set_current_state(TASK_UNINTERRUPTIBLE); mutex_unlock(&mem_hotplug.lock); schedule(); }} void mem_hotplug_done(void){ mem_hotplug.active_writer = NULL; mutex_unlock(&mem_hotplug.lock); memhp_lock_release(); mutex_unlock(&memory_add_remove_lock);} /* add this memory to iomem resource */static struct resource *register_memory_resource(u64 start, u64 size){ struct resource *res; res = kzalloc(sizeof(struct resource), GFP_KERNEL); if (!res) return ERR_PTR(-ENOMEM); res->name = "System RAM"; res->start = start; res->end = start + size - 1; res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; if (request_resource(&iomem_resource, res) < 0) { pr_debug("System RAM resource %pR cannot be added\n", res); kfree(res); return ERR_PTR(-EEXIST); } return res;} static void release_memory_resource(struct resource *res){ if (!res) return; release_resource(res); kfree(res); return;} #ifdef CONFIG_MEMORY_HOTPLUG_SPARSEvoid get_page_bootmem(unsigned long info, struct page *page, unsigned long type){ page->freelist = (void *)type; SetPagePrivate(page); set_page_private(page, info); page_ref_inc(page);} void put_page_bootmem(struct page *page){ unsigned long type; type = (unsigned long) page->freelist; BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); if (page_ref_dec_return(page) == 1) { page->freelist = NULL; ClearPagePrivate(page); set_page_private(page, 0); INIT_LIST_HEAD(&page->lru); free_reserved_page(page); }} #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE#ifndef CONFIG_SPARSEMEM_VMEMMAPstatic void register_page_bootmem_info_section(unsigned long start_pfn){ unsigned long *usemap, mapsize, section_nr, i; struct mem_section *ms; struct page *page, *memmap; section_nr = pfn_to_section_nr(start_pfn); ms = __nr_to_section(section_nr); /* Get section's memmap address */ memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); /* * Get page for the memmap's phys address * XXX: need more consideration for sparse_vmemmap... */ page = virt_to_page(memmap); mapsize = sizeof(struct page) * PAGES_PER_SECTION; mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; /* remember memmap's page */ for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, SECTION_INFO); usemap = __nr_to_section(section_nr)->pageblock_flags; page = virt_to_page(usemap); mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, MIX_SECTION_INFO); }#else /* CONFIG_SPARSEMEM_VMEMMAP */static void register_page_bootmem_info_section(unsigned long start_pfn){ unsigned long *usemap, mapsize, section_nr, i; struct mem_section *ms; struct page *page, *memmap; if (!pfn_valid(start_pfn)) return; section_nr = pfn_to_section_nr(start_pfn); ms = __nr_to_section(section_nr); memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); usemap = __nr_to_section(section_nr)->pageblock_flags; page = virt_to_page(usemap); mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, MIX_SECTION_INFO);}#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ void __init register_page_bootmem_info_node(struct pglist_data *pgdat){ unsigned long i, pfn, end_pfn, nr_pages; int node = pgdat->node_id; struct page *page; nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; page = virt_to_page(pgdat); for (i = 0; i < nr_pages; i++, page++) get_page_bootmem(node, page, NODE_INFO); pfn = pgdat->node_start_pfn; end_pfn = pgdat_end_pfn(pgdat); /* register section info */ for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { /* * Some platforms can assign the same pfn to multiple nodes - on * node0 as well as nodeN. To avoid registering a pfn against * multiple nodes we check that this pfn does not already * reside in some other nodes. */ if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node)) register_page_bootmem_info_section(pfn); }}#endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ static void __meminit grow_zone_span(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn){ unsigned long old_zone_end_pfn; zone_span_writelock(zone); old_zone_end_pfn = zone_end_pfn(zone); if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn) zone->zone_start_pfn = start_pfn; zone->spanned_pages = max(old_zone_end_pfn, end_pfn) - zone->zone_start_pfn; zone_span_writeunlock(zone);} static void resize_zone(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn){ zone_span_writelock(zone); if (end_pfn - start_pfn) { zone->zone_start_pfn = start_pfn; zone->spanned_pages = end_pfn - start_pfn; } else { /* * make it consist as free_area_init_core(), * if spanned_pages = 0, then keep start_pfn = 0 */ zone->zone_start_pfn = 0; zone->spanned_pages = 0; } zone_span_writeunlock(zone);} static void fix_zone_id(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn){ enum zone_type zid = zone_idx(zone); int nid = zone->zone_pgdat->node_id; unsigned long pfn; for (pfn = start_pfn; pfn < end_pfn; pfn++) set_page_links(pfn_to_page(pfn), zid, nid, pfn);} /* Can fail with -ENOMEM from allocating a wait table with vmalloc() or * alloc_bootmem_node_nopanic()/memblock_virt_alloc_node_nopanic() */static int __ref ensure_zone_is_initialized(struct zone *zone, unsigned long start_pfn, unsigned long num_pages){ if (!zone_is_initialized(zone)) return init_currently_empty_zone(zone, start_pfn, num_pages); return 0;} static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2, unsigned long start_pfn, unsigned long end_pfn){ int ret; unsigned long flags; unsigned long z1_start_pfn; ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn); if (ret) return ret; pgdat_resize_lock(z1->zone_pgdat, &flags); /* can't move pfns which are higher than @z2 */ if (end_pfn > zone_end_pfn(z2)) goto out_fail; /* the move out part must be at the left most of @z2 */ if (start_pfn > z2->zone_start_pfn) goto out_fail; /* must included/overlap */ if (end_pfn <= z2->zone_start_pfn) goto out_fail; /* use start_pfn for z1's start_pfn if z1 is empty */ if (!zone_is_empty(z1)) z1_start_pfn = z1->zone_start_pfn; else z1_start_pfn = start_pfn; resize_zone(z1, z1_start_pfn, end_pfn); resize_zone(z2, end_pfn, zone_end_pfn(z2)); pgdat_resize_unlock(z1->zone_pgdat, &flags); fix_zone_id(z1, start_pfn, end_pfn); return 0;out_fail: pgdat_resize_unlock(z1->zone_pgdat, &flags); return -1;} static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2, unsigned long start_pfn, unsigned long end_pfn){ int ret; unsigned long flags; unsigned long z2_end_pfn; ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn); if (ret) return ret; pgdat_resize_lock(z1->zone_pgdat, &flags); /* can't move pfns which are lower than @z1 */ if (z1->zone_start_pfn > start_pfn) goto out_fail; /* the move out part mast at the right most of @z1 */ if (zone_end_pfn(z1) > end_pfn) goto out_fail; /* must included/overlap */ if (start_pfn >= zone_end_pfn(z1)) goto out_fail; /* use end_pfn for z2's end_pfn if z2 is empty */ if (!zone_is_empty(z2)) z2_end_pfn = zone_end_pfn(z2); else z2_end_pfn = end_pfn; resize_zone(z1, z1->zone_start_pfn, start_pfn); resize_zone(z2, start_pfn, z2_end_pfn); pgdat_resize_unlock(z1->zone_pgdat, &flags); fix_zone_id(z2, start_pfn, end_pfn); return 0;out_fail: pgdat_resize_unlock(z1->zone_pgdat, &flags); return -1;} static struct zone * __meminit move_pfn_range(int zone_shift, unsigned long start_pfn, unsigned long end_pfn){ struct zone *zone = page_zone(pfn_to_page(start_pfn)); int ret = 0; if (zone_shift < 0) ret = move_pfn_range_left(zone + zone_shift, zone, start_pfn, end_pfn); else if (zone_shift) ret = move_pfn_range_right(zone, zone + zone_shift, start_pfn, end_pfn); if (ret) return NULL; return zone + zone_shift;} static void __meminit grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn, unsigned long end_pfn){ unsigned long old_pgdat_end_pfn = pgdat_end_pfn(pgdat); if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn) pgdat->node_start_pfn = start_pfn; pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) - pgdat->node_start_pfn;} static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn){ struct pglist_data *pgdat = zone->zone_pgdat; int nr_pages = PAGES_PER_SECTION; int nid = pgdat->node_id; int zone_type; unsigned long flags, pfn; int ret; zone_type = zone - pgdat->node_zones; ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages); if (ret) return ret; pgdat_resize_lock(zone->zone_pgdat, &flags); grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages); grow_pgdat_span(zone->zone_pgdat, phys_start_pfn, phys_start_pfn + nr_pages); pgdat_resize_unlock(zone->zone_pgdat, &flags); memmap_init_zone(nr_pages, nid, zone_type, phys_start_pfn, MEMMAP_HOTPLUG); /* online_page_range is called later and expects pages reserved */ for (pfn = phys_start_pfn; pfn < phys_start_pfn + nr_pages; pfn++) {Lines 1–500 of 2199. Load more
/* * linux/mm/memory_hotplug.c * * Copyright (C) */ #include <linux/stddef.h> #include <linux/mm.h> #include <linux/sched/signal.h> #include <linux/swap.h> #include <linux/interrupt.h> #include <linux/pagemap.h> #include <linux/compiler.h> #include <linux/export.h> #include <linux/pagevec.h> #include <linux/writeback.h> #include <linux/slab.h> #include <linux/sysctl.h> #include <linux/cpu.h> #include <linux/memory.h> #include <linux/memremap.h> #include <linux/memory_hotplug.h> #include <linux/highmem.h> #include <linux/vmalloc.h> #include <linux/ioport.h> #include <linux/delay.h> #include <linux/migrate.h> #include <linux/page-isolation.h> #include <linux/pfn.h> #include <linux/suspend.h> #include <linux/mm_inline.h> #include <linux/firmware-map.h> #include <linux/stop_machine.h> #include <linux/hugetlb.h> #include <linux/memblock.h> #include <linux/bootmem.h> #include <linux/compaction.h> #include <asm/tlbflush.h> #include "internal.h" /* * online_page_callback contains pointer to current page onlining function. * Initially it is generic_online_page(). If it is required it could be * changed by calling set_online_page_callback() for callback registration * and restore_online_page_callback() for generic callback restore. */ static void generic_online_page(struct page *page); static online_page_callback_t online_page_callback = generic_online_page; static DEFINE_MUTEX(online_page_callback_lock); /* The same as the cpu_hotplug lock, but for memory hotplug. */ static struct { struct task_struct *active_writer; struct mutex lock; /* Synchronizes accesses to refcount, */ /* * Also blocks the new readers during * an ongoing mem hotplug operation. */ int refcount; #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif } mem_hotplug = { .active_writer = NULL, .lock = __MUTEX_INITIALIZER(mem_hotplug.lock), .refcount = 0, #ifdef CONFIG_DEBUG_LOCK_ALLOC .dep_map = {.name = "mem_hotplug.lock" }, #endif }; /* Lockdep annotations for get/put_online_mems() and mem_hotplug_begin/end() */ #define memhp_lock_acquire_read() lock_map_acquire_read(&mem_hotplug.dep_map) #define memhp_lock_acquire() lock_map_acquire(&mem_hotplug.dep_map) #define memhp_lock_release() lock_map_release(&mem_hotplug.dep_map) #ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE bool memhp_auto_online; #else bool memhp_auto_online = true; #endif EXPORT_SYMBOL_GPL(memhp_auto_online); static int __init setup_memhp_default_state(char *str) { if (!strcmp(str, "online")) memhp_auto_online = true; else if (!strcmp(str, "offline")) memhp_auto_online = false; return 1; } __setup("memhp_default_state=", setup_memhp_default_state); void get_online_mems(void) { might_sleep(); if (mem_hotplug.active_writer == current) return; memhp_lock_acquire_read(); mutex_lock(&mem_hotplug.lock); mem_hotplug.refcount++; mutex_unlock(&mem_hotplug.lock); } void put_online_mems(void) { if (mem_hotplug.active_writer == current) return; mutex_lock(&mem_hotplug.lock); if (WARN_ON(!mem_hotplug.refcount)) mem_hotplug.refcount++; /* try to fix things up */ if (!--mem_hotplug.refcount && unlikely(mem_hotplug.active_writer)) wake_up_process(mem_hotplug.active_writer); mutex_unlock(&mem_hotplug.lock); memhp_lock_release(); } /* Serializes write accesses to mem_hotplug.active_writer. */ static DEFINE_MUTEX(memory_add_remove_lock); void mem_hotplug_begin(void) { mutex_lock(&memory_add_remove_lock); mem_hotplug.active_writer = current; memhp_lock_acquire(); for (;;) { mutex_lock(&mem_hotplug.lock); if (likely(!mem_hotplug.refcount)) break; __set_current_state(TASK_UNINTERRUPTIBLE); mutex_unlock(&mem_hotplug.lock); schedule(); } } void mem_hotplug_done(void) { mem_hotplug.active_writer = NULL; mutex_unlock(&mem_hotplug.lock); memhp_lock_release(); mutex_unlock(&memory_add_remove_lock); } /* add this memory to iomem resource */ static struct resource *register_memory_resource(u64 start, u64 size) { struct resource *res; res = kzalloc(sizeof(struct resource), GFP_KERNEL); if (!res) return ERR_PTR(-ENOMEM); res->name = "System RAM"; res->start = start; res->end = start + size - 1; res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; if (request_resource(&iomem_resource, res) < 0) { pr_debug("System RAM resource %pR cannot be added\n", res); kfree(res); return ERR_PTR(-EEXIST); } return res; } static void release_memory_resource(struct resource *res) { if (!res) return; release_resource(res); kfree(res); return; } #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE void get_page_bootmem(unsigned long info, struct page *page, unsigned long type) { page->freelist = (void *)type; SetPagePrivate(page); set_page_private(page, info); page_ref_inc(page); } void put_page_bootmem(struct page *page) { unsigned long type; type = (unsigned long) page->freelist; BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); if (page_ref_dec_return(page) == 1) { page->freelist = NULL; ClearPagePrivate(page); set_page_private(page, 0); INIT_LIST_HEAD(&page->lru); free_reserved_page(page); } } #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE #ifndef CONFIG_SPARSEMEM_VMEMMAP static void register_page_bootmem_info_section(unsigned long start_pfn) { unsigned long *usemap, mapsize, section_nr, i; struct mem_section *ms; struct page *page, *memmap; section_nr = pfn_to_section_nr(start_pfn); ms = __nr_to_section(section_nr); /* Get section's memmap address */ memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); /* * Get page for the memmap's phys address * XXX: need more consideration for sparse_vmemmap... */ page = virt_to_page(memmap); mapsize = sizeof(struct page) * PAGES_PER_SECTION; mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; /* remember memmap's page */ for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, SECTION_INFO); usemap = __nr_to_section(section_nr)->pageblock_flags; page = virt_to_page(usemap); mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, MIX_SECTION_INFO); } #else /* CONFIG_SPARSEMEM_VMEMMAP */ static void register_page_bootmem_info_section(unsigned long start_pfn) { unsigned long *usemap, mapsize, section_nr, i; struct mem_section *ms; struct page *page, *memmap; if (!pfn_valid(start_pfn)) return; section_nr = pfn_to_section_nr(start_pfn); ms = __nr_to_section(section_nr); memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); usemap = __nr_to_section(section_nr)->pageblock_flags; page = virt_to_page(usemap); mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, MIX_SECTION_INFO); } #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ void __init register_page_bootmem_info_node(struct pglist_data *pgdat) { unsigned long i, pfn, end_pfn, nr_pages; int node = pgdat->node_id; struct page *page; nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; page = virt_to_page(pgdat); for (i = 0; i < nr_pages; i++, page++) get_page_bootmem(node, page, NODE_INFO); pfn = pgdat->node_start_pfn; end_pfn = pgdat_end_pfn(pgdat); /* register section info */ for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { /* * Some platforms can assign the same pfn to multiple nodes - on * node0 as well as nodeN. To avoid registering a pfn against * multiple nodes we check that this pfn does not already * reside in some other nodes. */ if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node)) register_page_bootmem_info_section(pfn); } } #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ static void __meminit grow_zone_span(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn) { unsigned long old_zone_end_pfn; zone_span_writelock(zone); old_zone_end_pfn = zone_end_pfn(zone); if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn) zone->zone_start_pfn = start_pfn; zone->spanned_pages = max(old_zone_end_pfn, end_pfn) - zone->zone_start_pfn; zone_span_writeunlock(zone); } static void resize_zone(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn) { zone_span_writelock(zone); if (end_pfn - start_pfn) { zone->zone_start_pfn = start_pfn; zone->spanned_pages = end_pfn - start_pfn; } else { /* * make it consist as free_area_init_core(), * if spanned_pages = 0, then keep start_pfn = 0 */ zone->zone_start_pfn = 0; zone->spanned_pages = 0; } zone_span_writeunlock(zone); } static void fix_zone_id(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn) { enum zone_type zid = zone_idx(zone); int nid = zone->zone_pgdat->node_id; unsigned long pfn; for (pfn = start_pfn; pfn < end_pfn; pfn++) set_page_links(pfn_to_page(pfn), zid, nid, pfn); } /* Can fail with -ENOMEM from allocating a wait table with vmalloc() or * alloc_bootmem_node_nopanic()/memblock_virt_alloc_node_nopanic() */ static int __ref ensure_zone_is_initialized(struct zone *zone, unsigned long start_pfn, unsigned long num_pages) { if (!zone_is_initialized(zone)) return init_currently_empty_zone(zone, start_pfn, num_pages); return 0; } static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2, unsigned long start_pfn, unsigned long end_pfn) { int ret; unsigned long flags; unsigned long z1_start_pfn; ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn); if (ret) return ret; pgdat_resize_lock(z1->zone_pgdat, &flags); /* can't move pfns which are higher than @z2 */ if (end_pfn > zone_end_pfn(z2)) goto out_fail; /* the move out part must be at the left most of @z2 */ if (start_pfn > z2->zone_start_pfn) goto out_fail; /* must included/overlap */ if (end_pfn <= z2->zone_start_pfn) goto out_fail; /* use start_pfn for z1's start_pfn if z1 is empty */ if (!zone_is_empty(z1)) z1_start_pfn = z1->zone_start_pfn; else z1_start_pfn = start_pfn; resize_zone(z1, z1_start_pfn, end_pfn); resize_zone(z2, end_pfn, zone_end_pfn(z2)); pgdat_resize_unlock(z1->zone_pgdat, &flags); fix_zone_id(z1, start_pfn, end_pfn); return 0; out_fail: pgdat_resize_unlock(z1->zone_pgdat, &flags); return -1; } static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2, unsigned long start_pfn, unsigned long end_pfn) { int ret; unsigned long flags; unsigned long z2_end_pfn; ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn); if (ret) return ret; pgdat_resize_lock(z1->zone_pgdat, &flags); /* can't move pfns which are lower than @z1 */ if (z1->zone_start_pfn > start_pfn) goto out_fail; /* the move out part mast at the right most of @z1 */ if (zone_end_pfn(z1) > end_pfn) goto out_fail; /* must included/overlap */ if (start_pfn >= zone_end_pfn(z1)) goto out_fail; /* use end_pfn for z2's end_pfn if z2 is empty */ if (!zone_is_empty(z2)) z2_end_pfn = zone_end_pfn(z2); else z2_end_pfn = end_pfn; resize_zone(z1, z1->zone_start_pfn, start_pfn); resize_zone(z2, start_pfn, z2_end_pfn); pgdat_resize_unlock(z1->zone_pgdat, &flags); fix_zone_id(z2, start_pfn, end_pfn); return 0; out_fail: pgdat_resize_unlock(z1->zone_pgdat, &flags); return -1; } static struct zone * __meminit move_pfn_range(int zone_shift, unsigned long start_pfn, unsigned long end_pfn) { struct zone *zone = page_zone(pfn_to_page(start_pfn)); int ret = 0; if (zone_shift < 0) ret = move_pfn_range_left(zone + zone_shift, zone, start_pfn, end_pfn); else if (zone_shift) ret = move_pfn_range_right(zone, zone + zone_shift, start_pfn, end_pfn); if (ret) return NULL; return zone + zone_shift; } static void __meminit grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn, unsigned long end_pfn) { unsigned long old_pgdat_end_pfn = pgdat_end_pfn(pgdat); if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn) pgdat->node_start_pfn = start_pfn; pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) - pgdat->node_start_pfn; } static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn) { struct pglist_data *pgdat = zone->zone_pgdat; int nr_pages = PAGES_PER_SECTION; int nid = pgdat->node_id; int zone_type; unsigned long flags, pfn; int ret; zone_type = zone - pgdat->node_zones; ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages); if (ret) return ret; pgdat_resize_lock(zone->zone_pgdat, &flags); grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages); grow_pgdat_span(zone->zone_pgdat, phys_start_pfn, phys_start_pfn + nr_pages); pgdat_resize_unlock(zone->zone_pgdat, &flags); memmap_init_zone(nr_pages, nid, zone_type, phys_start_pfn, MEMMAP_HOTPLUG); /* online_page_range is called later and expects pages reserved */ for (pfn = phys_start_pfn; pfn < phys_start_pfn + nr_pages; pfn++) {