/* * linux/mm/memory_hotplug.c * * Copyright (C) */ #include <linux/stddef.h>#include <linux/mm.h>#include <linux/swap.h>#include <linux/interrupt.h>#include <linux/pagemap.h>#include <linux/bootmem.h>#include <linux/compiler.h>#include <linux/export.h>#include <linux/pagevec.h>#include <linux/writeback.h>#include <linux/slab.h>#include <linux/sysctl.h>#include <linux/cpu.h>#include <linux/memory.h>#include <linux/memory_hotplug.h>#include <linux/highmem.h>#include <linux/vmalloc.h>#include <linux/ioport.h>#include <linux/delay.h>#include <linux/migrate.h>#include <linux/page-isolation.h>#include <linux/pfn.h>#include <linux/suspend.h>#include <linux/mm_inline.h>#include <linux/firmware-map.h>#include <linux/stop_machine.h> #include <asm/tlbflush.h> #include "internal.h" /* * online_page_callback contains pointer to current page onlining function. * Initially it is generic_online_page(). If it is required it could be * changed by calling set_online_page_callback() for callback registration * and restore_online_page_callback() for generic callback restore. */ static void generic_online_page(struct page *page); static online_page_callback_t online_page_callback = generic_online_page; DEFINE_MUTEX(mem_hotplug_mutex); void lock_memory_hotplug(void){ mutex_lock(&mem_hotplug_mutex); /* for exclusive hibernation if CONFIG_HIBERNATION=y */ lock_system_sleep();} void unlock_memory_hotplug(void){ unlock_system_sleep(); mutex_unlock(&mem_hotplug_mutex);} /* add this memory to iomem resource */static struct resource *register_memory_resource(u64 start, u64 size){ struct resource *res; res = kzalloc(sizeof(struct resource), GFP_KERNEL); BUG_ON(!res); res->name = "System RAM"; res->start = start; res->end = start + size - 1; res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; if (request_resource(&iomem_resource, res) < 0) { printk("System RAM resource %pR cannot be added\n", res); kfree(res); res = NULL; } return res;} static void release_memory_resource(struct resource *res){ if (!res) return; release_resource(res); kfree(res); return;} #ifdef CONFIG_MEMORY_HOTPLUG_SPARSEvoid get_page_bootmem(unsigned long info, struct page *page, unsigned long type){ page->lru.next = (struct list_head *) type; SetPagePrivate(page); set_page_private(page, info); atomic_inc(&page->_count);} /* reference to __meminit __free_pages_bootmem is valid * so use __ref to tell modpost not to generate a warning */void __ref put_page_bootmem(struct page *page){ unsigned long type; static DEFINE_MUTEX(ppb_lock); type = (unsigned long) page->lru.next; BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); if (atomic_dec_return(&page->_count) == 1) { ClearPagePrivate(page); set_page_private(page, 0); INIT_LIST_HEAD(&page->lru); /* * Please refer to comment for __free_pages_bootmem() * for why we serialize here. */ mutex_lock(&ppb_lock); __free_pages_bootmem(page, 0); mutex_unlock(&ppb_lock); totalram_pages++; } } #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE#ifndef CONFIG_SPARSEMEM_VMEMMAPstatic void register_page_bootmem_info_section(unsigned long start_pfn){ unsigned long *usemap, mapsize, section_nr, i; struct mem_section *ms; struct page *page, *memmap; section_nr = pfn_to_section_nr(start_pfn); ms = __nr_to_section(section_nr); /* Get section's memmap address */ memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); /* * Get page for the memmap's phys address * XXX: need more consideration for sparse_vmemmap... */ page = virt_to_page(memmap); mapsize = sizeof(struct page) * PAGES_PER_SECTION; mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; /* remember memmap's page */ for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, SECTION_INFO); usemap = __nr_to_section(section_nr)->pageblock_flags; page = virt_to_page(usemap); mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, MIX_SECTION_INFO); }#else /* CONFIG_SPARSEMEM_VMEMMAP */static void register_page_bootmem_info_section(unsigned long start_pfn){ unsigned long *usemap, mapsize, section_nr, i; struct mem_section *ms; struct page *page, *memmap; if (!pfn_valid(start_pfn)) return; section_nr = pfn_to_section_nr(start_pfn); ms = __nr_to_section(section_nr); memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); usemap = __nr_to_section(section_nr)->pageblock_flags; page = virt_to_page(usemap); mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, MIX_SECTION_INFO);}#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ void register_page_bootmem_info_node(struct pglist_data *pgdat){ unsigned long i, pfn, end_pfn, nr_pages; int node = pgdat->node_id; struct page *page; struct zone *zone; nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; page = virt_to_page(pgdat); for (i = 0; i < nr_pages; i++, page++) get_page_bootmem(node, page, NODE_INFO); zone = &pgdat->node_zones[0]; for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) { if (zone->wait_table) { nr_pages = zone->wait_table_hash_nr_entries * sizeof(wait_queue_head_t); nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT; page = virt_to_page(zone->wait_table); for (i = 0; i < nr_pages; i++, page++) get_page_bootmem(node, page, NODE_INFO); } } pfn = pgdat->node_start_pfn; end_pfn = pgdat_end_pfn(pgdat); /* register_section info */ for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { /* * Some platforms can assign the same pfn to multiple nodes - on * node0 as well as nodeN. To avoid registering a pfn against * multiple nodes we check that this pfn does not already * reside in some other node. */ if (pfn_valid(pfn) && (pfn_to_nid(pfn) == node)) register_page_bootmem_info_section(pfn); }}#endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ static void grow_zone_span(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn){ unsigned long old_zone_end_pfn; zone_span_writelock(zone); old_zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages; if (!zone->spanned_pages || start_pfn < zone->zone_start_pfn) zone->zone_start_pfn = start_pfn; zone->spanned_pages = max(old_zone_end_pfn, end_pfn) - zone->zone_start_pfn; zone_span_writeunlock(zone);} static void resize_zone(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn){ zone_span_writelock(zone); if (end_pfn - start_pfn) { zone->zone_start_pfn = start_pfn; zone->spanned_pages = end_pfn - start_pfn; } else { /* * make it consist as free_area_init_core(), * if spanned_pages = 0, then keep start_pfn = 0 */ zone->zone_start_pfn = 0; zone->spanned_pages = 0; } zone_span_writeunlock(zone);} static void fix_zone_id(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn){ enum zone_type zid = zone_idx(zone); int nid = zone->zone_pgdat->node_id; unsigned long pfn; for (pfn = start_pfn; pfn < end_pfn; pfn++) set_page_links(pfn_to_page(pfn), zid, nid, pfn);} /* Can fail with -ENOMEM from allocating a wait table with vmalloc() or * alloc_bootmem_node_nopanic() */static int __ref ensure_zone_is_initialized(struct zone *zone, unsigned long start_pfn, unsigned long num_pages){ if (!zone_is_initialized(zone)) return init_currently_empty_zone(zone, start_pfn, num_pages, MEMMAP_HOTPLUG); return 0;} static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2, unsigned long start_pfn, unsigned long end_pfn){ int ret; unsigned long flags; unsigned long z1_start_pfn; ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn); if (ret) return ret; pgdat_resize_lock(z1->zone_pgdat, &flags); /* can't move pfns which are higher than @z2 */ if (end_pfn > zone_end_pfn(z2)) goto out_fail; /* the move out part mast at the left most of @z2 */ if (start_pfn > z2->zone_start_pfn) goto out_fail; /* must included/overlap */ if (end_pfn <= z2->zone_start_pfn) goto out_fail; /* use start_pfn for z1's start_pfn if z1 is empty */ if (z1->spanned_pages) z1_start_pfn = z1->zone_start_pfn; else z1_start_pfn = start_pfn; resize_zone(z1, z1_start_pfn, end_pfn); resize_zone(z2, end_pfn, zone_end_pfn(z2)); pgdat_resize_unlock(z1->zone_pgdat, &flags); fix_zone_id(z1, start_pfn, end_pfn); return 0;out_fail: pgdat_resize_unlock(z1->zone_pgdat, &flags); return -1;} static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2, unsigned long start_pfn, unsigned long end_pfn){ int ret; unsigned long flags; unsigned long z2_end_pfn; ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn); if (ret) return ret; pgdat_resize_lock(z1->zone_pgdat, &flags); /* can't move pfns which are lower than @z1 */ if (z1->zone_start_pfn > start_pfn) goto out_fail; /* the move out part mast at the right most of @z1 */ if (zone_end_pfn(z1) > end_pfn) goto out_fail; /* must included/overlap */ if (start_pfn >= zone_end_pfn(z1)) goto out_fail; /* use end_pfn for z2's end_pfn if z2 is empty */ if (z2->spanned_pages) z2_end_pfn = zone_end_pfn(z2); else z2_end_pfn = end_pfn; resize_zone(z1, z1->zone_start_pfn, start_pfn); resize_zone(z2, start_pfn, z2_end_pfn); pgdat_resize_unlock(z1->zone_pgdat, &flags); fix_zone_id(z2, start_pfn, end_pfn); return 0;out_fail: pgdat_resize_unlock(z1->zone_pgdat, &flags); return -1;} static void grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn, unsigned long end_pfn){ unsigned long old_pgdat_end_pfn = pgdat->node_start_pfn + pgdat->node_spanned_pages; if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn) pgdat->node_start_pfn = start_pfn; pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) - pgdat->node_start_pfn;} static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn){ struct pglist_data *pgdat = zone->zone_pgdat; int nr_pages = PAGES_PER_SECTION; int nid = pgdat->node_id; int zone_type; unsigned long flags; int ret; zone_type = zone - pgdat->node_zones; ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages); if (ret) return ret; pgdat_resize_lock(zone->zone_pgdat, &flags); grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages); grow_pgdat_span(zone->zone_pgdat, phys_start_pfn, phys_start_pfn + nr_pages); pgdat_resize_unlock(zone->zone_pgdat, &flags); memmap_init_zone(nr_pages, nid, zone_type, phys_start_pfn, MEMMAP_HOTPLUG); return 0;} static int __meminit __add_section(int nid, struct zone *zone, unsigned long phys_start_pfn){ int nr_pages = PAGES_PER_SECTION; int ret; if (pfn_valid(phys_start_pfn)) return -EEXIST; ret = sparse_add_one_section(zone, phys_start_pfn, nr_pages); if (ret < 0) return ret; ret = __add_zone(zone, phys_start_pfn); if (ret < 0) return ret; return register_new_memory(nid, __pfn_to_section(phys_start_pfn));} /* find the smallest valid pfn in the range [start_pfn, end_pfn) */static int find_smallest_section_pfn(int nid, struct zone *zone, unsigned long start_pfn, unsigned long end_pfn){ struct mem_section *ms; for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) { ms = __pfn_to_section(start_pfn); if (unlikely(!valid_section(ms))) continue; if (unlikely(pfn_to_nid(start_pfn) != nid)) continue; if (zone && zone != page_zone(pfn_to_page(start_pfn))) continue; return start_pfn; } return 0;} /* find the biggest valid pfn in the range [start_pfn, end_pfn). */static int find_biggest_section_pfn(int nid, struct zone *zone, unsigned long start_pfn, unsigned long end_pfn){ struct mem_section *ms; unsigned long pfn; /* pfn is the end pfn of a memory section. */ pfn = end_pfn - 1; for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) { ms = __pfn_to_section(pfn); if (unlikely(!valid_section(ms))) continue; if (unlikely(pfn_to_nid(pfn) != nid)) continue; if (zone && zone != page_zone(pfn_to_page(pfn))) continue; return pfn; } return 0;} static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn){ unsigned long zone_start_pfn = zone->zone_start_pfn; unsigned long zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages; unsigned long pfn; struct mem_section *ms; int nid = zone_to_nid(zone); Lines 1–500 of 1872. Load more
/* * linux/mm/memory_hotplug.c * * Copyright (C) */ #include <linux/stddef.h> #include <linux/mm.h> #include <linux/swap.h> #include <linux/interrupt.h> #include <linux/pagemap.h> #include <linux/bootmem.h> #include <linux/compiler.h> #include <linux/export.h> #include <linux/pagevec.h> #include <linux/writeback.h> #include <linux/slab.h> #include <linux/sysctl.h> #include <linux/cpu.h> #include <linux/memory.h> #include <linux/memory_hotplug.h> #include <linux/highmem.h> #include <linux/vmalloc.h> #include <linux/ioport.h> #include <linux/delay.h> #include <linux/migrate.h> #include <linux/page-isolation.h> #include <linux/pfn.h> #include <linux/suspend.h> #include <linux/mm_inline.h> #include <linux/firmware-map.h> #include <linux/stop_machine.h> #include <asm/tlbflush.h> #include "internal.h" /* * online_page_callback contains pointer to current page onlining function. * Initially it is generic_online_page(). If it is required it could be * changed by calling set_online_page_callback() for callback registration * and restore_online_page_callback() for generic callback restore. */ static void generic_online_page(struct page *page); static online_page_callback_t online_page_callback = generic_online_page; DEFINE_MUTEX(mem_hotplug_mutex); void lock_memory_hotplug(void) { mutex_lock(&mem_hotplug_mutex); /* for exclusive hibernation if CONFIG_HIBERNATION=y */ lock_system_sleep(); } void unlock_memory_hotplug(void) { unlock_system_sleep(); mutex_unlock(&mem_hotplug_mutex); } /* add this memory to iomem resource */ static struct resource *register_memory_resource(u64 start, u64 size) { struct resource *res; res = kzalloc(sizeof(struct resource), GFP_KERNEL); BUG_ON(!res); res->name = "System RAM"; res->start = start; res->end = start + size - 1; res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; if (request_resource(&iomem_resource, res) < 0) { printk("System RAM resource %pR cannot be added\n", res); kfree(res); res = NULL; } return res; } static void release_memory_resource(struct resource *res) { if (!res) return; release_resource(res); kfree(res); return; } #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE void get_page_bootmem(unsigned long info, struct page *page, unsigned long type) { page->lru.next = (struct list_head *) type; SetPagePrivate(page); set_page_private(page, info); atomic_inc(&page->_count); } /* reference to __meminit __free_pages_bootmem is valid * so use __ref to tell modpost not to generate a warning */ void __ref put_page_bootmem(struct page *page) { unsigned long type; static DEFINE_MUTEX(ppb_lock); type = (unsigned long) page->lru.next; BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); if (atomic_dec_return(&page->_count) == 1) { ClearPagePrivate(page); set_page_private(page, 0); INIT_LIST_HEAD(&page->lru); /* * Please refer to comment for __free_pages_bootmem() * for why we serialize here. */ mutex_lock(&ppb_lock); __free_pages_bootmem(page, 0); mutex_unlock(&ppb_lock); totalram_pages++; } } #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE #ifndef CONFIG_SPARSEMEM_VMEMMAP static void register_page_bootmem_info_section(unsigned long start_pfn) { unsigned long *usemap, mapsize, section_nr, i; struct mem_section *ms; struct page *page, *memmap; section_nr = pfn_to_section_nr(start_pfn); ms = __nr_to_section(section_nr); /* Get section's memmap address */ memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); /* * Get page for the memmap's phys address * XXX: need more consideration for sparse_vmemmap... */ page = virt_to_page(memmap); mapsize = sizeof(struct page) * PAGES_PER_SECTION; mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; /* remember memmap's page */ for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, SECTION_INFO); usemap = __nr_to_section(section_nr)->pageblock_flags; page = virt_to_page(usemap); mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, MIX_SECTION_INFO); } #else /* CONFIG_SPARSEMEM_VMEMMAP */ static void register_page_bootmem_info_section(unsigned long start_pfn) { unsigned long *usemap, mapsize, section_nr, i; struct mem_section *ms; struct page *page, *memmap; if (!pfn_valid(start_pfn)) return; section_nr = pfn_to_section_nr(start_pfn); ms = __nr_to_section(section_nr); memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); usemap = __nr_to_section(section_nr)->pageblock_flags; page = virt_to_page(usemap); mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, MIX_SECTION_INFO); } #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ void register_page_bootmem_info_node(struct pglist_data *pgdat) { unsigned long i, pfn, end_pfn, nr_pages; int node = pgdat->node_id; struct page *page; struct zone *zone; nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; page = virt_to_page(pgdat); for (i = 0; i < nr_pages; i++, page++) get_page_bootmem(node, page, NODE_INFO); zone = &pgdat->node_zones[0]; for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) { if (zone->wait_table) { nr_pages = zone->wait_table_hash_nr_entries * sizeof(wait_queue_head_t); nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT; page = virt_to_page(zone->wait_table); for (i = 0; i < nr_pages; i++, page++) get_page_bootmem(node, page, NODE_INFO); } } pfn = pgdat->node_start_pfn; end_pfn = pgdat_end_pfn(pgdat); /* register_section info */ for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { /* * Some platforms can assign the same pfn to multiple nodes - on * node0 as well as nodeN. To avoid registering a pfn against * multiple nodes we check that this pfn does not already * reside in some other node. */ if (pfn_valid(pfn) && (pfn_to_nid(pfn) == node)) register_page_bootmem_info_section(pfn); } } #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ static void grow_zone_span(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn) { unsigned long old_zone_end_pfn; zone_span_writelock(zone); old_zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages; if (!zone->spanned_pages || start_pfn < zone->zone_start_pfn) zone->zone_start_pfn = start_pfn; zone->spanned_pages = max(old_zone_end_pfn, end_pfn) - zone->zone_start_pfn; zone_span_writeunlock(zone); } static void resize_zone(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn) { zone_span_writelock(zone); if (end_pfn - start_pfn) { zone->zone_start_pfn = start_pfn; zone->spanned_pages = end_pfn - start_pfn; } else { /* * make it consist as free_area_init_core(), * if spanned_pages = 0, then keep start_pfn = 0 */ zone->zone_start_pfn = 0; zone->spanned_pages = 0; } zone_span_writeunlock(zone); } static void fix_zone_id(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn) { enum zone_type zid = zone_idx(zone); int nid = zone->zone_pgdat->node_id; unsigned long pfn; for (pfn = start_pfn; pfn < end_pfn; pfn++) set_page_links(pfn_to_page(pfn), zid, nid, pfn); } /* Can fail with -ENOMEM from allocating a wait table with vmalloc() or * alloc_bootmem_node_nopanic() */ static int __ref ensure_zone_is_initialized(struct zone *zone, unsigned long start_pfn, unsigned long num_pages) { if (!zone_is_initialized(zone)) return init_currently_empty_zone(zone, start_pfn, num_pages, MEMMAP_HOTPLUG); return 0; } static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2, unsigned long start_pfn, unsigned long end_pfn) { int ret; unsigned long flags; unsigned long z1_start_pfn; ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn); if (ret) return ret; pgdat_resize_lock(z1->zone_pgdat, &flags); /* can't move pfns which are higher than @z2 */ if (end_pfn > zone_end_pfn(z2)) goto out_fail; /* the move out part mast at the left most of @z2 */ if (start_pfn > z2->zone_start_pfn) goto out_fail; /* must included/overlap */ if (end_pfn <= z2->zone_start_pfn) goto out_fail; /* use start_pfn for z1's start_pfn if z1 is empty */ if (z1->spanned_pages) z1_start_pfn = z1->zone_start_pfn; else z1_start_pfn = start_pfn; resize_zone(z1, z1_start_pfn, end_pfn); resize_zone(z2, end_pfn, zone_end_pfn(z2)); pgdat_resize_unlock(z1->zone_pgdat, &flags); fix_zone_id(z1, start_pfn, end_pfn); return 0; out_fail: pgdat_resize_unlock(z1->zone_pgdat, &flags); return -1; } static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2, unsigned long start_pfn, unsigned long end_pfn) { int ret; unsigned long flags; unsigned long z2_end_pfn; ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn); if (ret) return ret; pgdat_resize_lock(z1->zone_pgdat, &flags); /* can't move pfns which are lower than @z1 */ if (z1->zone_start_pfn > start_pfn) goto out_fail; /* the move out part mast at the right most of @z1 */ if (zone_end_pfn(z1) > end_pfn) goto out_fail; /* must included/overlap */ if (start_pfn >= zone_end_pfn(z1)) goto out_fail; /* use end_pfn for z2's end_pfn if z2 is empty */ if (z2->spanned_pages) z2_end_pfn = zone_end_pfn(z2); else z2_end_pfn = end_pfn; resize_zone(z1, z1->zone_start_pfn, start_pfn); resize_zone(z2, start_pfn, z2_end_pfn); pgdat_resize_unlock(z1->zone_pgdat, &flags); fix_zone_id(z2, start_pfn, end_pfn); return 0; out_fail: pgdat_resize_unlock(z1->zone_pgdat, &flags); return -1; } static void grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn, unsigned long end_pfn) { unsigned long old_pgdat_end_pfn = pgdat->node_start_pfn + pgdat->node_spanned_pages; if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn) pgdat->node_start_pfn = start_pfn; pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) - pgdat->node_start_pfn; } static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn) { struct pglist_data *pgdat = zone->zone_pgdat; int nr_pages = PAGES_PER_SECTION; int nid = pgdat->node_id; int zone_type; unsigned long flags; int ret; zone_type = zone - pgdat->node_zones; ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages); if (ret) return ret; pgdat_resize_lock(zone->zone_pgdat, &flags); grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages); grow_pgdat_span(zone->zone_pgdat, phys_start_pfn, phys_start_pfn + nr_pages); pgdat_resize_unlock(zone->zone_pgdat, &flags); memmap_init_zone(nr_pages, nid, zone_type, phys_start_pfn, MEMMAP_HOTPLUG); return 0; } static int __meminit __add_section(int nid, struct zone *zone, unsigned long phys_start_pfn) { int nr_pages = PAGES_PER_SECTION; int ret; if (pfn_valid(phys_start_pfn)) return -EEXIST; ret = sparse_add_one_section(zone, phys_start_pfn, nr_pages); if (ret < 0) return ret; ret = __add_zone(zone, phys_start_pfn); if (ret < 0) return ret; return register_new_memory(nid, __pfn_to_section(phys_start_pfn)); } /* find the smallest valid pfn in the range [start_pfn, end_pfn) */ static int find_smallest_section_pfn(int nid, struct zone *zone, unsigned long start_pfn, unsigned long end_pfn) { struct mem_section *ms; for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) { ms = __pfn_to_section(start_pfn); if (unlikely(!valid_section(ms))) continue; if (unlikely(pfn_to_nid(start_pfn) != nid)) continue; if (zone && zone != page_zone(pfn_to_page(start_pfn))) continue; return start_pfn; } return 0; } /* find the biggest valid pfn in the range [start_pfn, end_pfn). */ static int find_biggest_section_pfn(int nid, struct zone *zone, unsigned long start_pfn, unsigned long end_pfn) { struct mem_section *ms; unsigned long pfn; /* pfn is the end pfn of a memory section. */ pfn = end_pfn - 1; for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) { ms = __pfn_to_section(pfn); if (unlikely(!valid_section(ms))) continue; if (unlikely(pfn_to_nid(pfn) != nid)) continue; if (zone && zone != page_zone(pfn_to_page(pfn))) continue; return pfn; } return 0; } static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn) { unsigned long zone_start_pfn = zone->zone_start_pfn; unsigned long zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages; unsigned long pfn; struct mem_section *ms; int nid = zone_to_nid(zone);