| @@ -83,8 +83,15 @@ static int __meminit sparse_index_init(unsigned long section_nr, int nid) · 82 unmodified lines |
| 83 | unsigned long root = SECTION_NR_TO_ROOT(section_nr); | 83 | unsigned long root = SECTION_NR_TO_ROOT(section_nr); |
| 84 | struct mem_section *section; | 84 | struct mem_section *section; |
| 85 | | 85 | |
| | 86 | /* |
| | 87 | * An existing section is possible in the sub-section hotplug |
| | 88 | * case. First hot-add instantiates, follow-on hot-add reuses |
| | 89 | * the existing section. |
| | 90 | * |
| | 91 | * The mem_hotplug_lock resolves the apparent race below. |
| | 92 | */ |
| 86 | if (mem_section[root]) | 93 | if (mem_section[root]) |
| 87 | return -EEXIST; | 94 | return 0; |
| 88 | | 95 | |
| 89 | section = sparse_index_alloc(nid); | 96 | section = sparse_index_alloc(nid); |
| 90 | if (!section) | 97 | if (!section) |
| @@ -715,10 +722,120 @@ static void free_map_bootmem(struct page *memmap) · 624 unmodified lines |
| 715 | } | 722 | } |
| 716 | #endif /* CONFIG_SPARSEMEM_VMEMMAP */ | 723 | #endif /* CONFIG_SPARSEMEM_VMEMMAP */ |
| 717 | | 724 | |
| | 725 | static void section_deactivate(unsigned long pfn, unsigned long nr_pages, |
| | 726 | struct vmem_altmap *altmap) |
| | 727 | { |
| | 728 | DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; |
| | 729 | DECLARE_BITMAP(tmp, SUBSECTIONS_PER_SECTION) = { 0 }; |
| | 730 | struct mem_section *ms = __pfn_to_section(pfn); |
| | 731 | bool section_is_early = early_section(ms); |
| | 732 | struct page *memmap = NULL; |
| | 733 | unsigned long *subsection_map = ms->usage |
| | 734 | ? &ms->usage->subsection_map[0] : NULL; |
| | 735 | |
| | 736 | subsection_mask_set(map, pfn, nr_pages); |
| | 737 | if (subsection_map) |
| | 738 | bitmap_and(tmp, map, subsection_map, SUBSECTIONS_PER_SECTION); |
| | 739 | |
| | 740 | if (WARN(!subsection_map || !bitmap_equal(tmp, map, SUBSECTIONS_PER_SECTION), |
| | 741 | "section already deactivated (%#lx + %ld)\n", |
| | 742 | pfn, nr_pages)) |
| | 743 | return; |
| | 744 | |
| | 745 | /* |
| | 746 | * There are 3 cases to handle across two configurations |
| | 747 | * (SPARSEMEM_VMEMMAP={y,n}): |
| | 748 | * |
| | 749 | * 1/ deactivation of a partial hot-added section (only possible |
| | 750 | * in the SPARSEMEM_VMEMMAP=y case). |
| | 751 | * a/ section was present at memory init |
| | 752 | * b/ section was hot-added post memory init |
| | 753 | * 2/ deactivation of a complete hot-added section |
| | 754 | * 3/ deactivation of a complete section from memory init |
| | 755 | * |
| | 756 | * For 1/, when subsection_map does not empty we will not be |
| | 757 | * freeing the usage map, but still need to free the vmemmap |
| | 758 | * range. |
| | 759 | * |
| | 760 | * For 2/ and 3/ the SPARSEMEM_VMEMMAP={y,n} cases are unified |
| | 761 | */ |
| | 762 | bitmap_xor(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION); |
| | 763 | if (bitmap_empty(subsection_map, SUBSECTIONS_PER_SECTION)) { |
| | 764 | unsigned long section_nr = pfn_to_section_nr(pfn); |
| | 765 | |
| | 766 | if (!section_is_early) { |
| | 767 | kfree(ms->usage); |
| | 768 | ms->usage = NULL; |
| | 769 | } |
| | 770 | memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); |
| | 771 | ms->section_mem_map = sparse_encode_mem_map(NULL, section_nr); |
| | 772 | } |
| | 773 | |
| | 774 | if (section_is_early && memmap) |
| | 775 | free_map_bootmem(memmap); |
| | 776 | else |
| | 777 | depopulate_section_memmap(pfn, nr_pages, altmap); |
| | 778 | } |
| | 779 | |
| | 780 | static struct page * __meminit section_activate(int nid, unsigned long pfn, |
| | 781 | unsigned long nr_pages, struct vmem_altmap *altmap) |
| | 782 | { |
| | 783 | DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; |
| | 784 | struct mem_section *ms = __pfn_to_section(pfn); |
| | 785 | struct mem_section_usage *usage = NULL; |
| | 786 | unsigned long *subsection_map; |
| | 787 | struct page *memmap; |
| | 788 | int rc = 0; |
| | 789 | |
| | 790 | subsection_mask_set(map, pfn, nr_pages); |
| | 791 | |
| | 792 | if (!ms->usage) { |
| | 793 | usage = kzalloc(mem_section_usage_size(), GFP_KERNEL); |
| | 794 | if (!usage) |
| | 795 | return ERR_PTR(-ENOMEM); |
| | 796 | ms->usage = usage; |
| | 797 | } |
| | 798 | subsection_map = &ms->usage->subsection_map[0]; |
| | 799 | |
| | 800 | if (bitmap_empty(map, SUBSECTIONS_PER_SECTION)) |
| | 801 | rc = -EINVAL; |
| | 802 | else if (bitmap_intersects(map, subsection_map, SUBSECTIONS_PER_SECTION)) |
| | 803 | rc = -EEXIST; |
| | 804 | else |
| | 805 | bitmap_or(subsection_map, map, subsection_map, |
| | 806 | SUBSECTIONS_PER_SECTION); |
| | 807 | |
| | 808 | if (rc) { |
| | 809 | if (usage) |
| | 810 | ms->usage = NULL; |
| | 811 | kfree(usage); |
| | 812 | return ERR_PTR(rc); |
| | 813 | } |
| | 814 | |
| | 815 | /* |
| | 816 | * The early init code does not consider partially populated |
| | 817 | * initial sections, it simply assumes that memory will never be |
| | 818 | * referenced. If we hot-add memory into such a section then we |
| | 819 | * do not need to populate the memmap and can simply reuse what |
| | 820 | * is already there. |
| | 821 | */ |
| | 822 | if (nr_pages < PAGES_PER_SECTION && early_section(ms)) |
| | 823 | return pfn_to_page(pfn); |
| | 824 | |
| | 825 | memmap = populate_section_memmap(pfn, nr_pages, nid, altmap); |
| | 826 | if (!memmap) { |
| | 827 | section_deactivate(pfn, nr_pages, altmap); |
| | 828 | return ERR_PTR(-ENOMEM); |
| | 829 | } |
| | 830 | |
| | 831 | return memmap; |
| | 832 | } |
| | 833 | |
| 718 | /** | 834 | /** |
| 719 | * sparse_add_one_section - add a memory section | 835 | * sparse_add_section - add a memory section, or populate an existing one |
| 720 | * @nid: The node to add section on | 836 | * @nid: The node to add section on |
| 721 | * @start_pfn: start pfn of the memory range | 837 | * @start_pfn: start pfn of the memory range |
| | 838 | * @nr_pages: number of pfns to add in the section |
| 722 | * @altmap: device page map | 839 | * @altmap: device page map |
| 723 | * | 840 | * |
| 724 | * This is only intended for hotplug. | 841 | * This is only intended for hotplug. |
| @@ -732,51 +849,34 @@ int __meminit sparse_add_section(int nid, unsigned long start_pfn, · 7 unmodified lines |
| 732 | unsigned long nr_pages, struct vmem_altmap *altmap) | 849 | unsigned long nr_pages, struct vmem_altmap *altmap) |
| 733 | { | 850 | { |
| 734 | unsigned long section_nr = pfn_to_section_nr(start_pfn); | 851 | unsigned long section_nr = pfn_to_section_nr(start_pfn); |
| 735 | struct mem_section_usage *usage; | | |
| 736 | struct mem_section *ms; | 852 | struct mem_section *ms; |
| 737 | struct page *memmap; | 853 | struct page *memmap; |
| 738 | int ret; | 854 | int ret; |
| 739 | | 855 | |
| 740 | /* | | |
| 741 | * no locking for this, because it does its own | | |
| 742 | * plus, it does a kmalloc | | |
| 743 | */ | | |
| 744 | ret = sparse_index_init(section_nr, nid); | 856 | ret = sparse_index_init(section_nr, nid); |
| 745 | if (ret < 0 && ret != -EEXIST) | 857 | if (ret < 0) |
| 746 | return ret; | 858 | return ret; |
| 747 | ret = 0; | | |
| 748 | memmap = populate_section_memmap(start_pfn, PAGES_PER_SECTION, nid, | | |
| 749 | altmap); | | |
| 750 | if (!memmap) | | |
| 751 | return -ENOMEM; | | |
| 752 | usage = kzalloc(mem_section_usage_size(), GFP_KERNEL); | | |
| 753 | if (!usage) { | | |
| 754 | depopulate_section_memmap(start_pfn, PAGES_PER_SECTION, altmap); | | |
| 755 | return -ENOMEM; | | |
| 756 | } | | |
| 757 | | 859 | |
| 758 | ms = __pfn_to_section(start_pfn); | 860 | memmap = section_activate(nid, start_pfn, nr_pages, altmap); |
| 759 | if (ms->section_mem_map & SECTION_MARKED_PRESENT) { | 861 | if (IS_ERR(memmap)) |
| 760 | ret = -EEXIST; | 862 | return PTR_ERR(memmap); |
| 761 | goto out; | | |
| 762 | } | | |
| 763 | | 863 | |
| 764 | /* | 864 | /* |
| 765 | * Poison uninitialized struct pages in order to catch invalid flags | 865 | * Poison uninitialized struct pages in order to catch invalid flags |
| 766 | * combinations. | 866 | * combinations. |
| 767 | */ | 867 | */ |
| 768 | page_init_poison(memmap, sizeof(struct page) * PAGES_PER_SECTION); | 868 | page_init_poison(pfn_to_page(start_pfn), sizeof(struct page) * nr_pages); |
| 769 | | 869 | |
| | 870 | ms = __pfn_to_section(start_pfn); |
| 770 | set_section_nid(section_nr, nid); | 871 | set_section_nid(section_nr, nid); |
| 771 | section_mark_present(ms); | 872 | section_mark_present(ms); |
| 772 | sparse_init_one_section(ms, section_nr, memmap, usage, 0); | | |
| 773 | | 873 | |
| 774 | out: | 874 | /* Align memmap to section boundary in the subsection case */ |
| 775 | if (ret < 0) { | 875 | if (section_nr_to_pfn(section_nr) != start_pfn) |
| 776 | kfree(usage); | 876 | memmap = pfn_to_kaddr(section_nr_to_pfn(section_nr)); |
| 777 | depopulate_section_memmap(start_pfn, PAGES_PER_SECTION, altmap); | 877 | sparse_init_one_section(ms, section_nr, memmap, ms->usage, 0); |
| 778 | } | 878 | |
| 779 | return ret; | 879 | return 0; |
| 780 | } | 880 | } |
| 781 | | 881 | |
| 782 | #ifdef CONFIG_MEMORY_FAILURE | 882 | #ifdef CONFIG_MEMORY_FAILURE |
| @@ -809,48 +909,12 @@ static inline void clear_hwpoisoned_pages(struct page *memmap, int nr_pages) · 26 unmodified lines |
| 809 | } | 909 | } |
| 810 | #endif | 910 | #endif |
| 811 | | 911 | |
| 812 | static void free_section_usage(struct mem_section *ms, struct page *memmap, | 912 | void sparse_remove_section(struct mem_section *ms, unsigned long pfn, |
| 813 | struct mem_section_usage *usage, unsigned long pfn, | | |
| 814 | unsigned long nr_pages, struct vmem_altmap *altmap) | | |
| 815 | { | | |
| 816 | if (!usage) | | |
| 817 | return; | | |
| 818 | | | |
| 819 | /* | | |
| 820 | * Check to see if allocation came from hot-plug-add | | |
| 821 | */ | | |
| 822 | if (!early_section(ms)) { | | |
| 823 | kfree(usage); | | |
| 824 | if (memmap) | | |
| 825 | depopulate_section_memmap(pfn, nr_pages, altmap); | | |
| 826 | return; | | |
| 827 | } | | |
| 828 | | | |
| 829 | /* | | |
| 830 | * The usemap came from bootmem. This is packed with other usemaps | | |
| 831 | * on the section which has pgdat at boot time. Just keep it as is now. | | |
| 832 | */ | | |
| 833 | | | |
| 834 | if (memmap) | | |
| 835 | free_map_bootmem(memmap); | | |
| 836 | } | | |
| 837 | | | |
| 838 | void sparse_remove_one_section(struct mem_section *ms, unsigned long pfn, | | |
| 839 | unsigned long nr_pages, unsigned long map_offset, | 913 | unsigned long nr_pages, unsigned long map_offset, |
| 840 | struct vmem_altmap *altmap) | 914 | struct vmem_altmap *altmap) |
| 841 | { | 915 | { |
| 842 | struct page *memmap = NULL; | 916 | clear_hwpoisoned_pages(pfn_to_page(pfn) + map_offset, |
| 843 | struct mem_section_usage *usage = NULL; | 917 | nr_pages - map_offset); |
| 844 | | 918 | section_deactivate(pfn, nr_pages, altmap); |
| 845 | if (ms->section_mem_map) { | | |
| 846 | usage = ms->usage; | | |
| 847 | memmap = sparse_decode_mem_map(ms->section_mem_map, | | |
| 848 | __section_nr(ms)); | | |
| 849 | ms->section_mem_map = 0; | | |
| 850 | ms->usage = NULL; | | |
| 851 | } | | |
| 852 | | | |
| 853 | clear_hwpoisoned_pages(memmap + map_offset, nr_pages - map_offset); | | |
| 854 | free_section_usage(ms, memmap, usage, pfn, nr_pages, altmap); | | |
| 855 | } | 919 | } |
| 856 | #endif /* CONFIG_MEMORY_HOTPLUG */ | 920 | #endif /* CONFIG_MEMORY_HOTPLUG */ |