| @@ -83,8 +83,15 @@ static int __meminit sparse_index_init(unsigned long section_nr, int nid) · 82 unmodified lines |
| 83 | 83 | unsigned long root = SECTION_NR_TO_ROOT(section_nr); |
| 84 | 84 | struct mem_section *section; |
| 85 | 85 | |
| 86 | + /* |
| 87 | + * An existing section is possible in the sub-section hotplug |
| 88 | + * case. First hot-add instantiates, follow-on hot-add reuses |
| 89 | + * the existing section. |
| 90 | + * |
| 91 | + * The mem_hotplug_lock resolves the apparent race below. |
| 92 | + */ |
| 86 | 93 | if (mem_section[root]) |
| 87 | | − return -EEXIST; |
| 94 | + return 0; |
| 88 | 95 | |
| 89 | 96 | section = sparse_index_alloc(nid); |
| 90 | 97 | if (!section) |
| @@ -715,10 +722,120 @@ static void free_map_bootmem(struct page *memmap) · 624 unmodified lines |
| 715 | 722 | } |
| 716 | 723 | #endif /* CONFIG_SPARSEMEM_VMEMMAP */ |
| 717 | 724 | |
| 725 | +static void section_deactivate(unsigned long pfn, unsigned long nr_pages, |
| 726 | + struct vmem_altmap *altmap) |
| 727 | +{ |
| 728 | + DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; |
| 729 | + DECLARE_BITMAP(tmp, SUBSECTIONS_PER_SECTION) = { 0 }; |
| 730 | + struct mem_section *ms = __pfn_to_section(pfn); |
| 731 | + bool section_is_early = early_section(ms); |
| 732 | + struct page *memmap = NULL; |
| 733 | + unsigned long *subsection_map = ms->usage |
| 734 | + ? &ms->usage->subsection_map[0] : NULL; |
| 735 | + |
| 736 | + subsection_mask_set(map, pfn, nr_pages); |
| 737 | + if (subsection_map) |
| 738 | + bitmap_and(tmp, map, subsection_map, SUBSECTIONS_PER_SECTION); |
| 739 | + |
| 740 | + if (WARN(!subsection_map || !bitmap_equal(tmp, map, SUBSECTIONS_PER_SECTION), |
| 741 | + "section already deactivated (%#lx + %ld)\n", |
| 742 | + pfn, nr_pages)) |
| 743 | + return; |
| 744 | + |
| 745 | + /* |
| 746 | + * There are 3 cases to handle across two configurations |
| 747 | + * (SPARSEMEM_VMEMMAP={y,n}): |
| 748 | + * |
| 749 | + * 1/ deactivation of a partial hot-added section (only possible |
| 750 | + * in the SPARSEMEM_VMEMMAP=y case). |
| 751 | + * a/ section was present at memory init |
| 752 | + * b/ section was hot-added post memory init |
| 753 | + * 2/ deactivation of a complete hot-added section |
| 754 | + * 3/ deactivation of a complete section from memory init |
| 755 | + * |
| 756 | + * For 1/, when subsection_map does not empty we will not be |
| 757 | + * freeing the usage map, but still need to free the vmemmap |
| 758 | + * range. |
| 759 | + * |
| 760 | + * For 2/ and 3/ the SPARSEMEM_VMEMMAP={y,n} cases are unified |
| 761 | + */ |
| 762 | + bitmap_xor(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION); |
| 763 | + if (bitmap_empty(subsection_map, SUBSECTIONS_PER_SECTION)) { |
| 764 | + unsigned long section_nr = pfn_to_section_nr(pfn); |
| 765 | + |
| 766 | + if (!section_is_early) { |
| 767 | + kfree(ms->usage); |
| 768 | + ms->usage = NULL; |
| 769 | + } |
| 770 | + memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); |
| 771 | + ms->section_mem_map = sparse_encode_mem_map(NULL, section_nr); |
| 772 | + } |
| 773 | + |
| 774 | + if (section_is_early && memmap) |
| 775 | + free_map_bootmem(memmap); |
| 776 | + else |
| 777 | + depopulate_section_memmap(pfn, nr_pages, altmap); |
| 778 | +} |
| 779 | + |
| 780 | +static struct page * __meminit section_activate(int nid, unsigned long pfn, |
| 781 | + unsigned long nr_pages, struct vmem_altmap *altmap) |
| 782 | +{ |
| 783 | + DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; |
| 784 | + struct mem_section *ms = __pfn_to_section(pfn); |
| 785 | + struct mem_section_usage *usage = NULL; |
| 786 | + unsigned long *subsection_map; |
| 787 | + struct page *memmap; |
| 788 | + int rc = 0; |
| 789 | + |
| 790 | + subsection_mask_set(map, pfn, nr_pages); |
| 791 | + |
| 792 | + if (!ms->usage) { |
| 793 | + usage = kzalloc(mem_section_usage_size(), GFP_KERNEL); |
| 794 | + if (!usage) |
| 795 | + return ERR_PTR(-ENOMEM); |
| 796 | + ms->usage = usage; |
| 797 | + } |
| 798 | + subsection_map = &ms->usage->subsection_map[0]; |
| 799 | + |
| 800 | + if (bitmap_empty(map, SUBSECTIONS_PER_SECTION)) |
| 801 | + rc = -EINVAL; |
| 802 | + else if (bitmap_intersects(map, subsection_map, SUBSECTIONS_PER_SECTION)) |
| 803 | + rc = -EEXIST; |
| 804 | + else |
| 805 | + bitmap_or(subsection_map, map, subsection_map, |
| 806 | + SUBSECTIONS_PER_SECTION); |
| 807 | + |
| 808 | + if (rc) { |
| 809 | + if (usage) |
| 810 | + ms->usage = NULL; |
| 811 | + kfree(usage); |
| 812 | + return ERR_PTR(rc); |
| 813 | + } |
| 814 | + |
| 815 | + /* |
| 816 | + * The early init code does not consider partially populated |
| 817 | + * initial sections, it simply assumes that memory will never be |
| 818 | + * referenced. If we hot-add memory into such a section then we |
| 819 | + * do not need to populate the memmap and can simply reuse what |
| 820 | + * is already there. |
| 821 | + */ |
| 822 | + if (nr_pages < PAGES_PER_SECTION && early_section(ms)) |
| 823 | + return pfn_to_page(pfn); |
| 824 | + |
| 825 | + memmap = populate_section_memmap(pfn, nr_pages, nid, altmap); |
| 826 | + if (!memmap) { |
| 827 | + section_deactivate(pfn, nr_pages, altmap); |
| 828 | + return ERR_PTR(-ENOMEM); |
| 829 | + } |
| 830 | + |
| 831 | + return memmap; |
| 832 | +} |
| 833 | + |
| 718 | 834 | /** |
| 719 | | − * sparse_add_one_section - add a memory section |
| 835 | + * sparse_add_section - add a memory section, or populate an existing one |
| 720 | 836 | * @nid: The node to add section on |
| 721 | 837 | * @start_pfn: start pfn of the memory range |
| 838 | + * @nr_pages: number of pfns to add in the section |
| 722 | 839 | * @altmap: device page map |
| 723 | 840 | * |
| 724 | 841 | * This is only intended for hotplug. |
| @@ -732,51 +849,34 @@ int __meminit sparse_add_section(int nid, unsigned long start_pfn, · 7 unmodified lines |
| 732 | 849 | unsigned long nr_pages, struct vmem_altmap *altmap) |
| 733 | 850 | { |
| 734 | 851 | unsigned long section_nr = pfn_to_section_nr(start_pfn); |
| 735 | | − struct mem_section_usage *usage; |
| 736 | 852 | struct mem_section *ms; |
| 737 | 853 | struct page *memmap; |
| 738 | 854 | int ret; |
| 739 | 855 | |
| 740 | | − /* |
| 741 | | − * no locking for this, because it does its own |
| 742 | | − * plus, it does a kmalloc |
| 743 | | − */ |
| 744 | 856 | ret = sparse_index_init(section_nr, nid); |
| 745 | | − if (ret < 0 && ret != -EEXIST) |
| 857 | + if (ret < 0) |
| 746 | 858 | return ret; |
| 747 | | − ret = 0; |
| 748 | | − memmap = populate_section_memmap(start_pfn, PAGES_PER_SECTION, nid, |
| 749 | | − altmap); |
| 750 | | − if (!memmap) |
| 751 | | − return -ENOMEM; |
| 752 | | − usage = kzalloc(mem_section_usage_size(), GFP_KERNEL); |
| 753 | | − if (!usage) { |
| 754 | | − depopulate_section_memmap(start_pfn, PAGES_PER_SECTION, altmap); |
| 755 | | − return -ENOMEM; |
| 756 | | − } |
| 757 | 859 | |
| 758 | | − ms = __pfn_to_section(start_pfn); |
| 759 | | − if (ms->section_mem_map & SECTION_MARKED_PRESENT) { |
| 760 | | − ret = -EEXIST; |
| 761 | | − goto out; |
| 762 | | − } |
| 860 | + memmap = section_activate(nid, start_pfn, nr_pages, altmap); |
| 861 | + if (IS_ERR(memmap)) |
| 862 | + return PTR_ERR(memmap); |
| 763 | 863 | |
| 764 | 864 | /* |
| 765 | 865 | * Poison uninitialized struct pages in order to catch invalid flags |
| 766 | 866 | * combinations. |
| 767 | 867 | */ |
| 768 | | − page_init_poison(memmap, sizeof(struct page) * PAGES_PER_SECTION); |
| 868 | + page_init_poison(pfn_to_page(start_pfn), sizeof(struct page) * nr_pages); |
| 769 | 869 | |
| 870 | + ms = __pfn_to_section(start_pfn); |
| 770 | 871 | set_section_nid(section_nr, nid); |
| 771 | 872 | section_mark_present(ms); |
| 772 | | − sparse_init_one_section(ms, section_nr, memmap, usage, 0); |
| 773 | 873 | |
| 774 | | −out: |
| 775 | | − if (ret < 0) { |
| 776 | | − kfree(usage); |
| 777 | | − depopulate_section_memmap(start_pfn, PAGES_PER_SECTION, altmap); |
| 778 | | − } |
| 779 | | − return ret; |
| 874 | + /* Align memmap to section boundary in the subsection case */ |
| 875 | + if (section_nr_to_pfn(section_nr) != start_pfn) |
| 876 | + memmap = pfn_to_kaddr(section_nr_to_pfn(section_nr)); |
| 877 | + sparse_init_one_section(ms, section_nr, memmap, ms->usage, 0); |
| 878 | + |
| 879 | + return 0; |
| 780 | 880 | } |
| 781 | 881 | |
| 782 | 882 | #ifdef CONFIG_MEMORY_FAILURE |
| @@ -809,48 +909,12 @@ static inline void clear_hwpoisoned_pages(struct page *memmap, int nr_pages) · 26 unmodified lines |
| 809 | 909 | } |
| 810 | 910 | #endif |
| 811 | 911 | |
| 812 | | −static void free_section_usage(struct mem_section *ms, struct page *memmap, |
| 813 | | − struct mem_section_usage *usage, unsigned long pfn, |
| 814 | | − unsigned long nr_pages, struct vmem_altmap *altmap) |
| 815 | | −{ |
| 816 | | − if (!usage) |
| 817 | | − return; |
| 818 | | − |
| 819 | | − /* |
| 820 | | − * Check to see if allocation came from hot-plug-add |
| 821 | | − */ |
| 822 | | − if (!early_section(ms)) { |
| 823 | | − kfree(usage); |
| 824 | | − if (memmap) |
| 825 | | − depopulate_section_memmap(pfn, nr_pages, altmap); |
| 826 | | − return; |
| 827 | | − } |
| 828 | | − |
| 829 | | − /* |
| 830 | | − * The usemap came from bootmem. This is packed with other usemaps |
| 831 | | − * on the section which has pgdat at boot time. Just keep it as is now. |
| 832 | | − */ |
| 833 | | − |
| 834 | | − if (memmap) |
| 835 | | − free_map_bootmem(memmap); |
| 836 | | −} |
| 837 | | − |
| 838 | | −void sparse_remove_one_section(struct mem_section *ms, unsigned long pfn, |
| 912 | +void sparse_remove_section(struct mem_section *ms, unsigned long pfn, |
| 839 | 913 | unsigned long nr_pages, unsigned long map_offset, |
| 840 | 914 | struct vmem_altmap *altmap) |
| 841 | 915 | { |
| 842 | | − struct page *memmap = NULL; |
| 843 | | − struct mem_section_usage *usage = NULL; |
| 844 | | − |
| 845 | | − if (ms->section_mem_map) { |
| 846 | | − usage = ms->usage; |
| 847 | | − memmap = sparse_decode_mem_map(ms->section_mem_map, |
| 848 | | − __section_nr(ms)); |
| 849 | | − ms->section_mem_map = 0; |
| 850 | | − ms->usage = NULL; |
| 851 | | − } |
| 852 | | − |
| 853 | | − clear_hwpoisoned_pages(memmap + map_offset, nr_pages - map_offset); |
| 854 | | − free_section_usage(ms, memmap, usage, pfn, nr_pages, altmap); |
| 916 | + clear_hwpoisoned_pages(pfn_to_page(pfn) + map_offset, |
| 917 | + nr_pages - map_offset); |
| 918 | + section_deactivate(pfn, nr_pages, altmap); |
| 855 | 919 | } |
| 856 | 920 | #endif /* CONFIG_MEMORY_HOTPLUG */ |