History

mm/memory.c: add apply_to_existing_page_range() helper

apply_to_page_range() takes an address range, and if any parts of it are not covered by the existing page table hierarchy, it allocates memory to fill them in. In some use cases, this is not what we want - we want to be able to operate exclusively on PTEs that are already in the tables. Add apply_to_existing_page_range() for this. Adjust the walker functions for apply_to_page_range to take 'create', which switches them between the old and new modes. This will be used in KASAN vmalloc. [akpm@linux-foundation.org: reduce code duplication] [akpm@linux-foundation.org: s/apply_to_existing_pages/apply_to_existing_page_range/] [akpm@linux-foundation.org: initialize __apply_to_page_range::err] Link: http://lkml.kernel.org/r/20191205140407.1874-1-dja@axtens.net Signed-off-by: Daniel Axtens <dja@axtens.net> Cc: Dmitry Vyukov <dvyukov@google.com> Cc: Uladzislau Rezki (Sony) <urezki@gmail.com> Cc: Alexander Potapenko <glider@google.com> Cc: Daniel Axtens <dja@axtens.net> Cc: Qian Cai <cai@lca.pw> Cc: Andrey Ryabinin <aryabinin@virtuozzo.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Daniel Axtens committed 6 years ago

Commit be1db471 parent d98c9e82 files · +97 −42Browse files

Unified · Split

2 files
  1. include/linux/mm.h +3 −0
  2. mm/memory.c +94 −42
include/linux/mm.h+3−0
@@ -2621,6 +2621,9 @@ static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags) · 2620 unmodified lines
26212621 typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data);
26222622 extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
26232623 unsigned long size, pte_fn_t fn, void *data);
2624+extern int apply_to_existing_page_range(struct mm_struct *mm,
2625+ unsigned long address, unsigned long size,
2626+ pte_fn_t fn, void *data);
26242627
26252628 #ifdef CONFIG_PAGE_POISONING
26262629 extern bool page_poisoning_enabled(void);
mm/memory.c+94−42
@@ -2021,26 +2021,34 @@ EXPORT_SYMBOL(vm_iomap_memory); · 2020 unmodified lines
20212021
20222022 static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
20232023 unsigned long addr, unsigned long end,
2024− pte_fn_t fn, void *data)
2024+ pte_fn_t fn, void *data, bool create)
20252025 {
20262026 pte_t *pte;
2027− int err;
2027+ int err = 0;
20282028 spinlock_t *uninitialized_var(ptl);
20292029
2030− pte = (mm == &init_mm) ?
2031− pte_alloc_kernel(pmd, addr) :
2032− pte_alloc_map_lock(mm, pmd, addr, &ptl);
2033− if (!pte)
2034− return -ENOMEM;
2030+ if (create) {
2031+ pte = (mm == &init_mm) ?
2032+ pte_alloc_kernel(pmd, addr) :
2033+ pte_alloc_map_lock(mm, pmd, addr, &ptl);
2034+ if (!pte)
2035+ return -ENOMEM;
2036+ } else {
2037+ pte = (mm == &init_mm) ?
2038+ pte_offset_kernel(pmd, addr) :
2039+ pte_offset_map_lock(mm, pmd, addr, &ptl);
2040+ }
20352041
20362042 BUG_ON(pmd_huge(*pmd));
20372043
20382044 arch_enter_lazy_mmu_mode();
20392045
20402046 do {
2041− err = fn(pte++, addr, data);
2042− if (err)
2043− break;
2047+ if (create || !pte_none(*pte)) {
2048+ err = fn(pte++, addr, data);
2049+ if (err)
2050+ break;
2051+ }
20442052 } while (addr += PAGE_SIZE, addr != end);
20452053
20462054 arch_leave_lazy_mmu_mode();
@@ -2052,77 +2060,95 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, · 5 unmodified lines
20522060
20532061 static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
20542062 unsigned long addr, unsigned long end,
2055− pte_fn_t fn, void *data)
2063+ pte_fn_t fn, void *data, bool create)
20562064 {
20572065 pmd_t *pmd;
20582066 unsigned long next;
2059− int err;
2067+ int err = 0;
20602068
20612069 BUG_ON(pud_huge(*pud));
20622070
2063− pmd = pmd_alloc(mm, pud, addr);
2064− if (!pmd)
2065− return -ENOMEM;
2071+ if (create) {
2072+ pmd = pmd_alloc(mm, pud, addr);
2073+ if (!pmd)
2074+ return -ENOMEM;
2075+ } else {
2076+ pmd = pmd_offset(pud, addr);
2077+ }
20662078 do {
20672079 next = pmd_addr_end(addr, end);
2068− err = apply_to_pte_range(mm, pmd, addr, next, fn, data);
2069− if (err)
2070− break;
2080+ if (create || !pmd_none_or_clear_bad(pmd)) {
2081+ err = apply_to_pte_range(mm, pmd, addr, next, fn, data,
2082+ create);
2083+ if (err)
2084+ break;
2085+ }
20712086 } while (pmd++, addr = next, addr != end);
20722087 return err;
20732088 }
20742089
20752090 static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
20762091 unsigned long addr, unsigned long end,
2077− pte_fn_t fn, void *data)
2092+ pte_fn_t fn, void *data, bool create)
20782093 {
20792094 pud_t *pud;
20802095 unsigned long next;
2081− int err;
2096+ int err = 0;
20822097
2083− pud = pud_alloc(mm, p4d, addr);
2084− if (!pud)
2085− return -ENOMEM;
2098+ if (create) {
2099+ pud = pud_alloc(mm, p4d, addr);
2100+ if (!pud)
2101+ return -ENOMEM;
2102+ } else {
2103+ pud = pud_offset(p4d, addr);
2104+ }
20862105 do {
20872106 next = pud_addr_end(addr, end);
2088− err = apply_to_pmd_range(mm, pud, addr, next, fn, data);
2089− if (err)
2090− break;
2107+ if (create || !pud_none_or_clear_bad(pud)) {
2108+ err = apply_to_pmd_range(mm, pud, addr, next, fn, data,
2109+ create);
2110+ if (err)
2111+ break;
2112+ }
20912113 } while (pud++, addr = next, addr != end);
20922114 return err;
20932115 }
20942116
20952117 static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
20962118 unsigned long addr, unsigned long end,
2097− pte_fn_t fn, void *data)
2119+ pte_fn_t fn, void *data, bool create)
20982120 {
20992121 p4d_t *p4d;
21002122 unsigned long next;
2101− int err;
2123+ int err = 0;
21022124
2103− p4d = p4d_alloc(mm, pgd, addr);
2104− if (!p4d)
2105− return -ENOMEM;
2125+ if (create) {
2126+ p4d = p4d_alloc(mm, pgd, addr);
2127+ if (!p4d)
2128+ return -ENOMEM;
2129+ } else {
2130+ p4d = p4d_offset(pgd, addr);
2131+ }
21062132 do {
21072133 next = p4d_addr_end(addr, end);
2108− err = apply_to_pud_range(mm, p4d, addr, next, fn, data);
2109− if (err)
2110− break;
2134+ if (create || !p4d_none_or_clear_bad(p4d)) {
2135+ err = apply_to_pud_range(mm, p4d, addr, next, fn, data,
2136+ create);
2137+ if (err)
2138+ break;
2139+ }
21112140 } while (p4d++, addr = next, addr != end);
21122141 return err;
21132142 }
21142143
2115−/*
2116− * Scan a region of virtual memory, filling in page tables as necessary
2117− * and calling a provided function on each leaf page table.
2118− */
2119−int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2120− unsigned long size, pte_fn_t fn, void *data)
2144+static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2145+ unsigned long size, pte_fn_t fn,
2146+ void *data, bool create)
21212147 {
21222148 pgd_t *pgd;
21232149 unsigned long next;
21242150 unsigned long end = addr + size;
2125− int err;
2151+ int err = 0;
21262152
21272153 if (WARN_ON(addr >= end))
21282154 return -EINVAL;
@@ -2130,15 +2156,41 @@ int apply_to_page_range(struct mm_struct *mm, unsigned long addr, · 1 unmodified line
21302156 pgd = pgd_offset(mm, addr);
21312157 do {
21322158 next = pgd_addr_end(addr, end);
2133− err = apply_to_p4d_range(mm, pgd, addr, next, fn, data);
2159+ if (!create && pgd_none_or_clear_bad(pgd))
2160+ continue;
2161+ err = apply_to_p4d_range(mm, pgd, addr, next, fn, data, create);
21342162 if (err)
21352163 break;
21362164 } while (pgd++, addr = next, addr != end);
21372165
21382166 return err;
21392167 }
2168+
2169+/*
2170+ * Scan a region of virtual memory, filling in page tables as necessary
2171+ * and calling a provided function on each leaf page table.
2172+ */
2173+int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2174+ unsigned long size, pte_fn_t fn, void *data)
2175+{
2176+ return __apply_to_page_range(mm, addr, size, fn, data, true);
2177+}
21402178 EXPORT_SYMBOL_GPL(apply_to_page_range);
21412179
2180+/*
2181+ * Scan a region of virtual memory, calling a provided function on
2182+ * each leaf page table where it exists.
2183+ *
2184+ * Unlike apply_to_page_range, this does _not_ fill in page tables
2185+ * where they are absent.
2186+ */
2187+int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
2188+ unsigned long size, pte_fn_t fn, void *data)
2189+{
2190+ return __apply_to_page_range(mm, addr, size, fn, data, false);
2191+}
2192+EXPORT_SYMBOL_GPL(apply_to_existing_page_range);
2193+
21422194 /*
21432195 * handle_pte_fault chooses page fault handler according to an entry which was
21442196 * read non-atomically. Before making any commitment, on those architectures