mm: kmsan: handle alloc failures in kmsan_ioremap_page_range()

Similarly to kmsan_vmap_pages_range_noflush(), kmsan_ioremap_page_range() must also properly handle allocation/mapping failures. In the case of such, it must clean up the already created metadata mappings and return an error code, so that the error can be propagated to ioremap_page_range(). Without doing so, KMSAN may silently fail to bring the metadata for the page range into a consistent state, which will result in user-visible crashes when trying to access them.

Alexander Potapenko committed 3 years ago

Commit fdea03e1 parent 47ebd033 files · +59 −19Browse files

Unified · Split

3 files

include/linux/kmsan.h +10 −9
mm/kmsan/hooks.c +47 −8
mm/vmalloc.c +2 −2

include/linux/kmsan.h+10−9

  * @page_shift:	page_shift argument passed to vmap_range_noflush().
  *
  * KMSAN creates new metadata pages for the physical pages mapped into the
- * virtual memory.
+ * virtual memory. Returns 0 on success, callers must check for non-zero return
+ * value.
  */
-void kmsan_ioremap_page_range(unsigned long addr, unsigned long end,
+int kmsan_ioremap_page_range(unsigned long addr, unsigned long end,
-			      phys_addr_t phys_addr, pgprot_t prot,
+			     phys_addr_t phys_addr, pgprot_t prot,
-			      unsigned int page_shift);
+			     unsigned int page_shift);
 /**
  * kmsan_iounmap_page_range() - Notify KMSAN about a iounmap_page_range() call.
 {
 }
-static inline void kmsan_ioremap_page_range(unsigned long start,
+static inline int kmsan_ioremap_page_range(unsigned long start,
-					    unsigned long end,
+					   unsigned long end,
-					    phys_addr_t phys_addr,
+					   phys_addr_t phys_addr, pgprot_t prot,
-					    pgprot_t prot,
+					   unsigned int page_shift)
-					    unsigned int page_shift)
 {
+	return 0;
 }
 static inline void kmsan_iounmap_page_range(unsigned long start,

mm/kmsan/hooks.c+47−8

@@ -148,35 +148,74 @@ void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end) · 147 unmodified lines
148	* into the virtual memory. If those physical pages already had shadow/origin,	148	* into the virtual memory. If those physical pages already had shadow/origin,
149	* those are ignored.	149	* those are ignored.
150	*/	150	*/
151	void kmsan_ioremap_page_range(unsigned long start, unsigned long end,	151	int kmsan_ioremap_page_range(unsigned long start, unsigned long end,
152	phys_addr_t phys_addr, pgprot_t prot,	152	phys_addr_t phys_addr, pgprot_t prot,
153	unsigned int page_shift)	153	unsigned int page_shift)
154	{	154	{
155	gfp_t gfp_mask = GFP_KERNEL \| __GFP_ZERO;	155	gfp_t gfp_mask = GFP_KERNEL \| __GFP_ZERO;
156	struct page shadow, origin;	156	struct page shadow, origin;
157	unsigned long off = 0;	157	unsigned long off = 0;
158	int nr;	158	int nr, err = 0, clean = 0, mapped;
159		159
160	if (!kmsan_enabled \|\| kmsan_in_runtime())	160	if (!kmsan_enabled \|\| kmsan_in_runtime())
161	return;	161	return 0;
162		162
163	nr = (end - start) / PAGE_SIZE;	163	nr = (end - start) / PAGE_SIZE;
164	kmsan_enter_runtime();	164	kmsan_enter_runtime();
165	for (int i = 0; i < nr; i++, off += PAGE_SIZE) {	165	for (int i = 0; i < nr; i++, off += PAGE_SIZE, clean = i) {
166	shadow = alloc_pages(gfp_mask, 1);	166	shadow = alloc_pages(gfp_mask, 1);
167	origin = alloc_pages(gfp_mask, 1);	167	origin = alloc_pages(gfp_mask, 1);
168	__vmap_pages_range_noflush(	168	if (!shadow \|\| !origin) {
		169	err = -ENOMEM;
		170	goto ret;
		171	}
		172	mapped = __vmap_pages_range_noflush(
169	vmalloc_shadow(start + off),	173	vmalloc_shadow(start + off),
170	vmalloc_shadow(start + off + PAGE_SIZE), prot, &shadow,	174	vmalloc_shadow(start + off + PAGE_SIZE), prot, &shadow,
171	PAGE_SHIFT);	175	PAGE_SHIFT);
172	__vmap_pages_range_noflush(	176	if (mapped) {
		177	err = mapped;
		178	goto ret;
		179	}
		180	shadow = NULL;
		181	mapped = __vmap_pages_range_noflush(
173	vmalloc_origin(start + off),	182	vmalloc_origin(start + off),
174	vmalloc_origin(start + off + PAGE_SIZE), prot, &origin,	183	vmalloc_origin(start + off + PAGE_SIZE), prot, &origin,
175	PAGE_SHIFT);	184	PAGE_SHIFT);
		185	if (mapped) {
		186	__vunmap_range_noflush(
		187	vmalloc_shadow(start + off),
		188	vmalloc_shadow(start + off + PAGE_SIZE));
		189	err = mapped;
		190	goto ret;
		191	}
		192	origin = NULL;
		193	}
		194	/* Page mapping loop finished normally, nothing to clean up. */
		195	clean = 0;
		196
		197	ret:
		198	if (clean > 0) {
		199	/*
		200	* Something went wrong. Clean up shadow/origin pages allocated
		201	* on the last loop iteration, then delete mappings created
		202	* during the previous iterations.
		203	*/
		204	if (shadow)
		205	__free_pages(shadow, 1);
		206	if (origin)
		207	__free_pages(origin, 1);
		208	__vunmap_range_noflush(
		209	vmalloc_shadow(start),
		210	vmalloc_shadow(start + clean * PAGE_SIZE));
		211	__vunmap_range_noflush(
		212	vmalloc_origin(start),
		213	vmalloc_origin(start + clean * PAGE_SIZE));
176	}	214	}
177	flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));	215	flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
178	flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));	216	flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
179	kmsan_leave_runtime();	217	kmsan_leave_runtime();
		218	return err;
180	}	219	}
181		220
182	void kmsan_iounmap_page_range(unsigned long start, unsigned long end)	221	void kmsan_iounmap_page_range(unsigned long start, unsigned long end)

mm/vmalloc.c+2−2

@@ -313,8 +313,8 @@ int ioremap_page_range(unsigned long addr, unsigned long end, · 312 unmodified lines
313	ioremap_max_page_shift);	313	ioremap_max_page_shift);
314	flush_cache_vmap(addr, end);	314	flush_cache_vmap(addr, end);
315	if (!err)	315	if (!err)
316	kmsan_ioremap_page_range(addr, end, phys_addr, prot,	316	err = kmsan_ioremap_page_range(addr, end, phys_addr, prot,
317	ioremap_max_page_shift);	317	ioremap_max_page_shift);
318	return err;	318	return err;
319	}	319	}
320		320