History

kho: warn and fail on metadata or preserved memory in scratch area

Patch series "KHO: kfence + KHO memory corruption fix", v3. This series fixes a memory corruption bug in KHO that occurs when KFENCE is enabled. The root cause is that KHO metadata, allocated via kzalloc(), can be randomly serviced by kfence_alloc(). When a kernel boots via KHO, the early memblock allocator is restricted to a "scratch area". This forces the KFENCE pool to be allocated within this scratch area, creating a conflict. If KHO metadata is subsequently placed in this pool, it gets corrupted during the next kexec operation. Google is using KHO and have had obscure crashes due to this memory corruption, with stacks all over the place. I would prefer this fix to be properly backported to stable so we can also automatically consume it once we switch to the upstream KHO. Patch 1/3 introduces a debug-only feature (CONFIG_KEXEC_HANDOVER_DEBUG) that adds checks to detect and fail any operation that attempts to place KHO metadata or preserved memory within the scratch area. This serves as a validation and diagnostic tool to confirm the problem without affecting production builds. Patch 2/3 Increases bitmap to PAGE_SIZE, so buddy allocator can be used. Patch 3/3 Provides the fix by modifying KHO to allocate its metadata directly from the buddy allocator instead of slab. This bypasses the KFENCE interception entirely. This patch (of 3): It is invalid for KHO metadata or preserved memory regions to be located within the KHO scratch area, as this area is overwritten when the next kernel is loaded, and used early in boot by the next kernel. This can lead to memory corruption. Add checks to kho_preserve_* and KHO's internal metadata allocators (xa_load_or_alloc, new_chunk) to verify that the physical address of the memory does not overlap with any defined scratch region. If an overlap is detected, the operation will fail and a WARN_ON is triggered. To avoid performance overhead in production kernels, these checks are enabled only when CONFIG_KEXEC_HANDOVER_DEBUG is selected. [rppt@kernel.org: fix KEXEC_HANDOVER_DEBUG Kconfig dependency] Link: https://lkml.kernel.org/r/aQHUyyFtiNZhx8jo@kernel.org [pasha.tatashin@soleen.com: build fix] Link: https://lkml.kernel.org/r/CA+CK2bBnorfsTymKtv4rKvqGBHs=y=MjEMMRg_tE-RME6n-zUw@mail.gmail.com Link: https://lkml.kernel.org/r/20251021000852.2924827-1-pasha.tatashin@soleen.com Link: https://lkml.kernel.org/r/20251021000852.2924827-2-pasha.tatashin@soleen.com Fixes: fc33e4b44b27 ("kexec: enable KHO support for memory preservation") Signed-off-by: Pasha Tatashin <pasha.tatashin@soleen.com> Signed-off-by: Mike Rapoport <rppt@kernel.org> Reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org> Reviewed-by: Pratyush Yadav <pratyush@kernel.org> Cc: Alexander Graf <graf@amazon.com> Cc: Christian Brauner <brauner@kernel.org> Cc: David Matlack <dmatlack@google.com> Cc: Jason Gunthorpe <jgg@ziepe.ca> Cc: Jonathan Corbet <corbet@lwn.net> Cc: Masahiro Yamada <masahiroy@kernel.org> Cc: Miguel Ojeda <ojeda@kernel.org> Cc: Randy Dunlap <rdunlap@infradead.org> Cc: Samiullah Khawaja <skhawaja@google.com> Cc: Tejun Heo <tj@kernel.org> Cc: <stable@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

Pasha Tatashin committed 1 year ago

Commit e38f65d1 parent 77008e15 files · +93 −19Browse files

Showing the changes to kernel (5 of 5 files). Show all

Unified · Split

5 files
  1. kernel/Kconfig.kexec +9 −0
  2. kernel/Makefile +1 −0
  3. kernel/kexec_handover.c +38 −19
  4. kernel/kexec_handover_debug.c +25 −0
  5. kernel/kexec_handover_internal.h +20 −0
kernel/Kconfig.kexec+9−0
@@ -109,6 +109,15 @@ config KEXEC_HANDOVER · 108 unmodified lines
109 to keep data or state alive across the kexec. For this to work,109 to keep data or state alive across the kexec. For this to work,
110 both source and target kernels need to have this option enabled.110 both source and target kernels need to have this option enabled.
111 111
112config KEXEC_HANDOVER_DEBUG
113 bool "Enable Kexec Handover debug checks"
114 depends on KEXEC_HANDOVER
115 help
116 This option enables extra sanity checks for the Kexec Handover
117 subsystem. Since, KHO performance is crucial in live update
118 scenarios and the extra code might be adding overhead it is
119 only optionally enabled.
120
112config CRASH_DUMP121config CRASH_DUMP
113 bool "kernel crash dumps"122 bool "kernel crash dumps"
114 default ARCH_DEFAULT_CRASH_DUMP123 default ARCH_DEFAULT_CRASH_DUMP
kernel/Makefile+1−0
@@ -83,6 +83,7 @@ obj-$(CONFIG_KEXEC) += kexec.o · 82 unmodified lines
83obj-$(CONFIG_KEXEC_FILE) += kexec_file.o83obj-$(CONFIG_KEXEC_FILE) += kexec_file.o
84obj-$(CONFIG_KEXEC_ELF) += kexec_elf.o84obj-$(CONFIG_KEXEC_ELF) += kexec_elf.o
85obj-$(CONFIG_KEXEC_HANDOVER) += kexec_handover.o85obj-$(CONFIG_KEXEC_HANDOVER) += kexec_handover.o
86obj-$(CONFIG_KEXEC_HANDOVER_DEBUG) += kexec_handover_debug.o
86obj-$(CONFIG_BACKTRACE_SELF_TEST) += backtracetest.o87obj-$(CONFIG_BACKTRACE_SELF_TEST) += backtracetest.o
87obj-$(CONFIG_COMPAT) += compat.o88obj-$(CONFIG_COMPAT) += compat.o
88obj-$(CONFIG_CGROUPS) += cgroup/89obj-$(CONFIG_CGROUPS) += cgroup/
kernel/kexec_handover.c+38−19
@@ -8,6 +8,7 @@ · 7 unmodified lines
8 8
9#define pr_fmt(fmt) "KHO: " fmt9#define pr_fmt(fmt) "KHO: " fmt
10 10
11#include <linux/cleanup.h>
11#include <linux/cma.h>12#include <linux/cma.h>
12#include <linux/count_zeros.h>13#include <linux/count_zeros.h>
13#include <linux/debugfs.h>14#include <linux/debugfs.h>
@@ -22,6 +23,7 @@ · 8 unmodified lines
22 23
23#include <asm/early_ioremap.h>24#include <asm/early_ioremap.h>
24 25
26#include "kexec_handover_internal.h"
25/*27/*
26 * KHO is tightly coupled with mm init and needs access to some of mm28 * KHO is tightly coupled with mm init and needs access to some of mm
27 * internal APIs.29 * internal APIs.
@@ -133,26 +135,26 @@ static struct kho_out kho_out = { · 105 unmodified lines
133 135
134static void *xa_load_or_alloc(struct xarray *xa, unsigned long index, size_t sz)136static void *xa_load_or_alloc(struct xarray *xa, unsigned long index, size_t sz)
135{137{
136 void *elm, *res;138 void *res = xa_load(xa, index);
137 139
138 elm = xa_load(xa, index);140 if (res)
139 if (elm)141 return res;
140 return elm;142
143 void *elm __free(kfree) = kzalloc(sz, GFP_KERNEL);
141 144
142 elm = kzalloc(sz, GFP_KERNEL);
143 if (!elm)145 if (!elm)
144 return ERR_PTR(-ENOMEM);146 return ERR_PTR(-ENOMEM);
145 147
148 if (WARN_ON(kho_scratch_overlap(virt_to_phys(elm), sz)))
149 return ERR_PTR(-EINVAL);
150
146 res = xa_cmpxchg(xa, index, NULL, elm, GFP_KERNEL);151 res = xa_cmpxchg(xa, index, NULL, elm, GFP_KERNEL);
147 if (xa_is_err(res))152 if (xa_is_err(res))
148 res = ERR_PTR(xa_err(res));153 return ERR_PTR(xa_err(res));
149 154 else if (res)
150 if (res) {
151 kfree(elm);
152 return res;155 return res;
153 }
154 156
155 return elm;157 return no_free_ptr(elm);
156}158}
157 159
158static void __kho_unpreserve(struct kho_mem_track *track, unsigned long pfn,160static void __kho_unpreserve(struct kho_mem_track *track, unsigned long pfn,
@@ -345,15 +347,19 @@ static_assert(sizeof(struct khoser_mem_chunk) == PAGE_SIZE); · 186 unmodified lines
345static struct khoser_mem_chunk *new_chunk(struct khoser_mem_chunk *cur_chunk,347static struct khoser_mem_chunk *new_chunk(struct khoser_mem_chunk *cur_chunk,
346 unsigned long order)348 unsigned long order)
347{349{
348 struct khoser_mem_chunk *chunk;350 struct khoser_mem_chunk *chunk __free(kfree) = NULL;
349 351
350 chunk = kzalloc(PAGE_SIZE, GFP_KERNEL);352 chunk = kzalloc(PAGE_SIZE, GFP_KERNEL);
351 if (!chunk)353 if (!chunk)
352 return NULL;354 return ERR_PTR(-ENOMEM);
355
356 if (WARN_ON(kho_scratch_overlap(virt_to_phys(chunk), PAGE_SIZE)))
357 return ERR_PTR(-EINVAL);
358
353 chunk->hdr.order = order;359 chunk->hdr.order = order;
354 if (cur_chunk)360 if (cur_chunk)
355 KHOSER_STORE_PTR(cur_chunk->hdr.next, chunk);361 KHOSER_STORE_PTR(cur_chunk->hdr.next, chunk);
356 return chunk;362 return no_free_ptr(chunk);
357}363}
358 364
359static void kho_mem_ser_free(struct khoser_mem_chunk *first_chunk)365static void kho_mem_ser_free(struct khoser_mem_chunk *first_chunk)
@@ -374,14 +380,17 @@ static int kho_mem_serialize(struct kho_serialization *ser) · 14 unmodified lines
374 struct khoser_mem_chunk *chunk = NULL;380 struct khoser_mem_chunk *chunk = NULL;
375 struct kho_mem_phys *physxa;381 struct kho_mem_phys *physxa;
376 unsigned long order;382 unsigned long order;
383 int err = -ENOMEM;
377 384
378 xa_for_each(&ser->track.orders, order, physxa) {385 xa_for_each(&ser->track.orders, order, physxa) {
379 struct kho_mem_phys_bits *bits;386 struct kho_mem_phys_bits *bits;
380 unsigned long phys;387 unsigned long phys;
381 388
382 chunk = new_chunk(chunk, order);389 chunk = new_chunk(chunk, order);
383 if (!chunk)390 if (IS_ERR(chunk)) {
391 err = PTR_ERR(chunk);
384 goto err_free;392 goto err_free;
393 }
385 394
386 if (!first_chunk)395 if (!first_chunk)
387 first_chunk = chunk;396 first_chunk = chunk;
@@ -391,8 +400,10 @@ static int kho_mem_serialize(struct kho_serialization *ser) · 3 unmodified lines
391 400
392 if (chunk->hdr.num_elms == ARRAY_SIZE(chunk->bitmaps)) {401 if (chunk->hdr.num_elms == ARRAY_SIZE(chunk->bitmaps)) {
393 chunk = new_chunk(chunk, order);402 chunk = new_chunk(chunk, order);
394 if (!chunk)403 if (IS_ERR(chunk)) {
404 err = PTR_ERR(chunk);
395 goto err_free;405 goto err_free;
406 }
396 }407 }
397 408
398 elm = &chunk->bitmaps[chunk->hdr.num_elms];409 elm = &chunk->bitmaps[chunk->hdr.num_elms];
@@ -409,7 +420,7 @@ static int kho_mem_serialize(struct kho_serialization *ser) · 10 unmodified lines
409 420
410err_free:421err_free:
411 kho_mem_ser_free(first_chunk);422 kho_mem_ser_free(first_chunk);
412 return -ENOMEM;423 return err;
413}424}
414 425
415static void __init deserialize_bitmap(unsigned int order,426static void __init deserialize_bitmap(unsigned int order,
@@ -465,8 +476,8 @@ static void __init kho_mem_deserialize(const void *fdt) · 49 unmodified lines
465 * area for early allocations that happen before page allocator is476 * area for early allocations that happen before page allocator is
466 * initialized.477 * initialized.
467 */478 */
468static struct kho_scratch *kho_scratch;479struct kho_scratch *kho_scratch;
469static unsigned int kho_scratch_cnt;480unsigned int kho_scratch_cnt;
470 481
471/*482/*
472 * The scratch areas are scaled by default as percent of memory allocated from483 * The scratch areas are scaled by default as percent of memory allocated from
@@ -752,6 +763,9 @@ int kho_preserve_folio(struct folio *folio) · 279 unmodified lines
752 const unsigned int order = folio_order(folio);763 const unsigned int order = folio_order(folio);
753 struct kho_mem_track *track = &kho_out.ser.track;764 struct kho_mem_track *track = &kho_out.ser.track;
754 765
766 if (WARN_ON(kho_scratch_overlap(pfn << PAGE_SHIFT, PAGE_SIZE << order)))
767 return -EINVAL;
768
755 return __kho_preserve_order(track, pfn, order);769 return __kho_preserve_order(track, pfn, order);
756}770}
757EXPORT_SYMBOL_GPL(kho_preserve_folio);771EXPORT_SYMBOL_GPL(kho_preserve_folio);
@@ -775,6 +789,11 @@ int kho_preserve_pages(struct page *page, unsigned int nr_pages) · 17 unmodified lines
775 unsigned long failed_pfn = 0;789 unsigned long failed_pfn = 0;
776 int err = 0;790 int err = 0;
777 791
792 if (WARN_ON(kho_scratch_overlap(start_pfn << PAGE_SHIFT,
793 nr_pages << PAGE_SHIFT))) {
794 return -EINVAL;
795 }
796
778 while (pfn < end_pfn) {797 while (pfn < end_pfn) {
779 const unsigned int order =798 const unsigned int order =
780 min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn));799 min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn));
kernel/kexec_handover_debug.c+25−0
@@ -0,0 +1,25 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * kexec_handover_debug.c - kexec handover optional debug functionality
4 * Copyright (C) 2025 Google LLC, Pasha Tatashin <pasha.tatashin@soleen.com>
5 */
6
7#define pr_fmt(fmt) "KHO: " fmt
8
9#include "kexec_handover_internal.h"
10
11bool kho_scratch_overlap(phys_addr_t phys, size_t size)
12{
13 phys_addr_t scratch_start, scratch_end;
14 unsigned int i;
15
16 for (i = 0; i < kho_scratch_cnt; i++) {
17 scratch_start = kho_scratch[i].addr;
18 scratch_end = kho_scratch[i].addr + kho_scratch[i].size;
19
20 if (phys < scratch_end && (phys + size) > scratch_start)
21 return true;
22 }
23
24 return false;
25}
kernel/kexec_handover_internal.h+20−0
@@ -0,0 +1,20 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef LINUX_KEXEC_HANDOVER_INTERNAL_H
3#define LINUX_KEXEC_HANDOVER_INTERNAL_H
4
5#include <linux/kexec_handover.h>
6#include <linux/types.h>
7
8extern struct kho_scratch *kho_scratch;
9extern unsigned int kho_scratch_cnt;
10
11#ifdef CONFIG_KEXEC_HANDOVER_DEBUG
12bool kho_scratch_overlap(phys_addr_t phys, size_t size);
13#else
14static inline bool kho_scratch_overlap(phys_addr_t phys, size_t size)
15{
16 return false;
17}
18#endif /* CONFIG_KEXEC_HANDOVER_DEBUG */
19
20#endif /* LINUX_KEXEC_HANDOVER_INTERNAL_H */