// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2008, 2009 Intel Corporation * Authors: Andi Kleen, Fengguang Wu * * High level machine check handler. Handles pages reported by the * hardware as being corrupted usually due to a multi-bit ECC memory or cache * failure. * * In addition there is a "soft offline" entry point that allows stop using * not-yet-corrupted-by-suspicious pages without killing anything. * * Handles page cache pages in various states. The tricky part * here is that we can access any page asynchronously in respect to * other VM users, because memory failures could happen anytime and * anywhere. This could violate some of their assumptions. This is why * this code has to be extremely careful. Generally it tries to use * normal locking rules, as in get the standard locks, even if that means * the error handling takes potentially a long time. * * It can be very tempting to add handling for obscure cases here. * In general any code for handling new cases should only be added iff: * - You know how to test it. * - You have a test that can be added to mce-test * https://git.kernel.org/cgit/utils/cpu/mce/mce-test.git/ * - The case actually shows up as a frequent (top 10) page state in * tools/mm/page-types when running a real workload. * * There are several operations here with exponential complexity because * of unsuitable VM data structures. For example the operation to map back * from RMAP chains to processes has to walk the complete process list and * has non linear complexity with the number. But since memory corruptions * are rare we hope to get away with this. This avoids impacting the core * VM. */ #define pr_fmt(fmt) "Memory failure: " fmt #include <linux/kernel.h>#include <linux/mm.h>#include <linux/memory-failure.h>#include <linux/page-flags.h>#include <linux/sched/signal.h>#include <linux/sched/task.h>#include <linux/dax.h>#include <linux/ksm.h>#include <linux/rmap.h>#include <linux/export.h>#include <linux/pagemap.h>#include <linux/swap.h>#include <linux/backing-dev.h>#include <linux/migrate.h>#include <linux/slab.h>#include <linux/leafops.h>#include <linux/hugetlb.h>#include <linux/memory_hotplug.h>#include <linux/mm_inline.h>#include <linux/memremap.h>#include <linux/kfifo.h>#include <linux/ratelimit.h>#include <linux/pagewalk.h>#include <linux/shmem_fs.h>#include <linux/sysctl.h> #define CREATE_TRACE_POINTS#include <trace/events/memory-failure.h> #include "swap.h"#include "page_alloc.h"#include "internal.h" static int sysctl_memory_failure_early_kill __read_mostly; static int sysctl_memory_failure_recovery __read_mostly = 1; static int sysctl_enable_soft_offline __read_mostly = 1; static int sysctl_panic_on_unrecoverable_mf __read_mostly; atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); static bool hw_memory_failure __read_mostly; static DEFINE_MUTEX(mf_mutex); void num_poisoned_pages_inc(unsigned long pfn){ atomic_long_inc(&num_poisoned_pages); memblk_nr_poison_inc(pfn);} void num_poisoned_pages_sub(unsigned long pfn, long i){ atomic_long_sub(i, &num_poisoned_pages); if (pfn != -1UL) memblk_nr_poison_sub(pfn, i);} /** * MF_ATTR_RO - Create sysfs entry for each memory failure statistics. * @_name: name of the file in the per NUMA sysfs directory. */#define MF_ATTR_RO(_name) \static ssize_t _name##_show(struct device *dev, \ struct device_attribute *attr, \ char *buf) \{ \ struct memory_failure_stats *mf_stats = \ &NODE_DATA(dev->id)->mf_stats; \ return sysfs_emit(buf, "%lu\n", mf_stats->_name); \} \static DEVICE_ATTR_RO(_name) MF_ATTR_RO(total);MF_ATTR_RO(ignored);MF_ATTR_RO(failed);MF_ATTR_RO(delayed);MF_ATTR_RO(recovered); static struct attribute *memory_failure_attr[] = { &dev_attr_total.attr, &dev_attr_ignored.attr, &dev_attr_failed.attr, &dev_attr_delayed.attr, &dev_attr_recovered.attr, NULL,}; const struct attribute_group memory_failure_attr_group = { .name = "memory_failure", .attrs = memory_failure_attr,}; static const struct ctl_table memory_failure_table[] = { { .procname = "memory_failure_early_kill", .data = &sysctl_memory_failure_early_kill, .maxlen = sizeof(sysctl_memory_failure_early_kill), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, { .procname = "memory_failure_recovery", .data = &sysctl_memory_failure_recovery, .maxlen = sizeof(sysctl_memory_failure_recovery), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, { .procname = "enable_soft_offline", .data = &sysctl_enable_soft_offline, .maxlen = sizeof(sysctl_enable_soft_offline), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, { .procname = "panic_on_unrecoverable_memory_failure", .data = &sysctl_panic_on_unrecoverable_mf, .maxlen = sizeof(sysctl_panic_on_unrecoverable_mf), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }}; static struct rb_root_cached pfn_space_itree = RB_ROOT_CACHED; static DEFINE_MUTEX(pfn_space_lock); /* * Return values: * 1: the page is dissolved (if needed) and taken off from buddy, * 0: the page is dissolved (if needed) and not taken off from buddy, * < 0: failed to dissolve. */static int __page_handle_poison(struct page *page){ int ret; zone_pcp_disable(page_zone(page)); ret = dissolve_free_hugetlb_folio(page_folio(page)); if (!ret) ret = take_page_off_buddy(page); zone_pcp_enable(page_zone(page)); return ret;} static bool page_handle_poison(struct page *page, bool hugepage_or_freepage, bool release){ if (hugepage_or_freepage) { /* * Doing this check for free pages is also fine since * dissolve_free_hugetlb_folio() returns 0 for non-hugetlb folios as well. */ if (__page_handle_poison(page) <= 0) /* * We could fail to take off the target page from buddy * for example due to racy page allocation, but that's * acceptable because soft-offlined page is not broken * and if someone really want to use it, they should * take it. */ return false; } SetPageHWPoison(page); if (release) put_page(page); page_ref_inc(page); num_poisoned_pages_inc(page_to_pfn(page)); return true;} static hwpoison_filter_func_t __rcu *hwpoison_filter_func __read_mostly; void hwpoison_filter_register(hwpoison_filter_func_t *filter){ rcu_assign_pointer(hwpoison_filter_func, filter);}EXPORT_SYMBOL_GPL(hwpoison_filter_register); void hwpoison_filter_unregister(void){ RCU_INIT_POINTER(hwpoison_filter_func, NULL); synchronize_rcu();}EXPORT_SYMBOL_GPL(hwpoison_filter_unregister); static int hwpoison_filter(struct page *p){ int ret = 0; hwpoison_filter_func_t *filter; rcu_read_lock(); filter = rcu_dereference(hwpoison_filter_func); if (filter) ret = filter(p); rcu_read_unlock(); return ret;} /* * Kill all processes that have a poisoned page mapped and then isolate * the page. * * General strategy: * Find all processes having the page mapped and kill them. * But we keep a page reference around so that the page is not * actually freed yet. * Then stash the page away * * There's no convenient way to get back to mapped processes * from the VMAs. So do a brute-force search over all * running processes. * * Remember that machine checks are not common (or rather * if they are common you have other problems), so this shouldn't * be a performance issue. * * Also there are some races possible while we get from the * error detection to actually handle it. */ struct to_kill { struct list_head nd; struct task_struct *tsk; unsigned long addr; short size_shift;}; /* * Send all the processes who have the page mapped a signal. * ``action optional'' if they are not immediately affected by the error * ``action required'' if error happened in current execution context */static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags){ struct task_struct *t = tk->tsk; short addr_lsb = tk->size_shift; int ret = 0; pr_err("%#lx: Sending SIGBUS to %s:%d due to hardware memory corruption\n", pfn, t->comm, task_pid_nr(t)); if ((flags & MF_ACTION_REQUIRED) && (t == current)) ret = force_sig_mceerr(BUS_MCEERR_AR, (void __user *)tk->addr, addr_lsb); else /* * Signal other processes sharing the page if they have * PF_MCE_EARLY set. * Don't use force here, it's convenient if the signal * can be temporarily blocked. */ ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr, addr_lsb, t); if (ret < 0) pr_info("Error sending signal to %s:%d: %d\n", t->comm, task_pid_nr(t), ret); return ret;} /* * Unknown page type encountered. Try to check whether it can turn PageLRU by * lru_add_drain_all. */void shake_folio(struct folio *folio){ if (folio_test_hugetlb(folio)) return; /* * TODO: Could shrink slab caches here if a lightweight range-based * shrinker will be available. */ if (folio_test_slab(folio)) return; lru_add_drain_all();}EXPORT_SYMBOL_GPL(shake_folio); static void shake_page(struct page *page){ shake_folio(page_folio(page));} static unsigned long dev_pagemap_mapping_shift(struct vm_area_struct *vma, unsigned long address){ unsigned long ret = 0; pgd_t *pgd; p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte; pte_t ptent; VM_BUG_ON_VMA(address == -EFAULT, vma); pgd = pgd_offset(vma->vm_mm, address); if (!pgd_present(*pgd)) return 0; p4d = p4d_offset(pgd, address); if (!p4d_present(*p4d)) return 0; pud = pud_offset(p4d, address); if (!pud_present(*pud)) return 0; if (pud_trans_huge(*pud)) return PUD_SHIFT; pmd = pmd_offset(pud, address); if (!pmd_present(*pmd)) return 0; if (pmd_trans_huge(*pmd)) return PMD_SHIFT; pte = pte_offset_map(pmd, address); if (!pte) return 0; ptent = ptep_get(pte); if (pte_present(ptent)) ret = PAGE_SHIFT; pte_unmap(pte); return ret;} /* * Failure handling: if we can't find or can't kill a process there's * not much we can do. We just print a message and ignore otherwise. */ /* * Schedule a process for later kill. * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM. */static void __add_to_kill(struct task_struct *tsk, const struct page *p, struct vm_area_struct *vma, struct list_head *to_kill, unsigned long addr){ struct to_kill *tk; tk = kmalloc_obj(struct to_kill, GFP_ATOMIC); if (!tk) { pr_err("Out of memory while machine check handling\n"); return; } tk->addr = addr; if (is_zone_device_page(p)) tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr); else tk->size_shift = folio_shift(page_folio(p)); /* * Send SIGKILL if "tk->addr == -EFAULT". Also, as * "tk->size_shift" is always non-zero for !is_zone_device_page(), * so "tk->size_shift == 0" effectively checks no mapping on * ZONE_DEVICE. Indeed, when a devdax page is mmapped N times * to a process' address space, it's possible not all N VMAs * contain mappings for the page, but at least one VMA does. * Only deliver SIGBUS with payload derived from the VMA that * has a mapping for the page. */ if (tk->addr == -EFAULT) { pr_info("Unable to find user space address %lx in %s\n", page_to_pfn(p), tsk->comm); } else if (tk->size_shift == 0) { kfree(tk); return; } get_task_struct(tsk); tk->tsk = tsk; list_add_tail(&tk->nd, to_kill);} static void add_to_kill_anon_file(struct task_struct *tsk, const struct page *p, struct vm_area_struct *vma, struct list_head *to_kill, unsigned long addr){ if (addr == -EFAULT) return; __add_to_kill(tsk, p, vma, to_kill, addr);} #ifdef CONFIG_KSMstatic bool task_in_to_kill_list(struct list_head *to_kill, struct task_struct *tsk){ struct to_kill *tk, *next; list_for_each_entry_safe(tk, next, to_kill, nd) { if (tk->tsk == tsk) return true; } return false;} void add_to_kill_ksm(struct task_struct *tsk, const struct page *p, struct vm_area_struct *vma, struct list_head *to_kill, unsigned long addr){ if (!task_in_to_kill_list(to_kill, tsk)) __add_to_kill(tsk, p, vma, to_kill, addr);}#endif/* * Kill the processes that have been collected earlier. * * Only do anything when FORCEKILL is set, otherwise just free the * list (this is used for clean pages which do not need killing) */static void kill_procs(struct list_head *to_kill, bool forcekill, unsigned long pfn, int flags){ struct to_kill *tk, *next; list_for_each_entry_safe(tk, next, to_kill, nd) { if (forcekill) { if (tk->addr == -EFAULT) { pr_err("%#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n", pfn, tk->tsk->comm, task_pid_nr(tk->tsk)); do_send_sig_info(SIGKILL, SEND_SIG_PRIV, tk->tsk, PIDTYPE_PID); } /* * In theory the process could have mapped * something else on the address in-between. We could * check for that, but we need to tell the * process anyways. */ else if (kill_proc(tk, pfn, flags) < 0) pr_err("%#lx: Cannot send advisory machine check signal to %s:%d\n", pfn, tk->tsk->comm, task_pid_nr(tk->tsk)); } list_del(&tk->nd); put_task_struct(tk->tsk); kfree(tk); }} /* * Find a dedicated thread which is supposed to handle SIGBUS(BUS_MCEERR_AO) * on behalf of the thread group. Return task_struct of the (first found) * dedicated thread if found, and return NULL otherwise. * * We already hold rcu lock in the caller, so we don't have to call * rcu_read_lock/unlock() in this function. */static struct task_struct *find_early_kill_thread(struct task_struct *tsk)Lines 1–500 of 3029. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2008, 2009 Intel Corporation * Authors: Andi Kleen, Fengguang Wu * * High level machine check handler. Handles pages reported by the * hardware as being corrupted usually due to a multi-bit ECC memory or cache * failure. * * In addition there is a "soft offline" entry point that allows stop using * not-yet-corrupted-by-suspicious pages without killing anything. * * Handles page cache pages in various states. The tricky part * here is that we can access any page asynchronously in respect to * other VM users, because memory failures could happen anytime and * anywhere. This could violate some of their assumptions. This is why * this code has to be extremely careful. Generally it tries to use * normal locking rules, as in get the standard locks, even if that means * the error handling takes potentially a long time. * * It can be very tempting to add handling for obscure cases here. * In general any code for handling new cases should only be added iff: * - You know how to test it. * - You have a test that can be added to mce-test * https://git.kernel.org/cgit/utils/cpu/mce/mce-test.git/ * - The case actually shows up as a frequent (top 10) page state in * tools/mm/page-types when running a real workload. * * There are several operations here with exponential complexity because * of unsuitable VM data structures. For example the operation to map back * from RMAP chains to processes has to walk the complete process list and * has non linear complexity with the number. But since memory corruptions * are rare we hope to get away with this. This avoids impacting the core * VM. */ #define pr_fmt(fmt) "Memory failure: " fmt #include <linux/kernel.h> #include <linux/mm.h> #include <linux/memory-failure.h> #include <linux/page-flags.h> #include <linux/sched/signal.h> #include <linux/sched/task.h> #include <linux/dax.h> #include <linux/ksm.h> #include <linux/rmap.h> #include <linux/export.h> #include <linux/pagemap.h> #include <linux/swap.h> #include <linux/backing-dev.h> #include <linux/migrate.h> #include <linux/slab.h> #include <linux/leafops.h> #include <linux/hugetlb.h> #include <linux/memory_hotplug.h> #include <linux/mm_inline.h> #include <linux/memremap.h> #include <linux/kfifo.h> #include <linux/ratelimit.h> #include <linux/pagewalk.h> #include <linux/shmem_fs.h> #include <linux/sysctl.h> #define CREATE_TRACE_POINTS #include <trace/events/memory-failure.h> #include "swap.h" #include "page_alloc.h" #include "internal.h" static int sysctl_memory_failure_early_kill __read_mostly; static int sysctl_memory_failure_recovery __read_mostly = 1; static int sysctl_enable_soft_offline __read_mostly = 1; static int sysctl_panic_on_unrecoverable_mf __read_mostly; atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); static bool hw_memory_failure __read_mostly; static DEFINE_MUTEX(mf_mutex); void num_poisoned_pages_inc(unsigned long pfn) { atomic_long_inc(&num_poisoned_pages); memblk_nr_poison_inc(pfn); } void num_poisoned_pages_sub(unsigned long pfn, long i) { atomic_long_sub(i, &num_poisoned_pages); if (pfn != -1UL) memblk_nr_poison_sub(pfn, i); } /** * MF_ATTR_RO - Create sysfs entry for each memory failure statistics. * @_name: name of the file in the per NUMA sysfs directory. */ #define MF_ATTR_RO(_name) \ static ssize_t _name##_show(struct device *dev, \ struct device_attribute *attr, \ char *buf) \ { \ struct memory_failure_stats *mf_stats = \ &NODE_DATA(dev->id)->mf_stats; \ return sysfs_emit(buf, "%lu\n", mf_stats->_name); \ } \ static DEVICE_ATTR_RO(_name) MF_ATTR_RO(total); MF_ATTR_RO(ignored); MF_ATTR_RO(failed); MF_ATTR_RO(delayed); MF_ATTR_RO(recovered); static struct attribute *memory_failure_attr[] = { &dev_attr_total.attr, &dev_attr_ignored.attr, &dev_attr_failed.attr, &dev_attr_delayed.attr, &dev_attr_recovered.attr, NULL, }; const struct attribute_group memory_failure_attr_group = { .name = "memory_failure", .attrs = memory_failure_attr, }; static const struct ctl_table memory_failure_table[] = { { .procname = "memory_failure_early_kill", .data = &sysctl_memory_failure_early_kill, .maxlen = sizeof(sysctl_memory_failure_early_kill), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, { .procname = "memory_failure_recovery", .data = &sysctl_memory_failure_recovery, .maxlen = sizeof(sysctl_memory_failure_recovery), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, { .procname = "enable_soft_offline", .data = &sysctl_enable_soft_offline, .maxlen = sizeof(sysctl_enable_soft_offline), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, { .procname = "panic_on_unrecoverable_memory_failure", .data = &sysctl_panic_on_unrecoverable_mf, .maxlen = sizeof(sysctl_panic_on_unrecoverable_mf), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, } }; static struct rb_root_cached pfn_space_itree = RB_ROOT_CACHED; static DEFINE_MUTEX(pfn_space_lock); /* * Return values: * 1: the page is dissolved (if needed) and taken off from buddy, * 0: the page is dissolved (if needed) and not taken off from buddy, * < 0: failed to dissolve. */ static int __page_handle_poison(struct page *page) { int ret; zone_pcp_disable(page_zone(page)); ret = dissolve_free_hugetlb_folio(page_folio(page)); if (!ret) ret = take_page_off_buddy(page); zone_pcp_enable(page_zone(page)); return ret; } static bool page_handle_poison(struct page *page, bool hugepage_or_freepage, bool release) { if (hugepage_or_freepage) { /* * Doing this check for free pages is also fine since * dissolve_free_hugetlb_folio() returns 0 for non-hugetlb folios as well. */ if (__page_handle_poison(page) <= 0) /* * We could fail to take off the target page from buddy * for example due to racy page allocation, but that's * acceptable because soft-offlined page is not broken * and if someone really want to use it, they should * take it. */ return false; } SetPageHWPoison(page); if (release) put_page(page); page_ref_inc(page); num_poisoned_pages_inc(page_to_pfn(page)); return true; } static hwpoison_filter_func_t __rcu *hwpoison_filter_func __read_mostly; void hwpoison_filter_register(hwpoison_filter_func_t *filter) { rcu_assign_pointer(hwpoison_filter_func, filter); } EXPORT_SYMBOL_GPL(hwpoison_filter_register); void hwpoison_filter_unregister(void) { RCU_INIT_POINTER(hwpoison_filter_func, NULL); synchronize_rcu(); } EXPORT_SYMBOL_GPL(hwpoison_filter_unregister); static int hwpoison_filter(struct page *p) { int ret = 0; hwpoison_filter_func_t *filter; rcu_read_lock(); filter = rcu_dereference(hwpoison_filter_func); if (filter) ret = filter(p); rcu_read_unlock(); return ret; } /* * Kill all processes that have a poisoned page mapped and then isolate * the page. * * General strategy: * Find all processes having the page mapped and kill them. * But we keep a page reference around so that the page is not * actually freed yet. * Then stash the page away * * There's no convenient way to get back to mapped processes * from the VMAs. So do a brute-force search over all * running processes. * * Remember that machine checks are not common (or rather * if they are common you have other problems), so this shouldn't * be a performance issue. * * Also there are some races possible while we get from the * error detection to actually handle it. */ struct to_kill { struct list_head nd; struct task_struct *tsk; unsigned long addr; short size_shift; }; /* * Send all the processes who have the page mapped a signal. * ``action optional'' if they are not immediately affected by the error * ``action required'' if error happened in current execution context */ static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags) { struct task_struct *t = tk->tsk; short addr_lsb = tk->size_shift; int ret = 0; pr_err("%#lx: Sending SIGBUS to %s:%d due to hardware memory corruption\n", pfn, t->comm, task_pid_nr(t)); if ((flags & MF_ACTION_REQUIRED) && (t == current)) ret = force_sig_mceerr(BUS_MCEERR_AR, (void __user *)tk->addr, addr_lsb); else /* * Signal other processes sharing the page if they have * PF_MCE_EARLY set. * Don't use force here, it's convenient if the signal * can be temporarily blocked. */ ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr, addr_lsb, t); if (ret < 0) pr_info("Error sending signal to %s:%d: %d\n", t->comm, task_pid_nr(t), ret); return ret; } /* * Unknown page type encountered. Try to check whether it can turn PageLRU by * lru_add_drain_all. */ void shake_folio(struct folio *folio) { if (folio_test_hugetlb(folio)) return; /* * TODO: Could shrink slab caches here if a lightweight range-based * shrinker will be available. */ if (folio_test_slab(folio)) return; lru_add_drain_all(); } EXPORT_SYMBOL_GPL(shake_folio); static void shake_page(struct page *page) { shake_folio(page_folio(page)); } static unsigned long dev_pagemap_mapping_shift(struct vm_area_struct *vma, unsigned long address) { unsigned long ret = 0; pgd_t *pgd; p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte; pte_t ptent; VM_BUG_ON_VMA(address == -EFAULT, vma); pgd = pgd_offset(vma->vm_mm, address); if (!pgd_present(*pgd)) return 0; p4d = p4d_offset(pgd, address); if (!p4d_present(*p4d)) return 0; pud = pud_offset(p4d, address); if (!pud_present(*pud)) return 0; if (pud_trans_huge(*pud)) return PUD_SHIFT; pmd = pmd_offset(pud, address); if (!pmd_present(*pmd)) return 0; if (pmd_trans_huge(*pmd)) return PMD_SHIFT; pte = pte_offset_map(pmd, address); if (!pte) return 0; ptent = ptep_get(pte); if (pte_present(ptent)) ret = PAGE_SHIFT; pte_unmap(pte); return ret; } /* * Failure handling: if we can't find or can't kill a process there's * not much we can do. We just print a message and ignore otherwise. */ /* * Schedule a process for later kill. * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM. */ static void __add_to_kill(struct task_struct *tsk, const struct page *p, struct vm_area_struct *vma, struct list_head *to_kill, unsigned long addr) { struct to_kill *tk; tk = kmalloc_obj(struct to_kill, GFP_ATOMIC); if (!tk) { pr_err("Out of memory while machine check handling\n"); return; } tk->addr = addr; if (is_zone_device_page(p)) tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr); else tk->size_shift = folio_shift(page_folio(p)); /* * Send SIGKILL if "tk->addr == -EFAULT". Also, as * "tk->size_shift" is always non-zero for !is_zone_device_page(), * so "tk->size_shift == 0" effectively checks no mapping on * ZONE_DEVICE. Indeed, when a devdax page is mmapped N times * to a process' address space, it's possible not all N VMAs * contain mappings for the page, but at least one VMA does. * Only deliver SIGBUS with payload derived from the VMA that * has a mapping for the page. */ if (tk->addr == -EFAULT) { pr_info("Unable to find user space address %lx in %s\n", page_to_pfn(p), tsk->comm); } else if (tk->size_shift == 0) { kfree(tk); return; } get_task_struct(tsk); tk->tsk = tsk; list_add_tail(&tk->nd, to_kill); } static void add_to_kill_anon_file(struct task_struct *tsk, const struct page *p, struct vm_area_struct *vma, struct list_head *to_kill, unsigned long addr) { if (addr == -EFAULT) return; __add_to_kill(tsk, p, vma, to_kill, addr); } #ifdef CONFIG_KSM static bool task_in_to_kill_list(struct list_head *to_kill, struct task_struct *tsk) { struct to_kill *tk, *next; list_for_each_entry_safe(tk, next, to_kill, nd) { if (tk->tsk == tsk) return true; } return false; } void add_to_kill_ksm(struct task_struct *tsk, const struct page *p, struct vm_area_struct *vma, struct list_head *to_kill, unsigned long addr) { if (!task_in_to_kill_list(to_kill, tsk)) __add_to_kill(tsk, p, vma, to_kill, addr); } #endif /* * Kill the processes that have been collected earlier. * * Only do anything when FORCEKILL is set, otherwise just free the * list (this is used for clean pages which do not need killing) */ static void kill_procs(struct list_head *to_kill, bool forcekill, unsigned long pfn, int flags) { struct to_kill *tk, *next; list_for_each_entry_safe(tk, next, to_kill, nd) { if (forcekill) { if (tk->addr == -EFAULT) { pr_err("%#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n", pfn, tk->tsk->comm, task_pid_nr(tk->tsk)); do_send_sig_info(SIGKILL, SEND_SIG_PRIV, tk->tsk, PIDTYPE_PID); } /* * In theory the process could have mapped * something else on the address in-between. We could * check for that, but we need to tell the * process anyways. */ else if (kill_proc(tk, pfn, flags) < 0) pr_err("%#lx: Cannot send advisory machine check signal to %s:%d\n", pfn, tk->tsk->comm, task_pid_nr(tk->tsk)); } list_del(&tk->nd); put_task_struct(tk->tsk); kfree(tk); } } /* * Find a dedicated thread which is supposed to handle SIGBUS(BUS_MCEERR_AO) * on behalf of the thread group. Return task_struct of the (first found) * dedicated thread if found, and return NULL otherwise. * * We already hold rcu lock in the caller, so we don't have to call * rcu_read_lock/unlock() in this function. */ static struct task_struct *find_early_kill_thread(struct task_struct *tsk)