// SPDX-License-Identifier: GPL-2.0/* * KCSAN core runtime. * * Copyright (C) 2019, Google LLC. */ #define pr_fmt(fmt) "kcsan: " fmt #include <linux/atomic.h>#include <linux/bug.h>#include <linux/delay.h>#include <linux/export.h>#include <linux/init.h>#include <linux/kernel.h>#include <linux/list.h>#include <linux/minmax.h>#include <linux/moduleparam.h>#include <linux/percpu.h>#include <linux/preempt.h>#include <linux/sched.h>#include <linux/string.h>#include <linux/uaccess.h> #include "encoding.h"#include "kcsan.h"#include "permissive.h" static bool kcsan_early_enable = IS_ENABLED(CONFIG_KCSAN_EARLY_ENABLE);unsigned int kcsan_udelay_task = CONFIG_KCSAN_UDELAY_TASK;unsigned int kcsan_udelay_interrupt = CONFIG_KCSAN_UDELAY_INTERRUPT;static long kcsan_skip_watch = CONFIG_KCSAN_SKIP_WATCH;static bool kcsan_interrupt_watcher = IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER); #ifdef MODULE_PARAM_PREFIX#undef MODULE_PARAM_PREFIX#endif#define MODULE_PARAM_PREFIX "kcsan."module_param_named(early_enable, kcsan_early_enable, bool, 0);module_param_named(udelay_task, kcsan_udelay_task, uint, 0644);module_param_named(udelay_interrupt, kcsan_udelay_interrupt, uint, 0644);module_param_named(skip_watch, kcsan_skip_watch, long, 0644);module_param_named(interrupt_watcher, kcsan_interrupt_watcher, bool, 0444); #ifdef CONFIG_KCSAN_WEAK_MEMORYstatic bool kcsan_weak_memory = true;module_param_named(weak_memory, kcsan_weak_memory, bool, 0644);#else#define kcsan_weak_memory false#endif bool kcsan_enabled; /* Per-CPU kcsan_ctx for interrupts */static DEFINE_PER_CPU(struct kcsan_ctx, kcsan_cpu_ctx) = {	.scoped_accesses	= {LIST_POISON1, NULL},}; /* * Helper macros to index into adjacent slots, starting from address slot * itself, followed by the right and left slots. * * The purpose is 2-fold: * *	1. if during insertion the address slot is already occupied, check if *	   any adjacent slots are free; *	2. accesses that straddle a slot boundary due to size that exceeds a *	   slot's range may check adjacent slots if any watchpoint matches. * * Note that accesses with very large size may still miss a watchpoint; however, * given this should be rare, this is a reasonable trade-off to make, since this * will avoid: * *	1. excessive contention between watchpoint checks and setup; *	2. larger number of simultaneous watchpoints without sacrificing *	   performance. * * Example: SLOT_IDX values for KCSAN_CHECK_ADJACENT=1, where i is [0, 1, 2]: * *   slot=0:  [ 1,  2,  0] *   slot=9:  [10, 11,  9] *   slot=63: [64, 65, 63] */#define SLOT_IDX(slot, i) (slot + ((i + KCSAN_CHECK_ADJACENT) % NUM_SLOTS)) /* * SLOT_IDX_FAST is used in the fast-path. Not first checking the address's primary * slot (middle) is fine if we assume that races occur rarely. The set of * indices {SLOT_IDX(slot, i) | i in [0, NUM_SLOTS)} is equivalent to * {SLOT_IDX_FAST(slot, i) | i in [0, NUM_SLOTS)}. */#define SLOT_IDX_FAST(slot, i) (slot + i) /* * Watchpoints, with each entry encoded as defined in encoding.h: in order to be * able to safely update and access a watchpoint without introducing locking * overhead, we encode each watchpoint as a single atomic long. The initial * zero-initialized state matches INVALID_WATCHPOINT. * * Add NUM_SLOTS-1 entries to account for overflow; this helps avoid having to * use more complicated SLOT_IDX_FAST calculation with modulo in the fast-path. */static atomic_long_t watchpoints[CONFIG_KCSAN_NUM_WATCHPOINTS + NUM_SLOTS-1]; /* * Instructions to skip watching counter, used in should_watch(). We use a * per-CPU counter to avoid excessive contention. */static DEFINE_PER_CPU(long, kcsan_skip); /* For kcsan_prandom_u32_max(). */static DEFINE_PER_CPU(u32, kcsan_rand_state); static __always_inline atomic_long_t *find_watchpoint(unsigned long addr,						      size_t size,						      bool expect_write,						      long *encoded_watchpoint){	const int slot = watchpoint_slot(addr);	const unsigned long addr_masked = addr & WATCHPOINT_ADDR_MASK;	atomic_long_t *watchpoint;	unsigned long wp_addr_masked;	size_t wp_size;	bool is_write;	int i; 	BUILD_BUG_ON(CONFIG_KCSAN_NUM_WATCHPOINTS < NUM_SLOTS); 	for (i = 0; i < NUM_SLOTS; ++i) {		watchpoint = &watchpoints[SLOT_IDX_FAST(slot, i)];		*encoded_watchpoint = atomic_long_read(watchpoint);		if (!decode_watchpoint(*encoded_watchpoint, &wp_addr_masked,				       &wp_size, &is_write))			continue; 		if (expect_write && !is_write)			continue; 		/* Check if the watchpoint matches the access. */		if (matching_access(wp_addr_masked, wp_size, addr_masked, size))			return watchpoint;	} 	return NULL;} static inline atomic_long_t *insert_watchpoint(unsigned long addr, size_t size, bool is_write){	const int slot = watchpoint_slot(addr);	const long encoded_watchpoint = encode_watchpoint(addr, size, is_write);	atomic_long_t *watchpoint;	int i; 	/* Check slot index logic, ensuring we stay within array bounds. */	BUILD_BUG_ON(SLOT_IDX(0, 0) != KCSAN_CHECK_ADJACENT);	BUILD_BUG_ON(SLOT_IDX(0, KCSAN_CHECK_ADJACENT+1) != 0);	BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS-1, KCSAN_CHECK_ADJACENT) != ARRAY_SIZE(watchpoints)-1);	BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS-1, KCSAN_CHECK_ADJACENT+1) != ARRAY_SIZE(watchpoints) - NUM_SLOTS); 	for (i = 0; i < NUM_SLOTS; ++i) {		long expect_val = INVALID_WATCHPOINT; 		/* Try to acquire this slot. */		watchpoint = &watchpoints[SLOT_IDX(slot, i)];		if (atomic_long_try_cmpxchg_relaxed(watchpoint, &expect_val, encoded_watchpoint))			return watchpoint;	} 	return NULL;} /* * Return true if watchpoint was successfully consumed, false otherwise. * * This may return false if: * *	1. another thread already consumed the watchpoint; *	2. the thread that set up the watchpoint already removed it; *	3. the watchpoint was removed and then re-used. */static __always_inline booltry_consume_watchpoint(atomic_long_t *watchpoint, long encoded_watchpoint){	return atomic_long_try_cmpxchg_relaxed(watchpoint, &encoded_watchpoint, CONSUMED_WATCHPOINT);} /* Return true if watchpoint was not touched, false if already consumed. */static inline bool consume_watchpoint(atomic_long_t *watchpoint){	return atomic_long_xchg_relaxed(watchpoint, CONSUMED_WATCHPOINT) != CONSUMED_WATCHPOINT;} /* Remove the watchpoint -- its slot may be reused after. */static inline void remove_watchpoint(atomic_long_t *watchpoint){	atomic_long_set(watchpoint, INVALID_WATCHPOINT);} static __always_inline struct kcsan_ctx *get_ctx(void){	/*	 * In interrupts, use raw_cpu_ptr to avoid unnecessary checks, that would	 * also result in calls that generate warnings in uaccess regions.	 */	return in_task() ? &current->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx);} static __always_inline voidcheck_access(const volatile void *ptr, size_t size, int type, unsigned long ip); /* Check scoped accesses; never inline because this is a slow-path! */static noinline void kcsan_check_scoped_accesses(void){	struct kcsan_ctx *ctx = get_ctx();	struct kcsan_scoped_access *scoped_access; 	if (ctx->disable_scoped)		return; 	ctx->disable_scoped++;	list_for_each_entry(scoped_access, &ctx->scoped_accesses, list) {		check_access(scoped_access->ptr, scoped_access->size,			     scoped_access->type, scoped_access->ip);	}	ctx->disable_scoped--;} /* Rules for generic atomic accesses. Called from fast-path. */static __always_inline boolis_atomic(struct kcsan_ctx *ctx, const volatile void *ptr, size_t size, int type){	if (type & KCSAN_ACCESS_ATOMIC)		return true; 	/*	 * Unless explicitly declared atomic, never consider an assertion access	 * as atomic. This allows using them also in atomic regions, such as	 * seqlocks, without implicitly changing their semantics.	 */	if (type & KCSAN_ACCESS_ASSERT)		return false; 	if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC) &&	    (type & KCSAN_ACCESS_WRITE) && size <= sizeof(long) &&	    !(type & KCSAN_ACCESS_COMPOUND) && IS_ALIGNED((unsigned long)ptr, size))		return true; /* Assume aligned writes up to word size are atomic. */ 	if (ctx->atomic_next > 0) {		/*		 * Because we do not have separate contexts for nested		 * interrupts, in case atomic_next is set, we simply assume that		 * the outer interrupt set atomic_next. In the worst case, we		 * will conservatively consider operations as atomic. This is a		 * reasonable trade-off to make, since this case should be		 * extremely rare; however, even if extremely rare, it could		 * lead to false positives otherwise.		 */		if ((hardirq_count() >> HARDIRQ_SHIFT) < 2)			--ctx->atomic_next; /* in task, or outer interrupt */		return true;	} 	return ctx->atomic_nest_count > 0 || ctx->in_flat_atomic;} static __always_inline boolshould_watch(struct kcsan_ctx *ctx, const volatile void *ptr, size_t size, int type){	/*	 * Never set up watchpoints when memory operations are atomic.	 *	 * Need to check this first, before kcsan_skip check below: (1) atomics	 * should not count towards skipped instructions, and (2) to actually	 * decrement kcsan_atomic_next for consecutive instruction stream.	 */	if (is_atomic(ctx, ptr, size, type))		return false; 	if (this_cpu_dec_return(kcsan_skip) >= 0)		return false; 	/*	 * NOTE: If we get here, kcsan_skip must always be reset in slow path	 * via reset_kcsan_skip() to avoid underflow.	 */ 	/* this operation should be watched */	return true;} /* * Returns a pseudo-random number in interval [0, ep_ro). Simple linear * congruential generator, using constants from "Numerical Recipes". */static u32 kcsan_prandom_u32_max(u32 ep_ro){	u32 state = this_cpu_read(kcsan_rand_state); 	state = 1664525 * state + 1013904223;	this_cpu_write(kcsan_rand_state, state); 	return state % ep_ro;} static inline void reset_kcsan_skip(void){	long skip_count = kcsan_skip_watch -			  (IS_ENABLED(CONFIG_KCSAN_SKIP_WATCH_RANDOMIZE) ?				   kcsan_prandom_u32_max(kcsan_skip_watch) :				   0);	this_cpu_write(kcsan_skip, skip_count);} static __always_inline bool kcsan_is_enabled(struct kcsan_ctx *ctx){	return READ_ONCE(kcsan_enabled) && !ctx->disable_count;} /* Introduce delay depending on context and configuration. */static void delay_access(int type){	unsigned int delay = in_task() ? kcsan_udelay_task : kcsan_udelay_interrupt;	/* For certain access types, skew the random delay to be longer. */	unsigned int skew_delay_order =		(type & (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_ASSERT)) ? 1 : 0; 	delay -= IS_ENABLED(CONFIG_KCSAN_DELAY_RANDOMIZE) ?			       kcsan_prandom_u32_max(delay >> skew_delay_order) :			       0;	udelay(delay);} /* * Reads the instrumented memory for value change detection; value change * detection is currently done for accesses up to a size of 8 bytes. */static __always_inline u64 read_instrumented_memory(const volatile void *ptr, size_t size){	/*	 * In the below we don't necessarily need the read of the location to	 * be atomic, and we don't use READ_ONCE(), since all we need for race	 * detection is to observe 2 different values.	 *	 * Furthermore, on certain architectures (such as arm64), READ_ONCE()	 * may turn into more complex instructions than a plain load that cannot	 * do unaligned accesses.	 */	switch (size) {	case 1:  return *(const volatile u8 *)ptr;	case 2:  return *(const volatile u16 *)ptr;	case 4:  return *(const volatile u32 *)ptr;	case 8:  return *(const volatile u64 *)ptr;	default: return 0; /* Ignore; we do not diff the values. */	}} void kcsan_save_irqtrace(struct task_struct *task){#ifdef CONFIG_TRACE_IRQFLAGS	task->kcsan_save_irqtrace = task->irqtrace;#endif} void kcsan_restore_irqtrace(struct task_struct *task){#ifdef CONFIG_TRACE_IRQFLAGS	task->irqtrace = task->kcsan_save_irqtrace;#endif} static __always_inline int get_kcsan_stack_depth(void){#ifdef CONFIG_KCSAN_WEAK_MEMORY	return current->kcsan_stack_depth;#else	BUILD_BUG();	return 0;#endif} static __always_inline void add_kcsan_stack_depth(int val){#ifdef CONFIG_KCSAN_WEAK_MEMORY	current->kcsan_stack_depth += val;#else	BUILD_BUG();#endif} static __always_inline struct kcsan_scoped_access *get_reorder_access(struct kcsan_ctx *ctx){#ifdef CONFIG_KCSAN_WEAK_MEMORY	return ctx->disable_scoped ? NULL : &ctx->reorder_access;#else	return NULL;#endif} static __always_inline boolfind_reorder_access(struct kcsan_ctx *ctx, const volatile void *ptr, size_t size,		    int type, unsigned long ip){	struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx); 	if (!reorder_access)		return false; 	/*	 * Note: If accesses are repeated while reorder_access is identical,	 * never matches the new access, because !(type & KCSAN_ACCESS_SCOPED).	 */	return reorder_access->ptr == ptr && reorder_access->size == size &&	       reorder_access->type == type && reorder_access->ip == ip;} static inline voidset_reorder_access(struct kcsan_ctx *ctx, const volatile void *ptr, size_t size,		   int type, unsigned long ip){	struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx); 	if (!reorder_access || !kcsan_weak_memory)		return; 	/*	 * To avoid nested interrupts or scheduler (which share kcsan_ctx)	 * reading an inconsistent reorder_access, ensure that the below has	 * exclusive access to reorder_access by disallowing concurrent use.	 */	ctx->disable_scoped++;	barrier();	reorder_access->ptr		= ptr;	reorder_access->size		= size;	reorder_access->type		= type | KCSAN_ACCESS_SCOPED;	reorder_access->ip		= ip;	reorder_access->stack_depth	= get_kcsan_stack_depth();	barrier();	ctx->disable_scoped--;} /* * Pull everything together: check_access() below contains the performance * critical operations; the fast-path (including check_access) functions should * all be inlinable by the instrumentation functions. * * The slow-path (kcsan_found_watchpoint, kcsan_setup_watchpoint) are * non-inlinable -- note that, we prefix these with "kcsan_" to ensure they can * be filtered from the stacktrace, as well as give them unique names for the * UACCESS whitelist of objtool. Each function uses user_access_save/restore(), * since they do not access any user memory, but instrumentation is still * emitted in UACCESS regions. */ static noinline void kcsan_found_watchpoint(const volatile void *ptr,					    size_t size,					    int type,					    unsigned long ip,					    atomic_long_t *watchpoint,					    long encoded_watchpoint){	const bool is_assert = (type & KCSAN_ACCESS_ASSERT) != 0;	struct kcsan_ctx *ctx = get_ctx();	unsigned long flags;	bool consumed; 	/*	 * We know a watchpoint exists. Let's try to keep the race-window	 * between here and finally consuming the watchpoint below as small as	 * possible -- avoid unneccessarily complex code until consumed.	 */ 	if (!kcsan_is_enabled(ctx))		return; 	/*	 * The access_mask check relies on value-change comparison. To avoid	 * reporting a race where e.g. the writer set up the watchpoint, but the	 * reader has access_mask!=0, we have to ignore the found watchpoint.	 *	 * reorder_access is never created from an access with access_mask set.	 */	if (ctx->access_mask && !find_reorder_access(ctx, ptr, size, type, ip))		return; 	/*	 * If the other thread does not want to ignore the access, and there was	 * a value change as a result of this thread's operation, we will still	 * generate a report of unknown origin.	 *	 * Use CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN=n to filter.	 */	if (!is_assert && kcsan_ignore_address(ptr))		return; 	/*	 * Consuming the watchpoint must be guarded by kcsan_is_enabled() to	 * avoid erroneously triggering reports if the context is disabled.	 */	consumed = try_consume_watchpoint(watchpoint, encoded_watchpoint);

Lines 1–500 of 1379. Load more