// SPDX-License-Identifier: GPL-2.0+/* * Copyright (C) 2007 Alan Stern * Copyright (C) IBM Corporation, 2009 * Copyright (C) 2009, Frederic Weisbecker <fweisbec@gmail.com> * * Thanks to Ingo Molnar for his many suggestions. * * Authors: Alan Stern <stern@rowland.harvard.edu> *          K.Prasad <prasad@linux.vnet.ibm.com> *          Frederic Weisbecker <fweisbec@gmail.com> */ /* * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, * using the CPU's debug registers. * This file contains the arch-independent routines. */ #include <linux/hw_breakpoint.h> #include <linux/atomic.h>#include <linux/bug.h>#include <linux/cpu.h>#include <linux/export.h>#include <linux/init.h>#include <linux/irqflags.h>#include <linux/kdebug.h>#include <linux/kernel.h>#include <linux/mutex.h>#include <linux/notifier.h>#include <linux/percpu-rwsem.h>#include <linux/percpu.h>#include <linux/rhashtable.h>#include <linux/sched.h>#include <linux/slab.h> /* * Datastructure to track the total uses of N slots across tasks or CPUs; * bp_slots_histogram::count[N] is the number of assigned N+1 breakpoint slots. */struct bp_slots_histogram {#ifdef hw_breakpoint_slots	atomic_t count[hw_breakpoint_slots(0)];#else	atomic_t *count;#endif}; /* * Per-CPU constraints data. */struct bp_cpuinfo {	/* Number of pinned CPU breakpoints in a CPU. */	unsigned int			cpu_pinned;	/* Histogram of pinned task breakpoints in a CPU. */	struct bp_slots_histogram	tsk_pinned;}; static DEFINE_PER_CPU(struct bp_cpuinfo, bp_cpuinfo[TYPE_MAX]); static struct bp_cpuinfo *get_bp_info(int cpu, enum bp_type_idx type){	return per_cpu_ptr(bp_cpuinfo + type, cpu);} /* Number of pinned CPU breakpoints globally. */static struct bp_slots_histogram cpu_pinned[TYPE_MAX];/* Number of pinned CPU-independent task breakpoints. */static struct bp_slots_histogram tsk_pinned_all[TYPE_MAX]; /* Keep track of the breakpoints attached to tasks */static struct rhltable task_bps_ht;static const struct rhashtable_params task_bps_ht_params = {	.head_offset = offsetof(struct hw_perf_event, bp_list),	.key_offset = offsetof(struct hw_perf_event, target),	.key_len = sizeof_field(struct hw_perf_event, target),	.automatic_shrinking = true,}; static bool constraints_initialized __ro_after_init; /* * Synchronizes accesses to the per-CPU constraints; the locking rules are: * *  1. Atomic updates to bp_cpuinfo::tsk_pinned only require a held read-lock *     (due to bp_slots_histogram::count being atomic, no update are lost). * *  2. Holding a write-lock is required for computations that require a *     stable snapshot of all bp_cpuinfo::tsk_pinned. * *  3. In all other cases, non-atomic accesses require the appropriately held *     lock (read-lock for read-only accesses; write-lock for reads/writes). */DEFINE_STATIC_PERCPU_RWSEM(bp_cpuinfo_sem); /* * Return mutex to serialize accesses to per-task lists in task_bps_ht. Since * rhltable synchronizes concurrent insertions/deletions, independent tasks may * insert/delete concurrently; therefore, a mutex per task is sufficient. * * Uses task_struct::perf_event_mutex, to avoid extending task_struct with a * hw_breakpoint-only mutex, which may be infrequently used. The caveat here is * that hw_breakpoint may contend with per-task perf event list management. The * assumption is that perf usecases involving hw_breakpoints are very unlikely * to result in unnecessary contention. */static inline struct mutex *get_task_bps_mutex(struct perf_event *bp){	struct task_struct *tsk = bp->hw.target; 	return tsk ? &tsk->perf_event_mutex : NULL;} static struct mutex *bp_constraints_lock(struct perf_event *bp){	struct mutex *tsk_mtx = get_task_bps_mutex(bp); 	if (tsk_mtx) {		/*		 * Fully analogous to the perf_try_init_event() nesting		 * argument in the comment near perf_event_ctx_lock_nested();		 * this child->perf_event_mutex cannot ever deadlock against		 * the parent->perf_event_mutex usage from		 * perf_event_task_{en,dis}able().		 *		 * Specifically, inherited events will never occur on		 * ->perf_event_list.		 */		mutex_lock_nested(tsk_mtx, SINGLE_DEPTH_NESTING);		percpu_down_read(&bp_cpuinfo_sem);	} else {		percpu_down_write(&bp_cpuinfo_sem);	} 	return tsk_mtx;} static void bp_constraints_unlock(struct mutex *tsk_mtx){	if (tsk_mtx) {		percpu_up_read(&bp_cpuinfo_sem);		mutex_unlock(tsk_mtx);	} else {		percpu_up_write(&bp_cpuinfo_sem);	}} static bool bp_constraints_is_locked(struct perf_event *bp){	struct mutex *tsk_mtx = get_task_bps_mutex(bp); 	return percpu_is_write_locked(&bp_cpuinfo_sem) ||	       (tsk_mtx ? mutex_is_locked(tsk_mtx) :			  percpu_is_read_locked(&bp_cpuinfo_sem));} static inline void assert_bp_constraints_lock_held(struct perf_event *bp){	struct mutex *tsk_mtx = get_task_bps_mutex(bp); 	if (tsk_mtx)		lockdep_assert_held(tsk_mtx);	lockdep_assert_held(&bp_cpuinfo_sem);} #ifdef hw_breakpoint_slots/* * Number of breakpoint slots is constant, and the same for all types. */static_assert(hw_breakpoint_slots(TYPE_INST) == hw_breakpoint_slots(TYPE_DATA));static inline int hw_breakpoint_slots_cached(int type)	{ return hw_breakpoint_slots(type); }static inline int init_breakpoint_slots(void)		{ return 0; }#else/* * Dynamic number of breakpoint slots. */static int __nr_bp_slots[TYPE_MAX] __ro_after_init; static inline int hw_breakpoint_slots_cached(int type){	return __nr_bp_slots[type];} static __init boolbp_slots_histogram_alloc(struct bp_slots_histogram *hist, enum bp_type_idx type){	hist->count = kzalloc_objs(*hist->count,				   hw_breakpoint_slots_cached(type));	return hist->count;} static __init void bp_slots_histogram_free(struct bp_slots_histogram *hist){	kfree(hist->count);} static __init int init_breakpoint_slots(void){	int i, cpu, err_cpu; 	for (i = 0; i < TYPE_MAX; i++)		__nr_bp_slots[i] = hw_breakpoint_slots(i); 	for_each_possible_cpu(cpu) {		for (i = 0; i < TYPE_MAX; i++) {			struct bp_cpuinfo *info = get_bp_info(cpu, i); 			if (!bp_slots_histogram_alloc(&info->tsk_pinned, i))				goto err;		}	}	for (i = 0; i < TYPE_MAX; i++) {		if (!bp_slots_histogram_alloc(&cpu_pinned[i], i))			goto err;		if (!bp_slots_histogram_alloc(&tsk_pinned_all[i], i))			goto err;	} 	return 0;err:	for_each_possible_cpu(err_cpu) {		for (i = 0; i < TYPE_MAX; i++)			bp_slots_histogram_free(&get_bp_info(err_cpu, i)->tsk_pinned);		if (err_cpu == cpu)			break;	}	for (i = 0; i < TYPE_MAX; i++) {		bp_slots_histogram_free(&cpu_pinned[i]);		bp_slots_histogram_free(&tsk_pinned_all[i]);	} 	return -ENOMEM;}#endif static inline voidbp_slots_histogram_add(struct bp_slots_histogram *hist, int old, int val){	const int old_idx = old - 1;	const int new_idx = old_idx + val; 	if (old_idx >= 0)		WARN_ON(atomic_dec_return_relaxed(&hist->count[old_idx]) < 0);	if (new_idx >= 0)		WARN_ON(atomic_inc_return_relaxed(&hist->count[new_idx]) < 0);} static intbp_slots_histogram_max(struct bp_slots_histogram *hist, enum bp_type_idx type){	for (int i = hw_breakpoint_slots_cached(type) - 1; i >= 0; i--) {		const int count = atomic_read(&hist->count[i]); 		/* Catch unexpected writers; we want a stable snapshot. */		ASSERT_EXCLUSIVE_WRITER(hist->count[i]);		if (count > 0)			return i + 1;		WARN(count < 0, "inconsistent breakpoint slots histogram");	} 	return 0;} static intbp_slots_histogram_max_merge(struct bp_slots_histogram *hist1, struct bp_slots_histogram *hist2,			     enum bp_type_idx type){	for (int i = hw_breakpoint_slots_cached(type) - 1; i >= 0; i--) {		const int count1 = atomic_read(&hist1->count[i]);		const int count2 = atomic_read(&hist2->count[i]); 		/* Catch unexpected writers; we want a stable snapshot. */		ASSERT_EXCLUSIVE_WRITER(hist1->count[i]);		ASSERT_EXCLUSIVE_WRITER(hist2->count[i]);		if (count1 + count2 > 0)			return i + 1;		WARN(count1 < 0, "inconsistent breakpoint slots histogram");		WARN(count2 < 0, "inconsistent breakpoint slots histogram");	} 	return 0;} #ifndef hw_breakpoint_weightstatic inline int hw_breakpoint_weight(struct perf_event *bp){	return 1;}#endif static inline enum bp_type_idx find_slot_idx(u64 bp_type){	if (bp_type & HW_BREAKPOINT_RW)		return TYPE_DATA; 	return TYPE_INST;} /* * Return the maximum number of pinned breakpoints a task has in this CPU. */static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type){	struct bp_slots_histogram *tsk_pinned = &get_bp_info(cpu, type)->tsk_pinned; 	/*	 * At this point we want to have acquired the bp_cpuinfo_sem as a	 * writer to ensure that there are no concurrent writers in	 * toggle_bp_task_slot() to tsk_pinned, and we get a stable snapshot.	 */	lockdep_assert_held_write(&bp_cpuinfo_sem);	return bp_slots_histogram_max_merge(tsk_pinned, &tsk_pinned_all[type], type);} /* * Count the number of breakpoints of the same type and same task. * The given event must be not on the list. * * If @cpu is -1, but the result of task_bp_pinned() is not CPU-independent, * returns a negative value. */static int task_bp_pinned(int cpu, struct perf_event *bp, enum bp_type_idx type){	struct rhlist_head *head, *pos;	struct perf_event *iter;	int count = 0; 	/*	 * We need a stable snapshot of the per-task breakpoint list.	 */	assert_bp_constraints_lock_held(bp); 	rcu_read_lock();	head = rhltable_lookup(&task_bps_ht, &bp->hw.target, task_bps_ht_params);	if (!head)		goto out; 	rhl_for_each_entry_rcu(iter, pos, head, hw.bp_list) {		if (find_slot_idx(iter->attr.bp_type) != type)			continue; 		if (iter->cpu >= 0) {			if (cpu == -1) {				count = -1;				goto out;			} else if (cpu != iter->cpu)				continue;		} 		count += hw_breakpoint_weight(iter);	} out:	rcu_read_unlock();	return count;} static const struct cpumask *cpumask_of_bp(struct perf_event *bp){	if (bp->cpu >= 0)		return cpumask_of(bp->cpu);	return cpu_possible_mask;} /* * Returns the max pinned breakpoint slots in a given * CPU (cpu > -1) or across all of them (cpu = -1). */static intmax_bp_pinned_slots(struct perf_event *bp, enum bp_type_idx type){	const struct cpumask *cpumask = cpumask_of_bp(bp);	int pinned_slots = 0;	int cpu; 	if (bp->hw.target && bp->cpu < 0) {		int max_pinned = task_bp_pinned(-1, bp, type); 		if (max_pinned >= 0) {			/*			 * Fast path: task_bp_pinned() is CPU-independent and			 * returns the same value for any CPU.			 */			max_pinned += bp_slots_histogram_max(&cpu_pinned[type], type);			return max_pinned;		}	} 	for_each_cpu(cpu, cpumask) {		struct bp_cpuinfo *info = get_bp_info(cpu, type);		int nr; 		nr = info->cpu_pinned;		if (!bp->hw.target)			nr += max_task_bp_pinned(cpu, type);		else			nr += task_bp_pinned(cpu, bp, type); 		pinned_slots = max(nr, pinned_slots);	} 	return pinned_slots;} /* * Add/remove the given breakpoint in our constraint table */static inttoggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type, int weight){	int cpu, next_tsk_pinned; 	if (!enable)		weight = -weight; 	if (!bp->hw.target) {		/*		 * Update the pinned CPU slots, in per-CPU bp_cpuinfo and in the		 * global histogram.		 */		struct bp_cpuinfo *info = get_bp_info(bp->cpu, type); 		lockdep_assert_held_write(&bp_cpuinfo_sem);		bp_slots_histogram_add(&cpu_pinned[type], info->cpu_pinned, weight);		info->cpu_pinned += weight;		return 0;	} 	/*	 * If bp->hw.target, tsk_pinned is only modified, but not used	 * otherwise. We can permit concurrent updates as long as there are no	 * other uses: having acquired bp_cpuinfo_sem as a reader allows	 * concurrent updates here. Uses of tsk_pinned will require acquiring	 * bp_cpuinfo_sem as a writer to stabilize tsk_pinned's value.	 */	lockdep_assert_held_read(&bp_cpuinfo_sem); 	/*	 * Update the pinned task slots, in per-CPU bp_cpuinfo and in the global	 * histogram. We need to take care of 4 cases:	 *	 *  1. This breakpoint targets all CPUs (cpu < 0), and there may only	 *     exist other task breakpoints targeting all CPUs. In this case we	 *     can simply update the global slots histogram.	 *	 *  2. This breakpoint targets a specific CPU (cpu >= 0), but there may	 *     only exist other task breakpoints targeting all CPUs.	 *	 *     a. On enable: remove the existing breakpoints from the global	 *        slots histogram and use the per-CPU histogram.	 *	 *     b. On disable: re-insert the existing breakpoints into the global	 *        slots histogram and remove from per-CPU histogram.	 *	 *  3. Some other existing task breakpoints target specific CPUs. Only	 *     update the per-CPU slots histogram.	 */ 	if (!enable) {		/*		 * Remove before updating histograms so we can determine if this		 * was the last task breakpoint for a specific CPU.		 */		int ret = rhltable_remove(&task_bps_ht, &bp->hw.bp_list, task_bps_ht_params); 		if (ret)			return ret;	}	/*	 * Note: If !enable, next_tsk_pinned will not count the to-be-removed breakpoint.	 */	next_tsk_pinned = task_bp_pinned(-1, bp, type); 	if (next_tsk_pinned >= 0) {		if (bp->cpu < 0) { /* Case 1: fast path */			if (!enable)				next_tsk_pinned += hw_breakpoint_weight(bp);			bp_slots_histogram_add(&tsk_pinned_all[type], next_tsk_pinned, weight);		} else if (enable) { /* Case 2.a: slow path */			/* Add existing to per-CPU histograms. */			for_each_possible_cpu(cpu) {				bp_slots_histogram_add(&get_bp_info(cpu, type)->tsk_pinned,						       0, next_tsk_pinned);			}			/* Add this first CPU-pinned task breakpoint. */			bp_slots_histogram_add(&get_bp_info(bp->cpu, type)->tsk_pinned,					       next_tsk_pinned, weight);			/* Rebalance global task pinned histogram. */			bp_slots_histogram_add(&tsk_pinned_all[type], next_tsk_pinned,					       -next_tsk_pinned);		} else { /* Case 2.b: slow path */			/* Remove this last CPU-pinned task breakpoint. */			bp_slots_histogram_add(&get_bp_info(bp->cpu, type)->tsk_pinned,					       next_tsk_pinned + hw_breakpoint_weight(bp), weight);			/* Remove all from per-CPU histograms. */			for_each_possible_cpu(cpu) {				bp_slots_histogram_add(&get_bp_info(cpu, type)->tsk_pinned,						       next_tsk_pinned, -next_tsk_pinned);			}

Lines 1–500 of 1025. Load more