// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar * Copyright (C) 2005-2006 Thomas Gleixner * * This file contains driver APIs to the irq subsystem. */ #define pr_fmt(fmt) "genirq: " fmt #include <linux/irq.h>#include <linux/kthread.h>#include <linux/module.h>#include <linux/random.h>#include <linux/interrupt.h>#include <linux/irqdomain.h>#include <linux/slab.h>#include <linux/sched.h>#include <linux/sched/rt.h>#include <linux/sched/task.h>#include <linux/sched/isolation.h>#include <uapi/linux/sched/types.h>#include <linux/task_work.h> #include "internals.h" #if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)DEFINE_STATIC_KEY_FALSE(force_irqthreads_key); static int __init setup_forced_irqthreads(char *arg){	static_branch_enable(&force_irqthreads_key);	return 0;}early_param("threadirqs", setup_forced_irqthreads);#endif #ifdef CONFIG_SMPstatic inline void synchronize_irqwork(struct irq_desc *desc){	/* Synchronize pending or on the fly redirect work */	irq_work_sync(&desc->redirect.work);}#elsestatic inline void synchronize_irqwork(struct irq_desc *desc) { }#endif static int __irq_get_irqchip_state(struct irq_data *d, enum irqchip_irq_state which, bool *state); static void __synchronize_hardirq(struct irq_desc *desc, bool sync_chip){	struct irq_data *irqd = irq_desc_get_irq_data(desc);	bool inprogress; 	do {		/*		 * Wait until we're out of the critical section.  This might		 * give the wrong answer due to the lack of memory barriers.		 */		while (irqd_irq_inprogress(&desc->irq_data))			cpu_relax(); 		/* Ok, that indicated we're done: double-check carefully. */		guard(raw_spinlock_irqsave)(&desc->lock);		inprogress = irqd_irq_inprogress(&desc->irq_data); 		/*		 * If requested and supported, check at the chip whether it		 * is in flight at the hardware level, i.e. already pending		 * in a CPU and waiting for service and acknowledge.		 */		if (!inprogress && sync_chip) {			/*			 * Ignore the return code. inprogress is only updated			 * when the chip supports it.			 */			__irq_get_irqchip_state(irqd, IRQCHIP_STATE_ACTIVE,						&inprogress);		}		/* Oops, that failed? */	} while (inprogress);} /** * synchronize_hardirq - wait for pending hard IRQ handlers (on other CPUs) * @irq: interrupt number to wait for * * This function waits for any pending hard IRQ handlers for this interrupt * to complete before returning. If you use this function while holding a * resource the IRQ handler may need you will deadlock. It does not take * associated threaded handlers into account. * * Do not use this for shutdown scenarios where you must be sure that all * parts (hardirq and threaded handler) have completed. * * Returns: false if a threaded handler is active. * * This function may be called - with care - from IRQ context. * * It does not check whether there is an interrupt in flight at the * hardware level, but not serviced yet, as this might deadlock when called * with interrupts disabled and the target CPU of the interrupt is the * current CPU. */bool synchronize_hardirq(unsigned int irq){	struct irq_desc *desc = irq_to_desc(irq); 	if (desc) {		__synchronize_hardirq(desc, false);		return !atomic_read(&desc->threads_active);	} 	return true;}EXPORT_SYMBOL(synchronize_hardirq); static void __synchronize_irq(struct irq_desc *desc){	synchronize_irqwork(desc);	__synchronize_hardirq(desc, true); 	/*	 * We made sure that no hardirq handler is running. Now verify that no	 * threaded handlers are active.	 */	wait_event(desc->wait_for_threads, !atomic_read(&desc->threads_active));} /** * synchronize_irq - wait for pending IRQ handlers (on other CPUs) * @irq: interrupt number to wait for * * This function waits for any pending IRQ handlers for this interrupt to * complete before returning. If you use this function while holding a * resource the IRQ handler may need you will deadlock. * * Can only be called from preemptible code as it might sleep when * an interrupt thread is associated to @irq. * * It optionally makes sure (when the irq chip supports that method) * that the interrupt is not pending in any CPU and waiting for * service. */void synchronize_irq(unsigned int irq){	struct irq_desc *desc = irq_to_desc(irq); 	if (desc)		__synchronize_irq(desc);}EXPORT_SYMBOL(synchronize_irq); #ifdef CONFIG_SMPcpumask_var_t irq_default_affinity; static bool __irq_can_set_affinity(struct irq_desc *desc){	if (!desc || !irqd_can_balance(&desc->irq_data) ||	    !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)		return false;	return true;} /** * irq_can_set_affinity - Check if the affinity of a given irq can be set * @irq:	Interrupt to check * */int irq_can_set_affinity(unsigned int irq){	return __irq_can_set_affinity(irq_to_desc(irq));}EXPORT_SYMBOL_GPL(irq_can_set_affinity); /** * irq_can_set_affinity_usr - Check if affinity of a irq can be set from user space * @irq:	Interrupt to check * * Like irq_can_set_affinity() above, but additionally checks for the * AFFINITY_MANAGED flag. */bool irq_can_set_affinity_usr(unsigned int irq){	struct irq_desc *desc = irq_to_desc(irq); 	return __irq_can_set_affinity(desc) &&		!irqd_affinity_is_managed(&desc->irq_data);} /** * irq_set_thread_affinity - Notify irq threads to adjust affinity * @desc:	irq descriptor which has affinity changed * * Just set IRQTF_AFFINITY and delegate the affinity setting to the * interrupt thread itself. We can not call set_cpus_allowed_ptr() here as * we hold desc->lock and this code can be called from hard interrupt * context. */static void irq_set_thread_affinity(struct irq_desc *desc){	struct irqaction *action; 	for_each_action_of_desc(desc, action) {		if (action->thread) {			set_bit(IRQTF_AFFINITY, &action->thread_flags);			wake_up_process(action->thread);		}		if (action->secondary && action->secondary->thread) {			set_bit(IRQTF_AFFINITY, &action->secondary->thread_flags);			wake_up_process(action->secondary->thread);		}	}} #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASKstatic void irq_validate_effective_affinity(struct irq_data *data){	const struct cpumask *m = irq_data_get_effective_affinity_mask(data);	struct irq_chip *chip = irq_data_get_irq_chip(data); 	if (!cpumask_empty(m))		return;	pr_warn_once("irq_chip %s did not update eff. affinity mask of irq %u\n",		     chip->name, data->irq);}#elsestatic inline void irq_validate_effective_affinity(struct irq_data *data) { }#endif static DEFINE_PER_CPU(struct cpumask, __tmp_mask); int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force){	struct cpumask *tmp_mask = this_cpu_ptr(&__tmp_mask);	struct irq_desc *desc = irq_data_to_desc(data);	struct irq_chip *chip = irq_data_get_irq_chip(data);	const struct cpumask  *prog_mask;	int ret; 	if (!chip || !chip->irq_set_affinity)		return -EINVAL; 	/*	 * If this is a managed interrupt and housekeeping is enabled on	 * it check whether the requested affinity mask intersects with	 * a housekeeping CPU. If so, then remove the isolated CPUs from	 * the mask and just keep the housekeeping CPU(s). This prevents	 * the affinity setter from routing the interrupt to an isolated	 * CPU to avoid that I/O submitted from a housekeeping CPU causes	 * interrupts on an isolated one.	 *	 * If the masks do not intersect or include online CPU(s) then	 * keep the requested mask. The isolated target CPUs are only	 * receiving interrupts when the I/O operation was submitted	 * directly from them.	 *	 * If all housekeeping CPUs in the affinity mask are offline, the	 * interrupt will be migrated by the CPU hotplug code once a	 * housekeeping CPU which belongs to the affinity mask comes	 * online.	 */	if (irqd_affinity_is_managed(data) &&	    housekeeping_enabled(HK_TYPE_MANAGED_IRQ)) {		const struct cpumask *hk_mask; 		hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ); 		cpumask_and(tmp_mask, mask, hk_mask);		if (!cpumask_intersects(tmp_mask, cpu_online_mask))			prog_mask = mask;		else			prog_mask = tmp_mask;	} else {		prog_mask = mask;	} 	/*	 * Make sure we only provide online CPUs to the irqchip,	 * unless we are being asked to force the affinity (in which	 * case we do as we are told).	 */	cpumask_and(tmp_mask, prog_mask, cpu_online_mask);	if (!force && !cpumask_empty(tmp_mask))		ret = chip->irq_set_affinity(data, tmp_mask, force);	else if (force)		ret = chip->irq_set_affinity(data, mask, force);	else		ret = -EINVAL; 	switch (ret) {	case IRQ_SET_MASK_OK:	case IRQ_SET_MASK_OK_DONE:		cpumask_copy(desc->irq_common_data.affinity, mask);		fallthrough;	case IRQ_SET_MASK_OK_NOCOPY:		irq_validate_effective_affinity(data);		irq_set_thread_affinity(desc);		ret = 0;	} 	return ret;} #ifdef CONFIG_GENERIC_PENDING_IRQstatic inline int irq_set_affinity_pending(struct irq_data *data,					   const struct cpumask *dest){	struct irq_desc *desc = irq_data_to_desc(data); 	irqd_set_move_pending(data);	irq_copy_pending(desc, dest);	return 0;}#elsestatic inline int irq_set_affinity_pending(struct irq_data *data,					   const struct cpumask *dest){	return -EBUSY;}#endif static int irq_try_set_affinity(struct irq_data *data,				const struct cpumask *dest, bool force){	int ret = irq_do_set_affinity(data, dest, force); 	/*	 * In case that the underlying vector management is busy and the	 * architecture supports the generic pending mechanism then utilize	 * this to avoid returning an error to user space.	 */	if (ret == -EBUSY && !force)		ret = irq_set_affinity_pending(data, dest);	return ret;} static bool irq_set_affinity_deactivated(struct irq_data *data,					 const struct cpumask *mask){	struct irq_desc *desc = irq_data_to_desc(data); 	/*	 * Handle irq chips which can handle affinity only in activated	 * state correctly	 *	 * If the interrupt is not yet activated, just store the affinity	 * mask and do not call the chip driver at all. On activation the	 * driver has to make sure anyway that the interrupt is in a	 * usable state so startup works.	 */	if (!IS_ENABLED(CONFIG_IRQ_DOMAIN_HIERARCHY) ||	    irqd_is_activated(data) || !irqd_affinity_on_activate(data))		return false; 	cpumask_copy(desc->irq_common_data.affinity, mask);	irq_data_update_effective_affinity(data, mask);	irqd_set(data, IRQD_AFFINITY_SET);	return true;} /** * irq_affinity_schedule_notify_work - Schedule work to notify about affinity change * @desc:  Interrupt descriptor whose affinity changed */void irq_affinity_schedule_notify_work(struct irq_desc *desc){	lockdep_assert_held(&desc->lock); 	kref_get(&desc->affinity_notify->kref);	if (!schedule_work(&desc->affinity_notify->work)) {		/* Work was already scheduled, drop our extra ref */		kref_put(&desc->affinity_notify->kref, desc->affinity_notify->release);	}} int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,			    bool force){	struct irq_chip *chip = irq_data_get_irq_chip(data);	struct irq_desc *desc = irq_data_to_desc(data);	int ret = 0; 	if (!chip || !chip->irq_set_affinity)		return -EINVAL; 	if (irq_set_affinity_deactivated(data, mask))		return 0; 	if (irq_can_move_pcntxt(data) && !irqd_is_setaffinity_pending(data)) {		ret = irq_try_set_affinity(data, mask, force);	} else {		irqd_set_move_pending(data);		irq_copy_pending(desc, mask);	} 	if (desc->affinity_notify)		irq_affinity_schedule_notify_work(desc); 	irqd_set(data, IRQD_AFFINITY_SET); 	return ret;} /** * irq_update_affinity_desc - Update affinity management for an interrupt * @irq:	The interrupt number to update * @affinity:	Pointer to the affinity descriptor * * This interface can be used to configure the affinity management of * interrupts which have been allocated already. * * There are certain limitations on when it may be used - attempts to use it * for when the kernel is configured for generic IRQ reservation mode (in * config GENERIC_IRQ_RESERVATION_MODE) will fail, as it may conflict with * managed/non-managed interrupt accounting. In addition, attempts to use it on * an interrupt which is already started or which has already been configured * as managed will also fail, as these mean invalid init state or double init. */int irq_update_affinity_desc(unsigned int irq, struct irq_affinity_desc *affinity){	/*	 * Supporting this with the reservation scheme used by x86 needs	 * some more thought. Fail it for now.	 */	if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE))		return -EOPNOTSUPP; 	scoped_irqdesc_get_and_buslock(irq, 0) {		struct irq_desc *desc = scoped_irqdesc;		bool activated; 		/* Requires the interrupt to be shut down */		if (irqd_is_started(&desc->irq_data))			return -EBUSY; 		/* Interrupts which are already managed cannot be modified */		if (irqd_affinity_is_managed(&desc->irq_data))			return -EBUSY;		/*		 * Deactivate the interrupt. That's required to undo		 * anything an earlier activation has established.		 */		activated = irqd_is_activated(&desc->irq_data);		if (activated)			irq_domain_deactivate_irq(&desc->irq_data); 		if (affinity->is_managed) {			irqd_set(&desc->irq_data, IRQD_AFFINITY_MANAGED);			irqd_set(&desc->irq_data, IRQD_MANAGED_SHUTDOWN);		} 		cpumask_copy(desc->irq_common_data.affinity, &affinity->mask); 		/* Restore the activation state */		if (activated)			irq_domain_activate_irq(&desc->irq_data, false);		return 0;	}	return -EINVAL;} static int __irq_set_affinity(unsigned int irq, const struct cpumask *mask,			      bool force){	struct irq_desc *desc = irq_to_desc(irq); 	if (!desc)		return -EINVAL; 	guard(raw_spinlock_irqsave)(&desc->lock);	return irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask, force);} /** * irq_set_affinity - Set the irq affinity of a given irq * @irq:	Interrupt to set affinity * @cpumask:	cpumask * * Fails if cpumask does not contain an online CPU */int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask){	return __irq_set_affinity(irq, cpumask, false);}EXPORT_SYMBOL_GPL(irq_set_affinity); /** * irq_force_affinity - Force the irq affinity of a given irq * @irq:	Interrupt to set affinity * @cpumask:	cpumask * * Same as irq_set_affinity, but without checking the mask against * online cpus. * * Solely for low level cpu hotplug code, where we need to make per * cpu interrupts affine before the cpu becomes online. */int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask){

Lines 1–500 of 2811. Load more