/* SPDX-License-Identifier: GPL-2.0 *//* * IRQ subsystem internal functions and variables: * * Do not ever include this file from anything else than * kernel/irq/. Do not even think about using any information outside * of this file for your non core code. */#include <linux/irqdesc.h>#include <linux/kernel_stat.h>#include <linux/pm_runtime.h>#include <linux/rcuref.h>#include <linux/sched/clock.h> #include "debugfs.h"#include "proc.h" #ifdef CONFIG_SPARSE_IRQ# define MAX_SPARSE_IRQS	INT_MAX#else# define MAX_SPARSE_IRQS	NR_IRQS#endif #define istate core_internal_state__do_not_mess_with_it extern bool noirqdebug;extern int irq_poll_cpu;extern unsigned int total_nr_irqs; extern struct irqaction chained_action; /* * Bits used by threaded handlers: * IRQTF_RUNTHREAD - signals that the interrupt handler thread should run * IRQTF_WARNED    - warning "IRQ_WAKE_THREAD w/o thread_fn" has been printed * IRQTF_AFFINITY  - irq thread is requested to adjust affinity * IRQTF_FORCED_THREAD  - irq action is force threaded * IRQTF_READY     - signals that irq thread is ready */enum {	IRQTF_RUNTHREAD,	IRQTF_WARNED,	IRQTF_AFFINITY,	IRQTF_FORCED_THREAD,	IRQTF_READY,}; /* * Bit masks for desc->core_internal_state__do_not_mess_with_it * * IRQS_AUTODETECT		- autodetection in progress * IRQS_SPURIOUS_DISABLED	- was disabled due to spurious interrupt *				  detection * IRQS_POLL_INPROGRESS		- polling in progress * IRQS_ONESHOT			- irq is not unmasked in primary handler * IRQS_REPLAY			- irq has been resent and will not be resent * 				  again until the handler has run and cleared * 				  this flag. * IRQS_WAITING			- irq is waiting * IRQS_PENDING			- irq needs to be resent and should be resent * 				  at the next available opportunity. * IRQS_SUSPENDED		- irq is suspended * IRQS_NMI			- irq line is used to deliver NMIs * IRQS_SYSFS			- descriptor has been added to sysfs */enum {	IRQS_AUTODETECT		= 0x00000001,	IRQS_SPURIOUS_DISABLED	= 0x00000002,	IRQS_POLL_INPROGRESS	= 0x00000008,	IRQS_ONESHOT		= 0x00000020,	IRQS_REPLAY		= 0x00000040,	IRQS_WAITING		= 0x00000080,	IRQS_PENDING		= 0x00000200,	IRQS_SUSPENDED		= 0x00000800,	IRQS_TIMINGS		= 0x00001000,	IRQS_NMI		= 0x00002000,	IRQS_SYSFS		= 0x00004000,}; #include "debug.h"#include "settings.h" extern int __irq_set_trigger(struct irq_desc *desc, unsigned long flags);extern void __disable_irq(struct irq_desc *desc);extern void __enable_irq(struct irq_desc *desc); #define IRQ_RESEND	true#define IRQ_NORESEND	false #define IRQ_START_FORCE	true#define IRQ_START_COND	false extern int irq_activate(struct irq_desc *desc);extern int irq_activate_and_startup(struct irq_desc *desc, bool resend);extern int irq_startup(struct irq_desc *desc, bool resend, bool force);extern void irq_startup_managed(struct irq_desc *desc); extern void irq_shutdown(struct irq_desc *desc);extern void irq_shutdown_and_deactivate(struct irq_desc *desc);extern void irq_disable(struct irq_desc *desc);extern void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu);extern void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu);extern void mask_irq(struct irq_desc *desc);extern void unmask_irq(struct irq_desc *desc);extern void unmask_threaded_irq(struct irq_desc *desc); #ifdef CONFIG_SPARSE_IRQstatic __always_inline void irq_mark_irq(unsigned int irq) { }void irq_desc_free_rcu(struct irq_desc *desc); static __always_inline bool irq_desc_get_ref(struct irq_desc *desc){	return rcuref_get(&desc->refcnt);} static __always_inline void irq_desc_put_ref(struct irq_desc *desc){	if (rcuref_put(&desc->refcnt))		irq_desc_free_rcu(desc);}#elseextern void irq_mark_irq(unsigned int irq);static __always_inline bool irq_desc_get_ref(struct irq_desc *desc) { return true; }static __always_inline void irq_desc_put_ref(struct irq_desc *desc) { }#endif irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc);irqreturn_t handle_irq_event_percpu(struct irq_desc *desc);irqreturn_t handle_irq_event(struct irq_desc *desc); /* Resending of interrupts :*/int check_irq_resend(struct irq_desc *desc, bool inject);void clear_irq_resend(struct irq_desc *desc);void irq_resend_init(struct irq_desc *desc);void __irq_wake_thread(struct irq_desc *desc, struct irqaction *action); void wake_threads_waitq(struct irq_desc *desc); #ifdef CONFIG_PROC_FSextern void register_irq_proc(unsigned int irq, struct irq_desc *desc);extern void unregister_irq_proc(unsigned int irq, struct irq_desc *desc);extern void register_handler_proc(unsigned int irq, struct irqaction *action);extern void unregister_handler_proc(unsigned int irq, struct irqaction *action);void irq_proc_update_valid(struct irq_desc *desc);#elsestatic inline void register_irq_proc(unsigned int irq, struct irq_desc *desc) { }static inline void unregister_irq_proc(unsigned int irq, struct irq_desc *desc) { }static inline void register_handler_proc(unsigned int irq,					 struct irqaction *action) { }static inline void unregister_handler_proc(unsigned int irq,					   struct irqaction *action) { }static inline void irq_proc_update_valid(struct irq_desc *desc) { }#endif struct irq_desc *irq_find_desc_at_or_after(unsigned int offset); extern bool irq_can_set_affinity_usr(unsigned int irq); extern int irq_do_set_affinity(struct irq_data *data,			       const struct cpumask *dest, bool force);extern void irq_affinity_schedule_notify_work(struct irq_desc *desc); #ifdef CONFIG_SMPextern int irq_setup_affinity(struct irq_desc *desc);#elsestatic inline int irq_setup_affinity(struct irq_desc *desc) { return 0; }#endif #define for_each_action_of_desc(desc, act)			\	for (act = desc->action; act; act = act->next) /* Inline functions for support of irq chips on slow busses */static inline void chip_bus_lock(struct irq_desc *desc){	if (unlikely(desc->irq_data.chip->irq_bus_lock))		desc->irq_data.chip->irq_bus_lock(&desc->irq_data);} static inline void chip_bus_sync_unlock(struct irq_desc *desc){	if (unlikely(desc->irq_data.chip->irq_bus_sync_unlock))		desc->irq_data.chip->irq_bus_sync_unlock(&desc->irq_data);} #define _IRQ_DESC_CHECK		(1 << 0)#define _IRQ_DESC_PERCPU	(1 << 1) #define IRQ_GET_DESC_CHECK_GLOBAL	(_IRQ_DESC_CHECK)#define IRQ_GET_DESC_CHECK_PERCPU	(_IRQ_DESC_CHECK | _IRQ_DESC_PERCPU) struct irq_desc *__irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,				     unsigned int check);void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus); __DEFINE_CLASS_IS_CONDITIONAL(irqdesc_lock, true);__DEFINE_UNLOCK_GUARD(irqdesc_lock, struct irq_desc,		      if (_T->lock) __irq_put_desc_unlock(_T->lock, _T->flags, _T->bus),		      unsigned long flags; bool bus); static inline class_irqdesc_lock_t class_irqdesc_lock_constructor(unsigned int irq, bool bus,								  unsigned int check){	class_irqdesc_lock_t _t = { .bus = bus, }; 	_t.lock = __irq_get_desc_lock(irq, &_t.flags, bus, check); 	return _t;} #define scoped_irqdesc_get_and_lock(_irq, _check)		\	scoped_guard(irqdesc_lock, _irq, false, _check) #define scoped_irqdesc_get_and_buslock(_irq, _check)		\	scoped_guard(irqdesc_lock, _irq, true, _check) #define scoped_irqdesc		((struct irq_desc *)(__guard_ptr(irqdesc_lock)(&scope))) #define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors) static inline unsigned int irqd_get(struct irq_data *d){	return __irqd_to_state(d);} /* * Manipulation functions for irq_data.state */static inline void irqd_set_move_pending(struct irq_data *d){	__irqd_to_state(d) |= IRQD_SETAFFINITY_PENDING;} static inline void irqd_clr_move_pending(struct irq_data *d){	__irqd_to_state(d) &= ~IRQD_SETAFFINITY_PENDING;} static inline void irqd_set_managed_shutdown(struct irq_data *d){	__irqd_to_state(d) |= IRQD_MANAGED_SHUTDOWN;} static inline void irqd_clr_managed_shutdown(struct irq_data *d){	__irqd_to_state(d) &= ~IRQD_MANAGED_SHUTDOWN;} static inline void irqd_clear(struct irq_data *d, unsigned int mask){	__irqd_to_state(d) &= ~mask;} static inline void irqd_set(struct irq_data *d, unsigned int mask){	__irqd_to_state(d) |= mask;} static inline bool irqd_has_set(struct irq_data *d, unsigned int mask){	return __irqd_to_state(d) & mask;} static inline void irq_state_set_disabled(struct irq_desc *desc){	irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);} static inline void irq_state_set_masked(struct irq_desc *desc){	irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);} #undef __irqd_to_state static inline void __kstat_incr_irqs_this_cpu(struct irq_desc *desc){	__this_cpu_inc(desc->kstat_irqs->cnt);	__this_cpu_inc(kstat.irqs_sum);} static inline void kstat_incr_irqs_this_cpu(struct irq_desc *desc){	__kstat_incr_irqs_this_cpu(desc);	desc->tot_count++;} static inline int irq_desc_get_node(struct irq_desc *desc){	return irq_common_data_get_node(&desc->irq_common_data);} static inline int irq_desc_is_chained(struct irq_desc *desc){	return (desc->action && desc->action == &chained_action);} static inline bool irq_is_nmi(struct irq_desc *desc){	return desc->istate & IRQS_NMI;} #ifdef CONFIG_PM_SLEEPvoid irq_pm_handle_wakeup(struct irq_desc *desc);void irq_pm_install_action(struct irq_desc *desc, struct irqaction *action);void irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action);#elsestatic inline void irq_pm_handle_wakeup(struct irq_desc *desc) { }static inline voidirq_pm_install_action(struct irq_desc *desc, struct irqaction *action) { }static inline voidirq_pm_remove_action(struct irq_desc *desc, struct irqaction *action) { }#endif #ifdef CONFIG_GENERIC_IRQ_CHIPvoid irq_init_generic_chip(struct irq_chip_generic *gc, const char *name,			   int num_ct, unsigned int irq_base,			   void __iomem *reg_base, irq_flow_handler_t handler);#elsestatic inline voidirq_init_generic_chip(struct irq_chip_generic *gc, const char *name,		      int num_ct, unsigned int irq_base,		      void __iomem *reg_base, irq_flow_handler_t handler) { }#endif /* CONFIG_GENERIC_IRQ_CHIP */ #ifdef CONFIG_GENERIC_PENDING_IRQstatic inline bool irq_can_move_pcntxt(struct irq_data *data){	return !(data->chip->flags & IRQCHIP_MOVE_DEFERRED);}static inline bool irq_move_pending(struct irq_data *data){	return irqd_is_setaffinity_pending(data);}static inline voidirq_copy_pending(struct irq_desc *desc, const struct cpumask *mask){	cpumask_copy(desc->pending_mask, mask);}static inline voidirq_get_pending(struct cpumask *mask, struct irq_desc *desc){	cpumask_copy(mask, desc->pending_mask);}static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc){	return desc->pending_mask;}bool irq_fixup_move_pending(struct irq_desc *desc, bool force_clear);void irq_force_complete_move(struct irq_desc *desc);#else /* CONFIG_GENERIC_PENDING_IRQ */static inline bool irq_can_move_pcntxt(struct irq_data *data){	return true;}static inline bool irq_move_pending(struct irq_data *data){	return false;}static inline voidirq_copy_pending(struct irq_desc *desc, const struct cpumask *mask){}static inline voidirq_get_pending(struct cpumask *mask, struct irq_desc *desc){}static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc){	return NULL;}static inline bool irq_fixup_move_pending(struct irq_desc *desc, bool fclear){	return false;}static inline void irq_force_complete_move(struct irq_desc *desc) { }#endif /* !CONFIG_GENERIC_PENDING_IRQ */ #if !defined(CONFIG_IRQ_DOMAIN) || !defined(CONFIG_IRQ_DOMAIN_HIERARCHY)static inline int irq_domain_activate_irq(struct irq_data *data, bool reserve){	irqd_set_activated(data);	return 0;}static inline void irq_domain_deactivate_irq(struct irq_data *data){	irqd_clr_activated(data);}#endif static inline struct irq_data *irqd_get_parent_data(struct irq_data *irqd){#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY	return irqd->parent_data;#else	return NULL;#endif}