// SPDX-License-Identifier: GPL-2.0/* * This file contains functions which emulate a local clock-event * device via a broadcast event source. * * Copyright(C) 2005-2006, Linutronix GmbH, Thomas Gleixner <tglx@kernel.org> * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner */#include <linux/cpu.h>#include <linux/err.h>#include <linux/hrtimer.h>#include <linux/interrupt.h>#include <linux/percpu.h>#include <linux/profile.h>#include <linux/sched.h>#include <linux/smp.h>#include <linux/module.h> #include "tick-internal.h" /* * Broadcast support for broken x86 hardware, where the local apic * timer stops in C3 state. */ static struct tick_device tick_broadcast_device;static cpumask_var_t tick_broadcast_mask __cpumask_var_read_mostly;static cpumask_var_t tick_broadcast_on __cpumask_var_read_mostly;static cpumask_var_t tmpmask __cpumask_var_read_mostly;static int tick_broadcast_forced; static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(tick_broadcast_lock); #ifdef CONFIG_TICK_ONESHOTstatic DEFINE_PER_CPU(struct clock_event_device *, tick_oneshot_wakeup_device); static void tick_broadcast_setup_oneshot(struct clock_event_device *bc, bool from_periodic);static void tick_broadcast_clear_oneshot(int cpu);static void tick_resume_broadcast_oneshot(struct clock_event_device *bc);# ifdef CONFIG_HOTPLUG_CPUstatic void tick_broadcast_oneshot_offline(unsigned int cpu);# endif#elsestatic inline voidtick_broadcast_setup_oneshot(struct clock_event_device *bc, bool from_periodic) { BUG(); }static inline void tick_broadcast_clear_oneshot(int cpu) { }static inline void tick_resume_broadcast_oneshot(struct clock_event_device *bc) { }# ifdef CONFIG_HOTPLUG_CPUstatic inline void tick_broadcast_oneshot_offline(unsigned int cpu) { }# endif#endif /* * Debugging: see timer_list.c */struct tick_device *tick_get_broadcast_device(void){	return &tick_broadcast_device;} struct cpumask *tick_get_broadcast_mask(void){	return tick_broadcast_mask;} static struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu); const struct clock_event_device *tick_get_wakeup_device(int cpu){	return tick_get_oneshot_wakeup_device(cpu);} /* * Start the device in periodic mode */static void tick_broadcast_start_periodic(struct clock_event_device *bc){	if (bc) {		bc->next_event_forced = 0;		tick_setup_periodic(bc, 1);	}} /* * Check, if the device can be utilized as broadcast device: */static bool tick_check_broadcast_device(struct clock_event_device *curdev,					struct clock_event_device *newdev){	if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||	    (newdev->features & CLOCK_EVT_FEAT_PERCPU) ||	    (newdev->features & CLOCK_EVT_FEAT_C3STOP))		return false; 	if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT &&	    !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))		return false; 	return !curdev || newdev->rating > curdev->rating;} #ifdef CONFIG_TICK_ONESHOTstatic struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu){	return per_cpu(tick_oneshot_wakeup_device, cpu);} static void tick_oneshot_wakeup_handler(struct clock_event_device *wd){	wd->next_event_forced = 0;	/*	 * If we woke up early and the tick was reprogrammed in the	 * meantime then this may be spurious but harmless.	 */	tick_receive_broadcast();} static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev,					   int cpu){	struct clock_event_device *curdev = tick_get_oneshot_wakeup_device(cpu); 	if (!newdev)		goto set_device; 	if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||	    (newdev->features & CLOCK_EVT_FEAT_C3STOP))		 return false; 	if (!(newdev->features & CLOCK_EVT_FEAT_PERCPU) ||	    !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))		return false; 	if (!cpumask_equal(newdev->cpumask, cpumask_of(cpu)))		return false; 	if (curdev && newdev->rating <= curdev->rating)		return false; 	if (!try_module_get(newdev->owner))		return false; 	newdev->event_handler = tick_oneshot_wakeup_handler;set_device:	clockevents_exchange_device(curdev, newdev);	per_cpu(tick_oneshot_wakeup_device, cpu) = newdev;	return true;}#elsestatic struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu){	return NULL;} static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev,					   int cpu){	return false;}#endif /* * Conditionally install/replace broadcast device */void tick_install_broadcast_device(struct clock_event_device *dev, int cpu){	struct clock_event_device *cur = tick_broadcast_device.evtdev; 	if (tick_set_oneshot_wakeup_device(dev, cpu))		return; 	if (!tick_check_broadcast_device(cur, dev))		return; 	if (!try_module_get(dev->owner))		return; 	clockevents_exchange_device(cur, dev);	if (cur)		cur->event_handler = clockevents_handle_noop;	tick_broadcast_device.evtdev = dev;	if (!cpumask_empty(tick_broadcast_mask))		tick_broadcast_start_periodic(dev); 	if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT))		return; 	/*	 * If the system already runs in oneshot mode, switch the newly	 * registered broadcast device to oneshot mode explicitly.	 */	if (tick_broadcast_oneshot_active()) {		tick_broadcast_switch_to_oneshot();		return;	} 	/*	 * Inform all cpus about this. We might be in a situation	 * where we did not switch to oneshot mode because the per cpu	 * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack	 * of a oneshot capable broadcast device. Without that	 * notification the systems stays stuck in periodic mode	 * forever.	 */	tick_clock_notify();} /* * Check, if the device is the broadcast device */int tick_is_broadcast_device(struct clock_event_device *dev){	return (dev && tick_broadcast_device.evtdev == dev);} int tick_broadcast_update_freq(struct clock_event_device *dev, u32 freq){	int ret = -ENODEV; 	if (tick_is_broadcast_device(dev)) {		raw_spin_lock(&tick_broadcast_lock);		ret = __clockevents_update_freq(dev, freq);		raw_spin_unlock(&tick_broadcast_lock);	}	return ret;}  static void err_broadcast(const struct cpumask *mask){	pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");} static void tick_device_setup_broadcast_func(struct clock_event_device *dev){	if (!dev->broadcast)		dev->broadcast = tick_broadcast;	if (!dev->broadcast) {		pr_warn_once("%s depends on broadcast, but no broadcast function available\n",			     dev->name);		dev->broadcast = err_broadcast;	}} /* * Check, if the device is dysfunctional and a placeholder, which * needs to be handled by the broadcast device. */int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu){	struct clock_event_device *bc = tick_broadcast_device.evtdev;	unsigned long flags;	int ret = 0; 	raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 	/*	 * Devices might be registered with both periodic and oneshot	 * mode disabled. This signals, that the device needs to be	 * operated from the broadcast device and is a placeholder for	 * the cpu local device.	 */	if (!tick_device_is_functional(dev)) {		dev->event_handler = tick_handle_periodic;		tick_device_setup_broadcast_func(dev);		cpumask_set_cpu(cpu, tick_broadcast_mask);		if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)			tick_broadcast_start_periodic(bc);		else			tick_broadcast_setup_oneshot(bc, false);		ret = 1;	} else {		/*		 * Clear the broadcast bit for this cpu if the		 * device is not power state affected.		 */		if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))			cpumask_clear_cpu(cpu, tick_broadcast_mask);		else			tick_device_setup_broadcast_func(dev); 		/*		 * Clear the broadcast bit if the CPU is not in		 * periodic broadcast on state.		 */		if (!cpumask_test_cpu(cpu, tick_broadcast_on))			cpumask_clear_cpu(cpu, tick_broadcast_mask); 		switch (tick_broadcast_device.mode) {		case TICKDEV_MODE_ONESHOT:			/*			 * If the system is in oneshot mode we can			 * unconditionally clear the oneshot mask bit,			 * because the CPU is running and therefore			 * not in an idle state which causes the power			 * state affected device to stop. Let the			 * caller initialize the device.			 */			tick_broadcast_clear_oneshot(cpu);			ret = 0;			break; 		case TICKDEV_MODE_PERIODIC:			/*			 * If the system is in periodic mode, check			 * whether the broadcast device can be			 * switched off now.			 */			if (cpumask_empty(tick_broadcast_mask) && bc)				clockevents_shutdown(bc);			/*			 * If we kept the cpu in the broadcast mask,			 * tell the caller to leave the per cpu device			 * in shutdown state. The periodic interrupt			 * is delivered by the broadcast device, if			 * the broadcast device exists and is not			 * hrtimer based.			 */			if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER))				ret = cpumask_test_cpu(cpu, tick_broadcast_mask);			break;		default:			break;		}	}	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);	return ret;} int tick_receive_broadcast(void){	struct tick_device *td = this_cpu_ptr(&tick_cpu_device);	struct clock_event_device *evt = td->evtdev; 	if (!evt)		return -ENODEV; 	if (!evt->event_handler)		return -EINVAL; 	evt->event_handler(evt);	return 0;} /* * Broadcast the event to the cpus, which are set in the mask (mangled). */static bool tick_do_broadcast(struct cpumask *mask){	int cpu = smp_processor_id();	struct tick_device *td;	bool local = false; 	/*	 * Check, if the current cpu is in the mask	 */	if (cpumask_test_cpu(cpu, mask)) {		struct clock_event_device *bc = tick_broadcast_device.evtdev; 		cpumask_clear_cpu(cpu, mask);		/*		 * We only run the local handler, if the broadcast		 * device is not hrtimer based. Otherwise we run into		 * a hrtimer recursion.		 *		 * local timer_interrupt()		 *   local_handler()		 *     expire_hrtimers()		 *       bc_handler()		 *         local_handler()		 *	     expire_hrtimers()		 */		local = !(bc->features & CLOCK_EVT_FEAT_HRTIMER);	} 	if (!cpumask_empty(mask)) {		/*		 * It might be necessary to actually check whether the devices		 * have different broadcast functions. For now, just use the		 * one of the first device. This works as long as we have this		 * misfeature only on x86 (lapic)		 */		td = &per_cpu(tick_cpu_device, cpumask_first(mask));		td->evtdev->broadcast(mask);	}	return local;} /* * Periodic broadcast: * - invoke the broadcast handlers */static bool tick_do_periodic_broadcast(void){	cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);	return tick_do_broadcast(tmpmask);} /* * Event handler for periodic broadcast ticks */static void tick_handle_periodic_broadcast(struct clock_event_device *dev){	struct tick_device *td = this_cpu_ptr(&tick_cpu_device);	bool bc_local; 	raw_spin_lock(&tick_broadcast_lock);	tick_broadcast_device.evtdev->next_event_forced = 0; 	/* Handle spurious interrupts gracefully */	if (clockevent_state_shutdown(tick_broadcast_device.evtdev)) {		raw_spin_unlock(&tick_broadcast_lock);		return;	} 	bc_local = tick_do_periodic_broadcast(); 	if (clockevent_state_oneshot(dev)) {		ktime_t next = ktime_add_ns(dev->next_event, TICK_NSEC); 		clockevents_program_event(dev, next, true);	}	raw_spin_unlock(&tick_broadcast_lock); 	/*	 * We run the handler of the local cpu after dropping	 * tick_broadcast_lock because the handler might deadlock when	 * trying to switch to oneshot mode.	 */	if (bc_local)		td->evtdev->event_handler(td->evtdev);} /** * tick_broadcast_control - Enable/disable or force broadcast mode * @mode:	The selected broadcast mode * * Called when the system enters a state where affected tick devices * might stop. Note: TICK_BROADCAST_FORCE cannot be undone. */void tick_broadcast_control(enum tick_broadcast_mode mode){	struct clock_event_device *bc, *dev;	struct tick_device *td;	int cpu, bc_stopped;	unsigned long flags; 	/* Protects also the local clockevent device. */	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);	td = this_cpu_ptr(&tick_cpu_device);	dev = td->evtdev; 	/*	 * Is the device not affected by the powerstate ?	 */	if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))		goto out; 	if (!tick_device_is_functional(dev))		goto out; 	cpu = smp_processor_id();	bc = tick_broadcast_device.evtdev;	bc_stopped = cpumask_empty(tick_broadcast_mask); 	switch (mode) {	case TICK_BROADCAST_FORCE:		tick_broadcast_forced = 1;		fallthrough;	case TICK_BROADCAST_ON:		cpumask_set_cpu(cpu, tick_broadcast_on);		if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {			/*			 * Only shutdown the cpu local device, if:			 *			 * - the broadcast device exists			 * - the broadcast device is not a hrtimer based one			 * - the broadcast device is in periodic mode to			 *   avoid a hiccup during switch to oneshot mode			 */			if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER) &&			    tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)				clockevents_shutdown(dev);		}		break; 	case TICK_BROADCAST_OFF:		if (tick_broadcast_forced)			break;		cpumask_clear_cpu(cpu, tick_broadcast_on);		if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {			if (tick_broadcast_device.mode ==			    TICKDEV_MODE_PERIODIC)				tick_setup_periodic(dev, 0);		}		break;	} 	if (bc) {

Lines 1–500 of 1248. Load more