// SPDX-License-Identifier: GPL-2.0-only/* * intel_pstate.c: Native P state management for Intel processors * * (C) Copyright 2012 Intel Corporation * Author: Dirk Brandewie <dirk.j.brandewie@intel.com> */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/kernel.h>#include <linux/kernel_stat.h>#include <linux/module.h>#include <linux/ktime.h>#include <linux/hrtimer.h>#include <linux/tick.h>#include <linux/slab.h>#include <linux/sched/cpufreq.h>#include <linux/sched/smt.h>#include <linux/list.h>#include <linux/cpu.h>#include <linux/cpufreq.h>#include <linux/sysfs.h>#include <linux/types.h>#include <linux/fs.h>#include <linux/acpi.h>#include <linux/vmalloc.h>#include <linux/pm_qos.h>#include <linux/bitfield.h>#include <trace/events/power.h>#include <linux/units.h> #include <asm/cpu.h>#include <asm/div64.h>#include <asm/msr.h>#include <asm/cpu_device_id.h>#include <asm/cpufeature.h>#include <asm/intel-family.h>#include "../drivers/thermal/intel/thermal_interrupt.h" #define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC) #define INTEL_CPUFREQ_TRANSITION_LATENCY 20000#define INTEL_CPUFREQ_TRANSITION_DELAY_HWP 5000#define INTEL_CPUFREQ_TRANSITION_DELAY 500 #ifdef CONFIG_ACPI#include <acpi/processor.h>#include <acpi/cppc_acpi.h>#endif #define FRAC_BITS 8#define int_tofp(X) ((int64_t)(X) << FRAC_BITS)#define fp_toint(X) ((X) >> FRAC_BITS) #define ONE_EIGHTH_FP ((int64_t)1 << (FRAC_BITS - 3)) #define EXT_BITS 6#define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS)#define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS)#define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS) static inline int32_t mul_fp(int32_t x, int32_t y){ return ((int64_t)x * (int64_t)y) >> FRAC_BITS;} static inline int32_t div_fp(s64 x, s64 y){ return div64_s64((int64_t)x << FRAC_BITS, y);} static inline int ceiling_fp(int32_t x){ int mask, ret; ret = fp_toint(x); mask = (1 << FRAC_BITS) - 1; if (x & mask) ret += 1; return ret;} static inline u64 mul_ext_fp(u64 x, u64 y){ return (x * y) >> EXT_FRAC_BITS;} static inline u64 div_ext_fp(u64 x, u64 y){ return div64_u64(x << EXT_FRAC_BITS, y);} /** * struct sample - Store performance sample * @core_avg_perf: Ratio of APERF/MPERF which is the actual average * performance during last sample period * @busy_scaled: Scaled busy value which is used to calculate next * P state. This can be different than core_avg_perf * to account for cpu idle period * @aperf: Difference of actual performance frequency clock count * read from APERF MSR between last and current sample * @mperf: Difference of maximum performance frequency clock count * read from MPERF MSR between last and current sample * @tsc: Difference of time stamp counter between last and * current sample * @time: Current time from scheduler * * This structure is used in the cpudata structure to store performance sample * data for choosing next P State. */struct sample { int32_t core_avg_perf; int32_t busy_scaled; u64 aperf; u64 mperf; u64 tsc; u64 time;}; /** * struct pstate_data - Store P state data * @current_pstate: Current requested P state * @min_pstate: Min P state possible for this platform * @max_pstate: Max P state possible for this platform * @max_pstate_physical:This is physical Max P state for a processor * This can be higher than the max_pstate which can * be limited by platform thermal design power limits * @perf_ctl_scaling: PERF_CTL P-state to frequency scaling factor * @scaling: Scaling factor between performance and frequency * @turbo_pstate: Max Turbo P state possible for this platform * @min_freq: @min_pstate frequency in cpufreq units * @max_freq: @max_pstate frequency in cpufreq units * @turbo_freq: @turbo_pstate frequency in cpufreq units * * Stores the per cpu model P state limits and current P state. */struct pstate_data { int current_pstate; int min_pstate; int max_pstate; int max_pstate_physical; int perf_ctl_scaling; int scaling; int turbo_pstate; unsigned int min_freq; unsigned int max_freq; unsigned int turbo_freq;}; /** * struct vid_data - Stores voltage information data * @min: VID data for this platform corresponding to * the lowest P state * @max: VID data corresponding to the highest P State. * @turbo: VID data for turbo P state * @ratio: Ratio of (vid max - vid min) / * (max P state - Min P State) * * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling) * This data is used in Atom platforms, where in addition to target P state, * the voltage data needs to be specified to select next P State. */struct vid_data { int min; int max; int turbo; int32_t ratio;}; /** * struct global_params - Global parameters, mostly tunable via sysfs. * @no_turbo: Whether or not to use turbo P-states. * @turbo_disabled: Whether or not turbo P-states are available at all, * based on the MSR_IA32_MISC_ENABLE value and whether or * not the maximum reported turbo P-state is different from * the maximum reported non-turbo one. * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo * P-state capacity. * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo * P-state capacity. */struct global_params { bool no_turbo; bool turbo_disabled; int max_perf_pct; int min_perf_pct;}; /** * struct cpudata - Per CPU instance data storage * @cpu: CPU number for this instance data * @policy: CPUFreq policy value * @update_util: CPUFreq utility callback information * @update_util_set: CPUFreq utility callback is set * @iowait_boost: iowait-related boost fraction * @last_update: Time of the last update. * @pstate: Stores P state limits for this CPU * @vid: Stores VID limits for this CPU * @last_sample_time: Last Sample time * @aperf_mperf_shift: APERF vs MPERF counting frequency difference * @prev_aperf: Last APERF value read from APERF MSR * @prev_mperf: Last MPERF value read from MPERF MSR * @prev_tsc: Last timestamp counter (TSC) value * @sample: Storage for storing last Sample data * @min_perf_ratio: Minimum capacity in terms of PERF or HWP ratios * @max_perf_ratio: Maximum capacity in terms of PERF or HWP ratios * @acpi_perf_data: Stores ACPI perf information read from _PSS * @valid_pss_table: Set to true for valid ACPI _PSS entries found * @epp_powersave: Last saved HWP energy performance preference * (EPP) or energy performance bias (EPB), * when policy switched to performance * @epp_policy: Last saved policy used to set EPP/EPB * @epp_default: Power on default HWP energy performance * preference/bias * @epp_cached: Cached HWP energy-performance preference value * @hwp_req_cached: Cached value of the last HWP Request MSR * @hwp_cap_cached: Cached value of the last HWP Capabilities MSR * @last_io_update: Last time when IO wake flag was set * @capacity_perf: Highest perf used for scale invariance * @sched_flags: Store scheduler flags for possible cross CPU update * @hwp_boost_min: Last HWP boosted min performance * @suspended: Whether or not the driver has been suspended. * @pd_registered: Set when a perf domain is registered for this CPU. * @hwp_notify_work: workqueue for HWP notifications. * * This structure stores per CPU instance data for all CPUs. */struct cpudata { int cpu; unsigned int policy; struct update_util_data update_util; bool update_util_set; struct pstate_data pstate; struct vid_data vid; u64 last_update; u64 last_sample_time; u64 aperf_mperf_shift; u64 prev_aperf; u64 prev_mperf; u64 prev_tsc; struct sample sample; int32_t min_perf_ratio; int32_t max_perf_ratio;#ifdef CONFIG_ACPI struct acpi_processor_performance acpi_perf_data; bool valid_pss_table;#endif unsigned int iowait_boost; s16 epp_powersave; s16 epp_policy; s16 epp_default; s16 epp_cached; u64 hwp_req_cached; u64 hwp_cap_cached; u64 last_io_update; unsigned int capacity_perf; unsigned int sched_flags; u32 hwp_boost_min; bool suspended;#ifdef CONFIG_ENERGY_MODEL bool pd_registered;#endif struct delayed_work hwp_notify_work;}; static struct cpudata **all_cpu_data; /** * struct pstate_funcs - Per CPU model specific callbacks * @get_max: Callback to get maximum non turbo effective P state * @get_max_physical: Callback to get maximum non turbo physical P state * @get_min: Callback to get minimum P state * @get_turbo: Callback to get turbo P state * @get_scaling: Callback to get frequency scaling factor * @get_cpu_scaling: Get frequency scaling factor for a given cpu * @get_aperf_mperf_shift: Callback to get the APERF vs MPERF frequency difference * @get_val: Callback to convert P state to actual MSR write value * @get_vid: Callback to get VID data for Atom platforms * * Core and Atom CPU models have different way to get P State limits. This * structure is used to store those callbacks. */struct pstate_funcs { int (*get_max)(int cpu); int (*get_max_physical)(int cpu); int (*get_min)(int cpu); int (*get_turbo)(int cpu); int (*get_scaling)(void); int (*get_cpu_scaling)(int cpu); int (*get_aperf_mperf_shift)(void); u64 (*get_val)(struct cpudata*, int pstate); void (*get_vid)(struct cpudata *);}; static struct pstate_funcs pstate_funcs __read_mostly; static bool hwp_active __ro_after_init;static bool per_cpu_limits __ro_after_init;static bool hwp_forced __ro_after_init;static bool hwp_boost __read_mostly;static bool hwp_is_hybrid;static u32 hwp_desired_mask __read_mostly = ~0U; static struct cpufreq_driver *intel_pstate_driver __read_mostly; #define INTEL_PSTATE_CORE_SCALING 100000#define HYBRID_SCALING_FACTOR_ADL 78741#define HYBRID_SCALING_FACTOR_MTL 80000#define HYBRID_SCALING_FACTOR_LNL 86957 static int hybrid_scaling_factor; static inline int core_get_scaling(void){ return INTEL_PSTATE_CORE_SCALING;} #ifdef CONFIG_ACPIstatic bool acpi_ppc;#endif static struct global_params global; static DEFINE_MUTEX(intel_pstate_driver_lock);static DEFINE_MUTEX(intel_pstate_limits_lock); #ifdef CONFIG_ACPI static bool intel_pstate_acpi_pm_profile_server(void){ if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER || acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER) return true; return false;} static bool intel_pstate_get_ppc_enable_status(void){ if (intel_pstate_acpi_pm_profile_server()) return true; return acpi_ppc;} #ifdef CONFIG_ACPI_CPPC_LIB /* The work item is needed to avoid CPU hotplug locking issues */static void intel_pstste_sched_itmt_work_fn(struct work_struct *work){ sched_set_itmt_support();} static DECLARE_WORK(sched_itmt_work, intel_pstste_sched_itmt_work_fn); #define CPPC_MAX_PERF U8_MAX static void intel_pstate_set_itmt_prio(int cpu){ struct cppc_perf_caps cppc_perf; static u32 max_highest_perf = 0, min_highest_perf = U32_MAX; int ret; ret = cppc_get_perf_caps(cpu, &cppc_perf); /* * If CPPC is not available, fall back to MSR_HWP_CAPABILITIES bits [8:0]. * * Also, on some systems with overclocking enabled, CPPC.highest_perf is * hardcoded to 0xff, so CPPC.highest_perf cannot be used to enable ITMT. * Fall back to MSR_HWP_CAPABILITIES then too. */ if (ret || cppc_perf.highest_perf == CPPC_MAX_PERF) cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached)); /* * The priorities can be set regardless of whether or not * sched_set_itmt_support(true) has been called and it is valid to * update them at any time after it has been called. */ sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu); if (max_highest_perf <= min_highest_perf) { if (cppc_perf.highest_perf > max_highest_perf) max_highest_perf = cppc_perf.highest_perf; if (cppc_perf.highest_perf < min_highest_perf) min_highest_perf = cppc_perf.highest_perf; if (max_highest_perf > min_highest_perf) { /* * This code can be run during CPU online under the * CPU hotplug locks, so sched_set_itmt_support() * cannot be called from here. Queue up a work item * to invoke it. */ schedule_work(&sched_itmt_work); } }} static int intel_pstate_get_cppc_guaranteed(int cpu){ struct cppc_perf_caps cppc_perf; int ret; ret = cppc_get_perf_caps(cpu, &cppc_perf); if (ret) return ret; if (cppc_perf.guaranteed_perf) return cppc_perf.guaranteed_perf; return cppc_perf.nominal_perf;} static int intel_pstate_cppc_get_scaling(int cpu){ struct cppc_perf_caps cppc_perf; /* * Compute the perf-to-frequency scaling factor for the given CPU if * possible, unless it would be 0. */ if (!cppc_get_perf_caps(cpu, &cppc_perf) && cppc_perf.nominal_perf && cppc_perf.nominal_freq) return div_u64(cppc_perf.nominal_freq * KHZ_PER_MHZ, cppc_perf.nominal_perf); return core_get_scaling();} #else /* CONFIG_ACPI_CPPC_LIB */static inline void intel_pstate_set_itmt_prio(int cpu){}#endif /* CONFIG_ACPI_CPPC_LIB */ static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy){ struct cpudata *cpu; int ret; int i; if (hwp_active) { intel_pstate_set_itmt_prio(policy->cpu); return; } if (!intel_pstate_get_ppc_enable_status()) return; cpu = all_cpu_data[policy->cpu]; ret = acpi_processor_register_performance(&cpu->acpi_perf_data, policy->cpu); if (ret) return; /* * Check if the control value in _PSS is for PERF_CTL MSR, which should * guarantee that the states returned by it map to the states in our * list directly. */ if (cpu->acpi_perf_data.control_register.space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) goto err; /* * If there is only one entry _PSS, simply ignore _PSS and continue as * usual without taking _PSS into account */ if (cpu->acpi_perf_data.state_count < 2) goto err; pr_debug("CPU%u - ACPI _PSS perf data\n", policy->cpu); for (i = 0; i < cpu->acpi_perf_data.state_count; i++) { pr_debug(" %cP%d: %u MHz, %u mW, 0x%x\n", (i == cpu->acpi_perf_data.state ? '*' : ' '), i, (u32) cpu->acpi_perf_data.states[i].core_frequency, (u32) cpu->acpi_perf_data.states[i].power, (u32) cpu->acpi_perf_data.states[i].control); } cpu->valid_pss_table = true; pr_debug("_PPC limits will be enforced\n"); return; err: cpu->valid_pss_table = false; acpi_processor_unregister_performance(policy->cpu);} static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy){ struct cpudata *cpu;Lines 1–500 of 3965. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * intel_pstate.c: Native P state management for Intel processors * * (C) Copyright 2012 Intel Corporation * Author: Dirk Brandewie <dirk.j.brandewie@intel.com> */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/kernel.h> #include <linux/kernel_stat.h> #include <linux/module.h> #include <linux/ktime.h> #include <linux/hrtimer.h> #include <linux/tick.h> #include <linux/slab.h> #include <linux/sched/cpufreq.h> #include <linux/sched/smt.h> #include <linux/list.h> #include <linux/cpu.h> #include <linux/cpufreq.h> #include <linux/sysfs.h> #include <linux/types.h> #include <linux/fs.h> #include <linux/acpi.h> #include <linux/vmalloc.h> #include <linux/pm_qos.h> #include <linux/bitfield.h> #include <trace/events/power.h> #include <linux/units.h> #include <asm/cpu.h> #include <asm/div64.h> #include <asm/msr.h> #include <asm/cpu_device_id.h> #include <asm/cpufeature.h> #include <asm/intel-family.h> #include "../drivers/thermal/intel/thermal_interrupt.h" #define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC) #define INTEL_CPUFREQ_TRANSITION_LATENCY 20000 #define INTEL_CPUFREQ_TRANSITION_DELAY_HWP 5000 #define INTEL_CPUFREQ_TRANSITION_DELAY 500 #ifdef CONFIG_ACPI #include <acpi/processor.h> #include <acpi/cppc_acpi.h> #endif #define FRAC_BITS 8 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS) #define fp_toint(X) ((X) >> FRAC_BITS) #define ONE_EIGHTH_FP ((int64_t)1 << (FRAC_BITS - 3)) #define EXT_BITS 6 #define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS) #define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS) #define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS) static inline int32_t mul_fp(int32_t x, int32_t y) { return ((int64_t)x * (int64_t)y) >> FRAC_BITS; } static inline int32_t div_fp(s64 x, s64 y) { return div64_s64((int64_t)x << FRAC_BITS, y); } static inline int ceiling_fp(int32_t x) { int mask, ret; ret = fp_toint(x); mask = (1 << FRAC_BITS) - 1; if (x & mask) ret += 1; return ret; } static inline u64 mul_ext_fp(u64 x, u64 y) { return (x * y) >> EXT_FRAC_BITS; } static inline u64 div_ext_fp(u64 x, u64 y) { return div64_u64(x << EXT_FRAC_BITS, y); } /** * struct sample - Store performance sample * @core_avg_perf: Ratio of APERF/MPERF which is the actual average * performance during last sample period * @busy_scaled: Scaled busy value which is used to calculate next * P state. This can be different than core_avg_perf * to account for cpu idle period * @aperf: Difference of actual performance frequency clock count * read from APERF MSR between last and current sample * @mperf: Difference of maximum performance frequency clock count * read from MPERF MSR between last and current sample * @tsc: Difference of time stamp counter between last and * current sample * @time: Current time from scheduler * * This structure is used in the cpudata structure to store performance sample * data for choosing next P State. */ struct sample { int32_t core_avg_perf; int32_t busy_scaled; u64 aperf; u64 mperf; u64 tsc; u64 time; }; /** * struct pstate_data - Store P state data * @current_pstate: Current requested P state * @min_pstate: Min P state possible for this platform * @max_pstate: Max P state possible for this platform * @max_pstate_physical:This is physical Max P state for a processor * This can be higher than the max_pstate which can * be limited by platform thermal design power limits * @perf_ctl_scaling: PERF_CTL P-state to frequency scaling factor * @scaling: Scaling factor between performance and frequency * @turbo_pstate: Max Turbo P state possible for this platform * @min_freq: @min_pstate frequency in cpufreq units * @max_freq: @max_pstate frequency in cpufreq units * @turbo_freq: @turbo_pstate frequency in cpufreq units * * Stores the per cpu model P state limits and current P state. */ struct pstate_data { int current_pstate; int min_pstate; int max_pstate; int max_pstate_physical; int perf_ctl_scaling; int scaling; int turbo_pstate; unsigned int min_freq; unsigned int max_freq; unsigned int turbo_freq; }; /** * struct vid_data - Stores voltage information data * @min: VID data for this platform corresponding to * the lowest P state * @max: VID data corresponding to the highest P State. * @turbo: VID data for turbo P state * @ratio: Ratio of (vid max - vid min) / * (max P state - Min P State) * * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling) * This data is used in Atom platforms, where in addition to target P state, * the voltage data needs to be specified to select next P State. */ struct vid_data { int min; int max; int turbo; int32_t ratio; }; /** * struct global_params - Global parameters, mostly tunable via sysfs. * @no_turbo: Whether or not to use turbo P-states. * @turbo_disabled: Whether or not turbo P-states are available at all, * based on the MSR_IA32_MISC_ENABLE value and whether or * not the maximum reported turbo P-state is different from * the maximum reported non-turbo one. * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo * P-state capacity. * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo * P-state capacity. */ struct global_params { bool no_turbo; bool turbo_disabled; int max_perf_pct; int min_perf_pct; }; /** * struct cpudata - Per CPU instance data storage * @cpu: CPU number for this instance data * @policy: CPUFreq policy value * @update_util: CPUFreq utility callback information * @update_util_set: CPUFreq utility callback is set * @iowait_boost: iowait-related boost fraction * @last_update: Time of the last update. * @pstate: Stores P state limits for this CPU * @vid: Stores VID limits for this CPU * @last_sample_time: Last Sample time * @aperf_mperf_shift: APERF vs MPERF counting frequency difference * @prev_aperf: Last APERF value read from APERF MSR * @prev_mperf: Last MPERF value read from MPERF MSR * @prev_tsc: Last timestamp counter (TSC) value * @sample: Storage for storing last Sample data * @min_perf_ratio: Minimum capacity in terms of PERF or HWP ratios * @max_perf_ratio: Maximum capacity in terms of PERF or HWP ratios * @acpi_perf_data: Stores ACPI perf information read from _PSS * @valid_pss_table: Set to true for valid ACPI _PSS entries found * @epp_powersave: Last saved HWP energy performance preference * (EPP) or energy performance bias (EPB), * when policy switched to performance * @epp_policy: Last saved policy used to set EPP/EPB * @epp_default: Power on default HWP energy performance * preference/bias * @epp_cached: Cached HWP energy-performance preference value * @hwp_req_cached: Cached value of the last HWP Request MSR * @hwp_cap_cached: Cached value of the last HWP Capabilities MSR * @last_io_update: Last time when IO wake flag was set * @capacity_perf: Highest perf used for scale invariance * @sched_flags: Store scheduler flags for possible cross CPU update * @hwp_boost_min: Last HWP boosted min performance * @suspended: Whether or not the driver has been suspended. * @pd_registered: Set when a perf domain is registered for this CPU. * @hwp_notify_work: workqueue for HWP notifications. * * This structure stores per CPU instance data for all CPUs. */ struct cpudata { int cpu; unsigned int policy; struct update_util_data update_util; bool update_util_set; struct pstate_data pstate; struct vid_data vid; u64 last_update; u64 last_sample_time; u64 aperf_mperf_shift; u64 prev_aperf; u64 prev_mperf; u64 prev_tsc; struct sample sample; int32_t min_perf_ratio; int32_t max_perf_ratio; #ifdef CONFIG_ACPI struct acpi_processor_performance acpi_perf_data; bool valid_pss_table; #endif unsigned int iowait_boost; s16 epp_powersave; s16 epp_policy; s16 epp_default; s16 epp_cached; u64 hwp_req_cached; u64 hwp_cap_cached; u64 last_io_update; unsigned int capacity_perf; unsigned int sched_flags; u32 hwp_boost_min; bool suspended; #ifdef CONFIG_ENERGY_MODEL bool pd_registered; #endif struct delayed_work hwp_notify_work; }; static struct cpudata **all_cpu_data; /** * struct pstate_funcs - Per CPU model specific callbacks * @get_max: Callback to get maximum non turbo effective P state * @get_max_physical: Callback to get maximum non turbo physical P state * @get_min: Callback to get minimum P state * @get_turbo: Callback to get turbo P state * @get_scaling: Callback to get frequency scaling factor * @get_cpu_scaling: Get frequency scaling factor for a given cpu * @get_aperf_mperf_shift: Callback to get the APERF vs MPERF frequency difference * @get_val: Callback to convert P state to actual MSR write value * @get_vid: Callback to get VID data for Atom platforms * * Core and Atom CPU models have different way to get P State limits. This * structure is used to store those callbacks. */ struct pstate_funcs { int (*get_max)(int cpu); int (*get_max_physical)(int cpu); int (*get_min)(int cpu); int (*get_turbo)(int cpu); int (*get_scaling)(void); int (*get_cpu_scaling)(int cpu); int (*get_aperf_mperf_shift)(void); u64 (*get_val)(struct cpudata*, int pstate); void (*get_vid)(struct cpudata *); }; static struct pstate_funcs pstate_funcs __read_mostly; static bool hwp_active __ro_after_init; static bool per_cpu_limits __ro_after_init; static bool hwp_forced __ro_after_init; static bool hwp_boost __read_mostly; static bool hwp_is_hybrid; static u32 hwp_desired_mask __read_mostly = ~0U; static struct cpufreq_driver *intel_pstate_driver __read_mostly; #define INTEL_PSTATE_CORE_SCALING 100000 #define HYBRID_SCALING_FACTOR_ADL 78741 #define HYBRID_SCALING_FACTOR_MTL 80000 #define HYBRID_SCALING_FACTOR_LNL 86957 static int hybrid_scaling_factor; static inline int core_get_scaling(void) { return INTEL_PSTATE_CORE_SCALING; } #ifdef CONFIG_ACPI static bool acpi_ppc; #endif static struct global_params global; static DEFINE_MUTEX(intel_pstate_driver_lock); static DEFINE_MUTEX(intel_pstate_limits_lock); #ifdef CONFIG_ACPI static bool intel_pstate_acpi_pm_profile_server(void) { if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER || acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER) return true; return false; } static bool intel_pstate_get_ppc_enable_status(void) { if (intel_pstate_acpi_pm_profile_server()) return true; return acpi_ppc; } #ifdef CONFIG_ACPI_CPPC_LIB /* The work item is needed to avoid CPU hotplug locking issues */ static void intel_pstste_sched_itmt_work_fn(struct work_struct *work) { sched_set_itmt_support(); } static DECLARE_WORK(sched_itmt_work, intel_pstste_sched_itmt_work_fn); #define CPPC_MAX_PERF U8_MAX static void intel_pstate_set_itmt_prio(int cpu) { struct cppc_perf_caps cppc_perf; static u32 max_highest_perf = 0, min_highest_perf = U32_MAX; int ret; ret = cppc_get_perf_caps(cpu, &cppc_perf); /* * If CPPC is not available, fall back to MSR_HWP_CAPABILITIES bits [8:0]. * * Also, on some systems with overclocking enabled, CPPC.highest_perf is * hardcoded to 0xff, so CPPC.highest_perf cannot be used to enable ITMT. * Fall back to MSR_HWP_CAPABILITIES then too. */ if (ret || cppc_perf.highest_perf == CPPC_MAX_PERF) cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached)); /* * The priorities can be set regardless of whether or not * sched_set_itmt_support(true) has been called and it is valid to * update them at any time after it has been called. */ sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu); if (max_highest_perf <= min_highest_perf) { if (cppc_perf.highest_perf > max_highest_perf) max_highest_perf = cppc_perf.highest_perf; if (cppc_perf.highest_perf < min_highest_perf) min_highest_perf = cppc_perf.highest_perf; if (max_highest_perf > min_highest_perf) { /* * This code can be run during CPU online under the * CPU hotplug locks, so sched_set_itmt_support() * cannot be called from here. Queue up a work item * to invoke it. */ schedule_work(&sched_itmt_work); } } } static int intel_pstate_get_cppc_guaranteed(int cpu) { struct cppc_perf_caps cppc_perf; int ret; ret = cppc_get_perf_caps(cpu, &cppc_perf); if (ret) return ret; if (cppc_perf.guaranteed_perf) return cppc_perf.guaranteed_perf; return cppc_perf.nominal_perf; } static int intel_pstate_cppc_get_scaling(int cpu) { struct cppc_perf_caps cppc_perf; /* * Compute the perf-to-frequency scaling factor for the given CPU if * possible, unless it would be 0. */ if (!cppc_get_perf_caps(cpu, &cppc_perf) && cppc_perf.nominal_perf && cppc_perf.nominal_freq) return div_u64(cppc_perf.nominal_freq * KHZ_PER_MHZ, cppc_perf.nominal_perf); return core_get_scaling(); } #else /* CONFIG_ACPI_CPPC_LIB */ static inline void intel_pstate_set_itmt_prio(int cpu) { } #endif /* CONFIG_ACPI_CPPC_LIB */ static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy) { struct cpudata *cpu; int ret; int i; if (hwp_active) { intel_pstate_set_itmt_prio(policy->cpu); return; } if (!intel_pstate_get_ppc_enable_status()) return; cpu = all_cpu_data[policy->cpu]; ret = acpi_processor_register_performance(&cpu->acpi_perf_data, policy->cpu); if (ret) return; /* * Check if the control value in _PSS is for PERF_CTL MSR, which should * guarantee that the states returned by it map to the states in our * list directly. */ if (cpu->acpi_perf_data.control_register.space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) goto err; /* * If there is only one entry _PSS, simply ignore _PSS and continue as * usual without taking _PSS into account */ if (cpu->acpi_perf_data.state_count < 2) goto err; pr_debug("CPU%u - ACPI _PSS perf data\n", policy->cpu); for (i = 0; i < cpu->acpi_perf_data.state_count; i++) { pr_debug(" %cP%d: %u MHz, %u mW, 0x%x\n", (i == cpu->acpi_perf_data.state ? '*' : ' '), i, (u32) cpu->acpi_perf_data.states[i].core_frequency, (u32) cpu->acpi_perf_data.states[i].power, (u32) cpu->acpi_perf_data.states[i].control); } cpu->valid_pss_table = true; pr_debug("_PPC limits will be enforced\n"); return; err: cpu->valid_pss_table = false; acpi_processor_unregister_performance(policy->cpu); } static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy) { struct cpudata *cpu;