| @@ -36,6 +36,7 @@ · 35 unmodified lines |
| 36 | #define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC) | 36 | #define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC) |
| 37 | | 37 | |
| 38 | #define INTEL_CPUFREQ_TRANSITION_LATENCY 20000 | 38 | #define INTEL_CPUFREQ_TRANSITION_LATENCY 20000 |
| | 39 | #define INTEL_CPUFREQ_TRANSITION_DELAY_HWP 5000 |
| 39 | #define INTEL_CPUFREQ_TRANSITION_DELAY 500 | 40 | #define INTEL_CPUFREQ_TRANSITION_DELAY 500 |
| 40 | | 41 | |
| 41 | #ifdef CONFIG_ACPI | 42 | #ifdef CONFIG_ACPI |
| @@ -220,6 +221,7 @@ struct global_params { · 178 unmodified lines |
| 220 | * preference/bias | 221 | * preference/bias |
| 221 | * @epp_saved: Saved EPP/EPB during system suspend or CPU offline | 222 | * @epp_saved: Saved EPP/EPB during system suspend or CPU offline |
| 222 | * operation | 223 | * operation |
| | 224 | * @epp_cached Cached HWP energy-performance preference value |
| 223 | * @hwp_req_cached: Cached value of the last HWP Request MSR | 225 | * @hwp_req_cached: Cached value of the last HWP Request MSR |
| 224 | * @hwp_cap_cached: Cached value of the last HWP Capabilities MSR | 226 | * @hwp_cap_cached: Cached value of the last HWP Capabilities MSR |
| 225 | * @last_io_update: Last time when IO wake flag was set | 227 | * @last_io_update: Last time when IO wake flag was set |
| @@ -257,6 +259,7 @@ struct cpudata { · 31 unmodified lines |
| 257 | s16 epp_policy; | 259 | s16 epp_policy; |
| 258 | s16 epp_default; | 260 | s16 epp_default; |
| 259 | s16 epp_saved; | 261 | s16 epp_saved; |
| | 262 | s16 epp_cached; |
| 260 | u64 hwp_req_cached; | 263 | u64 hwp_req_cached; |
| 261 | u64 hwp_cap_cached; | 264 | u64 hwp_cap_cached; |
| 262 | u64 last_io_update; | 265 | u64 last_io_update; |
| @@ -639,6 +642,26 @@ static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data, int *raw · 376 unmodified lines |
| 639 | return index; | 642 | return index; |
| 640 | } | 643 | } |
| 641 | | 644 | |
| | 645 | static int intel_pstate_set_epp(struct cpudata *cpu, u32 epp) |
| | 646 | { |
| | 647 | /* |
| | 648 | * Use the cached HWP Request MSR value, because in the active mode the |
| | 649 | * register itself may be updated by intel_pstate_hwp_boost_up() or |
| | 650 | * intel_pstate_hwp_boost_down() at any time. |
| | 651 | */ |
| | 652 | u64 value = READ_ONCE(cpu->hwp_req_cached); |
| | 653 | |
| | 654 | value &= ~GENMASK_ULL(31, 24); |
| | 655 | value |= (u64)epp << 24; |
| | 656 | /* |
| | 657 | * The only other updater of hwp_req_cached in the active mode, |
| | 658 | * intel_pstate_hwp_set(), is called under the same lock as this |
| | 659 | * function, so it cannot run in parallel with the update below. |
| | 660 | */ |
| | 661 | WRITE_ONCE(cpu->hwp_req_cached, value); |
| | 662 | return wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); |
| | 663 | } |
| | 664 | |
| 642 | static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data, | 665 | static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data, |
| 643 | int pref_index, bool use_raw, | 666 | int pref_index, bool use_raw, |
| 644 | u32 raw_epp) | 667 | u32 raw_epp) |
| @@ -650,28 +673,12 @@ static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data, · 5 unmodified lines |
| 650 | epp = cpu_data->epp_default; | 673 | epp = cpu_data->epp_default; |
| 651 | | 674 | |
| 652 | if (boot_cpu_has(X86_FEATURE_HWP_EPP)) { | 675 | if (boot_cpu_has(X86_FEATURE_HWP_EPP)) { |
| 653 | /* | | |
| 654 | * Use the cached HWP Request MSR value, because the register | | |
| 655 | * itself may be updated by intel_pstate_hwp_boost_up() or | | |
| 656 | * intel_pstate_hwp_boost_down() at any time. | | |
| 657 | */ | | |
| 658 | u64 value = READ_ONCE(cpu_data->hwp_req_cached); | | |
| 659 | | | |
| 660 | value &= ~GENMASK_ULL(31, 24); | | |
| 661 | | | |
| 662 | if (use_raw) | 676 | if (use_raw) |
| 663 | epp = raw_epp; | 677 | epp = raw_epp; |
| 664 | else if (epp == -EINVAL) | 678 | else if (epp == -EINVAL) |
| 665 | epp = epp_values[pref_index - 1]; | 679 | epp = epp_values[pref_index - 1]; |
| 666 | | 680 | |
| 667 | value |= (u64)epp << 24; | 681 | ret = intel_pstate_set_epp(cpu_data, epp); |
| 668 | /* | | |
| 669 | * The only other updater of hwp_req_cached in the active mode, | | |
| 670 | * intel_pstate_hwp_set(), is called under the same lock as this | | |
| 671 | * function, so it cannot run in parallel with the update below. | | |
| 672 | */ | | |
| 673 | WRITE_ONCE(cpu_data->hwp_req_cached, value); | | |
| 674 | ret = wrmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, value); | | |
| 675 | } else { | 682 | } else { |
| 676 | if (epp == -EINVAL) | 683 | if (epp == -EINVAL) |
| 677 | epp = (pref_index - 1) << 2; | 684 | epp = (pref_index - 1) << 2; |
| @@ -697,10 +704,12 @@ static ssize_t show_energy_performance_available_preferences( · 19 unmodified lines |
| 697 | | 704 | |
| 698 | cpufreq_freq_attr_ro(energy_performance_available_preferences); | 705 | cpufreq_freq_attr_ro(energy_performance_available_preferences); |
| 699 | | 706 | |
| | 707 | static struct cpufreq_driver intel_pstate; |
| | 708 | |
| 700 | static ssize_t store_energy_performance_preference( | 709 | static ssize_t store_energy_performance_preference( |
| 701 | struct cpufreq_policy *policy, const char *buf, size_t count) | 710 | struct cpufreq_policy *policy, const char *buf, size_t count) |
| 702 | { | 711 | { |
| 703 | struct cpudata *cpu_data = all_cpu_data[policy->cpu]; | 712 | struct cpudata *cpu = all_cpu_data[policy->cpu]; |
| 704 | char str_preference[21]; | 713 | char str_preference[21]; |
| 705 | bool raw = false; | 714 | bool raw = false; |
| 706 | ssize_t ret; | 715 | ssize_t ret; |
| @@ -725,15 +734,44 @@ static ssize_t store_energy_performance_preference( · 18 unmodified lines |
| 725 | raw = true; | 734 | raw = true; |
| 726 | } | 735 | } |
| 727 | | 736 | |
| | 737 | /* |
| | 738 | * This function runs with the policy R/W semaphore held, which |
| | 739 | * guarantees that the driver pointer will not change while it is |
| | 740 | * running. |
| | 741 | */ |
| | 742 | if (!intel_pstate_driver) |
| | 743 | return -EAGAIN; |
| | 744 | |
| 728 | mutex_lock(&intel_pstate_limits_lock); | 745 | mutex_lock(&intel_pstate_limits_lock); |
| 729 | | 746 | |
| 730 | ret = intel_pstate_set_energy_pref_index(cpu_data, ret, raw, epp); | 747 | if (intel_pstate_driver == &intel_pstate) { |
| 731 | if (!ret) | 748 | ret = intel_pstate_set_energy_pref_index(cpu, ret, raw, epp); |
| 732 | ret = count; | 749 | } else { |
| | 750 | /* |
| | 751 | * In the passive mode the governor needs to be stopped on the |
| | 752 | * target CPU before the EPP update and restarted after it, |
| | 753 | * which is super-heavy-weight, so make sure it is worth doing |
| | 754 | * upfront. |
| | 755 | */ |
| | 756 | if (!raw) |
| | 757 | epp = ret ? epp_values[ret - 1] : cpu->epp_default; |
| | 758 | |
| | 759 | if (cpu->epp_cached != epp) { |
| | 760 | int err; |
| | 761 | |
| | 762 | cpufreq_stop_governor(policy); |
| | 763 | ret = intel_pstate_set_epp(cpu, epp); |
| | 764 | err = cpufreq_start_governor(policy); |
| | 765 | if (!ret) { |
| | 766 | cpu->epp_cached = epp; |
| | 767 | ret = err; |
| | 768 | } |
| | 769 | } |
| | 770 | } |
| 733 | | 771 | |
| 734 | mutex_unlock(&intel_pstate_limits_lock); | 772 | mutex_unlock(&intel_pstate_limits_lock); |
| 735 | | 773 | |
| 736 | return ret; | 774 | return ret ?: count; |
| 737 | } | 775 | } |
| 738 | | 776 | |
| 739 | static ssize_t show_energy_performance_preference( | 777 | static ssize_t show_energy_performance_preference( |
| @@ -1145,8 +1183,6 @@ static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b, · 405 unmodified lines |
| 1145 | return count; | 1183 | return count; |
| 1146 | } | 1184 | } |
| 1147 | | 1185 | |
| 1148 | static struct cpufreq_driver intel_pstate; | | |
| 1149 | | | |
| 1150 | static void update_qos_request(enum freq_qos_req_type type) | 1186 | static void update_qos_request(enum freq_qos_req_type type) |
| 1151 | { | 1187 | { |
| 1152 | int max_state, turbo_max, freq, i, perf_pct; | 1188 | int max_state, turbo_max, freq, i, perf_pct; |
| @@ -1330,9 +1366,10 @@ static const struct attribute_group intel_pstate_attr_group = { · 177 unmodified lines |
| 1330 | | 1366 | |
| 1331 | static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[]; | 1367 | static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[]; |
| 1332 | | 1368 | |
| | 1369 | static struct kobject *intel_pstate_kobject; |
| | 1370 | |
| 1333 | static void __init intel_pstate_sysfs_expose_params(void) | 1371 | static void __init intel_pstate_sysfs_expose_params(void) |
| 1334 | { | 1372 | { |
| 1335 | struct kobject *intel_pstate_kobject; | | |
| 1336 | int rc; | 1373 | int rc; |
| 1337 | | 1374 | |
| 1338 | intel_pstate_kobject = kobject_create_and_add("intel_pstate", | 1375 | intel_pstate_kobject = kobject_create_and_add("intel_pstate", |
| @@ -1357,17 +1394,31 @@ static void __init intel_pstate_sysfs_expose_params(void) · 18 unmodified lines |
| 1357 | rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr); | 1394 | rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr); |
| 1358 | WARN_ON(rc); | 1395 | WARN_ON(rc); |
| 1359 | | 1396 | |
| 1360 | if (hwp_active) { | | |
| 1361 | rc = sysfs_create_file(intel_pstate_kobject, | | |
| 1362 | &hwp_dynamic_boost.attr); | | |
| 1363 | WARN_ON(rc); | | |
| 1364 | } | | |
| 1365 | | | |
| 1366 | if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids)) { | 1397 | if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids)) { |
| 1367 | rc = sysfs_create_file(intel_pstate_kobject, &energy_efficiency.attr); | 1398 | rc = sysfs_create_file(intel_pstate_kobject, &energy_efficiency.attr); |
| 1368 | WARN_ON(rc); | 1399 | WARN_ON(rc); |
| 1369 | } | 1400 | } |
| 1370 | } | 1401 | } |
| | 1402 | |
| | 1403 | static void intel_pstate_sysfs_expose_hwp_dynamic_boost(void) |
| | 1404 | { |
| | 1405 | int rc; |
| | 1406 | |
| | 1407 | if (!hwp_active) |
| | 1408 | return; |
| | 1409 | |
| | 1410 | rc = sysfs_create_file(intel_pstate_kobject, &hwp_dynamic_boost.attr); |
| | 1411 | WARN_ON_ONCE(rc); |
| | 1412 | } |
| | 1413 | |
| | 1414 | static void intel_pstate_sysfs_hide_hwp_dynamic_boost(void) |
| | 1415 | { |
| | 1416 | if (!hwp_active) |
| | 1417 | return; |
| | 1418 | |
| | 1419 | sysfs_remove_file(intel_pstate_kobject, &hwp_dynamic_boost.attr); |
| | 1420 | } |
| | 1421 | |
| 1371 | /************************** sysfs end ************************/ | 1422 | /************************** sysfs end ************************/ |
| 1372 | | 1423 | |
| 1373 | static void intel_pstate_hwp_enable(struct cpudata *cpudata) | 1424 | static void intel_pstate_hwp_enable(struct cpudata *cpudata) |
| @@ -2247,7 +2298,10 @@ static int intel_pstate_verify_policy(struct cpufreq_policy_data *policy) · 873 unmodified lines |
| 2247 | | 2298 | |
| 2248 | static void intel_cpufreq_stop_cpu(struct cpufreq_policy *policy) | 2299 | static void intel_cpufreq_stop_cpu(struct cpufreq_policy *policy) |
| 2249 | { | 2300 | { |
| 2250 | intel_pstate_set_min_pstate(all_cpu_data[policy->cpu]); | 2301 | if (hwp_active) |
| | 2302 | intel_pstate_hwp_force_min_perf(policy->cpu); |
| | 2303 | else |
| | 2304 | intel_pstate_set_min_pstate(all_cpu_data[policy->cpu]); |
| 2251 | } | 2305 | } |
| 2252 | | 2306 | |
| 2253 | static void intel_pstate_stop_cpu(struct cpufreq_policy *policy) | 2307 | static void intel_pstate_stop_cpu(struct cpufreq_policy *policy) |
| @@ -2255,12 +2309,10 @@ static void intel_pstate_stop_cpu(struct cpufreq_policy *policy) · 1 unmodified line |
| 2255 | pr_debug("CPU %d exiting\n", policy->cpu); | 2309 | pr_debug("CPU %d exiting\n", policy->cpu); |
| 2256 | | 2310 | |
| 2257 | intel_pstate_clear_update_util_hook(policy->cpu); | 2311 | intel_pstate_clear_update_util_hook(policy->cpu); |
| 2258 | if (hwp_active) { | 2312 | if (hwp_active) |
| 2259 | intel_pstate_hwp_save_state(policy); | 2313 | intel_pstate_hwp_save_state(policy); |
| 2260 | intel_pstate_hwp_force_min_perf(policy->cpu); | 2314 | |
| 2261 | } else { | 2315 | intel_cpufreq_stop_cpu(policy); |
| 2262 | intel_cpufreq_stop_cpu(policy); | | |
| 2263 | } | | |
| 2264 | } | 2316 | } |
| 2265 | | 2317 | |
| 2266 | static int intel_pstate_cpu_exit(struct cpufreq_policy *policy) | 2318 | static int intel_pstate_cpu_exit(struct cpufreq_policy *policy) |
| @@ -2390,13 +2442,71 @@ static void intel_cpufreq_trace(struct cpudata *cpu, unsigned int trace_type, in · 123 unmodified lines |
| 2390 | fp_toint(cpu->iowait_boost * 100)); | 2442 | fp_toint(cpu->iowait_boost * 100)); |
| 2391 | } | 2443 | } |
| 2392 | | 2444 | |
| | 2445 | static void intel_cpufreq_adjust_hwp(struct cpudata *cpu, u32 target_pstate, |
| | 2446 | bool fast_switch) |
| | 2447 | { |
| | 2448 | u64 prev = READ_ONCE(cpu->hwp_req_cached), value = prev; |
| | 2449 | |
| | 2450 | value &= ~HWP_MIN_PERF(~0L); |
| | 2451 | value |= HWP_MIN_PERF(target_pstate); |
| | 2452 | |
| | 2453 | /* |
| | 2454 | * The entire MSR needs to be updated in order to update the HWP min |
| | 2455 | * field in it, so opportunistically update the max too if needed. |
| | 2456 | */ |
| | 2457 | value &= ~HWP_MAX_PERF(~0L); |
| | 2458 | value |= HWP_MAX_PERF(cpu->max_perf_ratio); |
| | 2459 | |
| | 2460 | if (value == prev) |
| | 2461 | return; |
| | 2462 | |
| | 2463 | WRITE_ONCE(cpu->hwp_req_cached, value); |
| | 2464 | if (fast_switch) |
| | 2465 | wrmsrl(MSR_HWP_REQUEST, value); |
| | 2466 | else |
| | 2467 | wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); |
| | 2468 | } |
| | 2469 | |
| | 2470 | static void intel_cpufreq_adjust_perf_ctl(struct cpudata *cpu, |
| | 2471 | u32 target_pstate, bool fast_switch) |
| | 2472 | { |
| | 2473 | if (fast_switch) |
| | 2474 | wrmsrl(MSR_IA32_PERF_CTL, |
| | 2475 | pstate_funcs.get_val(cpu, target_pstate)); |
| | 2476 | else |
| | 2477 | wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, |
| | 2478 | pstate_funcs.get_val(cpu, target_pstate)); |
| | 2479 | } |
| | 2480 | |
| | 2481 | static int intel_cpufreq_update_pstate(struct cpudata *cpu, int target_pstate, |
| | 2482 | bool fast_switch) |
| | 2483 | { |
| | 2484 | int old_pstate = cpu->pstate.current_pstate; |
| | 2485 | |
| | 2486 | target_pstate = intel_pstate_prepare_request(cpu, target_pstate); |
| | 2487 | if (target_pstate != old_pstate) { |
| | 2488 | cpu->pstate.current_pstate = target_pstate; |
| | 2489 | if (hwp_active) |
| | 2490 | intel_cpufreq_adjust_hwp(cpu, target_pstate, |
| | 2491 | fast_switch); |
| | 2492 | else |
| | 2493 | intel_cpufreq_adjust_perf_ctl(cpu, target_pstate, |
| | 2494 | fast_switch); |
| | 2495 | } |
| | 2496 | |
| | 2497 | intel_cpufreq_trace(cpu, fast_switch ? INTEL_PSTATE_TRACE_FAST_SWITCH : |
| | 2498 | INTEL_PSTATE_TRACE_TARGET, old_pstate); |
| | 2499 | |
| | 2500 | return target_pstate; |
| | 2501 | } |
| | 2502 | |
| 2393 | static int intel_cpufreq_target(struct cpufreq_policy *policy, | 2503 | static int intel_cpufreq_target(struct cpufreq_policy *policy, |
| 2394 | unsigned int target_freq, | 2504 | unsigned int target_freq, |
| 2395 | unsigned int relation) | 2505 | unsigned int relation) |
| 2396 | { | 2506 | { |
| 2397 | struct cpudata *cpu = all_cpu_data[policy->cpu]; | 2507 | struct cpudata *cpu = all_cpu_data[policy->cpu]; |
| 2398 | struct cpufreq_freqs freqs; | 2508 | struct cpufreq_freqs freqs; |
| 2399 | int target_pstate, old_pstate; | 2509 | int target_pstate; |
| 2400 | | 2510 | |
| 2401 | update_turbo_state(); | 2511 | update_turbo_state(); |
| 2402 | | 2512 | |
| @@ -2404,6 +2514,7 @@ static int intel_cpufreq_target(struct cpufreq_policy *policy, · 1 unmodified line |
| 2404 | freqs.new = target_freq; | 2514 | freqs.new = target_freq; |
| 2405 | | 2515 | |
| 2406 | cpufreq_freq_transition_begin(policy, &freqs); | 2516 | cpufreq_freq_transition_begin(policy, &freqs); |
| | 2517 | |
| 2407 | switch (relation) { | 2518 | switch (relation) { |
| 2408 | case CPUFREQ_RELATION_L: | 2519 | case CPUFREQ_RELATION_L: |
| 2409 | target_pstate = DIV_ROUND_UP(freqs.new, cpu->pstate.scaling); | 2520 | target_pstate = DIV_ROUND_UP(freqs.new, cpu->pstate.scaling); |
| @@ -2415,15 +2526,11 @@ static int intel_cpufreq_target(struct cpufreq_policy *policy, · 5 unmodified lines |
| 2415 | target_pstate = DIV_ROUND_CLOSEST(freqs.new, cpu->pstate.scaling); | 2526 | target_pstate = DIV_ROUND_CLOSEST(freqs.new, cpu->pstate.scaling); |
| 2416 | break; | 2527 | break; |
| 2417 | } | 2528 | } |
| 2418 | target_pstate = intel_pstate_prepare_request(cpu, target_pstate); | 2529 | |
| 2419 | old_pstate = cpu->pstate.current_pstate; | 2530 | target_pstate = intel_cpufreq_update_pstate(cpu, target_pstate, false); |
| 2420 | if (target_pstate != cpu->pstate.current_pstate) { | 2531 | |
| 2421 | cpu->pstate.current_pstate = target_pstate; | | |
| 2422 | wrmsrl_on_cpu(policy->cpu, MSR_IA32_PERF_CTL, | | |
| 2423 | pstate_funcs.get_val(cpu, target_pstate)); | | |
| 2424 | } | | |
| 2425 | freqs.new = target_pstate * cpu->pstate.scaling; | 2532 | freqs.new = target_pstate * cpu->pstate.scaling; |
| 2426 | intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_TARGET, old_pstate); | 2533 | |
| 2427 | cpufreq_freq_transition_end(policy, &freqs, false); | 2534 | cpufreq_freq_transition_end(policy, &freqs, false); |
| 2428 | | 2535 | |
| 2429 | return 0; | 2536 | return 0; |
| @@ -2433,15 +2540,14 @@ static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy, · 3 unmodified lines |
| 2433 | unsigned int target_freq) | 2540 | unsigned int target_freq) |
| 2434 | { | 2541 | { |
| 2435 | struct cpudata *cpu = all_cpu_data[policy->cpu]; | 2542 | struct cpudata *cpu = all_cpu_data[policy->cpu]; |
| 2436 | int target_pstate, old_pstate; | 2543 | int target_pstate; |
| 2437 | | 2544 | |
| 2438 | update_turbo_state(); | 2545 | update_turbo_state(); |
| 2439 | | 2546 | |
| 2440 | target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling); | 2547 | target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling); |
| 2441 | target_pstate = intel_pstate_prepare_request(cpu, target_pstate); | 2548 | |
| 2442 | old_pstate = cpu->pstate.current_pstate; | 2549 | target_pstate = intel_cpufreq_update_pstate(cpu, target_pstate, true); |
| 2443 | intel_pstate_update_pstate(cpu, target_pstate); | 2550 | |
| 2444 | intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_FAST_SWITCH, old_pstate); | | |
| 2445 | return target_pstate * cpu->pstate.scaling; | 2551 | return target_pstate * cpu->pstate.scaling; |
| 2446 | } | 2552 | } |
| 2447 | | 2553 | |
| @@ -2461,7 +2567,6 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) · 13 unmodified lines |
| 2461 | return ret; | 2567 | return ret; |
| 2462 | | 2568 | |
| 2463 | policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY; | 2569 | policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY; |
| 2464 | policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY; | | |
| 2465 | /* This reflects the intel_pstate_get_cpu_pstates() setting. */ | 2570 | /* This reflects the intel_pstate_get_cpu_pstates() setting. */ |
| 2466 | policy->cur = policy->cpuinfo.min_freq; | 2571 | policy->cur = policy->cpuinfo.min_freq; |
| 2467 | | 2572 | |
| @@ -2473,10 +2578,18 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) · 5 unmodified lines |
| 2473 | | 2578 | |
| 2474 | cpu = all_cpu_data[policy->cpu]; | 2579 | cpu = all_cpu_data[policy->cpu]; |
| 2475 | | 2580 | |
| 2476 | if (hwp_active) | 2581 | if (hwp_active) { |
| | 2582 | u64 value; |
| | 2583 | |
| 2477 | intel_pstate_get_hwp_max(policy->cpu, &turbo_max, &max_state); | 2584 | intel_pstate_get_hwp_max(policy->cpu, &turbo_max, &max_state); |
| 2478 | else | 2585 | policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY_HWP; |
| | 2586 | rdmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, &value); |
| | 2587 | WRITE_ONCE(cpu->hwp_req_cached, value); |
| | 2588 | cpu->epp_cached = (value & GENMASK_ULL(31, 24)) >> 24; |
| | 2589 | } else { |
| 2479 | turbo_max = cpu->pstate.turbo_pstate; | 2590 | turbo_max = cpu->pstate.turbo_pstate; |
| | 2591 | policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY; |
| | 2592 | } |
| 2480 | | 2593 | |
| 2481 | min_freq = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100); | 2594 | min_freq = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100); |
| 2482 | min_freq *= cpu->pstate.scaling; | 2595 | min_freq *= cpu->pstate.scaling; |
| @@ -2553,6 +2666,10 @@ static void intel_pstate_driver_cleanup(void) · 70 unmodified lines |
| 2553 | } | 2666 | } |
| 2554 | } | 2667 | } |
| 2555 | put_online_cpus(); | 2668 | put_online_cpus(); |
| | 2669 | |
| | 2670 | if (intel_pstate_driver == &intel_pstate) |
| | 2671 | intel_pstate_sysfs_hide_hwp_dynamic_boost(); |
| | 2672 | |
| 2556 | intel_pstate_driver = NULL; | 2673 | intel_pstate_driver = NULL; |
| 2557 | } | 2674 | } |
| 2558 | | 2675 | |
| @@ -2560,6 +2677,9 @@ static int intel_pstate_register_driver(struct cpufreq_driver *driver) · 1 unmodified line |
| 2560 | { | 2677 | { |
| 2561 | int ret; | 2678 | int ret; |
| 2562 | | 2679 | |
| | 2680 | if (driver == &intel_pstate) |
| | 2681 | intel_pstate_sysfs_expose_hwp_dynamic_boost(); |
| | 2682 | |
| 2563 | memset(&global, 0, sizeof(global)); | 2683 | memset(&global, 0, sizeof(global)); |
| 2564 | global.max_perf_pct = 100; | 2684 | global.max_perf_pct = 100; |
| 2565 | | 2685 | |
| @@ -2577,9 +2697,6 @@ static int intel_pstate_register_driver(struct cpufreq_driver *driver) · 11 unmodified lines |
| 2577 | | 2697 | |
| 2578 | static int intel_pstate_unregister_driver(void) | 2698 | static int intel_pstate_unregister_driver(void) |
| 2579 | { | 2699 | { |
| 2580 | if (hwp_active) | | |
| 2581 | return -EBUSY; | | |
| 2582 | | | |
| 2583 | cpufreq_unregister_driver(intel_pstate_driver); | 2700 | cpufreq_unregister_driver(intel_pstate_driver); |
| 2584 | intel_pstate_driver_cleanup(); | 2701 | intel_pstate_driver_cleanup(); |
| 2585 | | 2702 | |
| @@ -2835,7 +2952,10 @@ static int __init intel_pstate_init(void) · 249 unmodified lines |
| 2835 | hwp_active++; | 2952 | hwp_active++; |
| 2836 | hwp_mode_bdw = id->driver_data; | 2953 | hwp_mode_bdw = id->driver_data; |
| 2837 | intel_pstate.attr = hwp_cpufreq_attrs; | 2954 | intel_pstate.attr = hwp_cpufreq_attrs; |
| 2838 | default_driver = &intel_pstate; | 2955 | intel_cpufreq.attr = hwp_cpufreq_attrs; |
| | 2956 | if (!default_driver) |
| | 2957 | default_driver = &intel_pstate; |
| | 2958 | |
| 2839 | goto hwp_cpu_matched; | 2959 | goto hwp_cpu_matched; |
| 2840 | } | 2960 | } |
| 2841 | } else { | 2961 | } else { |
| @@ -2906,14 +3026,13 @@ static int __init intel_pstate_setup(char *str) · 64 unmodified lines |
| 2906 | if (!str) | 3026 | if (!str) |
| 2907 | return -EINVAL; | 3027 | return -EINVAL; |
| 2908 | | 3028 | |
| 2909 | if (!strcmp(str, "disable")) { | 3029 | if (!strcmp(str, "disable")) |
| 2910 | no_load = 1; | 3030 | no_load = 1; |
| 2911 | } else if (!strcmp(str, "active")) { | 3031 | else if (!strcmp(str, "active")) |
| 2912 | default_driver = &intel_pstate; | 3032 | default_driver = &intel_pstate; |
| 2913 | } else if (!strcmp(str, "passive")) { | 3033 | else if (!strcmp(str, "passive")) |
| 2914 | default_driver = &intel_cpufreq; | 3034 | default_driver = &intel_cpufreq; |
| 2915 | no_hwp = 1; | 3035 | |
| 2916 | } | | |
| 2917 | if (!strcmp(str, "no_hwp")) { | 3036 | if (!strcmp(str, "no_hwp")) { |
| 2918 | pr_info("HWP disabled\n"); | 3037 | pr_info("HWP disabled\n"); |
| 2919 | no_hwp = 1; | 3038 | no_hwp = 1; |