History

intel_idle: Disable IBRS during long idle

Having IBRS enabled while the SMT sibling is idle unnecessarily slows down the running sibling. OTOH, disabling IBRS around idle takes two MSR writes, which will increase the idle latency. Therefore, only disable IBRS around deeper idle states. Shallow idle states are bounded by the tick in duration, since NOHZ is not allowed for them by virtue of their short target residency. Only do this for mwait-driven idle, since that keeps interrupts disabled across idle, which makes disabling IBRS vs IRQ-entry a non-issue. Note: C6 is a random threshold, most importantly C1 probably shouldn't disable IBRS, benchmarking needed.

Peter Zijlstra committed 4 years ago

Commit bf5835b1 parent 6ad0ad23 files · +45 −6Browse files

Unified · Split

3 files
  1. arch/x86/include/asm/nospec-branch.h +1 −0
  2. arch/x86/kernel/cpu/bugs.c +6 −0
  3. drivers/idle/intel_idle.c +38 −6
arch/x86/include/asm/nospec-branch.h+1−0
@@ -255,6 +255,7 @@ static inline void indirect_branch_prediction_barrier(void) · 254 unmodified lines
255/* The Intel SPEC CTRL MSR base value cache */255/* The Intel SPEC CTRL MSR base value cache */
256extern u64 x86_spec_ctrl_base;256extern u64 x86_spec_ctrl_base;
257extern void write_spec_ctrl_current(u64 val, bool force);257extern void write_spec_ctrl_current(u64 val, bool force);
258extern u64 spec_ctrl_current(void);
258 259
259/*260/*
260 * With retpoline, we must use IBRS to restrict branch prediction261 * With retpoline, we must use IBRS to restrict branch prediction
arch/x86/kernel/cpu/bugs.c+6−0
@@ -79,6 +79,12 @@ void write_spec_ctrl_current(u64 val, bool force) · 78 unmodified lines
79 wrmsrl(MSR_IA32_SPEC_CTRL, val);79 wrmsrl(MSR_IA32_SPEC_CTRL, val);
80}80}
81 81
82u64 spec_ctrl_current(void)
83{
84 return this_cpu_read(x86_spec_ctrl_current);
85}
86EXPORT_SYMBOL_GPL(spec_ctrl_current);
87
82/*88/*
83 * The vendor and possibly platform specific bits which can be modified in89 * The vendor and possibly platform specific bits which can be modified in
84 * x86_spec_ctrl_base.90 * x86_spec_ctrl_base.
drivers/idle/intel_idle.c+38−6
@@ -47,11 +47,13 @@ · 46 unmodified lines
47#include <linux/tick.h>47#include <linux/tick.h>
48#include <trace/events/power.h>48#include <trace/events/power.h>
49#include <linux/sched.h>49#include <linux/sched.h>
50#include <linux/sched/smt.h>
50#include <linux/notifier.h>51#include <linux/notifier.h>
51#include <linux/cpu.h>52#include <linux/cpu.h>
52#include <linux/moduleparam.h>53#include <linux/moduleparam.h>
53#include <asm/cpu_device_id.h>54#include <asm/cpu_device_id.h>
54#include <asm/intel-family.h>55#include <asm/intel-family.h>
56#include <asm/nospec-branch.h>
55#include <asm/mwait.h>57#include <asm/mwait.h>
56#include <asm/msr.h>58#include <asm/msr.h>
57 59
@@ -105,6 +107,12 @@ static unsigned int mwait_substates __initdata; · 47 unmodified lines
105 */107 */
106#define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15)108#define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15)
107 109
110/*
111 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
112 * above.
113 */
114#define CPUIDLE_FLAG_IBRS BIT(16)
115
108/*116/*
109 * MWAIT takes an 8-bit "hint" in EAX "suggesting"117 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
110 * the C-state (top nibble) and sub-state (bottom nibble)118 * the C-state (top nibble) and sub-state (bottom nibble)
@@ -159,6 +167,24 @@ static __cpuidle int intel_idle_irq(struct cpuidle_device *dev, · 48 unmodified lines
159 return ret;167 return ret;
160}168}
161 169
170static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
171 struct cpuidle_driver *drv, int index)
172{
173 bool smt_active = sched_smt_active();
174 u64 spec_ctrl = spec_ctrl_current();
175 int ret;
176
177 if (smt_active)
178 wrmsrl(MSR_IA32_SPEC_CTRL, 0);
179
180 ret = __intel_idle(dev, drv, index);
181
182 if (smt_active)
183 wrmsrl(MSR_IA32_SPEC_CTRL, spec_ctrl);
184
185 return ret;
186}
187
162/**188/**
163 * intel_idle_s2idle - Ask the processor to enter the given idle state.189 * intel_idle_s2idle - Ask the processor to enter the given idle state.
164 * @dev: cpuidle device of the target CPU.190 * @dev: cpuidle device of the target CPU.
@@ -680,7 +706,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { · 515 unmodified lines
680 {706 {
681 .name = "C6",707 .name = "C6",
682 .desc = "MWAIT 0x20",708 .desc = "MWAIT 0x20",
683 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,709 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
684 .exit_latency = 85,710 .exit_latency = 85,
685 .target_residency = 200,711 .target_residency = 200,
686 .enter = &intel_idle,712 .enter = &intel_idle,
@@ -688,7 +714,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { · 1 unmodified line
688 {714 {
689 .name = "C7s",715 .name = "C7s",
690 .desc = "MWAIT 0x33",716 .desc = "MWAIT 0x33",
691 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED,717 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
692 .exit_latency = 124,718 .exit_latency = 124,
693 .target_residency = 800,719 .target_residency = 800,
694 .enter = &intel_idle,720 .enter = &intel_idle,
@@ -696,7 +722,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { · 1 unmodified line
696 {722 {
697 .name = "C8",723 .name = "C8",
698 .desc = "MWAIT 0x40",724 .desc = "MWAIT 0x40",
699 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,725 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
700 .exit_latency = 200,726 .exit_latency = 200,
701 .target_residency = 800,727 .target_residency = 800,
702 .enter = &intel_idle,728 .enter = &intel_idle,
@@ -704,7 +730,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { · 1 unmodified line
704 {730 {
705 .name = "C9",731 .name = "C9",
706 .desc = "MWAIT 0x50",732 .desc = "MWAIT 0x50",
707 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,733 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
708 .exit_latency = 480,734 .exit_latency = 480,
709 .target_residency = 5000,735 .target_residency = 5000,
710 .enter = &intel_idle,736 .enter = &intel_idle,
@@ -712,7 +738,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { · 1 unmodified line
712 {738 {
713 .name = "C10",739 .name = "C10",
714 .desc = "MWAIT 0x60",740 .desc = "MWAIT 0x60",
715 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,741 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
716 .exit_latency = 890,742 .exit_latency = 890,
717 .target_residency = 5000,743 .target_residency = 5000,
718 .enter = &intel_idle,744 .enter = &intel_idle,
@@ -741,7 +767,7 @@ static struct cpuidle_state skx_cstates[] __initdata = { · 22 unmodified lines
741 {767 {
742 .name = "C6",768 .name = "C6",
743 .desc = "MWAIT 0x20",769 .desc = "MWAIT 0x20",
744 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,770 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
745 .exit_latency = 133,771 .exit_latency = 133,
746 .target_residency = 600,772 .target_residency = 600,
747 .enter = &intel_idle,773 .enter = &intel_idle,
@@ -1819,6 +1845,12 @@ static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) · 1071 unmodified lines
1819 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_IRQ_ENABLE)1845 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_IRQ_ENABLE)
1820 drv->states[drv->state_count].enter = intel_idle_irq;1846 drv->states[drv->state_count].enter = intel_idle_irq;
1821 1847
1848 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) &&
1849 cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_IBRS) {
1850 WARN_ON_ONCE(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_IRQ_ENABLE);
1851 drv->states[drv->state_count].enter = intel_idle_ibrs;
1852 }
1853
1822 if ((disabled_states_mask & BIT(drv->state_count)) ||1854 if ((disabled_states_mask & BIT(drv->state_count)) ||
1823 ((icpu->use_acpi || force_use_acpi) &&1855 ((icpu->use_acpi || force_use_acpi) &&
1824 intel_idle_off_by_default(mwait_hint) &&1856 intel_idle_off_by_default(mwait_hint) &&