History

arm64/cpufeature: don't use mutex in bringup path

Currently, cpus_set_cap() calls static_branch_enable_cpuslocked(), which must take the jump_label mutex. We call cpus_set_cap() in the secondary bringup path, from the idle thread where interrupts are disabled. Taking a mutex in this path "is a NONO" regardless of whether it's contended, and something we must avoid. We didn't spot this until recently, as ___might_sleep() won't warn for this case until all CPUs have been brought up. This patch avoids taking the mutex in the secondary bringup path. The poking of static keys is deferred until enable_cpu_capabilities(), which runs in a suitable context on the boot CPU. To account for the static keys being set later, cpus_have_const_cap() is updated to use another static key to check whether the const cap keys have been initialised, falling back to the caps bitmap until this is the case. This means that users of cpus_have_const_cap() gain should only gain a single additional NOP in the fast path once the const caps are initialised, but should always see the current cap value. The hyp code should never dereference the caps array, since the caps are initialized before we run the module initcall to initialise hyp. A check is added to the hyp init code to document this requirement. This change will sidestep a number of issues when the upcoming hotplug locking rework is merged.

Mark Rutland committed 9 years ago

Commit 63a1e1c1 parent 78a19cf3 files · +37 −6Browse files

Unified · Split

3 files
  1. arch/arm64/include/asm/cpufeature.h +10 −2
  2. arch/arm64/include/asm/kvm_host.h +6 −2
  3. arch/arm64/kernel/cpufeature.c +21 −2
arch/arm64/include/asm/cpufeature.h+10−2
@@ -115,6 +115,7 @@ struct arm64_cpu_capabilities { · 114 unmodified lines
115115
116116 extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
117117 extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];
118+extern struct static_key_false arm64_const_caps_ready;
118119
119120 bool this_cpu_has_cap(unsigned int cap);
120121
@@ -124,7 +125,7 @@ static inline bool cpu_have_feature(unsigned int num) · 3 unmodified lines
124125 }
125126
126127 /* System capability check for constant caps */
127−static inline bool cpus_have_const_cap(int num)
128+static inline bool __cpus_have_const_cap(int num)
128129 {
129130 if (num >= ARM64_NCAPS)
130131 return false;
@@ -138,6 +139,14 @@ static inline bool cpus_have_cap(unsigned int num) · 7 unmodified lines
138139 return test_bit(num, cpu_hwcaps);
139140 }
140141
142+static inline bool cpus_have_const_cap(int num)
143+{
144+ if (static_branch_likely(&arm64_const_caps_ready))
145+ return __cpus_have_const_cap(num);
146+ else
147+ return cpus_have_cap(num);
148+}
149+
141150 static inline void cpus_set_cap(unsigned int num)
142151 {
143152 if (num >= ARM64_NCAPS) {
@@ -145,7 +154,6 @@ static inline void cpus_set_cap(unsigned int num) · 1 unmodified line
145154 num, ARM64_NCAPS);
146155 } else {
147156 __set_bit(num, cpu_hwcaps);
148− static_branch_enable(&cpu_hwcap_keys[num]);
149157 }
150158 }
151159
arch/arm64/include/asm/kvm_host.h+6−2
@@ -24,6 +24,7 @@ · 23 unmodified lines
2424
2525 #include <linux/types.h>
2626 #include <linux/kvm_types.h>
27+#include <asm/cpufeature.h>
2728 #include <asm/kvm.h>
2829 #include <asm/kvm_asm.h>
2930 #include <asm/kvm_mmio.h>
@@ -355,9 +356,12 @@ static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr, · 325 unmodified lines
355356 unsigned long vector_ptr)
356357 {
357358 /*
358− * Call initialization code, and switch to the full blown
359− * HYP code.
359+ * Call initialization code, and switch to the full blown HYP code.
360+ * If the cpucaps haven't been finalized yet, something has gone very
361+ * wrong, and hyp will crash and burn when it uses any
362+ * cpus_have_const_cap() wrapper.
360363 */
364+ BUG_ON(!static_branch_likely(&arm64_const_caps_ready));
361365 __kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr);
362366 }
363367
arch/arm64/kernel/cpufeature.c+21−2
@@ -985,8 +985,16 @@ void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, · 984 unmodified lines
985985 */
986986 void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps)
987987 {
988− for (; caps->matches; caps++)
989− if (caps->enable && cpus_have_cap(caps->capability))
988+ for (; caps->matches; caps++) {
989+ unsigned int num = caps->capability;
990+
991+ if (!cpus_have_cap(num))
992+ continue;
993+
994+ /* Ensure cpus_have_const_cap(num) works */
995+ static_branch_enable(&cpu_hwcap_keys[num]);
996+
997+ if (caps->enable) {
990998 /*
991999 * Use stop_machine() as it schedules the work allowing
9921000 * us to modify PSTATE, instead of on_each_cpu() which
@@ -994,6 +1002,8 @@ void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps) · 1 unmodified line
9941002 * we return.
9951003 */
9961004 stop_machine(caps->enable, NULL, cpu_online_mask);
1005+ }
1006+ }
9971007 }
9981008
9991009 /*
@@ -1096,6 +1106,14 @@ static void __init setup_feature_capabilities(void) · 96 unmodified lines
10961106 enable_cpu_capabilities(arm64_features);
10971107 }
10981108
1109+DEFINE_STATIC_KEY_FALSE(arm64_const_caps_ready);
1110+EXPORT_SYMBOL(arm64_const_caps_ready);
1111+
1112+static void __init mark_const_caps_ready(void)
1113+{
1114+ static_branch_enable(&arm64_const_caps_ready);
1115+}
1116+
10991117 /*
11001118 * Check if the current CPU has a given feature capability.
11011119 * Should be called from non-preemptible context.
@@ -1131,6 +1149,7 @@ void __init setup_cpu_features(void) · 29 unmodified lines
11311149 /* Set the CPU feature capabilies */
11321150 setup_feature_capabilities();
11331151 enable_errata_workarounds();
1152+ mark_const_caps_ready();
11341153 setup_elf_hwcaps(arm64_elf_hwcaps);
11351154
11361155 if (system_supports_32bit_el0())

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t