History

arm64/cpufeature: don't use mutex in bringup path

Currently, cpus_set_cap() calls static_branch_enable_cpuslocked(), which must take the jump_label mutex. We call cpus_set_cap() in the secondary bringup path, from the idle thread where interrupts are disabled. Taking a mutex in this path "is a NONO" regardless of whether it's contended, and something we must avoid. We didn't spot this until recently, as ___might_sleep() won't warn for this case until all CPUs have been brought up. This patch avoids taking the mutex in the secondary bringup path. The poking of static keys is deferred until enable_cpu_capabilities(), which runs in a suitable context on the boot CPU. To account for the static keys being set later, cpus_have_const_cap() is updated to use another static key to check whether the const cap keys have been initialised, falling back to the caps bitmap until this is the case. This means that users of cpus_have_const_cap() gain should only gain a single additional NOP in the fast path once the const caps are initialised, but should always see the current cap value. The hyp code should never dereference the caps array, since the caps are initialized before we run the module initcall to initialise hyp. A check is added to the hyp init code to document this requirement. This change will sidestep a number of issues when the upcoming hotplug locking rework is merged.

Mark Rutland committed 9 years ago

Commit 63a1e1c1 parent 78a19cf3 files · +37 −6Browse files

Unified · Split

3 files
  1. arch/arm64/include/asm/cpufeature.h +10 −2
  2. arch/arm64/include/asm/kvm_host.h +6 −2
  3. arch/arm64/kernel/cpufeature.c +21 −2
arch/arm64/include/asm/cpufeature.h+10−2
@@ -115,6 +115,7 @@ struct arm64_cpu_capabilities { · 114 unmodified lines
115 115
116extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);116extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
117extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];117extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];
118extern struct static_key_false arm64_const_caps_ready;
118 119
119bool this_cpu_has_cap(unsigned int cap);120bool this_cpu_has_cap(unsigned int cap);
120 121
@@ -124,7 +125,7 @@ static inline bool cpu_have_feature(unsigned int num) · 3 unmodified lines
124}125}
125 126
126/* System capability check for constant caps */127/* System capability check for constant caps */
127static inline bool cpus_have_const_cap(int num)128static inline bool __cpus_have_const_cap(int num)
128{129{
129 if (num >= ARM64_NCAPS)130 if (num >= ARM64_NCAPS)
130 return false;131 return false;
@@ -138,6 +139,14 @@ static inline bool cpus_have_cap(unsigned int num) · 7 unmodified lines
138 return test_bit(num, cpu_hwcaps);139 return test_bit(num, cpu_hwcaps);
139}140}
140 141
142static inline bool cpus_have_const_cap(int num)
143{
144 if (static_branch_likely(&arm64_const_caps_ready))
145 return __cpus_have_const_cap(num);
146 else
147 return cpus_have_cap(num);
148}
149
141static inline void cpus_set_cap(unsigned int num)150static inline void cpus_set_cap(unsigned int num)
142{151{
143 if (num >= ARM64_NCAPS) {152 if (num >= ARM64_NCAPS) {
@@ -145,7 +154,6 @@ static inline void cpus_set_cap(unsigned int num) · 1 unmodified line
145 num, ARM64_NCAPS);154 num, ARM64_NCAPS);
146 } else {155 } else {
147 __set_bit(num, cpu_hwcaps);156 __set_bit(num, cpu_hwcaps);
148 static_branch_enable(&cpu_hwcap_keys[num]);
149 }157 }
150}158}
151 159
arch/arm64/include/asm/kvm_host.h+6−2
@@ -24,6 +24,7 @@ · 23 unmodified lines
24 24
25#include <linux/types.h>25#include <linux/types.h>
26#include <linux/kvm_types.h>26#include <linux/kvm_types.h>
27#include <asm/cpufeature.h>
27#include <asm/kvm.h>28#include <asm/kvm.h>
28#include <asm/kvm_asm.h>29#include <asm/kvm_asm.h>
29#include <asm/kvm_mmio.h>30#include <asm/kvm_mmio.h>
@@ -355,9 +356,12 @@ static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr, · 325 unmodified lines
355 unsigned long vector_ptr)356 unsigned long vector_ptr)
356{357{
357 /*358 /*
358 * Call initialization code, and switch to the full blown359 * Call initialization code, and switch to the full blown HYP code.
359 * HYP code.360 * If the cpucaps haven't been finalized yet, something has gone very
361 * wrong, and hyp will crash and burn when it uses any
362 * cpus_have_const_cap() wrapper.
360 */363 */
364 BUG_ON(!static_branch_likely(&arm64_const_caps_ready));
361 __kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr);365 __kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr);
362}366}
363 367
arch/arm64/kernel/cpufeature.c+21−2
@@ -985,8 +985,16 @@ void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, · 984 unmodified lines
985 */985 */
986void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps)986void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps)
987{987{
988 for (; caps->matches; caps++)988 for (; caps->matches; caps++) {
989 if (caps->enable && cpus_have_cap(caps->capability))989 unsigned int num = caps->capability;
990
991 if (!cpus_have_cap(num))
992 continue;
993
994 /* Ensure cpus_have_const_cap(num) works */
995 static_branch_enable(&cpu_hwcap_keys[num]);
996
997 if (caps->enable) {
990 /*998 /*
991 * Use stop_machine() as it schedules the work allowing999 * Use stop_machine() as it schedules the work allowing
992 * us to modify PSTATE, instead of on_each_cpu() which1000 * us to modify PSTATE, instead of on_each_cpu() which
@@ -994,6 +1002,8 @@ void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps) · 1 unmodified line
994 * we return.1002 * we return.
995 */1003 */
996 stop_machine(caps->enable, NULL, cpu_online_mask);1004 stop_machine(caps->enable, NULL, cpu_online_mask);
1005 }
1006 }
997}1007}
998 1008
999/*1009/*
@@ -1096,6 +1106,14 @@ static void __init setup_feature_capabilities(void) · 96 unmodified lines
1096 enable_cpu_capabilities(arm64_features);1106 enable_cpu_capabilities(arm64_features);
1097}1107}
1098 1108
1109DEFINE_STATIC_KEY_FALSE(arm64_const_caps_ready);
1110EXPORT_SYMBOL(arm64_const_caps_ready);
1111
1112static void __init mark_const_caps_ready(void)
1113{
1114 static_branch_enable(&arm64_const_caps_ready);
1115}
1116
1099/*1117/*
1100 * Check if the current CPU has a given feature capability.1118 * Check if the current CPU has a given feature capability.
1101 * Should be called from non-preemptible context.1119 * Should be called from non-preemptible context.
@@ -1131,6 +1149,7 @@ void __init setup_cpu_features(void) · 29 unmodified lines
1131 /* Set the CPU feature capabilies */1149 /* Set the CPU feature capabilies */
1132 setup_feature_capabilities();1150 setup_feature_capabilities();
1133 enable_errata_workarounds();1151 enable_errata_workarounds();
1152 mark_const_caps_ready();
1134 setup_elf_hwcaps(arm64_elf_hwcaps);1153 setup_elf_hwcaps(arm64_elf_hwcaps);
1135 1154
1136 if (system_supports_32bit_el0())1155 if (system_supports_32bit_el0())

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t