Re-evaluating the bitmap wheight of the online cpus bitmap in every invocation of num_online_cpus() over and over is a pretty useless exercise. Especially when num_online_cpus() is used in code paths like the IPI delivery of x86 or the membarrier code. Cache the number of online CPUs in the core and just return the cached variable. The accessor function provides only a snapshot when used without protection against concurrent CPU hotplug. The storage needs to use an atomic_t because the kexec and reboot code (ab)use set_cpu_online() in their 'shutdown' handlers without any form of serialization as pointed out by Mathieu. Regular CPU hotplug usage is properly serialized.
Showing the changes to include/linux (1 of 2 files). Show all
Unified · Split
| @@ -10,6 +10,7 @@ · 9 unmodified lines | |||
| 10 | #include <linux/kernel.h> | 10 | #include <linux/kernel.h> |
| 11 | #include <linux/threads.h> | 11 | #include <linux/threads.h> |
| 12 | #include <linux/bitmap.h> | 12 | #include <linux/bitmap.h> |
| 13 | #include <linux/atomic.h> | ||
| 13 | #include <linux/bug.h> | 14 | #include <linux/bug.h> |
| 14 | 15 | ||
| 15 | /* Don't assign or return these: may not be this big! */ | 16 | /* Don't assign or return these: may not be this big! */ |
| @@ -95,8 +96,21 @@ extern struct cpumask __cpu_active_mask; · 79 unmodified lines | |||
| 95 | #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask) | 96 | #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask) |
| 96 | #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask) | 97 | #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask) |
| 97 | 98 | ||
| 99 | extern atomic_t __num_online_cpus; | ||
| 100 | |||
| 98 | #if NR_CPUS > 1 | 101 | #if NR_CPUS > 1 |
| 99 | #define num_online_cpus() cpumask_weight(cpu_online_mask) | 102 | /** |
| 103 | * num_online_cpus() - Read the number of online CPUs | ||
| 104 | * | ||
| 105 | * Despite the fact that __num_online_cpus is of type atomic_t, this | ||
| 106 | * interface gives only a momentary snapshot and is not protected against | ||
| 107 | * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held | ||
| 108 | * region. | ||
| 109 | */ | ||
| 110 | static inline unsigned int num_online_cpus(void) | ||
| 111 | { | ||
| 112 | return atomic_read(&__num_online_cpus); | ||
| 113 | } | ||
| 100 | #define num_possible_cpus() cpumask_weight(cpu_possible_mask) | 114 | #define num_possible_cpus() cpumask_weight(cpu_possible_mask) |
| 101 | #define num_present_cpus() cpumask_weight(cpu_present_mask) | 115 | #define num_present_cpus() cpumask_weight(cpu_present_mask) |
| 102 | #define num_active_cpus() cpumask_weight(cpu_active_mask) | 116 | #define num_active_cpus() cpumask_weight(cpu_active_mask) |
| @@ -821,14 +835,7 @@ set_cpu_present(unsigned int cpu, bool present) · 718 unmodified lines | |||
| 821 | cpumask_clear_cpu(cpu, &__cpu_present_mask); | 835 | cpumask_clear_cpu(cpu, &__cpu_present_mask); |
| 822 | } | 836 | } |
| 823 | 837 | ||
| 824 | static inline void | 838 | void set_cpu_online(unsigned int cpu, bool online); |
| 825 | set_cpu_online(unsigned int cpu, bool online) | ||
| 826 | { | ||
| 827 | if (online) | ||
| 828 | cpumask_set_cpu(cpu, &__cpu_online_mask); | ||
| 829 | else | ||
| 830 | cpumask_clear_cpu(cpu, &__cpu_online_mask); | ||
| 831 | } | ||
| 832 | 839 | ||
| 833 | static inline void | 840 | static inline void |
| 834 | set_cpu_active(unsigned int cpu, bool active) | 841 | set_cpu_active(unsigned int cpu, bool active) |