The percpu_counter is used for scenarios where performance is more important than the accuracy. For percpu_counter users, who want more accurate information in their slowpath, percpu_counter_sum is provided which traverses all the online CPUs to accumulate the data. The reason it only needs to traverse online CPUs is because percpu_counter does implement CPU offline callback which syncs the local data of the offlined CPU. However there is a small race window between the online CPUs traversal of percpu_counter_sum and the CPU offline callback. The offline callback has to traverse all the percpu_counters on the system to flush the CPU local data which can be a lot. During that time, the CPU which is going offline has already been published as offline to all the readers. So, as the offline callback is running, percpu_counter_sum can be called for one counter which has some state on the CPU going offline. Since percpu_counter_sum only traverses online CPUs, it will skip that specific CPU and the offline callback might not have flushed the state for that specific percpu_counter on that offlined CPU. Normally this is not an issue because percpu_counter users can deal with some inaccuracy for small time window. However a new user i.e. mm_struct on the cleanup path wants to check the exact state of the percpu_counter through check_mm(). For such users, this patch introduces percpu_counter_sum_all() which traverses all possible CPUs and it is used in fork.c:check_mm() to avoid the potential race. This issue is exposed by the later patch "mm: convert mm's rss stats into percpu_counter".
Showing the changes to lib (1 of 3 files). Show all
Unified · Split
| @@ -117,11 +117,8 @@ void percpu_counter_sync(struct percpu_counter *fbc) · 116 unmodified lines | |||
| 117 | } | 117 | } |
| 118 | EXPORT_SYMBOL(percpu_counter_sync); | 118 | EXPORT_SYMBOL(percpu_counter_sync); |
| 119 | 119 | ||
| 120 | /* | 120 | static s64 __percpu_counter_sum_mask(struct percpu_counter *fbc, |
| 121 | * Add up all the per-cpu counts, return the result. This is a more accurate | 121 | const struct cpumask *cpu_mask) |
| 122 | * but much slower version of percpu_counter_read_positive() | ||
| 123 | */ | ||
| 124 | s64 __percpu_counter_sum(struct percpu_counter *fbc) | ||
| 125 | { | 122 | { |
| 126 | s64 ret; | 123 | s64 ret; |
| 127 | int cpu; | 124 | int cpu; |
| @@ -129,15 +126,35 @@ s64 __percpu_counter_sum(struct percpu_counter *fbc) · 1 unmodified line | |||
| 129 | 126 | ||
| 130 | raw_spin_lock_irqsave(&fbc->lock, flags); | 127 | raw_spin_lock_irqsave(&fbc->lock, flags); |
| 131 | ret = fbc->count; | 128 | ret = fbc->count; |
| 132 | for_each_online_cpu(cpu) { | 129 | for_each_cpu(cpu, cpu_mask) { |
| 133 | s32 *pcount = per_cpu_ptr(fbc->counters, cpu); | 130 | s32 *pcount = per_cpu_ptr(fbc->counters, cpu); |
| 134 | ret += *pcount; | 131 | ret += *pcount; |
| 135 | } | 132 | } |
| 136 | raw_spin_unlock_irqrestore(&fbc->lock, flags); | 133 | raw_spin_unlock_irqrestore(&fbc->lock, flags); |
| 137 | return ret; | 134 | return ret; |
| 138 | } | 135 | } |
| 136 | |||
| 137 | /* | ||
| 138 | * Add up all the per-cpu counts, return the result. This is a more accurate | ||
| 139 | * but much slower version of percpu_counter_read_positive() | ||
| 140 | */ | ||
| 141 | s64 __percpu_counter_sum(struct percpu_counter *fbc) | ||
| 142 | { | ||
| 143 | return __percpu_counter_sum_mask(fbc, cpu_online_mask); | ||
| 144 | } | ||
| 139 | EXPORT_SYMBOL(__percpu_counter_sum); | 145 | EXPORT_SYMBOL(__percpu_counter_sum); |
| 140 | 146 | ||
| 147 | /* | ||
| 148 | * This is slower version of percpu_counter_sum as it traverses all possible | ||
| 149 | * cpus. Use this only in the cases where accurate data is needed in the | ||
| 150 | * presense of CPUs getting offlined. | ||
| 151 | */ | ||
| 152 | s64 percpu_counter_sum_all(struct percpu_counter *fbc) | ||
| 153 | { | ||
| 154 | return __percpu_counter_sum_mask(fbc, cpu_possible_mask); | ||
| 155 | } | ||
| 156 | EXPORT_SYMBOL(percpu_counter_sum_all); | ||
| 157 | |||
| 141 | int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp, | 158 | int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp, |
| 142 | struct lock_class_key *key) | 159 | struct lock_class_key *key) |
| 143 | { | 160 | { |