Now that explicit invocation of percpu_ref_exit() is necessary to free the percpu counter, we can implement percpu_ref_reinit() which reinitializes a released percpu_ref. This can be used implement scalable gating switch which can be drained and then re-opened without worrying about memory allocation failures. percpu_ref_is_zero() is added to be used in a sanity check in percpu_ref_exit(). As this function will be useful for other purposes too, make it a public interface. v2: Use smp_read_barrier_depends() instead of smp_load_acquire(). We only need data dep barrier and smp_load_acquire() is stronger and heavier on some archs. Spotted by Lai Jiangshan.
Unified · Split
| @@ -67,6 +67,7 @@ struct percpu_ref { · 66 unmodified lines | ||
| 67 | 67 | |
| 68 | 68 | int __must_check percpu_ref_init(struct percpu_ref *ref, |
| 69 | 69 | percpu_ref_func_t *release); |
| 70 | +void percpu_ref_reinit(struct percpu_ref *ref); | |
| 70 | 71 | void percpu_ref_exit(struct percpu_ref *ref); |
| 71 | 72 | void percpu_ref_kill_and_confirm(struct percpu_ref *ref, |
| 72 | 73 | percpu_ref_func_t *confirm_kill); |
| @@ -99,6 +100,9 @@ static inline bool __pcpu_ref_alive(struct percpu_ref *ref, · 26 unmodified lines | ||
| 99 | 100 | { |
| 100 | 101 | unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr); |
| 101 | 102 | |
| 103 | + /* paired with smp_store_release() in percpu_ref_reinit() */ | |
| 104 | + smp_read_barrier_depends(); | |
| 105 | + | |
| 102 | 106 | if (unlikely(pcpu_ptr & PCPU_REF_DEAD)) |
| 103 | 107 | return false; |
| 104 | 108 | |
| @@ -206,4 +210,19 @@ static inline void percpu_ref_put(struct percpu_ref *ref) · 101 unmodified lines | ||
| 206 | 210 | rcu_read_unlock_sched(); |
| 207 | 211 | } |
| 208 | 212 | |
| 213 | +/** | |
| 214 | + * percpu_ref_is_zero - test whether a percpu refcount reached zero | |
| 215 | + * @ref: percpu_ref to test | |
| 216 | + * | |
| 217 | + * Returns %true if @ref reached zero. | |
| 218 | + */ | |
| 219 | +static inline bool percpu_ref_is_zero(struct percpu_ref *ref) | |
| 220 | +{ | |
| 221 | + unsigned __percpu *pcpu_count; | |
| 222 | + | |
| 223 | + if (__pcpu_ref_alive(ref, &pcpu_count)) | |
| 224 | + return false; | |
| 225 | + return !atomic_read(&ref->count); | |
| 226 | +} | |
| 227 | + | |
| 209 | 228 | #endif |
| @@ -60,6 +60,41 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release) · 59 unmodified lines | ||
| 60 | 60 | } |
| 61 | 61 | EXPORT_SYMBOL_GPL(percpu_ref_init); |
| 62 | 62 | |
| 63 | +/** | |
| 64 | + * percpu_ref_reinit - re-initialize a percpu refcount | |
| 65 | + * @ref: perpcu_ref to re-initialize | |
| 66 | + * | |
| 67 | + * Re-initialize @ref so that it's in the same state as when it finished | |
| 68 | + * percpu_ref_init(). @ref must have been initialized successfully, killed | |
| 69 | + * and reached 0 but not exited. | |
| 70 | + * | |
| 71 | + * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while | |
| 72 | + * this function is in progress. | |
| 73 | + */ | |
| 74 | +void percpu_ref_reinit(struct percpu_ref *ref) | |
| 75 | +{ | |
| 76 | + unsigned __percpu *pcpu_count = pcpu_count_ptr(ref); | |
| 77 | + int cpu; | |
| 78 | + | |
| 79 | + BUG_ON(!pcpu_count); | |
| 80 | + WARN_ON(!percpu_ref_is_zero(ref)); | |
| 81 | + | |
| 82 | + atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS); | |
| 83 | + | |
| 84 | + /* | |
| 85 | + * Restore per-cpu operation. smp_store_release() is paired with | |
| 86 | + * smp_read_barrier_depends() in __pcpu_ref_alive() and guarantees | |
| 87 | + * that the zeroing is visible to all percpu accesses which can see | |
| 88 | + * the following PCPU_REF_DEAD clearing. | |
| 89 | + */ | |
| 90 | + for_each_possible_cpu(cpu) | |
| 91 | + *per_cpu_ptr(pcpu_count, cpu) = 0; | |
| 92 | + | |
| 93 | + smp_store_release(&ref->pcpu_count_ptr, | |
| 94 | + ref->pcpu_count_ptr & ~PCPU_REF_DEAD); | |
| 95 | +} | |
| 96 | +EXPORT_SYMBOL_GPL(percpu_ref_reinit); | |
| 97 | + | |
| 63 | 98 | /** |
| 64 | 99 | * percpu_ref_exit - undo percpu_ref_init() |
| 65 | 100 | * @ref: percpu_ref to exit |