Now that explicit invocation of percpu_ref_exit() is necessary to free the percpu counter, we can implement percpu_ref_reinit() which reinitializes a released percpu_ref. This can be used implement scalable gating switch which can be drained and then re-opened without worrying about memory allocation failures. percpu_ref_is_zero() is added to be used in a sanity check in percpu_ref_exit(). As this function will be useful for other purposes too, make it a public interface. v2: Use smp_read_barrier_depends() instead of smp_load_acquire(). We only need data dep barrier and smp_load_acquire() is stronger and heavier on some archs. Spotted by Lai Jiangshan.
Unified · Split
| @@ -67,6 +67,7 @@ struct percpu_ref { · 66 unmodified lines | |||
| 67 | 67 | ||
| 68 | int __must_check percpu_ref_init(struct percpu_ref *ref, | 68 | int __must_check percpu_ref_init(struct percpu_ref *ref, |
| 69 | percpu_ref_func_t *release); | 69 | percpu_ref_func_t *release); |
| 70 | void percpu_ref_reinit(struct percpu_ref *ref); | ||
| 70 | void percpu_ref_exit(struct percpu_ref *ref); | 71 | void percpu_ref_exit(struct percpu_ref *ref); |
| 71 | void percpu_ref_kill_and_confirm(struct percpu_ref *ref, | 72 | void percpu_ref_kill_and_confirm(struct percpu_ref *ref, |
| 72 | percpu_ref_func_t *confirm_kill); | 73 | percpu_ref_func_t *confirm_kill); |
| @@ -99,6 +100,9 @@ static inline bool __pcpu_ref_alive(struct percpu_ref *ref, · 26 unmodified lines | |||
| 99 | { | 100 | { |
| 100 | unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr); | 101 | unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr); |
| 101 | 102 | ||
| 103 | /* paired with smp_store_release() in percpu_ref_reinit() */ | ||
| 104 | smp_read_barrier_depends(); | ||
| 105 | |||
| 102 | if (unlikely(pcpu_ptr & PCPU_REF_DEAD)) | 106 | if (unlikely(pcpu_ptr & PCPU_REF_DEAD)) |
| 103 | return false; | 107 | return false; |
| 104 | 108 | ||
| @@ -206,4 +210,19 @@ static inline void percpu_ref_put(struct percpu_ref *ref) · 101 unmodified lines | |||
| 206 | rcu_read_unlock_sched(); | 210 | rcu_read_unlock_sched(); |
| 207 | } | 211 | } |
| 208 | 212 | ||
| 213 | /** | ||
| 214 | * percpu_ref_is_zero - test whether a percpu refcount reached zero | ||
| 215 | * @ref: percpu_ref to test | ||
| 216 | * | ||
| 217 | * Returns %true if @ref reached zero. | ||
| 218 | */ | ||
| 219 | static inline bool percpu_ref_is_zero(struct percpu_ref *ref) | ||
| 220 | { | ||
| 221 | unsigned __percpu *pcpu_count; | ||
| 222 | |||
| 223 | if (__pcpu_ref_alive(ref, &pcpu_count)) | ||
| 224 | return false; | ||
| 225 | return !atomic_read(&ref->count); | ||
| 226 | } | ||
| 227 | |||
| 209 | #endif | 228 | #endif |
| @@ -60,6 +60,41 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release) · 59 unmodified lines | |||
| 60 | } | 60 | } |
| 61 | EXPORT_SYMBOL_GPL(percpu_ref_init); | 61 | EXPORT_SYMBOL_GPL(percpu_ref_init); |
| 62 | 62 | ||
| 63 | /** | ||
| 64 | * percpu_ref_reinit - re-initialize a percpu refcount | ||
| 65 | * @ref: perpcu_ref to re-initialize | ||
| 66 | * | ||
| 67 | * Re-initialize @ref so that it's in the same state as when it finished | ||
| 68 | * percpu_ref_init(). @ref must have been initialized successfully, killed | ||
| 69 | * and reached 0 but not exited. | ||
| 70 | * | ||
| 71 | * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while | ||
| 72 | * this function is in progress. | ||
| 73 | */ | ||
| 74 | void percpu_ref_reinit(struct percpu_ref *ref) | ||
| 75 | { | ||
| 76 | unsigned __percpu *pcpu_count = pcpu_count_ptr(ref); | ||
| 77 | int cpu; | ||
| 78 | |||
| 79 | BUG_ON(!pcpu_count); | ||
| 80 | WARN_ON(!percpu_ref_is_zero(ref)); | ||
| 81 | |||
| 82 | atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS); | ||
| 83 | |||
| 84 | /* | ||
| 85 | * Restore per-cpu operation. smp_store_release() is paired with | ||
| 86 | * smp_read_barrier_depends() in __pcpu_ref_alive() and guarantees | ||
| 87 | * that the zeroing is visible to all percpu accesses which can see | ||
| 88 | * the following PCPU_REF_DEAD clearing. | ||
| 89 | */ | ||
| 90 | for_each_possible_cpu(cpu) | ||
| 91 | *per_cpu_ptr(pcpu_count, cpu) = 0; | ||
| 92 | |||
| 93 | smp_store_release(&ref->pcpu_count_ptr, | ||
| 94 | ref->pcpu_count_ptr & ~PCPU_REF_DEAD); | ||
| 95 | } | ||
| 96 | EXPORT_SYMBOL_GPL(percpu_ref_reinit); | ||
| 97 | |||
| 63 | /** | 98 | /** |
| 64 | * percpu_ref_exit - undo percpu_ref_init() | 99 | * percpu_ref_exit - undo percpu_ref_init() |
| 65 | * @ref: percpu_ref to exit | 100 | * @ref: percpu_ref to exit |