Update the kvfree_call_rcu() function with head-less support. This allows RCU to reclaim objects without an embedded rcu_head. tree-RCU: We introduce two chains of arrays to store SLAB-backed and vmalloc pointers, each. Storage in either of these arrays does not require embedding an rcu_head within the object. Maintaining the arrays may become impossible due to high memory pressure. For such cases there is an emergency path. Objects with rcu_head inside are just queued on a backup rcu_head list. Later on that list is drained. As for the head-less variant, as the current context can sleep, the following emergency measures are applied: a) Synchronously wait until a grace period has elapsed. b) Call kvfree(). tiny-RCU: For double argument calls, there are no new changes in behavior. For single argument call, kvfree() is directly inlined on the current stack after a synchronize_rcu() call. Note that for tiny-RCU, any call to synchronize_rcu() is actually a quiescent state, therefore it does nothing.
Unified · Split
| @@ -34,9 +34,25 @@ static inline void synchronize_rcu_expedited(void) · 33 unmodified lines | |||
| 34 | synchronize_rcu(); | 34 | synchronize_rcu(); |
| 35 | } | 35 | } |
| 36 | 36 | ||
| 37 | /* | ||
| 38 | * Add one more declaration of kvfree() here. It is | ||
| 39 | * not so straight forward to just include <linux/mm.h> | ||
| 40 | * where it is defined due to getting many compile | ||
| 41 | * errors caused by that include. | ||
| 42 | */ | ||
| 43 | extern void kvfree(const void *addr); | ||
| 44 | |||
| 37 | static inline void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) | 45 | static inline void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) |
| 38 | { | 46 | { |
| 39 | call_rcu(head, func); | 47 | if (head) { |
| 48 | call_rcu(head, func); | ||
| 49 | return; | ||
| 50 | } | ||
| 51 | |||
| 52 | // kvfree_rcu(one_arg) call. | ||
| 53 | might_sleep(); | ||
| 54 | synchronize_rcu(); | ||
| 55 | kvfree((void *) func); | ||
| 40 | } | 56 | } |
| 41 | 57 | ||
| 42 | void rcu_qs(void); | 58 | void rcu_qs(void); |
| @@ -3314,6 +3314,13 @@ kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr) · 3313 unmodified lines | |||
| 3314 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) | 3314 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) |
| 3315 | return false; | 3315 | return false; |
| 3316 | 3316 | ||
| 3317 | /* | ||
| 3318 | * NOTE: For one argument of kvfree_rcu() we can | ||
| 3319 | * drop the lock and get the page in sleepable | ||
| 3320 | * context. That would allow to maintain an array | ||
| 3321 | * for the CONFIG_PREEMPT_RT as well if no cached | ||
| 3322 | * pages are available. | ||
| 3323 | */ | ||
| 3317 | bnode = (struct kvfree_rcu_bulk_data *) | 3324 | bnode = (struct kvfree_rcu_bulk_data *) |
| 3318 | __get_free_page(GFP_NOWAIT | __GFP_NOWARN); | 3325 | __get_free_page(GFP_NOWAIT | __GFP_NOWARN); |
| 3319 | } | 3326 | } |
| @@ -3353,16 +3360,33 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) · 33 unmodified lines | |||
| 3353 | { | 3360 | { |
| 3354 | unsigned long flags; | 3361 | unsigned long flags; |
| 3355 | struct kfree_rcu_cpu *krcp; | 3362 | struct kfree_rcu_cpu *krcp; |
| 3363 | bool success; | ||
| 3356 | void *ptr; | 3364 | void *ptr; |
| 3357 | 3365 | ||
| 3366 | if (head) { | ||
| 3367 | ptr = (void *) head - (unsigned long) func; | ||
| 3368 | } else { | ||
| 3369 | /* | ||
| 3370 | * Please note there is a limitation for the head-less | ||
| 3371 | * variant, that is why there is a clear rule for such | ||
| 3372 | * objects: it can be used from might_sleep() context | ||
| 3373 | * only. For other places please embed an rcu_head to | ||
| 3374 | * your data. | ||
| 3375 | */ | ||
| 3376 | might_sleep(); | ||
| 3377 | ptr = (unsigned long *) func; | ||
| 3378 | } | ||
| 3379 | |||
| 3358 | krcp = krc_this_cpu_lock(&flags); | 3380 | krcp = krc_this_cpu_lock(&flags); |
| 3359 | ptr = (void *)head - (unsigned long)func; | ||
| 3360 | 3381 | ||
| 3361 | // Queue the object but don't yet schedule the batch. | 3382 | // Queue the object but don't yet schedule the batch. |
| 3362 | if (debug_rcu_head_queue(ptr)) { | 3383 | if (debug_rcu_head_queue(ptr)) { |
| 3363 | // Probable double kfree_rcu(), just leak. | 3384 | // Probable double kfree_rcu(), just leak. |
| 3364 | WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n", | 3385 | WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n", |
| 3365 | __func__, head); | 3386 | __func__, head); |
| 3387 | |||
| 3388 | // Mark as success and leave. | ||
| 3389 | success = true; | ||
| 3366 | goto unlock_return; | 3390 | goto unlock_return; |
| 3367 | } | 3391 | } |
| 3368 | 3392 | ||
| @@ -3370,10 +3394,16 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) · 1 unmodified line | |||
| 3370 | * Under high memory pressure GFP_NOWAIT can fail, | 3394 | * Under high memory pressure GFP_NOWAIT can fail, |
| 3371 | * in that case the emergency path is maintained. | 3395 | * in that case the emergency path is maintained. |
| 3372 | */ | 3396 | */ |
| 3373 | if (unlikely(!kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr))) { | 3397 | success = kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr); |
| 3398 | if (!success) { | ||
| 3399 | if (head == NULL) | ||
| 3400 | // Inline if kvfree_rcu(one_arg) call. | ||
| 3401 | goto unlock_return; | ||
| 3402 | |||
| 3374 | head->func = func; | 3403 | head->func = func; |
| 3375 | head->next = krcp->head; | 3404 | head->next = krcp->head; |
| 3376 | krcp->head = head; | 3405 | krcp->head = head; |
| 3406 | success = true; | ||
| 3377 | } | 3407 | } |
| 3378 | 3408 | ||
| 3379 | WRITE_ONCE(krcp->count, krcp->count + 1); | 3409 | WRITE_ONCE(krcp->count, krcp->count + 1); |
| @@ -3387,6 +3417,17 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) · 7 unmodified lines | |||
| 3387 | 3417 | ||
| 3388 | unlock_return: | 3418 | unlock_return: |
| 3389 | krc_this_cpu_unlock(krcp, flags); | 3419 | krc_this_cpu_unlock(krcp, flags); |
| 3420 | |||
| 3421 | /* | ||
| 3422 | * Inline kvfree() after synchronize_rcu(). We can do | ||
| 3423 | * it from might_sleep() context only, so the current | ||
| 3424 | * CPU can pass the QS state. | ||
| 3425 | */ | ||
| 3426 | if (!success) { | ||
| 3427 | debug_rcu_head_unqueue((struct rcu_head *) ptr); | ||
| 3428 | synchronize_rcu(); | ||
| 3429 | kvfree(ptr); | ||
| 3430 | } | ||
| 3390 | } | 3431 | } |
| 3391 | EXPORT_SYMBOL_GPL(kvfree_call_rcu); | 3432 | EXPORT_SYMBOL_GPL(kvfree_call_rcu); |
| 3392 | 3433 | ||