Update the kvfree_call_rcu() function with head-less support. This allows RCU to reclaim objects without an embedded rcu_head. tree-RCU: We introduce two chains of arrays to store SLAB-backed and vmalloc pointers, each. Storage in either of these arrays does not require embedding an rcu_head within the object. Maintaining the arrays may become impossible due to high memory pressure. For such cases there is an emergency path. Objects with rcu_head inside are just queued on a backup rcu_head list. Later on that list is drained. As for the head-less variant, as the current context can sleep, the following emergency measures are applied: a) Synchronously wait until a grace period has elapsed. b) Call kvfree(). tiny-RCU: For double argument calls, there are no new changes in behavior. For single argument call, kvfree() is directly inlined on the current stack after a synchronize_rcu() call. Note that for tiny-RCU, any call to synchronize_rcu() is actually a quiescent state, therefore it does nothing.
Unified · Split
| @@ -34,9 +34,25 @@ static inline void synchronize_rcu_expedited(void) · 33 unmodified lines | ||
| 34 | 34 | synchronize_rcu(); |
| 35 | 35 | } |
| 36 | 36 | |
| 37 | +/* | |
| 38 | + * Add one more declaration of kvfree() here. It is | |
| 39 | + * not so straight forward to just include <linux/mm.h> | |
| 40 | + * where it is defined due to getting many compile | |
| 41 | + * errors caused by that include. | |
| 42 | + */ | |
| 43 | +extern void kvfree(const void *addr); | |
| 44 | + | |
| 37 | 45 | static inline void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) |
| 38 | 46 | { |
| 39 | − call_rcu(head, func); | |
| 47 | + if (head) { | |
| 48 | + call_rcu(head, func); | |
| 49 | + return; | |
| 50 | + } | |
| 51 | + | |
| 52 | + // kvfree_rcu(one_arg) call. | |
| 53 | + might_sleep(); | |
| 54 | + synchronize_rcu(); | |
| 55 | + kvfree((void *) func); | |
| 40 | 56 | } |
| 41 | 57 | |
| 42 | 58 | void rcu_qs(void); |
| @@ -3314,6 +3314,13 @@ kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr) · 3313 unmodified lines | ||
| 3314 | 3314 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) |
| 3315 | 3315 | return false; |
| 3316 | 3316 | |
| 3317 | + /* | |
| 3318 | + * NOTE: For one argument of kvfree_rcu() we can | |
| 3319 | + * drop the lock and get the page in sleepable | |
| 3320 | + * context. That would allow to maintain an array | |
| 3321 | + * for the CONFIG_PREEMPT_RT as well if no cached | |
| 3322 | + * pages are available. | |
| 3323 | + */ | |
| 3317 | 3324 | bnode = (struct kvfree_rcu_bulk_data *) |
| 3318 | 3325 | __get_free_page(GFP_NOWAIT | __GFP_NOWARN); |
| 3319 | 3326 | } |
| @@ -3353,16 +3360,33 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) · 33 unmodified lines | ||
| 3353 | 3360 | { |
| 3354 | 3361 | unsigned long flags; |
| 3355 | 3362 | struct kfree_rcu_cpu *krcp; |
| 3363 | + bool success; | |
| 3356 | 3364 | void *ptr; |
| 3357 | 3365 | |
| 3366 | + if (head) { | |
| 3367 | + ptr = (void *) head - (unsigned long) func; | |
| 3368 | + } else { | |
| 3369 | + /* | |
| 3370 | + * Please note there is a limitation for the head-less | |
| 3371 | + * variant, that is why there is a clear rule for such | |
| 3372 | + * objects: it can be used from might_sleep() context | |
| 3373 | + * only. For other places please embed an rcu_head to | |
| 3374 | + * your data. | |
| 3375 | + */ | |
| 3376 | + might_sleep(); | |
| 3377 | + ptr = (unsigned long *) func; | |
| 3378 | + } | |
| 3379 | + | |
| 3358 | 3380 | krcp = krc_this_cpu_lock(&flags); |
| 3359 | − ptr = (void *)head - (unsigned long)func; | |
| 3360 | 3381 | |
| 3361 | 3382 | // Queue the object but don't yet schedule the batch. |
| 3362 | 3383 | if (debug_rcu_head_queue(ptr)) { |
| 3363 | 3384 | // Probable double kfree_rcu(), just leak. |
| 3364 | 3385 | WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n", |
| 3365 | 3386 | __func__, head); |
| 3387 | + | |
| 3388 | + // Mark as success and leave. | |
| 3389 | + success = true; | |
| 3366 | 3390 | goto unlock_return; |
| 3367 | 3391 | } |
| 3368 | 3392 | |
| @@ -3370,10 +3394,16 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) · 1 unmodified line | ||
| 3370 | 3394 | * Under high memory pressure GFP_NOWAIT can fail, |
| 3371 | 3395 | * in that case the emergency path is maintained. |
| 3372 | 3396 | */ |
| 3373 | − if (unlikely(!kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr))) { | |
| 3397 | + success = kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr); | |
| 3398 | + if (!success) { | |
| 3399 | + if (head == NULL) | |
| 3400 | + // Inline if kvfree_rcu(one_arg) call. | |
| 3401 | + goto unlock_return; | |
| 3402 | + | |
| 3374 | 3403 | head->func = func; |
| 3375 | 3404 | head->next = krcp->head; |
| 3376 | 3405 | krcp->head = head; |
| 3406 | + success = true; | |
| 3377 | 3407 | } |
| 3378 | 3408 | |
| 3379 | 3409 | WRITE_ONCE(krcp->count, krcp->count + 1); |
| @@ -3387,6 +3417,17 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) · 7 unmodified lines | ||
| 3387 | 3417 | |
| 3388 | 3418 | unlock_return: |
| 3389 | 3419 | krc_this_cpu_unlock(krcp, flags); |
| 3420 | + | |
| 3421 | + /* | |
| 3422 | + * Inline kvfree() after synchronize_rcu(). We can do | |
| 3423 | + * it from might_sleep() context only, so the current | |
| 3424 | + * CPU can pass the QS state. | |
| 3425 | + */ | |
| 3426 | + if (!success) { | |
| 3427 | + debug_rcu_head_unqueue((struct rcu_head *) ptr); | |
| 3428 | + synchronize_rcu(); | |
| 3429 | + kvfree(ptr); | |
| 3430 | + } | |
| 3390 | 3431 | } |
| 3391 | 3432 | EXPORT_SYMBOL_GPL(kvfree_call_rcu); |
| 3392 | 3433 | |