To do so, we use an array of kvfree_rcu_bulk_data structures. It consists of two elements: - index number 0 corresponds to slab pointers. - index number 1 corresponds to vmalloc pointers. Keeping vmalloc pointers separated from slab pointers makes it possible to invoke the right freeing API for the right kind of pointer. It also prepares us for future headless support for vmalloc and SLAB objects. Such objects cannot be queued on a linked list and are instead directly into an array.
Showing the changes to kernel (1 of 1 file). Show all
Unified · Split
| @@ -57,6 +57,8 @@ · 56 unmodified lines | |||
| 57 | #include <linux/slab.h> | 57 | #include <linux/slab.h> |
| 58 | #include <linux/sched/isolation.h> | 58 | #include <linux/sched/isolation.h> |
| 59 | #include <linux/sched/clock.h> | 59 | #include <linux/sched/clock.h> |
| 60 | #include <linux/vmalloc.h> | ||
| 61 | #include <linux/mm.h> | ||
| 60 | #include "../time/tick-internal.h" | 62 | #include "../time/tick-internal.h" |
| 61 | 63 | ||
| 62 | #include "tree.h" | 64 | #include "tree.h" |
| @@ -2966,46 +2968,47 @@ EXPORT_SYMBOL_GPL(call_rcu); · 2903 unmodified lines | |||
| 2966 | /* Maximum number of jiffies to wait before draining a batch. */ | 2968 | /* Maximum number of jiffies to wait before draining a batch. */ |
| 2967 | #define KFREE_DRAIN_JIFFIES (HZ / 50) | 2969 | #define KFREE_DRAIN_JIFFIES (HZ / 50) |
| 2968 | #define KFREE_N_BATCHES 2 | 2970 | #define KFREE_N_BATCHES 2 |
| 2971 | #define FREE_N_CHANNELS 2 | ||
| 2969 | 2972 | ||
| 2970 | /** | 2973 | /** |
| 2971 | * struct kfree_rcu_bulk_data - single block to store kfree_rcu() pointers | 2974 | * struct kvfree_rcu_bulk_data - single block to store kvfree_rcu() pointers |
| 2972 | * @nr_records: Number of active pointers in the array | 2975 | * @nr_records: Number of active pointers in the array |
| 2973 | * @records: Array of the kfree_rcu() pointers | ||
| 2974 | * @next: Next bulk object in the block chain | 2976 | * @next: Next bulk object in the block chain |
| 2977 | * @records: Array of the kvfree_rcu() pointers | ||
| 2975 | */ | 2978 | */ |
| 2976 | struct kfree_rcu_bulk_data { | 2979 | struct kvfree_rcu_bulk_data { |
| 2977 | unsigned long nr_records; | 2980 | unsigned long nr_records; |
| 2978 | struct kfree_rcu_bulk_data *next; | 2981 | struct kvfree_rcu_bulk_data *next; |
| 2979 | void *records[]; | 2982 | void *records[]; |
| 2980 | }; | 2983 | }; |
| 2981 | 2984 | ||
| 2982 | /* | 2985 | /* |
| 2983 | * This macro defines how many entries the "records" array | 2986 | * This macro defines how many entries the "records" array |
| 2984 | * will contain. It is based on the fact that the size of | 2987 | * will contain. It is based on the fact that the size of |
| 2985 | * kfree_rcu_bulk_data structure becomes exactly one page. | 2988 | * kvfree_rcu_bulk_data structure becomes exactly one page. |
| 2986 | */ | 2989 | */ |
| 2987 | #define KFREE_BULK_MAX_ENTR \ | 2990 | #define KVFREE_BULK_MAX_ENTR \ |
| 2988 | ((PAGE_SIZE - sizeof(struct kfree_rcu_bulk_data)) / sizeof(void *)) | 2991 | ((PAGE_SIZE - sizeof(struct kvfree_rcu_bulk_data)) / sizeof(void *)) |
| 2989 | 2992 | ||
| 2990 | /** | 2993 | /** |
| 2991 | * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests | 2994 | * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests |
| 2992 | * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period | 2995 | * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period |
| 2993 | * @head_free: List of kfree_rcu() objects waiting for a grace period | 2996 | * @head_free: List of kfree_rcu() objects waiting for a grace period |
| 2994 | * @bhead_free: Bulk-List of kfree_rcu() objects waiting for a grace period | 2997 | * @bkvhead_free: Bulk-List of kvfree_rcu() objects waiting for a grace period |
| 2995 | * @krcp: Pointer to @kfree_rcu_cpu structure | 2998 | * @krcp: Pointer to @kfree_rcu_cpu structure |
| 2996 | */ | 2999 | */ |
| 2997 | 3000 | ||
| 2998 | struct kfree_rcu_cpu_work { | 3001 | struct kfree_rcu_cpu_work { |
| 2999 | struct rcu_work rcu_work; | 3002 | struct rcu_work rcu_work; |
| 3000 | struct rcu_head *head_free; | 3003 | struct rcu_head *head_free; |
| 3001 | struct kfree_rcu_bulk_data *bhead_free; | 3004 | struct kvfree_rcu_bulk_data *bkvhead_free[FREE_N_CHANNELS]; |
| 3002 | struct kfree_rcu_cpu *krcp; | 3005 | struct kfree_rcu_cpu *krcp; |
| 3003 | }; | 3006 | }; |
| 3004 | 3007 | ||
| 3005 | /** | 3008 | /** |
| 3006 | * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period | 3009 | * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period |
| 3007 | * @head: List of kfree_rcu() objects not yet waiting for a grace period | 3010 | * @head: List of kfree_rcu() objects not yet waiting for a grace period |
| 3008 | * @bhead: Bulk-List of kfree_rcu() objects not yet waiting for a grace period | 3011 | * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period |
| 3009 | * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period | 3012 | * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period |
| 3010 | * @lock: Synchronize access to this structure | 3013 | * @lock: Synchronize access to this structure |
| 3011 | * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES | 3014 | * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES |
| @@ -3020,7 +3023,7 @@ struct kfree_rcu_cpu_work { · 8 unmodified lines | |||
| 3020 | */ | 3023 | */ |
| 3021 | struct kfree_rcu_cpu { | 3024 | struct kfree_rcu_cpu { |
| 3022 | struct rcu_head *head; | 3025 | struct rcu_head *head; |
| 3023 | struct kfree_rcu_bulk_data *bhead; | 3026 | struct kvfree_rcu_bulk_data *bkvhead[FREE_N_CHANNELS]; |
| 3024 | struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES]; | 3027 | struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES]; |
| 3025 | raw_spinlock_t lock; | 3028 | raw_spinlock_t lock; |
| 3026 | struct delayed_work monitor_work; | 3029 | struct delayed_work monitor_work; |
| @@ -3044,7 +3047,7 @@ static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc) = { · 17 unmodified lines | |||
| 3044 | }; | 3047 | }; |
| 3045 | 3048 | ||
| 3046 | static __always_inline void | 3049 | static __always_inline void |
| 3047 | debug_rcu_bhead_unqueue(struct kfree_rcu_bulk_data *bhead) | 3050 | debug_rcu_bhead_unqueue(struct kvfree_rcu_bulk_data *bhead) |
| 3048 | { | 3051 | { |
| 3049 | #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD | 3052 | #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD |
| 3050 | int i; | 3053 | int i; |
| @@ -3073,20 +3076,20 @@ krc_this_cpu_unlock(struct kfree_rcu_cpu *krcp, unsigned long flags) · 22 unmodified lines | |||
| 3073 | local_irq_restore(flags); | 3076 | local_irq_restore(flags); |
| 3074 | } | 3077 | } |
| 3075 | 3078 | ||
| 3076 | static inline struct kfree_rcu_bulk_data * | 3079 | static inline struct kvfree_rcu_bulk_data * |
| 3077 | get_cached_bnode(struct kfree_rcu_cpu *krcp) | 3080 | get_cached_bnode(struct kfree_rcu_cpu *krcp) |
| 3078 | { | 3081 | { |
| 3079 | if (!krcp->nr_bkv_objs) | 3082 | if (!krcp->nr_bkv_objs) |
| 3080 | return NULL; | 3083 | return NULL; |
| 3081 | 3084 | ||
| 3082 | krcp->nr_bkv_objs--; | 3085 | krcp->nr_bkv_objs--; |
| 3083 | return (struct kfree_rcu_bulk_data *) | 3086 | return (struct kvfree_rcu_bulk_data *) |
| 3084 | llist_del_first(&krcp->bkvcache); | 3087 | llist_del_first(&krcp->bkvcache); |
| 3085 | } | 3088 | } |
| 3086 | 3089 | ||
| 3087 | static inline bool | 3090 | static inline bool |
| 3088 | put_cached_bnode(struct kfree_rcu_cpu *krcp, | 3091 | put_cached_bnode(struct kfree_rcu_cpu *krcp, |
| 3089 | struct kfree_rcu_bulk_data *bnode) | 3092 | struct kvfree_rcu_bulk_data *bnode) |
| 3090 | { | 3093 | { |
| 3091 | // Check the limit. | 3094 | // Check the limit. |
| 3092 | if (krcp->nr_bkv_objs >= rcu_min_cached_objs) | 3095 | if (krcp->nr_bkv_objs >= rcu_min_cached_objs) |
| @@ -3105,43 +3108,63 @@ put_cached_bnode(struct kfree_rcu_cpu *krcp, · 12 unmodified lines | |||
| 3105 | static void kfree_rcu_work(struct work_struct *work) | 3108 | static void kfree_rcu_work(struct work_struct *work) |
| 3106 | { | 3109 | { |
| 3107 | unsigned long flags; | 3110 | unsigned long flags; |
| 3111 | struct kvfree_rcu_bulk_data *bkvhead[FREE_N_CHANNELS], *bnext; | ||
| 3108 | struct rcu_head *head, *next; | 3112 | struct rcu_head *head, *next; |
| 3109 | struct kfree_rcu_bulk_data *bhead, *bnext; | ||
| 3110 | struct kfree_rcu_cpu *krcp; | 3113 | struct kfree_rcu_cpu *krcp; |
| 3111 | struct kfree_rcu_cpu_work *krwp; | 3114 | struct kfree_rcu_cpu_work *krwp; |
| 3115 | int i, j; | ||
| 3112 | 3116 | ||
| 3113 | krwp = container_of(to_rcu_work(work), | 3117 | krwp = container_of(to_rcu_work(work), |
| 3114 | struct kfree_rcu_cpu_work, rcu_work); | 3118 | struct kfree_rcu_cpu_work, rcu_work); |
| 3115 | krcp = krwp->krcp; | 3119 | krcp = krwp->krcp; |
| 3120 | |||
| 3116 | raw_spin_lock_irqsave(&krcp->lock, flags); | 3121 | raw_spin_lock_irqsave(&krcp->lock, flags); |
| 3122 | // Channels 1 and 2. | ||
| 3123 | for (i = 0; i < FREE_N_CHANNELS; i++) { | ||
| 3124 | bkvhead[i] = krwp->bkvhead_free[i]; | ||
| 3125 | krwp->bkvhead_free[i] = NULL; | ||
| 3126 | } | ||
| 3127 | |||
| 3128 | // Channel 3. | ||
| 3117 | head = krwp->head_free; | 3129 | head = krwp->head_free; |
| 3118 | krwp->head_free = NULL; | 3130 | krwp->head_free = NULL; |
| 3119 | bhead = krwp->bhead_free; | ||
| 3120 | krwp->bhead_free = NULL; | ||
| 3121 | raw_spin_unlock_irqrestore(&krcp->lock, flags); | 3131 | raw_spin_unlock_irqrestore(&krcp->lock, flags); |
| 3122 | 3132 | ||
| 3123 | /* "bhead" is now private, so traverse locklessly. */ | 3133 | // Handle two first channels. |
| 3124 | for (; bhead; bhead = bnext) { | 3134 | for (i = 0; i < FREE_N_CHANNELS; i++) { |
| 3125 | bnext = bhead->next; | 3135 | for (; bkvhead[i]; bkvhead[i] = bnext) { |
| 3126 | 3136 | bnext = bkvhead[i]->next; | |
| 3127 | debug_rcu_bhead_unqueue(bhead); | 3137 | debug_rcu_bhead_unqueue(bkvhead[i]); |
| 3128 | 3138 | ||
| 3129 | rcu_lock_acquire(&rcu_callback_map); | 3139 | rcu_lock_acquire(&rcu_callback_map); |
| 3130 | trace_rcu_invoke_kfree_bulk_callback(rcu_state.name, | 3140 | if (i == 0) { // kmalloc() / kfree(). |
| 3131 | bhead->nr_records, bhead->records); | 3141 | trace_rcu_invoke_kfree_bulk_callback( |
| 3132 | 3142 | rcu_state.name, bkvhead[i]->nr_records, | |
| 3133 | kfree_bulk(bhead->nr_records, bhead->records); | 3143 | bkvhead[i]->records); |
| 3134 | rcu_lock_release(&rcu_callback_map); | 3144 | |
| 3145 | kfree_bulk(bkvhead[i]->nr_records, | ||
| 3146 | bkvhead[i]->records); | ||
| 3147 | } else { // vmalloc() / vfree(). | ||
| 3148 | for (j = 0; j < bkvhead[i]->nr_records; j++) { | ||
| 3149 | trace_rcu_invoke_kfree_callback( | ||
| 3150 | rcu_state.name, | ||
| 3151 | bkvhead[i]->records[j], 0); | ||
| 3152 | |||
| 3153 | vfree(bkvhead[i]->records[j]); | ||
| 3154 | } | ||
| 3155 | } | ||
| 3156 | rcu_lock_release(&rcu_callback_map); | ||
| 3135 | 3157 | ||
| 3136 | krcp = krc_this_cpu_lock(&flags); | 3158 | krcp = krc_this_cpu_lock(&flags); |
| 3137 | if (put_cached_bnode(krcp, bhead)) | 3159 | if (put_cached_bnode(krcp, bkvhead[i])) |
| 3138 | bhead = NULL; | 3160 | bkvhead[i] = NULL; |
| 3139 | krc_this_cpu_unlock(krcp, flags); | 3161 | krc_this_cpu_unlock(krcp, flags); |
| 3140 | 3162 | ||
| 3141 | if (bhead) | 3163 | if (bkvhead[i]) |
| 3142 | free_page((unsigned long) bhead); | 3164 | free_page((unsigned long) bkvhead[i]); |
| 3143 | 3165 | ||
| 3144 | cond_resched_tasks_rcu_qs(); | 3166 | cond_resched_tasks_rcu_qs(); |
| 3167 | } | ||
| 3145 | } | 3168 | } |
| 3146 | 3169 | ||
| 3147 | /* | 3170 | /* |
| @@ -3159,7 +3182,7 @@ static void kfree_rcu_work(struct work_struct *work) · 11 unmodified lines | |||
| 3159 | trace_rcu_invoke_kfree_callback(rcu_state.name, head, offset); | 3182 | trace_rcu_invoke_kfree_callback(rcu_state.name, head, offset); |
| 3160 | 3183 | ||
| 3161 | if (!WARN_ON_ONCE(!__is_kfree_rcu_offset(offset))) | 3184 | if (!WARN_ON_ONCE(!__is_kfree_rcu_offset(offset))) |
| 3162 | kfree(ptr); | 3185 | kvfree(ptr); |
| 3163 | 3186 | ||
| 3164 | rcu_lock_release(&rcu_callback_map); | 3187 | rcu_lock_release(&rcu_callback_map); |
| 3165 | cond_resched_tasks_rcu_qs(); | 3188 | cond_resched_tasks_rcu_qs(); |
| @@ -3176,7 +3199,7 @@ static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp) · 10 unmodified lines | |||
| 3176 | { | 3199 | { |
| 3177 | struct kfree_rcu_cpu_work *krwp; | 3200 | struct kfree_rcu_cpu_work *krwp; |
| 3178 | bool repeat = false; | 3201 | bool repeat = false; |
| 3179 | int i; | 3202 | int i, j; |
| 3180 | 3203 | ||
| 3181 | lockdep_assert_held(&krcp->lock); | 3204 | lockdep_assert_held(&krcp->lock); |
| 3182 | 3205 | ||
| @@ -3184,21 +3207,25 @@ static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp) · 1 unmodified line | |||
| 3184 | krwp = &(krcp->krw_arr[i]); | 3207 | krwp = &(krcp->krw_arr[i]); |
| 3185 | 3208 | ||
| 3186 | /* | 3209 | /* |
| 3187 | * Try to detach bhead or head and attach it over any | 3210 | * Try to detach bkvhead or head and attach it over any |
| 3188 | * available corresponding free channel. It can be that | 3211 | * available corresponding free channel. It can be that |
| 3189 | * a previous RCU batch is in progress, it means that | 3212 | * a previous RCU batch is in progress, it means that |
| 3190 | * immediately to queue another one is not possible so | 3213 | * immediately to queue another one is not possible so |
| 3191 | * return false to tell caller to retry. | 3214 | * return false to tell caller to retry. |
| 3192 | */ | 3215 | */ |
| 3193 | if ((krcp->bhead && !krwp->bhead_free) || | 3216 | if ((krcp->bkvhead[0] && !krwp->bkvhead_free[0]) || |
| 3217 | (krcp->bkvhead[1] && !krwp->bkvhead_free[1]) || | ||
| 3194 | (krcp->head && !krwp->head_free)) { | 3218 | (krcp->head && !krwp->head_free)) { |
| 3195 | /* Channel 1. */ | 3219 | // Channel 1 corresponds to SLAB ptrs. |
| 3196 | if (!krwp->bhead_free) { | 3220 | // Channel 2 corresponds to vmalloc ptrs. |
| 3197 | krwp->bhead_free = krcp->bhead; | 3221 | for (j = 0; j < FREE_N_CHANNELS; j++) { |
| 3198 | krcp->bhead = NULL; | 3222 | if (!krwp->bkvhead_free[j]) { |
| 3223 | krwp->bkvhead_free[j] = krcp->bkvhead[j]; | ||
| 3224 | krcp->bkvhead[j] = NULL; | ||
| 3225 | } | ||
| 3199 | } | 3226 | } |
| 3200 | 3227 | ||
| 3201 | /* Channel 2. */ | 3228 | // Channel 3 corresponds to emergency path. |
| 3202 | if (!krwp->head_free) { | 3229 | if (!krwp->head_free) { |
| 3203 | krwp->head_free = krcp->head; | 3230 | krwp->head_free = krcp->head; |
| 3204 | krcp->head = NULL; | 3231 | krcp->head = NULL; |
| @@ -3207,16 +3234,17 @@ static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp) · 2 unmodified lines | |||
| 3207 | WRITE_ONCE(krcp->count, 0); | 3234 | WRITE_ONCE(krcp->count, 0); |
| 3208 | 3235 | ||
| 3209 | /* | 3236 | /* |
| 3210 | * One work is per one batch, so there are two "free channels", | 3237 | * One work is per one batch, so there are three |
| 3211 | * "bhead_free" and "head_free" the batch can handle. It can be | 3238 | * "free channels", the batch can handle. It can |
| 3212 | * that the work is in the pending state when two channels have | 3239 | * be that the work is in the pending state when |
| 3213 | * been detached following each other, one by one. | 3240 | * channels have been detached following by each |
| 3241 | * other. | ||
| 3214 | */ | 3242 | */ |
| 3215 | queue_rcu_work(system_wq, &krwp->rcu_work); | 3243 | queue_rcu_work(system_wq, &krwp->rcu_work); |
| 3216 | } | 3244 | } |
| 3217 | 3245 | ||
| 3218 | /* Repeat if any "free" corresponding channel is still busy. */ | 3246 | // Repeat if any "free" corresponding channel is still busy. |
| 3219 | if (krcp->bhead || krcp->head) | 3247 | if (krcp->bkvhead[0] || krcp->bkvhead[1] || krcp->head) |
| 3220 | repeat = true; | 3248 | repeat = true; |
| 3221 | } | 3249 | } |
| 3222 | 3250 | ||
| @@ -3258,23 +3286,22 @@ static void kfree_rcu_monitor(struct work_struct *work) · 35 unmodified lines | |||
| 3258 | } | 3286 | } |
| 3259 | 3287 | ||
| 3260 | static inline bool | 3288 | static inline bool |
| 3261 | kfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, | 3289 | kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr) |
| 3262 | struct rcu_head *head, rcu_callback_t func) | ||
| 3263 | { | 3290 | { |
| 3264 | struct kfree_rcu_bulk_data *bnode; | 3291 | struct kvfree_rcu_bulk_data *bnode; |
| 3292 | int idx; | ||
| 3265 | 3293 | ||
| 3266 | if (unlikely(!krcp->initialized)) | 3294 | if (unlikely(!krcp->initialized)) |
| 3267 | return false; | 3295 | return false; |
| 3268 | 3296 | ||
| 3269 | lockdep_assert_held(&krcp->lock); | 3297 | lockdep_assert_held(&krcp->lock); |
| 3298 | idx = !!is_vmalloc_addr(ptr); | ||
| 3270 | 3299 | ||
| 3271 | /* Check if a new block is required. */ | 3300 | /* Check if a new block is required. */ |
| 3272 | if (!krcp->bhead || | 3301 | if (!krcp->bkvhead[idx] || |
| 3273 | krcp->bhead->nr_records == KFREE_BULK_MAX_ENTR) { | 3302 | krcp->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) { |
| 3274 | bnode = get_cached_bnode(krcp); | 3303 | bnode = get_cached_bnode(krcp); |
| 3275 | if (!bnode) { | 3304 | if (!bnode) { |
| 3276 | WARN_ON_ONCE(sizeof(struct kfree_rcu_bulk_data) > PAGE_SIZE); | ||
| 3277 | |||
| 3278 | /* | 3305 | /* |
| 3279 | * To keep this path working on raw non-preemptible | 3306 | * To keep this path working on raw non-preemptible |
| 3280 | * sections, prevent the optional entry into the | 3307 | * sections, prevent the optional entry into the |
| @@ -3287,7 +3314,7 @@ kfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, · 6 unmodified lines | |||
| 3287 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) | 3314 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) |
| 3288 | return false; | 3315 | return false; |
| 3289 | 3316 | ||
| 3290 | bnode = (struct kfree_rcu_bulk_data *) | 3317 | bnode = (struct kvfree_rcu_bulk_data *) |
| 3291 | __get_free_page(GFP_NOWAIT | __GFP_NOWARN); | 3318 | __get_free_page(GFP_NOWAIT | __GFP_NOWARN); |
| 3292 | } | 3319 | } |
| 3293 | 3320 | ||
| @@ -3297,30 +3324,30 @@ kfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, · 3 unmodified lines | |||
| 3297 | 3324 | ||
| 3298 | /* Initialize the new block. */ | 3325 | /* Initialize the new block. */ |
| 3299 | bnode->nr_records = 0; | 3326 | bnode->nr_records = 0; |
| 3300 | bnode->next = krcp->bhead; | 3327 | bnode->next = krcp->bkvhead[idx]; |
| 3301 | 3328 | ||
| 3302 | /* Attach it to the head. */ | 3329 | /* Attach it to the head. */ |
| 3303 | krcp->bhead = bnode; | 3330 | krcp->bkvhead[idx] = bnode; |
| 3304 | } | 3331 | } |
| 3305 | 3332 | ||
| 3306 | /* Finally insert. */ | 3333 | /* Finally insert. */ |
| 3307 | krcp->bhead->records[krcp->bhead->nr_records++] = | 3334 | krcp->bkvhead[idx]->records |
| 3308 | (void *) head - (unsigned long) func; | 3335 | [krcp->bkvhead[idx]->nr_records++] = ptr; |
| 3309 | 3336 | ||
| 3310 | return true; | 3337 | return true; |
| 3311 | } | 3338 | } |
| 3312 | 3339 | ||
| 3313 | /* | 3340 | /* |
| 3314 | * Queue a request for lazy invocation of kfree_bulk()/kfree() after a grace | 3341 | * Queue a request for lazy invocation of appropriate free routine after a |
| 3315 | * period. Please note there are two paths are maintained, one is the main one | 3342 | * grace period. Please note there are three paths are maintained, two are the |
| 3316 | * that uses kfree_bulk() interface and second one is emergency one, that is | 3343 | * main ones that use array of pointers interface and third one is emergency |
| 3317 | * used only when the main path can not be maintained temporary, due to memory | 3344 | * one, that is used only when the main path can not be maintained temporary, |
| 3318 | * pressure. | 3345 | * due to memory pressure. |
| 3319 | * | 3346 | * |
| 3320 | * Each kfree_call_rcu() request is added to a batch. The batch will be drained | 3347 | * Each kfree_call_rcu() request is added to a batch. The batch will be drained |
| 3321 | * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch will | 3348 | * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch will |
| 3322 | * be free'd in workqueue context. This allows us to: batch requests together to | 3349 | * be free'd in workqueue context. This allows us to: batch requests together to |
| 3323 | * reduce the number of grace periods during heavy kfree_rcu() load. | 3350 | * reduce the number of grace periods during heavy kfree_rcu()/kvfree_rcu() load. |
| 3324 | */ | 3351 | */ |
| 3325 | void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func) | 3352 | void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func) |
| 3326 | { | 3353 | { |
| @@ -3343,7 +3370,7 @@ void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func) · 16 unmodified lines | |||
| 3343 | * Under high memory pressure GFP_NOWAIT can fail, | 3370 | * Under high memory pressure GFP_NOWAIT can fail, |
| 3344 | * in that case the emergency path is maintained. | 3371 | * in that case the emergency path is maintained. |
| 3345 | */ | 3372 | */ |
| 3346 | if (unlikely(!kfree_call_rcu_add_ptr_to_bulk(krcp, head, func))) { | 3373 | if (unlikely(!kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr))) { |
| 3347 | head->func = func; | 3374 | head->func = func; |
| 3348 | head->next = krcp->head; | 3375 | head->next = krcp->head; |
| 3349 | krcp->head = head; | 3376 | krcp->head = head; |
| @@ -4324,7 +4351,7 @@ static void __init kfree_rcu_batch_init(void) · 974 unmodified lines | |||
| 4324 | 4351 | ||
| 4325 | for_each_possible_cpu(cpu) { | 4352 | for_each_possible_cpu(cpu) { |
| 4326 | struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu); | 4353 | struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu); |
| 4327 | struct kfree_rcu_bulk_data *bnode; | 4354 | struct kvfree_rcu_bulk_data *bnode; |
| 4328 | 4355 | ||
| 4329 | for (i = 0; i < KFREE_N_BATCHES; i++) { | 4356 | for (i = 0; i < KFREE_N_BATCHES; i++) { |
| 4330 | INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work); | 4357 | INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work); |
| @@ -4332,7 +4359,7 @@ static void __init kfree_rcu_batch_init(void) · 1 unmodified line | |||
| 4332 | } | 4359 | } |
| 4333 | 4360 | ||
| 4334 | for (i = 0; i < rcu_min_cached_objs; i++) { | 4361 | for (i = 0; i < rcu_min_cached_objs; i++) { |
| 4335 | bnode = (struct kfree_rcu_bulk_data *) | 4362 | bnode = (struct kvfree_rcu_bulk_data *) |
| 4336 | __get_free_page(GFP_NOWAIT | __GFP_NOWARN); | 4363 | __get_free_page(GFP_NOWAIT | __GFP_NOWARN); |
| 4337 | 4364 | ||
| 4338 | if (bnode) | 4365 | if (bnode) |