Pull slab fix from Vlastimil Babka: "A NULL pointer deref hotfix" * tag 'slab-for-6.18-rc1-hotfix' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab: slab: fix barn NULL pointer dereference on memoryless nodes
Unified · Split
| @@ -504,10 +504,18 @@ static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) · 503 unmodified lines | ||
| 504 | 504 | return s->node[node]; |
| 505 | 505 | } |
| 506 | 506 | |
| 507 | −/* Get the barn of the current cpu's memory node */ | |
| 507 | +/* | |
| 508 | + * Get the barn of the current cpu's closest memory node. It may not exist on | |
| 509 | + * systems with memoryless nodes but without CONFIG_HAVE_MEMORYLESS_NODES | |
| 510 | + */ | |
| 508 | 511 | static inline struct node_barn *get_barn(struct kmem_cache *s) |
| 509 | 512 | { |
| 510 | − return get_node(s, numa_mem_id())->barn; | |
| 513 | + struct kmem_cache_node *n = get_node(s, numa_mem_id()); | |
| 514 | + | |
| 515 | + if (!n) | |
| 516 | + return NULL; | |
| 517 | + | |
| 518 | + return n->barn; | |
| 511 | 519 | } |
| 512 | 520 | |
| 513 | 521 | /* |
| @@ -4982,6 +4990,10 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, · 4468 unmodified lines | ||
| 4982 | 4990 | } |
| 4983 | 4991 | |
| 4984 | 4992 | barn = get_barn(s); |
| 4993 | + if (!barn) { | |
| 4994 | + local_unlock(&s->cpu_sheaves->lock); | |
| 4995 | + return NULL; | |
| 4996 | + } | |
| 4985 | 4997 | |
| 4986 | 4998 | full = barn_replace_empty_sheaf(barn, pcs->main); |
| 4987 | 4999 | |
| @@ -5153,13 +5165,20 @@ next_batch: · 165 unmodified lines | ||
| 5153 | 5165 | if (unlikely(pcs->main->size == 0)) { |
| 5154 | 5166 | |
| 5155 | 5167 | struct slab_sheaf *full; |
| 5168 | + struct node_barn *barn; | |
| 5156 | 5169 | |
| 5157 | 5170 | if (pcs->spare && pcs->spare->size > 0) { |
| 5158 | 5171 | swap(pcs->main, pcs->spare); |
| 5159 | 5172 | goto do_alloc; |
| 5160 | 5173 | } |
| 5161 | 5174 | |
| 5162 | − full = barn_replace_empty_sheaf(get_barn(s), pcs->main); | |
| 5175 | + barn = get_barn(s); | |
| 5176 | + if (!barn) { | |
| 5177 | + local_unlock(&s->cpu_sheaves->lock); | |
| 5178 | + return allocated; | |
| 5179 | + } | |
| 5180 | + | |
| 5181 | + full = barn_replace_empty_sheaf(barn, pcs->main); | |
| 5163 | 5182 | |
| 5164 | 5183 | if (full) { |
| 5165 | 5184 | stat(s, BARN_GET); |
| @@ -5314,6 +5333,7 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size) · 148 unmodified lines | ||
| 5314 | 5333 | { |
| 5315 | 5334 | struct slub_percpu_sheaves *pcs; |
| 5316 | 5335 | struct slab_sheaf *sheaf = NULL; |
| 5336 | + struct node_barn *barn; | |
| 5317 | 5337 | |
| 5318 | 5338 | if (unlikely(size > s->sheaf_capacity)) { |
| 5319 | 5339 | |
| @@ -5355,8 +5375,11 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size) · 35 unmodified lines | ||
| 5355 | 5375 | pcs->spare = NULL; |
| 5356 | 5376 | stat(s, SHEAF_PREFILL_FAST); |
| 5357 | 5377 | } else { |
| 5378 | + barn = get_barn(s); | |
| 5379 | + | |
| 5358 | 5380 | stat(s, SHEAF_PREFILL_SLOW); |
| 5359 | − sheaf = barn_get_full_or_empty_sheaf(get_barn(s)); | |
| 5381 | + if (barn) | |
| 5382 | + sheaf = barn_get_full_or_empty_sheaf(barn); | |
| 5360 | 5383 | if (sheaf && sheaf->size) |
| 5361 | 5384 | stat(s, BARN_GET); |
| 5362 | 5385 | else |
| @@ -5426,7 +5449,7 @@ void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp, · 63 unmodified lines | ||
| 5426 | 5449 | * If the barn has too many full sheaves or we fail to refill the sheaf, |
| 5427 | 5450 | * simply flush and free it. |
| 5428 | 5451 | */ |
| 5429 | − if (data_race(barn->nr_full) >= MAX_FULL_SHEAVES || | |
| 5452 | + if (!barn || data_race(barn->nr_full) >= MAX_FULL_SHEAVES || | |
| 5430 | 5453 | refill_sheaf(s, sheaf, gfp)) { |
| 5431 | 5454 | sheaf_flush_unused(s, sheaf); |
| 5432 | 5455 | free_empty_sheaf(s, sheaf); |
| @@ -5943,10 +5966,9 @@ slab_empty: · 510 unmodified lines | ||
| 5943 | 5966 | * put the full sheaf there. |
| 5944 | 5967 | */ |
| 5945 | 5968 | static void __pcs_install_empty_sheaf(struct kmem_cache *s, |
| 5946 | − struct slub_percpu_sheaves *pcs, struct slab_sheaf *empty) | |
| 5969 | + struct slub_percpu_sheaves *pcs, struct slab_sheaf *empty, | |
| 5970 | + struct node_barn *barn) | |
| 5947 | 5971 | { |
| 5948 | − struct node_barn *barn; | |
| 5949 | − | |
| 5950 | 5972 | lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); |
| 5951 | 5973 | |
| 5952 | 5974 | /* This is what we expect to find if nobody interrupted us. */ |
| @@ -5956,8 +5978,6 @@ static void __pcs_install_empty_sheaf(struct kmem_cache *s, · 3 unmodified lines | ||
| 5956 | 5978 | return; |
| 5957 | 5979 | } |
| 5958 | 5980 | |
| 5959 | − barn = get_barn(s); | |
| 5960 | − | |
| 5961 | 5981 | /* |
| 5962 | 5982 | * Unlikely because if the main sheaf had space, we would have just |
| 5963 | 5983 | * freed to it. Get rid of our empty sheaf. |
| @@ -6002,6 +6022,11 @@ restart: · 38 unmodified lines | ||
| 6002 | 6022 | lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); |
| 6003 | 6023 | |
| 6004 | 6024 | barn = get_barn(s); |
| 6025 | + if (!barn) { | |
| 6026 | + local_unlock(&s->cpu_sheaves->lock); | |
| 6027 | + return NULL; | |
| 6028 | + } | |
| 6029 | + | |
| 6005 | 6030 | put_fail = false; |
| 6006 | 6031 | |
| 6007 | 6032 | if (!pcs->spare) { |
| @@ -6084,7 +6109,7 @@ got_empty: · 76 unmodified lines | ||
| 6084 | 6109 | } |
| 6085 | 6110 | |
| 6086 | 6111 | pcs = this_cpu_ptr(s->cpu_sheaves); |
| 6087 | − __pcs_install_empty_sheaf(s, pcs, empty); | |
| 6112 | + __pcs_install_empty_sheaf(s, pcs, empty, barn); | |
| 6088 | 6113 | |
| 6089 | 6114 | return pcs; |
| 6090 | 6115 | } |
| @@ -6121,8 +6146,9 @@ bool free_to_pcs(struct kmem_cache *s, void *object) · 30 unmodified lines | ||
| 6121 | 6146 | |
| 6122 | 6147 | static void rcu_free_sheaf(struct rcu_head *head) |
| 6123 | 6148 | { |
| 6149 | + struct kmem_cache_node *n; | |
| 6124 | 6150 | struct slab_sheaf *sheaf; |
| 6125 | − struct node_barn *barn; | |
| 6151 | + struct node_barn *barn = NULL; | |
| 6126 | 6152 | struct kmem_cache *s; |
| 6127 | 6153 | |
| 6128 | 6154 | sheaf = container_of(head, struct slab_sheaf, rcu_head); |
| @@ -6139,7 +6165,11 @@ static void rcu_free_sheaf(struct rcu_head *head) · 10 unmodified lines | ||
| 6139 | 6165 | */ |
| 6140 | 6166 | __rcu_free_sheaf_prepare(s, sheaf); |
| 6141 | 6167 | |
| 6142 | − barn = get_node(s, sheaf->node)->barn; | |
| 6168 | + n = get_node(s, sheaf->node); | |
| 6169 | + if (!n) | |
| 6170 | + goto flush; | |
| 6171 | + | |
| 6172 | + barn = n->barn; | |
| 6143 | 6173 | |
| 6144 | 6174 | /* due to slab_free_hook() */ |
| 6145 | 6175 | if (unlikely(sheaf->size == 0)) |
| @@ -6157,11 +6187,12 @@ static void rcu_free_sheaf(struct rcu_head *head) · 11 unmodified lines | ||
| 6157 | 6187 | return; |
| 6158 | 6188 | } |
| 6159 | 6189 | |
| 6190 | +flush: | |
| 6160 | 6191 | stat(s, BARN_PUT_FAIL); |
| 6161 | 6192 | sheaf_flush_unused(s, sheaf); |
| 6162 | 6193 | |
| 6163 | 6194 | empty: |
| 6164 | − if (data_race(barn->nr_empty) < MAX_EMPTY_SHEAVES) { | |
| 6195 | + if (barn && data_race(barn->nr_empty) < MAX_EMPTY_SHEAVES) { | |
| 6165 | 6196 | barn_put_empty_sheaf(barn, sheaf); |
| 6166 | 6197 | return; |
| 6167 | 6198 | } |
| @@ -6191,6 +6222,10 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) · 23 unmodified lines | ||
| 6191 | 6222 | } |
| 6192 | 6223 | |
| 6193 | 6224 | barn = get_barn(s); |
| 6225 | + if (!barn) { | |
| 6226 | + local_unlock(&s->cpu_sheaves->lock); | |
| 6227 | + goto fail; | |
| 6228 | + } | |
| 6194 | 6229 | |
| 6195 | 6230 | empty = barn_get_empty_sheaf(barn); |
| 6196 | 6231 | |
| @@ -6304,6 +6339,8 @@ next_batch: · 107 unmodified lines | ||
| 6304 | 6339 | goto do_free; |
| 6305 | 6340 | |
| 6306 | 6341 | barn = get_barn(s); |
| 6342 | + if (!barn) | |
| 6343 | + goto no_empty; | |
| 6307 | 6344 | |
| 6308 | 6345 | if (!pcs->spare) { |
| 6309 | 6346 | empty = barn_get_empty_sheaf(barn); |