Commit 284f17ac13fe ("mm/slub: handle bulk and single object freeing separately") splits single and bulk object freeing in two functions slab_free() and slab_free_bulk() which leads slab_free() to call slab_free_hook() directly instead of slab_free_freelist_hook(). If `init_on_free` is set, slab_free_hook() zeroes the object. Afterward, if `slub_debug=F` and `CONFIG_SLAB_FREELIST_HARDENED` are set, the do_slab_free() slowpath executes freelist consistency checks and try to decode a zeroed freepointer which leads to a "Freepointer corrupt" detection in check_object(). During bulk free, slab_free_freelist_hook() isn't affected as it always sets it objects freepointer using set_freepointer() to maintain its reconstructed freelist after `init_on_free`. For single free, object's freepointer thus needs to be avoided when stored outside the object if `init_on_free` is set. The freepointer left as is, check_object() may later detect an invalid pointer value due to objects overflow. To reproduce, set `slub_debug=FU init_on_free=1 log_level=7` on the command line of a kernel build with `CONFIG_SLAB_FREELIST_HARDENED=y`. dmesg sample log: [ 10.708715] ============================================================================= [ 10.710323] BUG kmalloc-rnd-05-32 (Tainted: G B T ): Freepointer corrupt [ 10.712695] ----------------------------------------------------------------------------- [ 10.712695] [ 10.712695] Slab 0xffffd8bdc400d580 objects=32 used=4 fp=0xffff9d9a80356f80 flags=0x200000000000a00(workingset|slab|node=0|zone=2) [ 10.716698] Object 0xffff9d9a80356600 @offset=1536 fp=0x7ee4f480ce0ecd7c [ 10.716698] [ 10.716698] Bytes b4 ffff9d9a803565f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ [ 10.720703] Object ffff9d9a80356600: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ [ 10.720703] Object ffff9d9a80356610: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ [ 10.724696] Padding ffff9d9a8035666c: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ [ 10.724696] Padding ffff9d9a8035667c: 00 00 00 00 .... [ 10.724696] FIX kmalloc-rnd-05-32: Object at 0xffff9d9a80356600 not freed
Unified · Split
| @@ -557,6 +557,26 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) · 556 unmodified lines | |||
| 557 | *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr); | 557 | *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr); |
| 558 | } | 558 | } |
| 559 | 559 | ||
| 560 | /* | ||
| 561 | * See comment in calculate_sizes(). | ||
| 562 | */ | ||
| 563 | static inline bool freeptr_outside_object(struct kmem_cache *s) | ||
| 564 | { | ||
| 565 | return s->offset >= s->inuse; | ||
| 566 | } | ||
| 567 | |||
| 568 | /* | ||
| 569 | * Return offset of the end of info block which is inuse + free pointer if | ||
| 570 | * not overlapping with object. | ||
| 571 | */ | ||
| 572 | static inline unsigned int get_info_end(struct kmem_cache *s) | ||
| 573 | { | ||
| 574 | if (freeptr_outside_object(s)) | ||
| 575 | return s->inuse + sizeof(void *); | ||
| 576 | else | ||
| 577 | return s->inuse; | ||
| 578 | } | ||
| 579 | |||
| 560 | /* Loop over all objects in a slab */ | 580 | /* Loop over all objects in a slab */ |
| 561 | #define for_each_object(__p, __s, __addr, __objects) \ | 581 | #define for_each_object(__p, __s, __addr, __objects) \ |
| 562 | for (__p = fixup_red_left(__s, __addr); \ | 582 | for (__p = fixup_red_left(__s, __addr); \ |
| @@ -845,26 +865,6 @@ static void print_section(char *level, char *text, u8 *addr, · 282 unmodified lines | |||
| 845 | metadata_access_disable(); | 865 | metadata_access_disable(); |
| 846 | } | 866 | } |
| 847 | 867 | ||
| 848 | /* | ||
| 849 | * See comment in calculate_sizes(). | ||
| 850 | */ | ||
| 851 | static inline bool freeptr_outside_object(struct kmem_cache *s) | ||
| 852 | { | ||
| 853 | return s->offset >= s->inuse; | ||
| 854 | } | ||
| 855 | |||
| 856 | /* | ||
| 857 | * Return offset of the end of info block which is inuse + free pointer if | ||
| 858 | * not overlapping with object. | ||
| 859 | */ | ||
| 860 | static inline unsigned int get_info_end(struct kmem_cache *s) | ||
| 861 | { | ||
| 862 | if (freeptr_outside_object(s)) | ||
| 863 | return s->inuse + sizeof(void *); | ||
| 864 | else | ||
| 865 | return s->inuse; | ||
| 866 | } | ||
| 867 | |||
| 868 | static struct track *get_track(struct kmem_cache *s, void *object, | 868 | static struct track *get_track(struct kmem_cache *s, void *object, |
| 869 | enum track_item alloc) | 869 | enum track_item alloc) |
| 870 | { | 870 | { |
| @@ -2092,15 +2092,20 @@ bool slab_free_hook(struct kmem_cache *s, void *x, bool init) · 1221 unmodified lines | |||
| 2092 | * | 2092 | * |
| 2093 | * The initialization memset's clear the object and the metadata, | 2093 | * The initialization memset's clear the object and the metadata, |
| 2094 | * but don't touch the SLAB redzone. | 2094 | * but don't touch the SLAB redzone. |
| 2095 | * | ||
| 2096 | * The object's freepointer is also avoided if stored outside the | ||
| 2097 | * object. | ||
| 2095 | */ | 2098 | */ |
| 2096 | if (unlikely(init)) { | 2099 | if (unlikely(init)) { |
| 2097 | int rsize; | 2100 | int rsize; |
| 2101 | unsigned int inuse; | ||
| 2098 | 2102 | ||
| 2103 | inuse = get_info_end(s); | ||
| 2099 | if (!kasan_has_integrated_init()) | 2104 | if (!kasan_has_integrated_init()) |
| 2100 | memset(kasan_reset_tag(x), 0, s->object_size); | 2105 | memset(kasan_reset_tag(x), 0, s->object_size); |
| 2101 | rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0; | 2106 | rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0; |
| 2102 | memset((char *)kasan_reset_tag(x) + s->inuse, 0, | 2107 | memset((char *)kasan_reset_tag(x) + inuse, 0, |
| 2103 | s->size - s->inuse - rsize); | 2108 | s->size - inuse - rsize); |
| 2104 | } | 2109 | } |
| 2105 | /* KASAN might put x into memory quarantine, delaying its reuse. */ | 2110 | /* KASAN might put x into memory quarantine, delaying its reuse. */ |
| 2106 | return !kasan_slab_free(s, x, init); | 2111 | return !kasan_slab_free(s, x, init); |
| @@ -3722,7 +3727,8 @@ static void *__slab_alloc_node(struct kmem_cache *s, · 1615 unmodified lines | |||
| 3722 | static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s, | 3727 | static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s, |
| 3723 | void *obj) | 3728 | void *obj) |
| 3724 | { | 3729 | { |
| 3725 | if (unlikely(slab_want_init_on_free(s)) && obj) | 3730 | if (unlikely(slab_want_init_on_free(s)) && obj && |
| 3731 | !freeptr_outside_object(s)) | ||
| 3726 | memset((void *)((char *)kasan_reset_tag(obj) + s->offset), | 3732 | memset((void *)((char *)kasan_reset_tag(obj) + s->offset), |
| 3727 | 0, sizeof(void *)); | 3733 | 0, sizeof(void *)); |
| 3728 | } | 3734 | } |