History

mm/kasan: Convert to struct folio and struct slab

KASAN accesses some slab related struct page fields so we need to convert it to struct slab. Some places are a bit simplified thanks to kasan_addr_to_slab() encapsulating the PageSlab flag check through virt_to_slab(). When resolving object address to either a real slab or a large kmalloc, use struct folio as the intermediate type for testing the slab flag to avoid unnecessary implicit compound_head(). [ vbabka@suse.cz: use struct folio, adjust to differences in previous patches ]

Matthew Wilcox (Oracle) committed 5 years ago

Commit 6e48a961 parent 50757019 files · +42 −28Browse files

Unified · Split

9 files
  1. include/linux/kasan.h +5 −4
  2. mm/kasan/common.c +13 −10
  3. mm/kasan/generic.c +4 −4
  4. mm/kasan/kasan.h +1 −0
  5. mm/kasan/quarantine.c +1 −1
  6. mm/kasan/report.c +11 −2
  7. mm/kasan/report_tags.c +5 −5
  8. mm/slab.c +1 −1
  9. mm/slub.c +1 −1
include/linux/kasan.h+5−4
@@ -9,6 +9,7 @@ · 8 unmodified lines
9 9
10struct kmem_cache;10struct kmem_cache;
11struct page;11struct page;
12struct slab;
12struct vm_struct;13struct vm_struct;
13struct task_struct;14struct task_struct;
14 15
@@ -193,11 +194,11 @@ static __always_inline size_t kasan_metadata_size(struct kmem_cache *cache) · 178 unmodified lines
193 return 0;194 return 0;
194}195}
195 196
196void __kasan_poison_slab(struct page *page);197void __kasan_poison_slab(struct slab *slab);
197static __always_inline void kasan_poison_slab(struct page *page)198static __always_inline void kasan_poison_slab(struct slab *slab)
198{199{
199 if (kasan_enabled())200 if (kasan_enabled())
200 __kasan_poison_slab(page);201 __kasan_poison_slab(slab);
201}202}
202 203
203void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object);204void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object);
@@ -322,7 +323,7 @@ static inline void kasan_cache_create(struct kmem_cache *cache, · 118 unmodified lines
322 slab_flags_t *flags) {}323 slab_flags_t *flags) {}
323static inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) {}324static inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) {}
324static inline size_t kasan_metadata_size(struct kmem_cache *cache) { return 0; }325static inline size_t kasan_metadata_size(struct kmem_cache *cache) { return 0; }
325static inline void kasan_poison_slab(struct page *page) {}326static inline void kasan_poison_slab(struct slab *slab) {}
326static inline void kasan_unpoison_object_data(struct kmem_cache *cache,327static inline void kasan_unpoison_object_data(struct kmem_cache *cache,
327 void *object) {}328 void *object) {}
328static inline void kasan_poison_object_data(struct kmem_cache *cache,329static inline void kasan_poison_object_data(struct kmem_cache *cache,
mm/kasan/common.c+13−10
@@ -247,8 +247,9 @@ struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache, · 246 unmodified lines
247}247}
248#endif248#endif
249 249
250void __kasan_poison_slab(struct page *page)250void __kasan_poison_slab(struct slab *slab)
251{251{
252 struct page *page = slab_page(slab);
252 unsigned long i;253 unsigned long i;
253 254
254 for (i = 0; i < compound_nr(page); i++)255 for (i = 0; i < compound_nr(page); i++)
@@ -401,9 +402,9 @@ void __kasan_kfree_large(void *ptr, unsigned long ip) · 146 unmodified lines
401 402
402void __kasan_slab_free_mempool(void *ptr, unsigned long ip)403void __kasan_slab_free_mempool(void *ptr, unsigned long ip)
403{404{
404 struct page *page;405 struct folio *folio;
405 406
406 page = virt_to_head_page(ptr);407 folio = virt_to_folio(ptr);
407 408
408 /*409 /*
409 * Even though this function is only called for kmem_cache_alloc and410 * Even though this function is only called for kmem_cache_alloc and
@@ -411,12 +412,14 @@ void __kasan_slab_free_mempool(void *ptr, unsigned long ip) · 1 unmodified line
411 * !PageSlab() when the size provided to kmalloc is larger than412 * !PageSlab() when the size provided to kmalloc is larger than
412 * KMALLOC_MAX_SIZE, and kmalloc falls back onto page_alloc.413 * KMALLOC_MAX_SIZE, and kmalloc falls back onto page_alloc.
413 */414 */
414 if (unlikely(!PageSlab(page))) {415 if (unlikely(!folio_test_slab(folio))) {
415 if (____kasan_kfree_large(ptr, ip))416 if (____kasan_kfree_large(ptr, ip))
416 return;417 return;
417 kasan_poison(ptr, page_size(page), KASAN_FREE_PAGE, false);418 kasan_poison(ptr, folio_size(folio), KASAN_FREE_PAGE, false);
418 } else {419 } else {
419 ____kasan_slab_free(page->slab_cache, ptr, ip, false, false);420 struct slab *slab = folio_slab(folio);
421
422 ____kasan_slab_free(slab->slab_cache, ptr, ip, false, false);
420 }423 }
421}424}
422 425
@@ -560,7 +563,7 @@ void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, · 137 unmodified lines
560 563
561void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flags)564void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flags)
562{565{
563 struct page *page;566 struct slab *slab;
564 567
565 if (unlikely(object == ZERO_SIZE_PTR))568 if (unlikely(object == ZERO_SIZE_PTR))
566 return (void *)object;569 return (void *)object;
@@ -572,13 +575,13 @@ void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flag · 5 unmodified lines
572 */575 */
573 kasan_unpoison(object, size, false);576 kasan_unpoison(object, size, false);
574 577
575 page = virt_to_head_page(object);578 slab = virt_to_slab(object);
576 579
577 /* Piggy-back on kmalloc() instrumentation to poison the redzone. */580 /* Piggy-back on kmalloc() instrumentation to poison the redzone. */
578 if (unlikely(!PageSlab(page)))581 if (unlikely(!slab))
579 return __kasan_kmalloc_large(object, size, flags);582 return __kasan_kmalloc_large(object, size, flags);
580 else583 else
581 return ____kasan_kmalloc(page->slab_cache, object, size, flags);584 return ____kasan_kmalloc(slab->slab_cache, object, size, flags);
582}585}
583 586
584bool __kasan_check_byte(const void *address, unsigned long ip)587bool __kasan_check_byte(const void *address, unsigned long ip)
mm/kasan/generic.c+4−4
@@ -330,16 +330,16 @@ DEFINE_ASAN_SET_SHADOW(f8); · 329 unmodified lines
330 330
331static void __kasan_record_aux_stack(void *addr, bool can_alloc)331static void __kasan_record_aux_stack(void *addr, bool can_alloc)
332{332{
333 struct page *page = kasan_addr_to_page(addr);333 struct slab *slab = kasan_addr_to_slab(addr);
334 struct kmem_cache *cache;334 struct kmem_cache *cache;
335 struct kasan_alloc_meta *alloc_meta;335 struct kasan_alloc_meta *alloc_meta;
336 void *object;336 void *object;
337 337
338 if (is_kfence_address(addr) || !(page && PageSlab(page)))338 if (is_kfence_address(addr) || !slab)
339 return;339 return;
340 340
341 cache = page->slab_cache;341 cache = slab->slab_cache;
342 object = nearest_obj(cache, page_slab(page), addr);342 object = nearest_obj(cache, slab, addr);
343 alloc_meta = kasan_get_alloc_meta(cache, object);343 alloc_meta = kasan_get_alloc_meta(cache, object);
344 if (!alloc_meta)344 if (!alloc_meta)
345 return;345 return;
mm/kasan/kasan.h+1−0
@@ -265,6 +265,7 @@ bool kasan_report(unsigned long addr, size_t size, · 264 unmodified lines
265void kasan_report_invalid_free(void *object, unsigned long ip);265void kasan_report_invalid_free(void *object, unsigned long ip);
266 266
267struct page *kasan_addr_to_page(const void *addr);267struct page *kasan_addr_to_page(const void *addr);
268struct slab *kasan_addr_to_slab(const void *addr);
268 269
269depot_stack_handle_t kasan_save_stack(gfp_t flags, bool can_alloc);270depot_stack_handle_t kasan_save_stack(gfp_t flags, bool can_alloc);
270void kasan_set_track(struct kasan_track *track, gfp_t flags);271void kasan_set_track(struct kasan_track *track, gfp_t flags);
mm/kasan/quarantine.c+1−1
@@ -117,7 +117,7 @@ static unsigned long quarantine_batch_size; · 116 unmodified lines
117 117
118static struct kmem_cache *qlink_to_cache(struct qlist_node *qlink)118static struct kmem_cache *qlink_to_cache(struct qlist_node *qlink)
119{119{
120 return virt_to_head_page(qlink)->slab_cache;120 return virt_to_slab(qlink)->slab_cache;
121}121}
122 122
123static void *qlink_to_object(struct qlist_node *qlink, struct kmem_cache *cache)123static void *qlink_to_object(struct qlist_node *qlink, struct kmem_cache *cache)
mm/kasan/report.c+11−2
@@ -150,6 +150,14 @@ struct page *kasan_addr_to_page(const void *addr) · 149 unmodified lines
150 return NULL;150 return NULL;
151}151}
152 152
153struct slab *kasan_addr_to_slab(const void *addr)
154{
155 if ((addr >= (void *)PAGE_OFFSET) &&
156 (addr < high_memory))
157 return virt_to_slab(addr);
158 return NULL;
159}
160
153static void describe_object_addr(struct kmem_cache *cache, void *object,161static void describe_object_addr(struct kmem_cache *cache, void *object,
154 const void *addr)162 const void *addr)
155{163{
@@ -248,8 +256,9 @@ static void print_address_description(void *addr, u8 tag) · 92 unmodified lines
248 pr_err("\n");256 pr_err("\n");
249 257
250 if (page && PageSlab(page)) {258 if (page && PageSlab(page)) {
251 struct kmem_cache *cache = page->slab_cache;259 struct slab *slab = page_slab(page);
252 void *object = nearest_obj(cache, page_slab(page), addr);260 struct kmem_cache *cache = slab->slab_cache;
261 void *object = nearest_obj(cache, slab, addr);
253 262
254 describe_object(cache, object, addr, tag);263 describe_object(cache, object, addr, tag);
255 }264 }
mm/kasan/report_tags.c+5−5
@@ -12,7 +12,7 @@ const char *kasan_get_bug_type(struct kasan_access_info *info) · 11 unmodified lines
12#ifdef CONFIG_KASAN_TAGS_IDENTIFY12#ifdef CONFIG_KASAN_TAGS_IDENTIFY
13 struct kasan_alloc_meta *alloc_meta;13 struct kasan_alloc_meta *alloc_meta;
14 struct kmem_cache *cache;14 struct kmem_cache *cache;
15 struct page *page;15 struct slab *slab;
16 const void *addr;16 const void *addr;
17 void *object;17 void *object;
18 u8 tag;18 u8 tag;
@@ -20,10 +20,10 @@ const char *kasan_get_bug_type(struct kasan_access_info *info) · 1 unmodified line
20 20
21 tag = get_tag(info->access_addr);21 tag = get_tag(info->access_addr);
22 addr = kasan_reset_tag(info->access_addr);22 addr = kasan_reset_tag(info->access_addr);
23 page = kasan_addr_to_page(addr);23 slab = kasan_addr_to_slab(addr);
24 if (page && PageSlab(page)) {24 if (slab) {
25 cache = page->slab_cache;25 cache = slab->slab_cache;
26 object = nearest_obj(cache, page_slab(page), (void *)addr);26 object = nearest_obj(cache, slab, (void *)addr);
27 alloc_meta = kasan_get_alloc_meta(cache, object);27 alloc_meta = kasan_get_alloc_meta(cache, object);
28 28
29 if (alloc_meta) {29 if (alloc_meta) {
mm/slab.c+1−1
@@ -2604,7 +2604,7 @@ static struct slab *cache_grow_begin(struct kmem_cache *cachep, · 2603 unmodified lines
2604 * page_address() in the latter returns a non-tagged pointer,2604 * page_address() in the latter returns a non-tagged pointer,
2605 * as it should be for slab pages.2605 * as it should be for slab pages.
2606 */2606 */
2607 kasan_poison_slab(slab_page(slab));2607 kasan_poison_slab(slab);
2608 2608
2609 /* Get slab management. */2609 /* Get slab management. */
2610 freelist = alloc_slabmgmt(cachep, slab, offset,2610 freelist = alloc_slabmgmt(cachep, slab, offset,
mm/slub.c+1−1
@@ -1961,7 +1961,7 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) · 1960 unmodified lines
1961 1961
1962 slab->slab_cache = s;1962 slab->slab_cache = s;
1963 1963
1964 kasan_poison_slab(slab_page(slab));1964 kasan_poison_slab(slab);
1965 1965
1966 start = slab_address(slab);1966 start = slab_address(slab);
1967 1967

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t