mm: memcg/slab: use a single set of kmem_caches for all allocations

Instead of having two sets of kmem_caches: one for system-wide and non-accounted allocations and the second one shared by all accounted allocations, we can use just one. The idea is simple: space for obj_cgroup metadata can be allocated on demand and filled only for accounted allocations. It allows to remove a bunch of code which is required to handle kmem_cache clones for accounted allocations. There is no more need to create them, accumulate statistics, propagate attributes, etc. It's a quite significant simplification. Also, because the total number of slab_caches is reduced almost twice (not all kmem_caches have a memcg clone), some additional memory savings are expected. On my devvm it additionally saves about 3.5% of slab memory. [guro@fb.com: fix build on MIPS] Link: http://lkml.kernel.org/r/20200717214810.3733082-1-guro@fb.com

Roman Gushchin committed 6 years ago

Commit 10befea1 parent 15999ee8 files · +78 −590Browse files

Unified · Split

8 files

include/linux/slab.h +0 −2
include/linux/slab_def.h +0 −3
include/linux/slub_def.h +0 −10
mm/memcontrol.c +24 −1
mm/slab.c +3 −38
mm/slab.h +42 −152
mm/slab_common.c +5 −225
mm/slub.c +4 −159

include/linux/slab.h+0−2

@@ -155,8 +155,6 @@ struct kmem_cache kmem_cache_create_usercopy(const char name, · 154 unmodified lines
155	155	void kmem_cache_destroy(struct kmem_cache *);
156	156	int kmem_cache_shrink(struct kmem_cache *);
157	157
158		−void memcg_create_kmem_cache(struct kmem_cache *cachep);
159		−
160	158	/*
161	159	* Please use this macro to create slab caches. Simply specify the
162	160	* name of the structure and maybe some flags that are listed above.

include/linux/slab_def.h+0−3

@@ -72,9 +72,6 @@ struct kmem_cache { · 71 unmodified lines
72	72	int obj_offset;
73	73	#endif /* CONFIG_DEBUG_SLAB */
74	74
75		−#ifdef CONFIG_MEMCG
76		− struct memcg_cache_params memcg_params;
77		−#endif
78	75	#ifdef CONFIG_KASAN
79	76	struct kasan_cache kasan_info;
80	77	#endif

include/linux/slub_def.h+0−10

@@ -108,17 +108,7 @@ struct kmem_cache { · 107 unmodified lines
108	108	struct list_head list; /* List of slab caches */
109	109	#ifdef CONFIG_SYSFS
110	110	struct kobject kobj; /* For sysfs */
111		− struct work_struct kobj_remove_work;
112	111	#endif
113		−#ifdef CONFIG_MEMCG
114		− struct memcg_cache_params memcg_params;
115		− /* For propagation, maximum size of a stored attr */
116		− unsigned int max_attr_size;
117		−#ifdef CONFIG_SYSFS
118		− struct kset *memcg_kset;
119		−#endif
120		−#endif
121		−
122	112	#ifdef CONFIG_SLAB_FREELIST_HARDENED
123	113	unsigned long random;
124	114	#endif

mm/memcontrol.c+24−1

@@ -2800,6 +2800,26 @@ static void commit_charge(struct page page, struct mem_cgroup memcg) · 2799 unmodified lines
2800	2800	}
2801	2801
2802	2802	#ifdef CONFIG_MEMCG_KMEM
	2803	+int memcg_alloc_page_obj_cgroups(struct page page, struct kmem_cache s,
	2804	+ gfp_t gfp)
	2805	+{
	2806	+ unsigned int objects = objs_per_slab_page(s, page);
	2807	+ void *vec;
	2808	+
	2809	+ vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
	2810	+ page_to_nid(page));
	2811	+ if (!vec)
	2812	+ return -ENOMEM;
	2813	+
	2814	+ if (cmpxchg(&page->obj_cgroups, NULL,
	2815	+ (struct obj_cgroup **) ((unsigned long)vec \| 0x1UL)))
	2816	+ kfree(vec);
	2817	+ else
	2818	+ kmemleak_not_leak(vec);
	2819	+
	2820	+ return 0;
	2821	+}
	2822	+
2803	2823	/*
2804	2824	* Returns a pointer to the memory cgroup to which the kernel object is charged.
2805	2825	*
@@ -2826,7 +2846,10 @@ struct mem_cgroup mem_cgroup_from_obj(void p) · 20 unmodified lines
2826	2846
2827	2847	off = obj_to_index(page->slab_cache, page, p);
2828	2848	objcg = page_obj_cgroups(page)[off];
2829		− return obj_cgroup_memcg(objcg);
	2849	+ if (objcg)
	2850	+ return obj_cgroup_memcg(objcg);
	2851	+
	2852	+ return NULL;
2830	2853	}
2831	2854
2832	2855	/* All other pages use page->mem_cgroup */

mm/slab.c+3−38

@@ -1379,11 +1379,7 @@ static struct page kmem_getpages(struct kmem_cache cachep, gfp_t flags, · 1378 unmodified lines
1379	1379	return NULL;
1380	1380	}
1381	1381
1382		− if (charge_slab_page(page, flags, cachep->gfporder, cachep)) {
1383		− __free_pages(page, cachep->gfporder);
1384		− return NULL;
1385		− }
1386		−
	1382	+ charge_slab_page(page, flags, cachep->gfporder, cachep);
1387	1383	__SetPageSlab(page);
1388	1384	/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
1389	1385	if (sk_memalloc_socks() && page_is_pfmemalloc(page))
@@ -3799,8 +3795,8 @@ fail: · 2409 unmodified lines
3799	3795	}
3800	3796
3801	3797	/* Always called with the slab_mutex held */
3802		−static int __do_tune_cpucache(struct kmem_cache *cachep, int limit,
3803		− int batchcount, int shared, gfp_t gfp)
	3798	+static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
	3799	+ int batchcount, int shared, gfp_t gfp)
3804	3800	{
3805	3801	struct array_cache __percpu cpu_cache, prev;
3806	3802	int cpu;
@@ -3845,30 +3841,6 @@ setup_node: · 38 unmodified lines
3845	3841	return setup_kmem_cache_nodes(cachep, gfp);
3846	3842	}
3847	3843
3848		−static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
3849		− int batchcount, int shared, gfp_t gfp)
3850		−{
3851		− int ret;
3852		− struct kmem_cache *c;
3853		−
3854		− ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
3855		−
3856		− if (slab_state < FULL)
3857		− return ret;
3858		−
3859		− if ((ret < 0) \|\| !is_root_cache(cachep))
3860		− return ret;
3861		−
3862		− lockdep_assert_held(&slab_mutex);
3863		− c = memcg_cache(cachep);
3864		− if (c) {
3865		− /* return value determined by the root cache only */
3866		− __do_tune_cpucache(c, limit, batchcount, shared, gfp);
3867		− }
3868		−
3869		− return ret;
3870		−}
3871		−
3872	3844	/* Called with slab_mutex held always */
3873	3845	static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
3874	3846	{
@@ -3881,13 +3853,6 @@ static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp) · 6 unmodified lines
3881	3853	if (err)
3882	3854	goto end;
3883	3855
3884		− if (!is_root_cache(cachep)) {
3885		− struct kmem_cache *root = memcg_root_cache(cachep);
3886		− limit = root->limit;
3887		− shared = root->shared;
3888		− batchcount = root->batchcount;
3889		− }
3890		−
3891	3856	if (limit && shared && batchcount)
3892	3857	goto skip_setup;
3893	3858	/*

mm/slab.h+42−152

@@ -30,28 +30,6 @@ struct kmem_cache { · 29 unmodified lines
30	30	struct list_head list; /* List of all slab caches on the system */
31	31	};
32	32
33		−#else /* !CONFIG_SLOB */
34		−
35		−/*
36		− * This is the main placeholder for memcg-related information in kmem caches.
37		− * Both the root cache and the child cache will have it. Some fields are used
38		− * in both cases, other are specific to root caches.
39		− *
40		− * @root_cache: Common to root and child caches. NULL for root, pointer to
41		− * the root cache for children.
42		− *
43		− * The following fields are specific to root caches.
44		− *
45		− * @memcg_cache: pointer to memcg kmem cache, used by all non-root memory
46		− * cgroups.
47		− * @work: work struct used to create the non-root cache.
48		− */
49		−struct memcg_cache_params {
50		− struct kmem_cache *root_cache;
51		−
52		− struct kmem_cache *memcg_cache;
53		− struct work_struct work;
54		−};
55	33	#endif /* CONFIG_SLOB */
56	34
57	35	#ifdef CONFIG_SLAB
@@ -196,7 +174,6 @@ int __kmem_cache_shutdown(struct kmem_cache *); · 138 unmodified lines
196	174	void __kmem_cache_release(struct kmem_cache *);
197	175	int __kmem_cache_shrink(struct kmem_cache *);
198	176	void slab_kmem_cache_release(struct kmem_cache *);
199		−void kmem_cache_shrink_all(struct kmem_cache *s);
200	177
201	178	struct seq_file;
202	179	struct file;
@@ -263,43 +240,6 @@ static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t fla · 60 unmodified lines
263	240	}
264	241
265	242	#ifdef CONFIG_MEMCG_KMEM
266		−static inline bool is_root_cache(struct kmem_cache *s)
267		−{
268		− return !s->memcg_params.root_cache;
269		−}
270		−
271		−static inline bool slab_equal_or_root(struct kmem_cache *s,
272		− struct kmem_cache *p)
273		−{
274		− return p == s \|\| p == s->memcg_params.root_cache;
275		−}
276		−
277		−/*
278		− * We use suffixes to the name in memcg because we can't have caches
279		− * created in the system with the same name. But when we print them
280		− * locally, better refer to them with the base name
281		− */
282		−static inline const char cache_name(struct kmem_cache s)
283		−{
284		− if (!is_root_cache(s))
285		− s = s->memcg_params.root_cache;
286		− return s->name;
287		−}
288		−
289		−static inline struct kmem_cache memcg_root_cache(struct kmem_cache s)
290		−{
291		− if (is_root_cache(s))
292		− return s;
293		− return s->memcg_params.root_cache;
294		−}
295		−
296		−static inline struct kmem_cache memcg_cache(struct kmem_cache s)
297		−{
298		− if (is_root_cache(s))
299		− return s->memcg_params.memcg_cache;
300		− return NULL;
301		−}
302		−
303	243	static inline struct obj_cgroup *page_obj_cgroups(struct page page)
304	244	{
305	245	/*
@@ -317,21 +257,8 @@ static inline bool page_has_obj_cgroups(struct page *page) · 11 unmodified lines
317	257	return ((unsigned long)page->obj_cgroups & 0x1UL);
318	258	}
319	259
320		−static inline int memcg_alloc_page_obj_cgroups(struct page *page,
321		− struct kmem_cache *s, gfp_t gfp)
322		−{
323		− unsigned int objects = objs_per_slab_page(s, page);
324		− void *vec;
325		−
326		− vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
327		− page_to_nid(page));
328		− if (!vec)
329		− return -ENOMEM;
330		−
331		− kmemleak_not_leak(vec);
332		− page->obj_cgroups = (struct obj_cgroup **) ((unsigned long)vec \| 0x1UL);
333		− return 0;
334		−}
	260	+int memcg_alloc_page_obj_cgroups(struct page page, struct kmem_cache s,
	261	+ gfp_t gfp);
335	262
336	263	static inline void memcg_free_page_obj_cgroups(struct page *page)
337	264	{
@@ -348,38 +275,25 @@ static inline size_t obj_full_size(struct kmem_cache *s) · 10 unmodified lines
348	275	return s->size + sizeof(struct obj_cgroup *);
349	276	}
350	277
351		−static inline struct kmem_cache memcg_slab_pre_alloc_hook(struct kmem_cache s,
352		− struct obj_cgroup **objcgp,
353		− size_t objects, gfp_t flags)
	278	+static inline struct obj_cgroup memcg_slab_pre_alloc_hook(struct kmem_cache s,
	279	+ size_t objects,
	280	+ gfp_t flags)
354	281	{
355		− struct kmem_cache *cachep;
356	282	struct obj_cgroup *objcg;
357	283
358	284	if (memcg_kmem_bypass())
359		− return s;
360		−
361		− cachep = READ_ONCE(s->memcg_params.memcg_cache);
362		− if (unlikely(!cachep)) {
363		− /*
364		− * If memcg cache does not exist yet, we schedule it's
365		− * asynchronous creation and let the current allocation
366		− * go through with the root cache.
367		− */
368		− queue_work(system_wq, &s->memcg_params.work);
369		− return s;
370		− }
	285	+ return NULL;
371	286
372	287	objcg = get_obj_cgroup_from_current();
373	288	if (!objcg)
374		− return s;
	289	+ return NULL;
375	290
376	291	if (obj_cgroup_charge(objcg, flags, objects * obj_full_size(s))) {
377	292	obj_cgroup_put(objcg);
378		− cachep = NULL;
	293	+ return NULL;
379	294	}
380	295
381		− *objcgp = objcg;
382		− return cachep;
	296	+ return objcg;
383	297	}
384	298
385	299	static inline void mod_objcg_state(struct obj_cgroup *objcg,
@@ -398,15 +312,27 @@ static inline void mod_objcg_state(struct obj_cgroup *objcg, · 12 unmodified lines
398	312
399	313	static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
400	314	struct obj_cgroup *objcg,
401		− size_t size, void **p)
	315	+ gfp_t flags, size_t size,
	316	+ void **p)
402	317	{
403	318	struct page *page;
404	319	unsigned long off;
405	320	size_t i;
406	321
	322	+ if (!objcg)
	323	+ return;
	324	+
	325	+ flags &= ~__GFP_ACCOUNT;
407	326	for (i = 0; i < size; i++) {
408	327	if (likely(p[i])) {
409	328	page = virt_to_head_page(p[i]);
	329	+
	330	+ if (!page_has_obj_cgroups(page) &&
	331	+ memcg_alloc_page_obj_cgroups(page, s, flags)) {
	332	+ obj_cgroup_uncharge(objcg, obj_full_size(s));
	333	+ continue;
	334	+ }
	335	+
410	336	off = obj_to_index(s, page, p[i]);
411	337	obj_cgroup_get(objcg);
412	338	page_obj_cgroups(page)[off] = objcg;
@@ -425,13 +351,19 @@ static inline void memcg_slab_free_hook(struct kmem_cache s, struct page page, · 12 unmodified lines
425	351	struct obj_cgroup *objcg;
426	352	unsigned int off;
427	353
428		− if (!memcg_kmem_enabled() \|\| is_root_cache(s))
	354	+ if (!memcg_kmem_enabled())
	355	+ return;
	356	+
	357	+ if (!page_has_obj_cgroups(page))
429	358	return;
430	359
431	360	off = obj_to_index(s, page, p);
432	361	objcg = page_obj_cgroups(page)[off];
433	362	page_obj_cgroups(page)[off] = NULL;
434	363
	364	+ if (!objcg)
	365	+ return;
	366	+
435	367	obj_cgroup_uncharge(objcg, obj_full_size(s));
436	368	mod_objcg_state(objcg, page_pgdat(page), cache_vmstat_idx(s),
437	369	-obj_full_size(s));
@@ -439,35 +371,7 @@ static inline void memcg_slab_free_hook(struct kmem_cache s, struct page page, · 1 unmodified line
439	371	obj_cgroup_put(objcg);
440	372	}
441	373
442		−extern void slab_init_memcg_params(struct kmem_cache *);
443		−
444	374	#else /* CONFIG_MEMCG_KMEM */
445		−static inline bool is_root_cache(struct kmem_cache *s)
446		−{
447		− return true;
448		−}
449		−
450		−static inline bool slab_equal_or_root(struct kmem_cache *s,
451		− struct kmem_cache *p)
452		−{
453		− return s == p;
454		−}
455		−
456		−static inline const char cache_name(struct kmem_cache s)
457		−{
458		− return s->name;
459		−}
460		−
461		−static inline struct kmem_cache memcg_root_cache(struct kmem_cache s)
462		−{
463		− return s;
464		−}
465		−
466		−static inline struct kmem_cache memcg_cache(struct kmem_cache s)
467		−{
468		− return NULL;
469		−}
470		−
471	375	static inline bool page_has_obj_cgroups(struct page *page)
472	376	{
473	377	return false;
@@ -488,16 +392,17 @@ static inline void memcg_free_page_obj_cgroups(struct page *page) · 14 unmodified lines
488	392	{
489	393	}
490	394
491		−static inline struct kmem_cache memcg_slab_pre_alloc_hook(struct kmem_cache s,
492		− struct obj_cgroup **objcgp,
493		− size_t objects, gfp_t flags)
	395	+static inline struct obj_cgroup memcg_slab_pre_alloc_hook(struct kmem_cache s,
	396	+ size_t objects,
	397	+ gfp_t flags)
494	398	{
495	399	return NULL;
496	400	}
497	401
498	402	static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
499	403	struct obj_cgroup *objcg,
500		− size_t size, void **p)
	404	+ gfp_t flags, size_t size,
	405	+ void **p)
501	406	{
502	407	}
503	408
@@ -505,11 +410,6 @@ static inline void memcg_slab_free_hook(struct kmem_cache s, struct page page, · 1 unmodified line
505	410	void *p)
506	411	{
507	412	}
508		−
509		−static inline void slab_init_memcg_params(struct kmem_cache *s)
510		−{
511		−}
512		−
513	413	#endif /* CONFIG_MEMCG_KMEM */
514	414
515	415	static inline struct kmem_cache virt_to_cache(const void obj)
@@ -523,27 +423,18 @@ static inline struct kmem_cache virt_to_cache(const void obj) · 7 unmodified lines
523	423	return page->slab_cache;
524	424	}
525	425
526		−static __always_inline int charge_slab_page(struct page *page,
527		− gfp_t gfp, int order,
528		− struct kmem_cache *s)
	426	+static __always_inline void charge_slab_page(struct page *page,
	427	+ gfp_t gfp, int order,
	428	+ struct kmem_cache *s)
529	429	{
530		− if (memcg_kmem_enabled() && !is_root_cache(s)) {
531		− int ret;
532		−
533		− ret = memcg_alloc_page_obj_cgroups(page, s, gfp);
534		− if (ret)
535		− return ret;
536		− }
537		−
538	430	mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),
539	431	PAGE_SIZE << order);
540		− return 0;
541	432	}
542	433
543	434	static __always_inline void uncharge_slab_page(struct page *page, int order,
544	435	struct kmem_cache *s)
545	436	{
546		− if (memcg_kmem_enabled() && !is_root_cache(s))
	437	+ if (memcg_kmem_enabled())
547	438	memcg_free_page_obj_cgroups(page);
548	439
549	440	mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),
@@ -555,12 +446,11 @@ static inline struct kmem_cache cache_from_obj(struct kmem_cache s, void *x) · 5 unmodified lines
555	446	struct kmem_cache *cachep;
556	447
557	448	if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) &&
558		− !memcg_kmem_enabled() &&
559	449	!kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS))
560	450	return s;
561	451
562	452	cachep = virt_to_cache(x);
563		− if (WARN(cachep && !slab_equal_or_root(cachep, s),
	453	+ if (WARN(cachep && cachep != s,
564	454	"%s: Wrong slab cache. %s but object is from %s\n",
565	455	__func__, s->name, cachep->name))
566	456	print_tracking(cachep, x);
@@ -613,7 +503,7 @@ static inline struct kmem_cache slab_pre_alloc_hook(struct kmem_cache s, · 46 unmodified lines
613	503
614	504	if (memcg_kmem_enabled() &&
615	505	((flags & __GFP_ACCOUNT) \|\| (s->flags & SLAB_ACCOUNT)))
616		− return memcg_slab_pre_alloc_hook(s, objcgp, size, flags);
	506	+ *objcgp = memcg_slab_pre_alloc_hook(s, size, flags);
617	507
618	508	return s;
619	509	}
@@ -632,8 +522,8 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s, · 12 unmodified lines
632	522	s->flags, flags);
633	523	}
634	524
635		− if (memcg_kmem_enabled() && !is_root_cache(s))
636		− memcg_slab_post_alloc_hook(s, objcg, size, p);
	525	+ if (memcg_kmem_enabled())
	526	+ memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
637	527	}
638	528
639	529	#ifndef CONFIG_SLOB

mm/slab_common.c+5−225

@@ -130,36 +130,6 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr, · 129 unmodified lines
130	130	return i;
131	131	}
132	132
133		−#ifdef CONFIG_MEMCG_KMEM
134		−static void memcg_kmem_cache_create_func(struct work_struct *work)
135		−{
136		− struct kmem_cache *cachep = container_of(work, struct kmem_cache,
137		− memcg_params.work);
138		− memcg_create_kmem_cache(cachep);
139		−}
140		−
141		−void slab_init_memcg_params(struct kmem_cache *s)
142		−{
143		− s->memcg_params.root_cache = NULL;
144		− s->memcg_params.memcg_cache = NULL;
145		− INIT_WORK(&s->memcg_params.work, memcg_kmem_cache_create_func);
146		−}
147		−
148		−static void init_memcg_params(struct kmem_cache *s,
149		− struct kmem_cache *root_cache)
150		−{
151		− if (root_cache)
152		− s->memcg_params.root_cache = root_cache;
153		− else
154		− slab_init_memcg_params(s);
155		−}
156		−#else
157		−static inline void init_memcg_params(struct kmem_cache *s,
158		− struct kmem_cache *root_cache)
159		−{
160		−}
161		−#endif /* CONFIG_MEMCG_KMEM */
162		−
163	133	/*
164	134	* Figure out what the alignment of the objects will be given a set of
165	135	* flags, a user specified alignment and the size of the objects.
@@ -197,9 +167,6 @@ int slab_unmergeable(struct kmem_cache *s) · 31 unmodified lines
197	167	if (slab_nomerge \|\| (s->flags & SLAB_NEVER_MERGE))
198	168	return 1;
199	169
200		− if (!is_root_cache(s))
201		− return 1;
202		−
203	170	if (s->ctor)
204	171	return 1;
205	172
@@ -286,7 +253,6 @@ static struct kmem_cache create_cache(const char name, · 80 unmodified lines
286	253	s->useroffset = useroffset;
287	254	s->usersize = usersize;
288	255
289		− init_memcg_params(s, root_cache);
290	256	err = __kmem_cache_create(s, flags);
291	257	if (err)
292	258	goto out_free_cache;
@@ -344,7 +310,6 @@ kmem_cache_create_usercopy(const char *name, · 51 unmodified lines
344	310
345	311	get_online_cpus();
346	312	get_online_mems();
347		− memcg_get_cache_ids();
348	313
349	314	mutex_lock(&slab_mutex);
350	315
@@ -394,7 +359,6 @@ kmem_cache_create_usercopy(const char *name, · 43 unmodified lines
394	359	out_unlock:
395	360	mutex_unlock(&slab_mutex);
396	361
397		− memcg_put_cache_ids();
398	362	put_online_mems();
399	363	put_online_cpus();
400	364
@@ -507,87 +471,6 @@ static int shutdown_cache(struct kmem_cache *s) · 106 unmodified lines
507	471	return 0;
508	472	}
509	473
510		−#ifdef CONFIG_MEMCG_KMEM
511		−/*
512		− * memcg_create_kmem_cache - Create a cache for non-root memory cgroups.
513		− * @root_cache: The parent of the new cache.
514		− *
515		− * This function attempts to create a kmem cache that will serve allocation
516		− * requests going all non-root memory cgroups to @root_cache. The new cache
517		− * inherits properties from its parent.
518		− */
519		−void memcg_create_kmem_cache(struct kmem_cache *root_cache)
520		−{
521		− struct kmem_cache *s = NULL;
522		− char *cache_name;
523		−
524		− get_online_cpus();
525		− get_online_mems();
526		−
527		− mutex_lock(&slab_mutex);
528		−
529		− if (root_cache->memcg_params.memcg_cache)
530		− goto out_unlock;
531		−
532		− cache_name = kasprintf(GFP_KERNEL, "%s-memcg", root_cache->name);
533		− if (!cache_name)
534		− goto out_unlock;
535		−
536		− s = create_cache(cache_name, root_cache->object_size,
537		− root_cache->align,
538		− root_cache->flags & CACHE_CREATE_MASK,
539		− root_cache->useroffset, root_cache->usersize,
540		− root_cache->ctor, root_cache);
541		− /*
542		− * If we could not create a memcg cache, do not complain, because
543		− * that's not critical at all as we can always proceed with the root
544		− * cache.
545		− */
546		− if (IS_ERR(s)) {
547		− kfree(cache_name);
548		− goto out_unlock;
549		− }
550		−
551		− /*
552		− * Since readers won't lock (see memcg_slab_pre_alloc_hook()), we need a
553		− * barrier here to ensure nobody will see the kmem_cache partially
554		− * initialized.
555		− */
556		− smp_wmb();
557		− root_cache->memcg_params.memcg_cache = s;
558		−
559		−out_unlock:
560		− mutex_unlock(&slab_mutex);
561		−
562		− put_online_mems();
563		− put_online_cpus();
564		−}
565		−
566		−static int shutdown_memcg_caches(struct kmem_cache *s)
567		−{
568		− BUG_ON(!is_root_cache(s));
569		−
570		− if (s->memcg_params.memcg_cache)
571		− WARN_ON(shutdown_cache(s->memcg_params.memcg_cache));
572		−
573		− return 0;
574		−}
575		−
576		−static void cancel_memcg_cache_creation(struct kmem_cache *s)
577		−{
578		− cancel_work_sync(&s->memcg_params.work);
579		−}
580		−#else
581		−static inline int shutdown_memcg_caches(struct kmem_cache *s)
582		−{
583		− return 0;
584		−}
585		−
586		−static inline void cancel_memcg_cache_creation(struct kmem_cache *s)
587		−{
588		−}
589		−#endif /* CONFIG_MEMCG_KMEM */
590		−
591	474	void slab_kmem_cache_release(struct kmem_cache *s)
592	475	{
593	476	__kmem_cache_release(s);
@@ -602,8 +485,6 @@ void kmem_cache_destroy(struct kmem_cache *s) · 8 unmodified lines
602	485	if (unlikely(!s))
603	486	return;
604	487
605		− cancel_memcg_cache_creation(s);
606		−
607	488	get_online_cpus();
608	489	get_online_mems();
609	490
@@ -613,10 +494,7 @@ void kmem_cache_destroy(struct kmem_cache *s) · 3 unmodified lines
613	494	if (s->refcount)
614	495	goto out_unlock;
615	496
616		− err = shutdown_memcg_caches(s);
617		− if (!err)
618		− err = shutdown_cache(s);
619		−
	497	+ err = shutdown_cache(s);
620	498	if (err) {
621	499	pr_err("kmem_cache_destroy %s: Slab cache still has objects\n",
622	500	s->name);
@@ -653,33 +531,6 @@ int kmem_cache_shrink(struct kmem_cache *cachep) · 30 unmodified lines
653	531	}
654	532	EXPORT_SYMBOL(kmem_cache_shrink);
655	533
656		−/**
657		− * kmem_cache_shrink_all - shrink root and memcg caches
658		− * @s: The cache pointer
659		− */
660		−void kmem_cache_shrink_all(struct kmem_cache *s)
661		−{
662		− struct kmem_cache *c;
663		−
664		− if (!IS_ENABLED(CONFIG_MEMCG_KMEM) \|\| !is_root_cache(s)) {
665		− kmem_cache_shrink(s);
666		− return;
667		− }
668		−
669		− get_online_cpus();
670		− get_online_mems();
671		− kasan_cache_shrink(s);
672		− __kmem_cache_shrink(s);
673		−
674		− c = memcg_cache(s);
675		− if (c) {
676		− kasan_cache_shrink(c);
677		− __kmem_cache_shrink(c);
678		− }
679		− put_online_mems();
680		− put_online_cpus();
681		−}
682		−
683	534	bool slab_is_available(void)
684	535	{
685	536	return slab_state >= UP;
@@ -708,8 +559,6 @@ void __init create_boot_cache(struct kmem_cache s, const char name, · 22 unmodified lines
708	559	s->useroffset = useroffset;
709	560	s->usersize = usersize;
710	561
711		− slab_init_memcg_params(s);
712		−
713	562	err = __kmem_cache_create(s, flags);
714	563
715	564	if (err)
@@ -1098,25 +947,6 @@ void slab_stop(struct seq_file m, void p) · 382 unmodified lines
1098	947	mutex_unlock(&slab_mutex);
1099	948	}
1100	949
1101		−static void
1102		−memcg_accumulate_slabinfo(struct kmem_cache s, struct slabinfo info)
1103		−{
1104		− struct kmem_cache *c;
1105		− struct slabinfo sinfo;
1106		−
1107		− c = memcg_cache(s);
1108		− if (c) {
1109		− memset(&sinfo, 0, sizeof(sinfo));
1110		− get_slabinfo(c, &sinfo);
1111		−
1112		− info->active_slabs += sinfo.active_slabs;
1113		− info->num_slabs += sinfo.num_slabs;
1114		− info->shared_avail += sinfo.shared_avail;
1115		− info->active_objs += sinfo.active_objs;
1116		− info->num_objs += sinfo.num_objs;
1117		− }
1118		−}
1119		−
1120	950	static void cache_show(struct kmem_cache s, struct seq_file m)
1121	951	{
1122	952	struct slabinfo sinfo;
@@ -1124,10 +954,8 @@ static void cache_show(struct kmem_cache s, struct seq_file m) · 1 unmodified line
1124	954	memset(&sinfo, 0, sizeof(sinfo));
1125	955	get_slabinfo(s, &sinfo);
1126	956
1127		− memcg_accumulate_slabinfo(s, &sinfo);
1128		−
1129	957	seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
1130		− cache_name(s), sinfo.active_objs, sinfo.num_objs, s->size,
	958	+ s->name, sinfo.active_objs, sinfo.num_objs, s->size,
1131	959	sinfo.objects_per_slab, (1 << sinfo.cache_order));
1132	960
1133	961	seq_printf(m, " : tunables %4u %4u %4u",
@@ -1144,8 +972,7 @@ static int slab_show(struct seq_file m, void p) · 10 unmodified lines
1144	972
1145	973	if (p == slab_caches.next)
1146	974	print_slabinfo_header(m);
1147		− if (is_root_cache(s))
1148		− cache_show(s, m);
	975	+ cache_show(s, m);
1149	976	return 0;
1150	977	}
1151	978
@@ -1170,13 +997,13 @@ void dump_unreclaimable_slab(void) · 18 unmodified lines
1170	997	pr_info("Name Used Total\n");
1171	998
1172	999	list_for_each_entry_safe(s, s2, &slab_caches, list) {
1173		− if (!is_root_cache(s) \|\| (s->flags & SLAB_RECLAIM_ACCOUNT))
	1000	+ if (s->flags & SLAB_RECLAIM_ACCOUNT)
1174	1001	continue;
1175	1002
1176	1003	get_slabinfo(s, &sinfo);
1177	1004
1178	1005	if (sinfo.num_objs > 0)
1179		− pr_info("%-17s %10luKB %10luKB\n", cache_name(s),
	1006	+ pr_info("%-17s %10luKB %10luKB\n", s->name,
1180	1007	(sinfo.active_objs * s->size) / 1024,
1181	1008	(sinfo.num_objs * s->size) / 1024);
1182	1009	}
@@ -1235,53 +1062,6 @@ static int __init slab_proc_init(void) · 52 unmodified lines
1235	1062	}
1236	1063	module_init(slab_proc_init);
1237	1064
1238		−#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_MEMCG_KMEM)
1239		−/*
1240		− * Display information about kmem caches that have memcg cache.
1241		− */
1242		−static int memcg_slabinfo_show(struct seq_file m, void unused)
1243		−{
1244		− struct kmem_cache s, c;
1245		− struct slabinfo sinfo;
1246		−
1247		− mutex_lock(&slab_mutex);
1248		− seq_puts(m, "# <name> <css_id[:dead\|deact]> <active_objs> <num_objs>");
1249		− seq_puts(m, " <active_slabs> <num_slabs>\n");
1250		− list_for_each_entry(s, &slab_caches, list) {
1251		− /*
1252		− * Skip kmem caches that don't have the memcg cache.
1253		− */
1254		− if (!s->memcg_params.memcg_cache)
1255		− continue;
1256		−
1257		− memset(&sinfo, 0, sizeof(sinfo));
1258		− get_slabinfo(s, &sinfo);
1259		− seq_printf(m, "%-17s root %6lu %6lu %6lu %6lu\n",
1260		− cache_name(s), sinfo.active_objs, sinfo.num_objs,
1261		− sinfo.active_slabs, sinfo.num_slabs);
1262		−
1263		− c = s->memcg_params.memcg_cache;
1264		− memset(&sinfo, 0, sizeof(sinfo));
1265		− get_slabinfo(c, &sinfo);
1266		− seq_printf(m, "%-17s %4d %6lu %6lu %6lu %6lu\n",
1267		− cache_name(c), root_mem_cgroup->css.id,
1268		− sinfo.active_objs, sinfo.num_objs,
1269		− sinfo.active_slabs, sinfo.num_slabs);
1270		− }
1271		− mutex_unlock(&slab_mutex);
1272		− return 0;
1273		−}
1274		−DEFINE_SHOW_ATTRIBUTE(memcg_slabinfo);
1275		−
1276		−static int __init memcg_slabinfo_init(void)
1277		−{
1278		− debugfs_create_file("memcg_slabinfo", S_IFREG \| S_IRUGO,
1279		− NULL, NULL, &memcg_slabinfo_fops);
1280		− return 0;
1281		−}
1282		−
1283		−late_initcall(memcg_slabinfo_init);
1284		−#endif /* CONFIG_DEBUG_FS && CONFIG_MEMCG_KMEM */
1285	1065	#endif /* CONFIG_SLAB \|\| CONFIG_SLUB_DEBUG */
1286	1066
1287	1067	static __always_inline void __do_krealloc(const void p, size_t new_size,

mm/slub.c+4−159

@@ -218,14 +218,10 @@ enum track_item { TRACK_ALLOC, TRACK_FREE }; · 217 unmodified lines
218	218	#ifdef CONFIG_SYSFS
219	219	static int sysfs_slab_add(struct kmem_cache *);
220	220	static int sysfs_slab_alias(struct kmem_cache , const char );
221		−static void memcg_propagate_slab_attrs(struct kmem_cache *s);
222		−static void sysfs_slab_remove(struct kmem_cache *s);
223	221	#else
224	222	static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
225	223	static inline int sysfs_slab_alias(struct kmem_cache s, const char p)
226	224	{ return 0; }
227		−static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { }
228		−static inline void sysfs_slab_remove(struct kmem_cache *s) { }
229	225	#endif
230	226
231	227	static inline void stat(const struct kmem_cache *s, enum stat_item si)
@@ -1624,10 +1620,8 @@ static inline struct page alloc_slab_page(struct kmem_cache s, · 1392 unmodified lines
1624	1620	else
1625	1621	page = __alloc_pages_node(node, flags, order);
1626	1622
1627		− if (page && charge_slab_page(page, flags, order, s)) {
1628		− __free_pages(page, order);
1629		− page = NULL;
1630		− }
	1623	+ if (page)
	1624	+ charge_slab_page(page, flags, order, s);
1631	1625
1632	1626	return page;
1633	1627	}
@@ -3920,7 +3914,6 @@ int __kmem_cache_shutdown(struct kmem_cache *s) · 2286 unmodified lines
3920	3914	if (n->nr_partial \|\| slabs_node(s, node))
3921	3915	return 1;
3922	3916	}
3923		− sysfs_slab_remove(s);
3924	3917	return 0;
3925	3918	}
3926	3919
@@ -4358,7 +4351,6 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) · 431 unmodified lines
4358	4351	p->slab_cache = s;
4359	4352	#endif
4360	4353	}
4361		− slab_init_memcg_params(s);
4362	4354	list_add(&s->list, &slab_caches);
4363	4355	return s;
4364	4356	}
@@ -4414,7 +4406,7 @@ struct kmem_cache * · 49 unmodified lines
4414	4406	__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
4415	4407	slab_flags_t flags, void (ctor)(void ))
4416	4408	{
4417		− struct kmem_cache s, c;
	4409	+ struct kmem_cache *s;
4418	4410
4419	4411	s = find_mergeable(size, align, flags, name, ctor);
4420	4412	if (s) {
@@ -4427,12 +4419,6 @@ __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, · 6 unmodified lines
4427	4419	s->object_size = max(s->object_size, size);
4428	4420	s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
4429	4421
4430		− c = memcg_cache(s);
4431		− if (c) {
4432		− c->object_size = s->object_size;
4433		− c->inuse = max(c->inuse, ALIGN(size, sizeof(void *)));
4434		− }
4435		−
4436	4422	if (sysfs_slab_alias(s, name)) {
4437	4423	s->refcount--;
4438	4424	s = NULL;
@@ -4454,7 +4440,6 @@ int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags) · 15 unmodified lines
4454	4440	if (slab_state <= UP)
4455	4441	return 0;
4456	4442
4457		− memcg_propagate_slab_attrs(s);
4458	4443	err = sysfs_slab_add(s);
4459	4444	if (err)
4460	4445	__kmem_cache_release(s);
@@ -5312,7 +5297,7 @@ static ssize_t shrink_store(struct kmem_cache *s, · 851 unmodified lines
5312	5297	const char *buf, size_t length)
5313	5298	{
5314	5299	if (buf[0] == '1')
5315		− kmem_cache_shrink_all(s);
	5300	+ kmem_cache_shrink(s);
5316	5301	else
5317	5302	return -EINVAL;
5318	5303	return length;
@@ -5536,99 +5521,9 @@ static ssize_t slab_attr_store(struct kobject *kobj, · 217 unmodified lines
5536	5521	return -EIO;
5537	5522
5538	5523	err = attribute->store(s, buf, len);
5539		−#ifdef CONFIG_MEMCG
5540		− if (slab_state >= FULL && err >= 0 && is_root_cache(s)) {
5541		− struct kmem_cache *c;
5542		−
5543		− mutex_lock(&slab_mutex);
5544		− if (s->max_attr_size < len)
5545		− s->max_attr_size = len;
5546		−
5547		− /*
5548		− * This is a best effort propagation, so this function's return
5549		− * value will be determined by the parent cache only. This is
5550		− * basically because not all attributes will have a well
5551		− * defined semantics for rollbacks - most of the actions will
5552		− * have permanent effects.
5553		− *
5554		− * Returning the error value of any of the children that fail
5555		− * is not 100 % defined, in the sense that users seeing the
5556		− * error code won't be able to know anything about the state of
5557		− * the cache.
5558		− *
5559		− * Only returning the error code for the parent cache at least
5560		− * has well defined semantics. The cache being written to
5561		− * directly either failed or succeeded, in which case we loop
5562		− * through the descendants with best-effort propagation.
5563		− */
5564		− c = memcg_cache(s);
5565		− if (c)
5566		− attribute->store(c, buf, len);
5567		− mutex_unlock(&slab_mutex);
5568		− }
5569		−#endif
5570	5524	return err;
5571	5525	}
5572	5526
5573		−static void memcg_propagate_slab_attrs(struct kmem_cache *s)
5574		−{
5575		−#ifdef CONFIG_MEMCG
5576		− int i;
5577		− char *buffer = NULL;
5578		− struct kmem_cache *root_cache;
5579		−
5580		− if (is_root_cache(s))
5581		− return;
5582		−
5583		− root_cache = s->memcg_params.root_cache;
5584		−
5585		− /*
5586		− * This mean this cache had no attribute written. Therefore, no point
5587		− * in copying default values around
5588		− */
5589		− if (!root_cache->max_attr_size)
5590		− return;
5591		−
5592		− for (i = 0; i < ARRAY_SIZE(slab_attrs); i++) {
5593		− char mbuf[64];
5594		− char *buf;
5595		− struct slab_attribute *attr = to_slab_attr(slab_attrs[i]);
5596		− ssize_t len;
5597		−
5598		− if (!attr \|\| !attr->store \|\| !attr->show)
5599		− continue;
5600		−
5601		− /*
5602		− * It is really bad that we have to allocate here, so we will
5603		− * do it only as a fallback. If we actually allocate, though,
5604		− * we can just use the allocated buffer until the end.
5605		− *
5606		− * Most of the slub attributes will tend to be very small in
5607		− * size, but sysfs allows buffers up to a page, so they can
5608		− * theoretically happen.
5609		− */
5610		− if (buffer)
5611		− buf = buffer;
5612		− else if (root_cache->max_attr_size < ARRAY_SIZE(mbuf) &&
5613		− !IS_ENABLED(CONFIG_SLUB_STATS))
5614		− buf = mbuf;
5615		− else {
5616		− buffer = (char *) get_zeroed_page(GFP_KERNEL);
5617		− if (WARN_ON(!buffer))
5618		− continue;
5619		− buf = buffer;
5620		− }
5621		−
5622		− len = attr->show(root_cache, buf);
5623		− if (len > 0)
5624		− attr->store(s, buf, len);
5625		− }
5626		−
5627		− if (buffer)
5628		− free_page((unsigned long)buffer);
5629		−#endif /* CONFIG_MEMCG */
5630		−}
5631		−
5632	5527	static void kmem_cache_release(struct kobject *k)
5633	5528	{
5634	5529	slab_kmem_cache_release(to_slab(k));
@@ -5648,10 +5543,6 @@ static struct kset *slab_kset; · 13 unmodified lines
5648	5543
5649	5544	static inline struct kset cache_kset(struct kmem_cache s)
5650	5545	{
5651		−#ifdef CONFIG_MEMCG
5652		− if (!is_root_cache(s))
5653		− return s->memcg_params.root_cache->memcg_kset;
5654		−#endif
5655	5546	return slab_kset;
5656	5547	}
5657	5548
@@ -5694,27 +5585,6 @@ static char create_unique_id(struct kmem_cache s) · 36 unmodified lines
5694	5585	return name;
5695	5586	}
5696	5587
5697		−static void sysfs_slab_remove_workfn(struct work_struct *work)
5698		−{
5699		− struct kmem_cache *s =
5700		− container_of(work, struct kmem_cache, kobj_remove_work);
5701		−
5702		− if (!s->kobj.state_in_sysfs)
5703		− /*
5704		− * For a memcg cache, this may be called during
5705		− * deactivation and again on shutdown. Remove only once.
5706		− * A cache is never shut down before deactivation is
5707		− * complete, so no need to worry about synchronization.
5708		− */
5709		− goto out;
5710		−
5711		−#ifdef CONFIG_MEMCG
5712		− kset_unregister(s->memcg_kset);
5713		−#endif
5714		−out:
5715		− kobject_put(&s->kobj);
5716		−}
5717		−
5718	5588	static int sysfs_slab_add(struct kmem_cache *s)
5719	5589	{
5720	5590	int err;
@@ -5722,8 +5592,6 @@ static int sysfs_slab_add(struct kmem_cache *s) · 1 unmodified line
5722	5592	struct kset *kset = cache_kset(s);
5723	5593	int unmergeable = slab_unmergeable(s);
5724	5594
5725		− INIT_WORK(&s->kobj_remove_work, sysfs_slab_remove_workfn);
5726		−
5727	5595	if (!kset) {
5728	5596	kobject_init(&s->kobj, &slab_ktype);
5729	5597	return 0;
@@ -5760,16 +5628,6 @@ static int sysfs_slab_add(struct kmem_cache *s) · 30 unmodified lines
5760	5628	if (err)
5761	5629	goto out_del_kobj;
5762	5630
5763		−#ifdef CONFIG_MEMCG
5764		− if (is_root_cache(s) && memcg_sysfs_enabled) {
5765		− s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj);
5766		− if (!s->memcg_kset) {
5767		− err = -ENOMEM;
5768		− goto out_del_kobj;
5769		− }
5770		− }
5771		−#endif
5772		−
5773	5631	if (!unmergeable) {
5774	5632	/* Setup first alias */
5775	5633	sysfs_slab_alias(s, s->name);
@@ -5783,19 +5641,6 @@ out_del_kobj: · 7 unmodified lines
5783	5641	goto out;
5784	5642	}
5785	5643
5786		−static void sysfs_slab_remove(struct kmem_cache *s)
5787		−{
5788		− if (slab_state < FULL)
5789		− /*
5790		− * Sysfs has not been setup yet so no need to remove the
5791		− * cache from sysfs.
5792		− */
5793		− return;
5794		−
5795		− kobject_get(&s->kobj);
5796		− schedule_work(&s->kobj_remove_work);
5797		−}
5798		−
5799	5644	void sysfs_slab_unlink(struct kmem_cache *s)
5800	5645	{
5801	5646	if (slab_state >= FULL)