// SPDX-License-Identifier: GPL-2.0#include <linux/memcontrol.h>#include <linux/rwsem.h>#include <linux/shrinker.h>#include <linux/rculist.h>#include <trace/events/vmscan.h> #include "internal.h" LIST_HEAD(shrinker_list);DEFINE_MUTEX(shrinker_mutex); #ifdef CONFIG_MEMCGstatic int shrinker_nr_max; static inline int shrinker_unit_size(int nr_items){	return (DIV_ROUND_UP(nr_items, SHRINKER_UNIT_BITS) * sizeof(struct shrinker_info_unit *));} static inline void shrinker_unit_free(struct shrinker_info *info, int start){	struct shrinker_info_unit **unit;	int nr, i; 	if (!info)		return; 	unit = info->unit;	nr = DIV_ROUND_UP(info->map_nr_max, SHRINKER_UNIT_BITS); 	for (i = start; i < nr; i++) {		if (!unit[i])			break; 		kfree(unit[i]);		unit[i] = NULL;	}} static inline int shrinker_unit_alloc(struct shrinker_info *new,				       struct shrinker_info *old, int nid){	struct shrinker_info_unit *unit;	int nr = DIV_ROUND_UP(new->map_nr_max, SHRINKER_UNIT_BITS);	int start = old ? DIV_ROUND_UP(old->map_nr_max, SHRINKER_UNIT_BITS) : 0;	int i; 	for (i = start; i < nr; i++) {		unit = kzalloc_node(sizeof(*unit), GFP_KERNEL, nid);		if (!unit) {			shrinker_unit_free(new, start);			return -ENOMEM;		} 		new->unit[i] = unit;	} 	return 0;} static void __free_shrinker_info(struct mem_cgroup *memcg){	struct mem_cgroup_per_node *pn;	struct shrinker_info *info;	int nid; 	lockdep_assert_held(&shrinker_mutex); 	for_each_node(nid) {		pn = memcg->nodeinfo[nid];		info = rcu_dereference_protected(pn->shrinker_info, true);		shrinker_unit_free(info, 0);		kvfree(info);		rcu_assign_pointer(pn->shrinker_info, NULL);	}} void free_shrinker_info(struct mem_cgroup *memcg){	mutex_lock(&shrinker_mutex);	__free_shrinker_info(memcg);	mutex_unlock(&shrinker_mutex);} int alloc_shrinker_info(struct mem_cgroup *memcg){	int nid, ret = 0;	int array_size = 0; 	mutex_lock(&shrinker_mutex);	array_size = shrinker_unit_size(shrinker_nr_max);	for_each_node(nid) {		struct shrinker_info *info = kvzalloc_node(sizeof(*info) + array_size,							   GFP_KERNEL, nid);		if (!info)			goto err;		info->map_nr_max = shrinker_nr_max;		if (shrinker_unit_alloc(info, NULL, nid)) {			kvfree(info);			goto err;		}		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info);	}	mutex_unlock(&shrinker_mutex); 	return ret; err:	__free_shrinker_info(memcg);	mutex_unlock(&shrinker_mutex);	return -ENOMEM;} static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg,						     int nid){	return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info,					 lockdep_is_held(&shrinker_mutex));} static int expand_one_shrinker_info(struct mem_cgroup *memcg, int new_size,				    int old_size, int new_nr_max){	struct shrinker_info *new, *old;	struct mem_cgroup_per_node *pn;	int nid; 	for_each_node(nid) {		pn = memcg->nodeinfo[nid];		old = shrinker_info_protected(memcg, nid);		/* Not yet online memcg */		if (!old)			return 0; 		/* Already expanded this shrinker_info */		if (new_nr_max <= old->map_nr_max)			continue; 		new = kvzalloc_node(sizeof(*new) + new_size, GFP_KERNEL, nid);		if (!new)			return -ENOMEM; 		new->map_nr_max = new_nr_max; 		memcpy(new->unit, old->unit, old_size);		if (shrinker_unit_alloc(new, old, nid)) {			kvfree(new);			return -ENOMEM;		} 		rcu_assign_pointer(pn->shrinker_info, new);		kvfree_rcu(old, rcu);	} 	return 0;} static int expand_shrinker_info(int new_id){	int ret = 0;	int new_nr_max = round_up(new_id + 1, SHRINKER_UNIT_BITS);	int new_size, old_size = 0;	struct mem_cgroup *memcg; 	if (!root_mem_cgroup)		goto out; 	lockdep_assert_held(&shrinker_mutex); 	new_size = shrinker_unit_size(new_nr_max);	old_size = shrinker_unit_size(shrinker_nr_max); 	memcg = mem_cgroup_iter(NULL, NULL, NULL);	do {		ret = expand_one_shrinker_info(memcg, new_size, old_size,					       new_nr_max);		if (ret) {			mem_cgroup_iter_break(NULL, memcg);			goto out;		}	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);out:	if (!ret)		shrinker_nr_max = new_nr_max; 	return ret;} static inline int shrinker_id_to_index(int shrinker_id){	return shrinker_id / SHRINKER_UNIT_BITS;} static inline int shrinker_id_to_offset(int shrinker_id){	return shrinker_id % SHRINKER_UNIT_BITS;} static inline int calc_shrinker_id(int index, int offset){	return index * SHRINKER_UNIT_BITS + offset;} void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id){	if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {		struct shrinker_info *info; 		rcu_read_lock();		info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);		if (!WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) {			struct shrinker_info_unit *unit; 			unit = info->unit[shrinker_id_to_index(shrinker_id)];			/* Pairs with smp mb in shrink_slab() */			smp_mb__before_atomic();			set_bit(shrinker_id_to_offset(shrinker_id), unit->map);		}		rcu_read_unlock();	}} static DEFINE_IDR(shrinker_idr); static int shrinker_memcg_alloc(struct shrinker *shrinker){	int id; 	if (mem_cgroup_disabled())		return -ENOSYS;	if (mem_cgroup_kmem_disabled() && !(shrinker->flags & SHRINKER_NONSLAB))		return -ENOSYS; 	guard(mutex)(&shrinker_mutex);	id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL);	if (id < 0)		return id; 	if (id >= shrinker_nr_max) {		if (expand_shrinker_info(id)) {			idr_remove(&shrinker_idr, id);			return -ENOMEM;		}	}	shrinker->id = id;	return 0;} static void shrinker_memcg_remove(struct shrinker *shrinker){	int id = shrinker->id; 	BUG_ON(id < 0); 	lockdep_assert_held(&shrinker_mutex); 	idr_remove(&shrinker_idr, id);} static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,				   struct mem_cgroup *memcg){	struct shrinker_info *info;	struct shrinker_info_unit *unit;	long nr_deferred; 	rcu_read_lock();	info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);	unit = info->unit[shrinker_id_to_index(shrinker->id)];	nr_deferred = atomic_long_xchg(&unit->nr_deferred[shrinker_id_to_offset(shrinker->id)], 0);	rcu_read_unlock(); 	return nr_deferred;} static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,				  struct mem_cgroup *memcg){	struct shrinker_info *info;	struct shrinker_info_unit *unit;	long nr_deferred; 	rcu_read_lock();	info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);	unit = info->unit[shrinker_id_to_index(shrinker->id)];	nr_deferred =		atomic_long_add_return(nr, &unit->nr_deferred[shrinker_id_to_offset(shrinker->id)]);	rcu_read_unlock(); 	return nr_deferred;} void reparent_shrinker_deferred(struct mem_cgroup *memcg){	int nid, index, offset;	long nr;	struct mem_cgroup *parent = parent_mem_cgroup(memcg);	struct shrinker_info *child_info, *parent_info;	struct shrinker_info_unit *child_unit, *parent_unit; 	/* Prevent from concurrent shrinker_info expand */	mutex_lock(&shrinker_mutex);	for_each_node(nid) {		child_info = shrinker_info_protected(memcg, nid);		parent_info = shrinker_info_protected(parent, nid);		for (index = 0; index < shrinker_id_to_index(child_info->map_nr_max); index++) {			child_unit = child_info->unit[index];			parent_unit = parent_info->unit[index];			for (offset = 0; offset < SHRINKER_UNIT_BITS; offset++) {				nr = atomic_long_read(&child_unit->nr_deferred[offset]);				atomic_long_add(nr, &parent_unit->nr_deferred[offset]);			}		}	}	mutex_unlock(&shrinker_mutex);}#elsestatic int shrinker_memcg_alloc(struct shrinker *shrinker){	return -ENOSYS;} static void shrinker_memcg_remove(struct shrinker *shrinker){} static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,				   struct mem_cgroup *memcg){	return 0;} static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,				  struct mem_cgroup *memcg){	return 0;}#endif /* CONFIG_MEMCG */ static long xchg_nr_deferred(struct shrinker *shrinker,			     struct shrink_control *sc){	int nid = sc->nid; 	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))		nid = 0; 	if (sc->memcg &&	    (shrinker->flags & SHRINKER_MEMCG_AWARE))		return xchg_nr_deferred_memcg(nid, shrinker,					      sc->memcg); 	return atomic_long_xchg(&shrinker->nr_deferred[nid], 0);}  static long add_nr_deferred(long nr, struct shrinker *shrinker,			    struct shrink_control *sc){	int nid = sc->nid; 	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))		nid = 0; 	if (sc->memcg &&	    (shrinker->flags & SHRINKER_MEMCG_AWARE))		return add_nr_deferred_memcg(nr, nid, shrinker,					     sc->memcg); 	return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]);} #define SHRINK_BATCH 128 static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,				    struct shrinker *shrinker, int priority){	unsigned long freed = 0;	unsigned long long delta;	long total_scan;	long freeable;	long nr;	long new_nr;	long batch_size = shrinker->batch ? shrinker->batch					  : SHRINK_BATCH;	long scanned = 0, next_deferred; 	freeable = shrinker->count_objects(shrinker, shrinkctl);	if (freeable == 0 || freeable == SHRINK_EMPTY)		return freeable; 	/*	 * copy the current shrinker scan count into a local variable	 * and zero it so that other concurrent shrinker invocations	 * don't also do this scanning work.	 */	nr = xchg_nr_deferred(shrinker, shrinkctl); 	if (shrinker->seeks) {		delta = freeable >> priority;		delta *= 4;		do_div(delta, shrinker->seeks);	} else {		/*		 * These objects don't require any IO to create. Trim		 * them aggressively under memory pressure to keep		 * them from causing refetches in the IO caches.		 */		delta = freeable / 2;	} 	total_scan = nr >> priority;	total_scan += delta;	total_scan = min(total_scan, (2 * freeable)); 	trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,				   freeable, delta, total_scan, priority,				   shrinkctl->memcg); 	/*	 * Normally, we should not scan less than batch_size objects in one	 * pass to avoid too frequent shrinker calls, but if the slab has less	 * than batch_size objects in total and we are really tight on memory,	 * we will try to reclaim all available objects, otherwise we can end	 * up failing allocations although there are plenty of reclaimable	 * objects spread over several slabs with usage less than the	 * batch_size.	 *	 * We detect the "tight on memory" situations by looking at the total	 * number of objects we want to scan (total_scan). If it is greater	 * than the total number of objects on slab (freeable), we must be	 * scanning at high prio and therefore should try to reclaim as much as	 * possible.	 */	while (total_scan >= batch_size ||	       total_scan >= freeable) {		unsigned long ret;		unsigned long nr_to_scan = min(batch_size, total_scan); 		shrinkctl->nr_to_scan = nr_to_scan;		shrinkctl->nr_scanned = nr_to_scan;		ret = shrinker->scan_objects(shrinker, shrinkctl);		if (ret == SHRINK_STOP)			break;		freed += ret; 		count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned);		total_scan -= shrinkctl->nr_scanned;		scanned += shrinkctl->nr_scanned; 		cond_resched();	} 	/*	 * The deferred work is increased by any new work (delta) that wasn't	 * done, decreased by old deferred work that was done now.	 *	 * And it is capped to two times of the freeable items.	 */	next_deferred = max_t(long, (nr + delta - scanned), 0);	next_deferred = min(next_deferred, (2 * freeable)); 	/*	 * move the unused scan count back into the shrinker in a	 * manner that handles concurrent updates.	 */	new_nr = add_nr_deferred(next_deferred, shrinker, shrinkctl); 	trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan,				 shrinkctl->memcg);	return freed;} #ifdef CONFIG_MEMCGstatic unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,			struct mem_cgroup *memcg, int priority){	struct shrinker_info *info;	unsigned long ret, freed = 0;	int offset, index = 0; 	if (!mem_cgroup_online(memcg))		return 0; 	/*	 * lockless algorithm of memcg shrink.	 *	 * The shrinker_info may be freed asynchronously via RCU in the	 * expand_one_shrinker_info(), so the rcu_read_lock() needs to be used	 * to ensure the existence of the shrinker_info.	 *	 * The shrinker_info_unit is never freed unless its corresponding memcg	 * is destroyed. Here we already hold the refcount of memcg, so the	 * memcg will not be destroyed, and of course shrinker_info_unit will	 * not be freed.	 *	 * So in the memcg shrink:	 *  step 1: use rcu_read_lock() to guarantee existence of the	 *          shrinker_info.

Lines 1–500 of 822. Load more