// SPDX-License-Identifier: GPL-2.0/* * linux/mm/compaction.c * * Memory compaction for the reduction of external fragmentation. Note that * this heavily depends upon page migration to do all the real heavy * lifting * * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie> */#include <linux/cpu.h>#include <linux/swap.h>#include <linux/migrate.h>#include <linux/compaction.h>#include <linux/mm_inline.h>#include <linux/sched/signal.h>#include <linux/backing-dev.h>#include <linux/sysctl.h>#include <linux/sysfs.h>#include <linux/page-isolation.h>#include <linux/kasan.h>#include <linux/kthread.h>#include <linux/freezer.h>#include <linux/page_owner.h>#include <linux/psi.h>#include "internal.h" #ifdef CONFIG_COMPACTIONstatic inline void count_compact_event(enum vm_event_item item){	count_vm_event(item);} static inline void count_compact_events(enum vm_event_item item, long delta){	count_vm_events(item, delta);}#else#define count_compact_event(item) do { } while (0)#define count_compact_events(item, delta) do { } while (0)#endif #if defined CONFIG_COMPACTION || defined CONFIG_CMA #define CREATE_TRACE_POINTS#include <trace/events/compaction.h> #define block_start_pfn(pfn, order)	round_down(pfn, 1UL << (order))#define block_end_pfn(pfn, order)	ALIGN((pfn) + 1, 1UL << (order))#define pageblock_start_pfn(pfn)	block_start_pfn(pfn, pageblock_order)#define pageblock_end_pfn(pfn)		block_end_pfn(pfn, pageblock_order) /* * Fragmentation score check interval for proactive compaction purposes. */static const unsigned int HPAGE_FRAG_CHECK_INTERVAL_MSEC = 500; /* * Page order with-respect-to which proactive compaction * calculates external fragmentation, which is used as * the "fragmentation score" of a node/zone. */#if defined CONFIG_TRANSPARENT_HUGEPAGE#define COMPACTION_HPAGE_ORDER	HPAGE_PMD_ORDER#elif defined CONFIG_HUGETLBFS#define COMPACTION_HPAGE_ORDER	HUGETLB_PAGE_ORDER#else#define COMPACTION_HPAGE_ORDER	(PMD_SHIFT - PAGE_SHIFT)#endif static unsigned long release_freepages(struct list_head *freelist){	struct page *page, *next;	unsigned long high_pfn = 0; 	list_for_each_entry_safe(page, next, freelist, lru) {		unsigned long pfn = page_to_pfn(page);		list_del(&page->lru);		__free_page(page);		if (pfn > high_pfn)			high_pfn = pfn;	} 	return high_pfn;} static void split_map_pages(struct list_head *list){	unsigned int i, order, nr_pages;	struct page *page, *next;	LIST_HEAD(tmp_list); 	list_for_each_entry_safe(page, next, list, lru) {		list_del(&page->lru); 		order = page_private(page);		nr_pages = 1 << order; 		post_alloc_hook(page, order, __GFP_MOVABLE);		if (order)			split_page(page, order); 		for (i = 0; i < nr_pages; i++) {			list_add(&page->lru, &tmp_list);			page++;		}	} 	list_splice(&tmp_list, list);} #ifdef CONFIG_COMPACTION int PageMovable(struct page *page){	struct address_space *mapping; 	VM_BUG_ON_PAGE(!PageLocked(page), page);	if (!__PageMovable(page))		return 0; 	mapping = page_mapping(page);	if (mapping && mapping->a_ops && mapping->a_ops->isolate_page)		return 1; 	return 0;}EXPORT_SYMBOL(PageMovable); void __SetPageMovable(struct page *page, struct address_space *mapping){	VM_BUG_ON_PAGE(!PageLocked(page), page);	VM_BUG_ON_PAGE((unsigned long)mapping & PAGE_MAPPING_MOVABLE, page);	page->mapping = (void *)((unsigned long)mapping | PAGE_MAPPING_MOVABLE);}EXPORT_SYMBOL(__SetPageMovable); void __ClearPageMovable(struct page *page){	VM_BUG_ON_PAGE(!PageLocked(page), page);	VM_BUG_ON_PAGE(!PageMovable(page), page);	/*	 * Clear registered address_space val with keeping PAGE_MAPPING_MOVABLE	 * flag so that VM can catch up released page by driver after isolation.	 * With it, VM migration doesn't try to put it back.	 */	page->mapping = (void *)((unsigned long)page->mapping &				PAGE_MAPPING_MOVABLE);}EXPORT_SYMBOL(__ClearPageMovable); /* Do not skip compaction more than 64 times */#define COMPACT_MAX_DEFER_SHIFT 6 /* * Compaction is deferred when compaction fails to result in a page * allocation success. 1 << compact_defer_shift, compactions are skipped up * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT */void defer_compaction(struct zone *zone, int order){	zone->compact_considered = 0;	zone->compact_defer_shift++; 	if (order < zone->compact_order_failed)		zone->compact_order_failed = order; 	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT; 	trace_mm_compaction_defer_compaction(zone, order);} /* Returns true if compaction should be skipped this time */bool compaction_deferred(struct zone *zone, int order){	unsigned long defer_limit = 1UL << zone->compact_defer_shift; 	if (order < zone->compact_order_failed)		return false; 	/* Avoid possible overflow */	if (++zone->compact_considered > defer_limit)		zone->compact_considered = defer_limit; 	if (zone->compact_considered >= defer_limit)		return false; 	trace_mm_compaction_deferred(zone, order); 	return true;} /* * Update defer tracking counters after successful compaction of given order, * which means an allocation either succeeded (alloc_success == true) or is * expected to succeed. */void compaction_defer_reset(struct zone *zone, int order,		bool alloc_success){	if (alloc_success) {		zone->compact_considered = 0;		zone->compact_defer_shift = 0;	}	if (order >= zone->compact_order_failed)		zone->compact_order_failed = order + 1; 	trace_mm_compaction_defer_reset(zone, order);} /* Returns true if restarting compaction after many failures */bool compaction_restarting(struct zone *zone, int order){	if (order < zone->compact_order_failed)		return false; 	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&		zone->compact_considered >= 1UL << zone->compact_defer_shift;} /* Returns true if the pageblock should be scanned for pages to isolate. */static inline bool isolation_suitable(struct compact_control *cc,					struct page *page){	if (cc->ignore_skip_hint)		return true; 	return !get_pageblock_skip(page);} static void reset_cached_positions(struct zone *zone){	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;	zone->compact_cached_free_pfn =				pageblock_start_pfn(zone_end_pfn(zone) - 1);} /* * Compound pages of >= pageblock_order should consistenly be skipped until * released. It is always pointless to compact pages of such order (if they are * migratable), and the pageblocks they occupy cannot contain any free pages. */static bool pageblock_skip_persistent(struct page *page){	if (!PageCompound(page))		return false; 	page = compound_head(page); 	if (compound_order(page) >= pageblock_order)		return true; 	return false;} static bool__reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,							bool check_target){	struct page *page = pfn_to_online_page(pfn);	struct page *block_page;	struct page *end_page;	unsigned long block_pfn; 	if (!page)		return false;	if (zone != page_zone(page))		return false;	if (pageblock_skip_persistent(page))		return false; 	/*	 * If skip is already cleared do no further checking once the	 * restart points have been set.	 */	if (check_source && check_target && !get_pageblock_skip(page))		return true; 	/*	 * If clearing skip for the target scanner, do not select a	 * non-movable pageblock as the starting point.	 */	if (!check_source && check_target &&	    get_pageblock_migratetype(page) != MIGRATE_MOVABLE)		return false; 	/* Ensure the start of the pageblock or zone is online and valid */	block_pfn = pageblock_start_pfn(pfn);	block_pfn = max(block_pfn, zone->zone_start_pfn);	block_page = pfn_to_online_page(block_pfn);	if (block_page) {		page = block_page;		pfn = block_pfn;	} 	/* Ensure the end of the pageblock or zone is online and valid */	block_pfn = pageblock_end_pfn(pfn) - 1;	block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);	end_page = pfn_to_online_page(block_pfn);	if (!end_page)		return false; 	/*	 * Only clear the hint if a sample indicates there is either a	 * free page or an LRU page in the block. One or other condition	 * is necessary for the block to be a migration source/target.	 */	do {		if (pfn_valid_within(pfn)) {			if (check_source && PageLRU(page)) {				clear_pageblock_skip(page);				return true;			} 			if (check_target && PageBuddy(page)) {				clear_pageblock_skip(page);				return true;			}		} 		page += (1 << PAGE_ALLOC_COSTLY_ORDER);		pfn += (1 << PAGE_ALLOC_COSTLY_ORDER);	} while (page <= end_page); 	return false;} /* * This function is called to clear all cached information on pageblocks that * should be skipped for page isolation when the migrate and free page scanner * meet. */static void __reset_isolation_suitable(struct zone *zone){	unsigned long migrate_pfn = zone->zone_start_pfn;	unsigned long free_pfn = zone_end_pfn(zone) - 1;	unsigned long reset_migrate = free_pfn;	unsigned long reset_free = migrate_pfn;	bool source_set = false;	bool free_set = false; 	if (!zone->compact_blockskip_flush)		return; 	zone->compact_blockskip_flush = false; 	/*	 * Walk the zone and update pageblock skip information. Source looks	 * for PageLRU while target looks for PageBuddy. When the scanner	 * is found, both PageBuddy and PageLRU are checked as the pageblock	 * is suitable as both source and target.	 */	for (; migrate_pfn < free_pfn; migrate_pfn += pageblock_nr_pages,					free_pfn -= pageblock_nr_pages) {		cond_resched(); 		/* Update the migrate PFN */		if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&		    migrate_pfn < reset_migrate) {			source_set = true;			reset_migrate = migrate_pfn;			zone->compact_init_migrate_pfn = reset_migrate;			zone->compact_cached_migrate_pfn[0] = reset_migrate;			zone->compact_cached_migrate_pfn[1] = reset_migrate;		} 		/* Update the free PFN */		if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&		    free_pfn > reset_free) {			free_set = true;			reset_free = free_pfn;			zone->compact_init_free_pfn = reset_free;			zone->compact_cached_free_pfn = reset_free;		}	} 	/* Leave no distance if no suitable block was reset */	if (reset_migrate >= reset_free) {		zone->compact_cached_migrate_pfn[0] = migrate_pfn;		zone->compact_cached_migrate_pfn[1] = migrate_pfn;		zone->compact_cached_free_pfn = free_pfn;	}} void reset_isolation_suitable(pg_data_t *pgdat){	int zoneid; 	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {		struct zone *zone = &pgdat->node_zones[zoneid];		if (!populated_zone(zone))			continue; 		/* Only flush if a full compaction finished recently */		if (zone->compact_blockskip_flush)			__reset_isolation_suitable(zone);	}} /* * Sets the pageblock skip bit if it was clear. Note that this is a hint as * locks are not required for read/writers. Returns true if it was already set. */static bool test_and_set_skip(struct compact_control *cc, struct page *page,							unsigned long pfn){	bool skip; 	/* Do no update if skip hint is being ignored */	if (cc->ignore_skip_hint)		return false; 	if (!IS_ALIGNED(pfn, pageblock_nr_pages))		return false; 	skip = get_pageblock_skip(page);	if (!skip && !cc->no_set_skip_hint)		set_pageblock_skip(page); 	return skip;} static void update_cached_migrate(struct compact_control *cc, unsigned long pfn){	struct zone *zone = cc->zone; 	pfn = pageblock_end_pfn(pfn); 	/* Set for isolation rather than compaction */	if (cc->no_set_skip_hint)		return; 	if (pfn > zone->compact_cached_migrate_pfn[0])		zone->compact_cached_migrate_pfn[0] = pfn;	if (cc->mode != MIGRATE_ASYNC &&	    pfn > zone->compact_cached_migrate_pfn[1])		zone->compact_cached_migrate_pfn[1] = pfn;} /* * If no pages were isolated then mark this pageblock to be skipped in the * future. The information is later cleared by __reset_isolation_suitable(). */static void update_pageblock_skip(struct compact_control *cc,			struct page *page, unsigned long pfn){	struct zone *zone = cc->zone; 	if (cc->no_set_skip_hint)		return; 	if (!page)		return; 	set_pageblock_skip(page); 	/* Update where async and sync compaction should restart */	if (pfn < zone->compact_cached_free_pfn)		zone->compact_cached_free_pfn = pfn;}#elsestatic inline bool isolation_suitable(struct compact_control *cc,					struct page *page){	return true;} static inline bool pageblock_skip_persistent(struct page *page){	return false;} static inline void update_pageblock_skip(struct compact_control *cc,			struct page *page, unsigned long pfn){} static void update_cached_migrate(struct compact_control *cc, unsigned long pfn){} static bool test_and_set_skip(struct compact_control *cc, struct page *page,							unsigned long pfn){	return false;}#endif /* CONFIG_COMPACTION */ /* * Compaction requires the taking of some coarse locks that are potentially * very heavily contended. For async compaction, trylock and record if the * lock is contended. The lock will still be acquired but compaction will * abort when the current block is finished regardless of success rate. * Sync compaction acquires the lock. * * Always returns true which makes it easier to track lock state in callers. */static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,

Lines 1–500 of 2925. Load more