// SPDX-License-Identifier: GPL-2.0/* * linux/mm/compaction.c * * Memory compaction for the reduction of external fragmentation. Note that * this heavily depends upon page migration to do all the real heavy * lifting * * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie> */#include <linux/cpu.h>#include <linux/swap.h>#include <linux/migrate.h>#include <linux/compaction.h>#include <linux/mm_inline.h>#include <linux/sched/signal.h>#include <linux/backing-dev.h>#include <linux/sysctl.h>#include <linux/sysfs.h>#include <linux/page-isolation.h>#include <linux/kasan.h>#include <linux/kthread.h>#include <linux/freezer.h>#include <linux/page_owner.h>#include <linux/psi.h>#include <linux/cpuset.h>#include "page_alloc.h"#include "internal.h" #ifdef CONFIG_COMPACTION/* * Fragmentation score check interval for proactive compaction purposes. */#define HPAGE_FRAG_CHECK_INTERVAL_MSEC	(500) static inline void count_compact_event(enum vm_event_item item){	count_vm_event(item);} static inline void count_compact_events(enum vm_event_item item, long delta){	count_vm_events(item, delta);} /* * order == -1 is expected when compacting proactively via * 1. /proc/sys/vm/compact_memory * 2. /sys/devices/system/node/nodex/compact * 3. /proc/sys/vm/compaction_proactiveness */static inline bool is_via_compact_memory(int order){	return order == -1;} #else#define count_compact_event(item) do { } while (0)#define count_compact_events(item, delta) do { } while (0)static inline bool is_via_compact_memory(int order) { return false; }#endif #if defined CONFIG_COMPACTION || defined CONFIG_CMA #define CREATE_TRACE_POINTS#include <trace/events/compaction.h> #define block_start_pfn(pfn, order)	round_down(pfn, 1UL << (order))#define block_end_pfn(pfn, order)	ALIGN((pfn) + 1, 1UL << (order)) /* * Page order with-respect-to which proactive compaction * calculates external fragmentation, which is used as * the "fragmentation score" of a node/zone. */#if defined CONFIG_TRANSPARENT_HUGEPAGE#define COMPACTION_HPAGE_ORDER	HPAGE_PMD_ORDER#elif defined CONFIG_HUGETLBFS#define COMPACTION_HPAGE_ORDER	HUGETLB_PAGE_ORDER#else#define COMPACTION_HPAGE_ORDER	(PMD_SHIFT - PAGE_SHIFT)#endif static struct page *mark_allocated_noprof(struct page *page, unsigned int order, gfp_t gfp_flags){	post_alloc_hook(page, order, __GFP_MOVABLE, ALLOC_DEFAULT);	set_page_refcounted(page);	return page;}#define mark_allocated(...)	alloc_hooks(mark_allocated_noprof(__VA_ARGS__)) static unsigned long release_free_list(struct list_head *freepages){	int order;	unsigned long high_pfn = 0; 	for (order = 0; order < NR_PAGE_ORDERS; order++) {		struct page *page, *next; 		list_for_each_entry_safe(page, next, &freepages[order], lru) {			unsigned long pfn = page_to_pfn(page); 			list_del(&page->lru);			/*			 * Convert free pages into post allocation pages, so			 * that we can free them via __free_page.			 */			mark_allocated(page, order, __GFP_MOVABLE);			__free_pages(page, order);			if (pfn > high_pfn)				high_pfn = pfn;		}	}	return high_pfn;} #ifdef CONFIG_COMPACTION /* Do not skip compaction more than 64 times */#define COMPACT_MAX_DEFER_SHIFT 6 /* * Compaction is deferred when compaction fails to result in a page * allocation success. 1 << compact_defer_shift, compactions are skipped up * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT */static void defer_compaction(struct zone *zone, int order){	zone->compact_considered = 0;	zone->compact_defer_shift++; 	if (order < zone->compact_order_failed)		zone->compact_order_failed = order; 	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT; 	trace_mm_compaction_defer_compaction(zone, order);} /* Returns true if compaction should be skipped this time */static bool compaction_deferred(struct zone *zone, int order){	unsigned long defer_limit = 1UL << zone->compact_defer_shift; 	if (order < zone->compact_order_failed)		return false; 	/* Avoid possible overflow */	if (++zone->compact_considered >= defer_limit) {		zone->compact_considered = defer_limit;		return false;	} 	trace_mm_compaction_deferred(zone, order); 	return true;} /* * Update defer tracking counters after successful compaction of given order, * which means an allocation either succeeded (alloc_success == true) or is * expected to succeed. */void compaction_defer_reset(struct zone *zone, int order,		bool alloc_success){	if (alloc_success) {		zone->compact_considered = 0;		zone->compact_defer_shift = 0;	}	if (order >= zone->compact_order_failed)		zone->compact_order_failed = order + 1; 	trace_mm_compaction_defer_reset(zone, order);} /* Returns true if restarting compaction after many failures */static bool compaction_restarting(struct zone *zone, int order){	if (order < zone->compact_order_failed)		return false; 	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&		zone->compact_considered >= 1UL << zone->compact_defer_shift;} /* Returns true if the pageblock should be scanned for pages to isolate. */static inline bool isolation_suitable(struct compact_control *cc,					struct page *page){	if (cc->ignore_skip_hint)		return true; 	return !get_pageblock_skip(page);} static void reset_cached_positions(struct zone *zone){	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;	zone->compact_cached_free_pfn =				pageblock_start_pfn(zone_end_pfn(zone) - 1);} #ifdef CONFIG_SPARSEMEM/* * If the PFN falls into an offline section, return the start PFN of the * next online section. If the PFN falls into an online section or if * there is no next online section, return 0. */static unsigned long skip_offline_sections(unsigned long start_pfn){	unsigned long start_nr = pfn_to_section_nr(start_pfn); 	if (online_section_nr(start_nr))		return 0; 	while (++start_nr <= __highest_present_section_nr) {		if (online_section_nr(start_nr))			return section_nr_to_pfn(start_nr);	} 	return 0;} /* * If the PFN falls into an offline section, return the end PFN of the * next online section in reverse. If the PFN falls into an online section * or if there is no next online section in reverse, return 0. */static unsigned long skip_offline_sections_reverse(unsigned long start_pfn){	unsigned long start_nr = pfn_to_section_nr(start_pfn); 	if (!start_nr || online_section_nr(start_nr))		return 0; 	while (start_nr-- > 0) {		if (online_section_nr(start_nr))			return section_nr_to_pfn(start_nr) + PAGES_PER_SECTION;	} 	return 0;}#elsestatic unsigned long skip_offline_sections(unsigned long start_pfn){	return 0;} static unsigned long skip_offline_sections_reverse(unsigned long start_pfn){	return 0;}#endif /* * Compound pages of >= pageblock_order should consistently be skipped until * released. It is always pointless to compact pages of such order (if they are * migratable), and the pageblocks they occupy cannot contain any free pages. */static bool pageblock_skip_persistent(struct page *page){	if (!PageCompound(page))		return false; 	page = compound_head(page); 	if (compound_order(page) >= pageblock_order)		return true; 	return false;} static bool__reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,							bool check_target){	struct page *page = pfn_to_online_page(pfn);	struct page *block_page;	struct page *end_page;	unsigned long block_pfn; 	if (!page)		return false;	if (zone != page_zone(page))		return false;	if (pageblock_skip_persistent(page))		return false; 	/*	 * If skip is already cleared do no further checking once the	 * restart points have been set.	 */	if (check_source && check_target && !get_pageblock_skip(page))		return true; 	/*	 * If clearing skip for the target scanner, do not select a	 * non-movable pageblock as the starting point.	 */	if (!check_source && check_target &&	    get_pageblock_migratetype(page) != MIGRATE_MOVABLE)		return false; 	/* Ensure the start of the pageblock or zone is online and valid */	block_pfn = pageblock_start_pfn(pfn);	block_pfn = max(block_pfn, zone->zone_start_pfn);	block_page = pfn_to_online_page(block_pfn);	if (block_page) {		page = block_page;		pfn = block_pfn;	} 	/* Ensure the end of the pageblock or zone is online and valid */	block_pfn = pageblock_end_pfn(pfn) - 1;	block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);	end_page = pfn_to_online_page(block_pfn);	if (!end_page)		return false; 	/*	 * Only clear the hint if a sample indicates there is either a	 * free page or an LRU page in the block. One or other condition	 * is necessary for the block to be a migration source/target.	 */	do {		if (check_source && PageLRU(page)) {			clear_pageblock_skip(page);			return true;		} 		if (check_target && PageBuddy(page)) {			clear_pageblock_skip(page);			return true;		} 		page += (1 << PAGE_ALLOC_COSTLY_ORDER);	} while (page <= end_page); 	return false;} /* * This function is called to clear all cached information on pageblocks that * should be skipped for page isolation when the migrate and free page scanner * meet. */static void __reset_isolation_suitable(struct zone *zone){	unsigned long migrate_pfn = zone->zone_start_pfn;	unsigned long free_pfn = zone_end_pfn(zone) - 1;	unsigned long reset_migrate = free_pfn;	unsigned long reset_free = migrate_pfn;	bool source_set = false;	bool free_set = false; 	/* Only flush if a full compaction finished recently */	if (!zone->compact_blockskip_flush)		return; 	zone->compact_blockskip_flush = false; 	/*	 * Walk the zone and update pageblock skip information. Source looks	 * for PageLRU while target looks for PageBuddy. When the scanner	 * is found, both PageBuddy and PageLRU are checked as the pageblock	 * is suitable as both source and target.	 */	for (; migrate_pfn < free_pfn; migrate_pfn += pageblock_nr_pages,					free_pfn -= pageblock_nr_pages) {		cond_resched(); 		/* Update the migrate PFN */		if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&		    migrate_pfn < reset_migrate) {			source_set = true;			reset_migrate = migrate_pfn;			zone->compact_init_migrate_pfn = reset_migrate;			zone->compact_cached_migrate_pfn[0] = reset_migrate;			zone->compact_cached_migrate_pfn[1] = reset_migrate;		} 		/* Update the free PFN */		if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&		    free_pfn > reset_free) {			free_set = true;			reset_free = free_pfn;			zone->compact_init_free_pfn = reset_free;			zone->compact_cached_free_pfn = reset_free;		}	} 	/* Leave no distance if no suitable block was reset */	if (reset_migrate >= reset_free) {		zone->compact_cached_migrate_pfn[0] = migrate_pfn;		zone->compact_cached_migrate_pfn[1] = migrate_pfn;		zone->compact_cached_free_pfn = free_pfn;	}} void reset_isolation_suitable(pg_data_t *pgdat){	int zoneid; 	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {		struct zone *zone = &pgdat->node_zones[zoneid];		if (!populated_zone(zone))			continue; 		__reset_isolation_suitable(zone);	}} /* * Sets the pageblock skip bit if it was clear. Note that this is a hint as * locks are not required for read/writers. Returns true if it was already set. */static bool test_and_set_skip(struct compact_control *cc, struct page *page){	bool skip; 	/* Do not update if skip hint is being ignored */	if (cc->ignore_skip_hint)		return false; 	skip = get_pageblock_skip(page);	if (!skip && !cc->no_set_skip_hint)		set_pageblock_skip(page); 	return skip;} static void update_cached_migrate(struct compact_control *cc, unsigned long pfn){	struct zone *zone = cc->zone; 	/* Set for isolation rather than compaction */	if (cc->no_set_skip_hint)		return; 	pfn = pageblock_end_pfn(pfn); 	/* Update where async and sync compaction should restart */	if (pfn > zone->compact_cached_migrate_pfn[0])		zone->compact_cached_migrate_pfn[0] = pfn;	if (cc->mode != MIGRATE_ASYNC &&	    pfn > zone->compact_cached_migrate_pfn[1])		zone->compact_cached_migrate_pfn[1] = pfn;} /* * If no pages were isolated then mark this pageblock to be skipped in the * future. The information is later cleared by __reset_isolation_suitable(). */static void update_pageblock_skip(struct compact_control *cc,			struct page *page, unsigned long pfn){	struct zone *zone = cc->zone; 	if (cc->no_set_skip_hint)		return; 	set_pageblock_skip(page); 	if (pfn < zone->compact_cached_free_pfn)		zone->compact_cached_free_pfn = pfn;}#elsestatic inline bool isolation_suitable(struct compact_control *cc,					struct page *page){	return true;} static inline bool pageblock_skip_persistent(struct page *page){	return false;} static inline void update_pageblock_skip(struct compact_control *cc,			struct page *page, unsigned long pfn){} static void update_cached_migrate(struct compact_control *cc, unsigned long pfn){} static bool test_and_set_skip(struct compact_control *cc, struct page *page){	return false;}#endif /* CONFIG_COMPACTION */ /* * Compaction requires the taking of some coarse locks that are potentially * very heavily contended. For async compaction, trylock and record if the * lock is contended. The lock will still be acquired but compaction will

Lines 1–500 of 3363. Load more