1717 lines · 47.3 KB · C
RawDownload
/* * linux/mm/compaction.c * * Memory compaction for the reduction of external fragmentation. Note that * this heavily depends upon page migration to do all the real heavy * lifting * * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie> */#include <linux/swap.h>#include <linux/migrate.h>#include <linux/compaction.h>#include <linux/mm_inline.h>#include <linux/backing-dev.h>#include <linux/sysctl.h>#include <linux/sysfs.h>#include <linux/balloon_compaction.h>#include <linux/page-isolation.h>#include <linux/kasan.h>#include "internal.h" #ifdef CONFIG_COMPACTIONstatic inline void count_compact_event(enum vm_event_item item){	count_vm_event(item);} static inline void count_compact_events(enum vm_event_item item, long delta){	count_vm_events(item, delta);}#else#define count_compact_event(item) do { } while (0)#define count_compact_events(item, delta) do { } while (0)#endif #if defined CONFIG_COMPACTION || defined CONFIG_CMA#ifdef CONFIG_TRACEPOINTSstatic const char *const compaction_status_string[] = {	"deferred",	"skipped",	"continue",	"partial",	"complete",	"no_suitable_page",	"not_suitable_zone",};#endif #define CREATE_TRACE_POINTS#include <trace/events/compaction.h> static unsigned long release_freepages(struct list_head *freelist){	struct page *page, *next;	unsigned long high_pfn = 0; 	list_for_each_entry_safe(page, next, freelist, lru) {		unsigned long pfn = page_to_pfn(page);		list_del(&page->lru);		__free_page(page);		if (pfn > high_pfn)			high_pfn = pfn;	} 	return high_pfn;} static void map_pages(struct list_head *list){	struct page *page; 	list_for_each_entry(page, list, lru) {		arch_alloc_page(page, 0);		kernel_map_pages(page, 1, 1);		kasan_alloc_pages(page, 0);	}} static inline bool migrate_async_suitable(int migratetype){	return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE;} /* * Check that the whole (or subset of) a pageblock given by the interval of * [start_pfn, end_pfn) is valid and within the same zone, before scanning it * with the migration of free compaction scanner. The scanners then need to * use only pfn_valid_within() check for arches that allow holes within * pageblocks. * * Return struct page pointer of start_pfn, or NULL if checks were not passed. * * It's possible on some configurations to have a setup like node0 node1 node0 * i.e. it's possible that all pages within a zones range of pages do not * belong to a single zone. We assume that a border between node0 and node1 * can occur within a single pageblock, but not a node0 node1 node0 * interleaving within a single pageblock. It is therefore sufficient to check * the first and last page of a pageblock and avoid checking each individual * page in a pageblock. */static struct page *pageblock_pfn_to_page(unsigned long start_pfn,				unsigned long end_pfn, struct zone *zone){	struct page *start_page;	struct page *end_page; 	/* end_pfn is one past the range we are checking */	end_pfn--; 	if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))		return NULL; 	start_page = pfn_to_page(start_pfn); 	if (page_zone(start_page) != zone)		return NULL; 	end_page = pfn_to_page(end_pfn); 	/* This gives a shorter code than deriving page_zone(end_page) */	if (page_zone_id(start_page) != page_zone_id(end_page))		return NULL; 	return start_page;} #ifdef CONFIG_COMPACTION /* Do not skip compaction more than 64 times */#define COMPACT_MAX_DEFER_SHIFT 6 /* * Compaction is deferred when compaction fails to result in a page * allocation success. 1 << compact_defer_limit compactions are skipped up * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT */void defer_compaction(struct zone *zone, int order){	zone->compact_considered = 0;	zone->compact_defer_shift++; 	if (order < zone->compact_order_failed)		zone->compact_order_failed = order; 	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT; 	trace_mm_compaction_defer_compaction(zone, order);} /* Returns true if compaction should be skipped this time */bool compaction_deferred(struct zone *zone, int order){	unsigned long defer_limit = 1UL << zone->compact_defer_shift; 	if (order < zone->compact_order_failed)		return false; 	/* Avoid possible overflow */	if (++zone->compact_considered > defer_limit)		zone->compact_considered = defer_limit; 	if (zone->compact_considered >= defer_limit)		return false; 	trace_mm_compaction_deferred(zone, order); 	return true;} /* * Update defer tracking counters after successful compaction of given order, * which means an allocation either succeeded (alloc_success == true) or is * expected to succeed. */void compaction_defer_reset(struct zone *zone, int order,		bool alloc_success){	if (alloc_success) {		zone->compact_considered = 0;		zone->compact_defer_shift = 0;	}	if (order >= zone->compact_order_failed)		zone->compact_order_failed = order + 1; 	trace_mm_compaction_defer_reset(zone, order);} /* Returns true if restarting compaction after many failures */bool compaction_restarting(struct zone *zone, int order){	if (order < zone->compact_order_failed)		return false; 	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&		zone->compact_considered >= 1UL << zone->compact_defer_shift;} /* Returns true if the pageblock should be scanned for pages to isolate. */static inline bool isolation_suitable(struct compact_control *cc,					struct page *page){	if (cc->ignore_skip_hint)		return true; 	return !get_pageblock_skip(page);} /* * This function is called to clear all cached information on pageblocks that * should be skipped for page isolation when the migrate and free page scanner * meet. */static void __reset_isolation_suitable(struct zone *zone){	unsigned long start_pfn = zone->zone_start_pfn;	unsigned long end_pfn = zone_end_pfn(zone);	unsigned long pfn; 	zone->compact_cached_migrate_pfn[0] = start_pfn;	zone->compact_cached_migrate_pfn[1] = start_pfn;	zone->compact_cached_free_pfn = end_pfn;	zone->compact_blockskip_flush = false; 	/* Walk the zone and mark every pageblock as suitable for isolation */	for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {		struct page *page; 		cond_resched(); 		if (!pfn_valid(pfn))			continue; 		page = pfn_to_page(pfn);		if (zone != page_zone(page))			continue; 		clear_pageblock_skip(page);	}} void reset_isolation_suitable(pg_data_t *pgdat){	int zoneid; 	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {		struct zone *zone = &pgdat->node_zones[zoneid];		if (!populated_zone(zone))			continue; 		/* Only flush if a full compaction finished recently */		if (zone->compact_blockskip_flush)			__reset_isolation_suitable(zone);	}} /* * If no pages were isolated then mark this pageblock to be skipped in the * future. The information is later cleared by __reset_isolation_suitable(). */static void update_pageblock_skip(struct compact_control *cc,			struct page *page, unsigned long nr_isolated,			bool migrate_scanner){	struct zone *zone = cc->zone;	unsigned long pfn; 	if (cc->ignore_skip_hint)		return; 	if (!page)		return; 	if (nr_isolated)		return; 	set_pageblock_skip(page); 	pfn = page_to_pfn(page); 	/* Update where async and sync compaction should restart */	if (migrate_scanner) {		if (pfn > zone->compact_cached_migrate_pfn[0])			zone->compact_cached_migrate_pfn[0] = pfn;		if (cc->mode != MIGRATE_ASYNC &&		    pfn > zone->compact_cached_migrate_pfn[1])			zone->compact_cached_migrate_pfn[1] = pfn;	} else {		if (pfn < zone->compact_cached_free_pfn)			zone->compact_cached_free_pfn = pfn;	}}#elsestatic inline bool isolation_suitable(struct compact_control *cc,					struct page *page){	return true;} static void update_pageblock_skip(struct compact_control *cc,			struct page *page, unsigned long nr_isolated,			bool migrate_scanner){}#endif /* CONFIG_COMPACTION */ /* * Compaction requires the taking of some coarse locks that are potentially * very heavily contended. For async compaction, back out if the lock cannot * be taken immediately. For sync compaction, spin on the lock if needed. * * Returns true if the lock is held * Returns false if the lock is not held and compaction should abort */static bool compact_trylock_irqsave(spinlock_t *lock, unsigned long *flags,						struct compact_control *cc){	if (cc->mode == MIGRATE_ASYNC) {		if (!spin_trylock_irqsave(lock, *flags)) {			cc->contended = COMPACT_CONTENDED_LOCK;			return false;		}	} else {		spin_lock_irqsave(lock, *flags);	} 	return true;} /* * Compaction requires the taking of some coarse locks that are potentially * very heavily contended. The lock should be periodically unlocked to avoid * having disabled IRQs for a long time, even when there is nobody waiting on * the lock. It might also be that allowing the IRQs will result in * need_resched() becoming true. If scheduling is needed, async compaction * aborts. Sync compaction schedules. * Either compaction type will also abort if a fatal signal is pending. * In either case if the lock was locked, it is dropped and not regained. * * Returns true if compaction should abort due to fatal signal pending, or *		async compaction due to need_resched() * Returns false when compaction can continue (sync compaction might have *		scheduled) */static bool compact_unlock_should_abort(spinlock_t *lock,		unsigned long flags, bool *locked, struct compact_control *cc){	if (*locked) {		spin_unlock_irqrestore(lock, flags);		*locked = false;	} 	if (fatal_signal_pending(current)) {		cc->contended = COMPACT_CONTENDED_SCHED;		return true;	} 	if (need_resched()) {		if (cc->mode == MIGRATE_ASYNC) {			cc->contended = COMPACT_CONTENDED_SCHED;			return true;		}		cond_resched();	} 	return false;} /* * Aside from avoiding lock contention, compaction also periodically checks * need_resched() and either schedules in sync compaction or aborts async * compaction. This is similar to what compact_unlock_should_abort() does, but * is used where no lock is concerned. * * Returns false when no scheduling was needed, or sync compaction scheduled. * Returns true when async compaction should abort. */static inline bool compact_should_abort(struct compact_control *cc){	/* async compaction aborts if contended */	if (need_resched()) {		if (cc->mode == MIGRATE_ASYNC) {			cc->contended = COMPACT_CONTENDED_SCHED;			return true;		} 		cond_resched();	} 	return false;} /* * Isolate free pages onto a private freelist. If @strict is true, will abort * returning 0 on any invalid PFNs or non-free pages inside of the pageblock * (even though it may still end up isolating some pages). */static unsigned long isolate_freepages_block(struct compact_control *cc,				unsigned long *start_pfn,				unsigned long end_pfn,				struct list_head *freelist,				bool strict){	int nr_scanned = 0, total_isolated = 0;	struct page *cursor, *valid_page = NULL;	unsigned long flags = 0;	bool locked = false;	unsigned long blockpfn = *start_pfn; 	cursor = pfn_to_page(blockpfn); 	/* Isolate free pages. */	for (; blockpfn < end_pfn; blockpfn++, cursor++) {		int isolated, i;		struct page *page = cursor; 		/*		 * Periodically drop the lock (if held) regardless of its		 * contention, to give chance to IRQs. Abort if fatal signal		 * pending or async compaction detects need_resched()		 */		if (!(blockpfn % SWAP_CLUSTER_MAX)		    && compact_unlock_should_abort(&cc->zone->lock, flags,								&locked, cc))			break; 		nr_scanned++;		if (!pfn_valid_within(blockpfn))			goto isolate_fail; 		if (!valid_page)			valid_page = page;		if (!PageBuddy(page))			goto isolate_fail; 		/*		 * If we already hold the lock, we can skip some rechecking.		 * Note that if we hold the lock now, checked_pageblock was		 * already set in some previous iteration (or strict is true),		 * so it is correct to skip the suitable migration target		 * recheck as well.		 */		if (!locked) {			/*			 * The zone lock must be held to isolate freepages.			 * Unfortunately this is a very coarse lock and can be			 * heavily contended if there are parallel allocations			 * or parallel compactions. For async compaction do not			 * spin on the lock and we acquire the lock as late as			 * possible.			 */			locked = compact_trylock_irqsave(&cc->zone->lock,								&flags, cc);			if (!locked)				break; 			/* Recheck this is a buddy page under lock */			if (!PageBuddy(page))				goto isolate_fail;		} 		/* Found a free page, break it into order-0 pages */		isolated = split_free_page(page);		total_isolated += isolated;		for (i = 0; i < isolated; i++) {			list_add(&page->lru, freelist);			page++;		} 		/* If a page was split, advance to the end of it */		if (isolated) {			cc->nr_freepages += isolated;			if (!strict &&				cc->nr_migratepages <= cc->nr_freepages) {				blockpfn += isolated;				break;			} 			blockpfn += isolated - 1;			cursor += isolated - 1;			continue;		} isolate_fail:		if (strict)			break;		else			continue; 	} 	trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn,					nr_scanned, total_isolated); 	/* Record how far we have got within the block */	*start_pfn = blockpfn; 	/*	 * If strict isolation is requested by CMA then check that all the

Lines 1–500 of 1717.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t