/* * linux/mm/compaction.c * * Memory compaction for the reduction of external fragmentation. Note that * this heavily depends upon page migration to do all the real heavy * lifting * * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie> */#include <linux/cpu.h>#include <linux/swap.h>#include <linux/migrate.h>#include <linux/compaction.h>#include <linux/mm_inline.h>#include <linux/sched/signal.h>#include <linux/backing-dev.h>#include <linux/sysctl.h>#include <linux/sysfs.h>#include <linux/page-isolation.h>#include <linux/kasan.h>#include <linux/kthread.h>#include <linux/freezer.h>#include <linux/page_owner.h>#include "internal.h" #ifdef CONFIG_COMPACTIONstatic inline void count_compact_event(enum vm_event_item item){	count_vm_event(item);} static inline void count_compact_events(enum vm_event_item item, long delta){	count_vm_events(item, delta);}#else#define count_compact_event(item) do { } while (0)#define count_compact_events(item, delta) do { } while (0)#endif #if defined CONFIG_COMPACTION || defined CONFIG_CMA #define CREATE_TRACE_POINTS#include <trace/events/compaction.h> #define block_start_pfn(pfn, order)	round_down(pfn, 1UL << (order))#define block_end_pfn(pfn, order)	ALIGN((pfn) + 1, 1UL << (order))#define pageblock_start_pfn(pfn)	block_start_pfn(pfn, pageblock_order)#define pageblock_end_pfn(pfn)		block_end_pfn(pfn, pageblock_order) static unsigned long release_freepages(struct list_head *freelist){	struct page *page, *next;	unsigned long high_pfn = 0; 	list_for_each_entry_safe(page, next, freelist, lru) {		unsigned long pfn = page_to_pfn(page);		list_del(&page->lru);		__free_page(page);		if (pfn > high_pfn)			high_pfn = pfn;	} 	return high_pfn;} static void map_pages(struct list_head *list){	unsigned int i, order, nr_pages;	struct page *page, *next;	LIST_HEAD(tmp_list); 	list_for_each_entry_safe(page, next, list, lru) {		list_del(&page->lru); 		order = page_private(page);		nr_pages = 1 << order; 		post_alloc_hook(page, order, __GFP_MOVABLE);		if (order)			split_page(page, order); 		for (i = 0; i < nr_pages; i++) {			list_add(&page->lru, &tmp_list);			page++;		}	} 	list_splice(&tmp_list, list);} #ifdef CONFIG_COMPACTION int PageMovable(struct page *page){	struct address_space *mapping; 	VM_BUG_ON_PAGE(!PageLocked(page), page);	if (!__PageMovable(page))		return 0; 	mapping = page_mapping(page);	if (mapping && mapping->a_ops && mapping->a_ops->isolate_page)		return 1; 	return 0;}EXPORT_SYMBOL(PageMovable); void __SetPageMovable(struct page *page, struct address_space *mapping){	VM_BUG_ON_PAGE(!PageLocked(page), page);	VM_BUG_ON_PAGE((unsigned long)mapping & PAGE_MAPPING_MOVABLE, page);	page->mapping = (void *)((unsigned long)mapping | PAGE_MAPPING_MOVABLE);}EXPORT_SYMBOL(__SetPageMovable); void __ClearPageMovable(struct page *page){	VM_BUG_ON_PAGE(!PageLocked(page), page);	VM_BUG_ON_PAGE(!PageMovable(page), page);	/*	 * Clear registered address_space val with keeping PAGE_MAPPING_MOVABLE	 * flag so that VM can catch up released page by driver after isolation.	 * With it, VM migration doesn't try to put it back.	 */	page->mapping = (void *)((unsigned long)page->mapping &				PAGE_MAPPING_MOVABLE);}EXPORT_SYMBOL(__ClearPageMovable); /* Do not skip compaction more than 64 times */#define COMPACT_MAX_DEFER_SHIFT 6 /* * Compaction is deferred when compaction fails to result in a page * allocation success. 1 << compact_defer_limit compactions are skipped up * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT */void defer_compaction(struct zone *zone, int order){	zone->compact_considered = 0;	zone->compact_defer_shift++; 	if (order < zone->compact_order_failed)		zone->compact_order_failed = order; 	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT; 	trace_mm_compaction_defer_compaction(zone, order);} /* Returns true if compaction should be skipped this time */bool compaction_deferred(struct zone *zone, int order){	unsigned long defer_limit = 1UL << zone->compact_defer_shift; 	if (order < zone->compact_order_failed)		return false; 	/* Avoid possible overflow */	if (++zone->compact_considered > defer_limit)		zone->compact_considered = defer_limit; 	if (zone->compact_considered >= defer_limit)		return false; 	trace_mm_compaction_deferred(zone, order); 	return true;} /* * Update defer tracking counters after successful compaction of given order, * which means an allocation either succeeded (alloc_success == true) or is * expected to succeed. */void compaction_defer_reset(struct zone *zone, int order,		bool alloc_success){	if (alloc_success) {		zone->compact_considered = 0;		zone->compact_defer_shift = 0;	}	if (order >= zone->compact_order_failed)		zone->compact_order_failed = order + 1; 	trace_mm_compaction_defer_reset(zone, order);} /* Returns true if restarting compaction after many failures */bool compaction_restarting(struct zone *zone, int order){	if (order < zone->compact_order_failed)		return false; 	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&		zone->compact_considered >= 1UL << zone->compact_defer_shift;} /* Returns true if the pageblock should be scanned for pages to isolate. */static inline bool isolation_suitable(struct compact_control *cc,					struct page *page){	if (cc->ignore_skip_hint)		return true; 	return !get_pageblock_skip(page);} static void reset_cached_positions(struct zone *zone){	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;	zone->compact_cached_free_pfn =				pageblock_start_pfn(zone_end_pfn(zone) - 1);} /* * This function is called to clear all cached information on pageblocks that * should be skipped for page isolation when the migrate and free page scanner * meet. */static void __reset_isolation_suitable(struct zone *zone){	unsigned long start_pfn = zone->zone_start_pfn;	unsigned long end_pfn = zone_end_pfn(zone);	unsigned long pfn; 	zone->compact_blockskip_flush = false; 	/* Walk the zone and mark every pageblock as suitable for isolation */	for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {		struct page *page; 		cond_resched(); 		page = pfn_to_online_page(pfn);		if (!page)			continue;		if (zone != page_zone(page))			continue; 		clear_pageblock_skip(page);	} 	reset_cached_positions(zone);} void reset_isolation_suitable(pg_data_t *pgdat){	int zoneid; 	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {		struct zone *zone = &pgdat->node_zones[zoneid];		if (!populated_zone(zone))			continue; 		/* Only flush if a full compaction finished recently */		if (zone->compact_blockskip_flush)			__reset_isolation_suitable(zone);	}} /* * If no pages were isolated then mark this pageblock to be skipped in the * future. The information is later cleared by __reset_isolation_suitable(). */static void update_pageblock_skip(struct compact_control *cc,			struct page *page, unsigned long nr_isolated,			bool migrate_scanner){	struct zone *zone = cc->zone;	unsigned long pfn; 	if (cc->ignore_skip_hint)		return; 	if (!page)		return; 	if (nr_isolated)		return; 	set_pageblock_skip(page); 	pfn = page_to_pfn(page); 	/* Update where async and sync compaction should restart */	if (migrate_scanner) {		if (pfn > zone->compact_cached_migrate_pfn[0])			zone->compact_cached_migrate_pfn[0] = pfn;		if (cc->mode != MIGRATE_ASYNC &&		    pfn > zone->compact_cached_migrate_pfn[1])			zone->compact_cached_migrate_pfn[1] = pfn;	} else {		if (pfn < zone->compact_cached_free_pfn)			zone->compact_cached_free_pfn = pfn;	}}#elsestatic inline bool isolation_suitable(struct compact_control *cc,					struct page *page){	return true;} static void update_pageblock_skip(struct compact_control *cc,			struct page *page, unsigned long nr_isolated,			bool migrate_scanner){}#endif /* CONFIG_COMPACTION */ /* * Compaction requires the taking of some coarse locks that are potentially * very heavily contended. For async compaction, back out if the lock cannot * be taken immediately. For sync compaction, spin on the lock if needed. * * Returns true if the lock is held * Returns false if the lock is not held and compaction should abort */static bool compact_trylock_irqsave(spinlock_t *lock, unsigned long *flags,						struct compact_control *cc){	if (cc->mode == MIGRATE_ASYNC) {		if (!spin_trylock_irqsave(lock, *flags)) {			cc->contended = true;			return false;		}	} else {		spin_lock_irqsave(lock, *flags);	} 	return true;} /* * Compaction requires the taking of some coarse locks that are potentially * very heavily contended. The lock should be periodically unlocked to avoid * having disabled IRQs for a long time, even when there is nobody waiting on * the lock. It might also be that allowing the IRQs will result in * need_resched() becoming true. If scheduling is needed, async compaction * aborts. Sync compaction schedules. * Either compaction type will also abort if a fatal signal is pending. * In either case if the lock was locked, it is dropped and not regained. * * Returns true if compaction should abort due to fatal signal pending, or *		async compaction due to need_resched() * Returns false when compaction can continue (sync compaction might have *		scheduled) */static bool compact_unlock_should_abort(spinlock_t *lock,		unsigned long flags, bool *locked, struct compact_control *cc){	if (*locked) {		spin_unlock_irqrestore(lock, flags);		*locked = false;	} 	if (fatal_signal_pending(current)) {		cc->contended = true;		return true;	} 	if (need_resched()) {		if (cc->mode == MIGRATE_ASYNC) {			cc->contended = true;			return true;		}		cond_resched();	} 	return false;} /* * Aside from avoiding lock contention, compaction also periodically checks * need_resched() and either schedules in sync compaction or aborts async * compaction. This is similar to what compact_unlock_should_abort() does, but * is used where no lock is concerned. * * Returns false when no scheduling was needed, or sync compaction scheduled. * Returns true when async compaction should abort. */static inline bool compact_should_abort(struct compact_control *cc){	/* async compaction aborts if contended */	if (need_resched()) {		if (cc->mode == MIGRATE_ASYNC) {			cc->contended = true;			return true;		} 		cond_resched();	} 	return false;} /* * Isolate free pages onto a private freelist. If @strict is true, will abort * returning 0 on any invalid PFNs or non-free pages inside of the pageblock * (even though it may still end up isolating some pages). */static unsigned long isolate_freepages_block(struct compact_control *cc,				unsigned long *start_pfn,				unsigned long end_pfn,				struct list_head *freelist,				bool strict){	int nr_scanned = 0, total_isolated = 0;	struct page *cursor, *valid_page = NULL;	unsigned long flags = 0;	bool locked = false;	unsigned long blockpfn = *start_pfn;	unsigned int order; 	cursor = pfn_to_page(blockpfn); 	/* Isolate free pages. */	for (; blockpfn < end_pfn; blockpfn++, cursor++) {		int isolated;		struct page *page = cursor; 		/*		 * Periodically drop the lock (if held) regardless of its		 * contention, to give chance to IRQs. Abort if fatal signal		 * pending or async compaction detects need_resched()		 */		if (!(blockpfn % SWAP_CLUSTER_MAX)		    && compact_unlock_should_abort(&cc->zone->lock, flags,								&locked, cc))			break; 		nr_scanned++;		if (!pfn_valid_within(blockpfn))			goto isolate_fail; 		if (!valid_page)			valid_page = page; 		/*		 * For compound pages such as THP and hugetlbfs, we can save		 * potentially a lot of iterations if we skip them at once.		 * The check is racy, but we can consider only valid values		 * and the only danger is skipping too much.		 */		if (PageCompound(page)) {			unsigned int comp_order = compound_order(page); 			if (likely(comp_order < MAX_ORDER)) {				blockpfn += (1UL << comp_order) - 1;				cursor += (1UL << comp_order) - 1;			} 			goto isolate_fail;		} 		if (!PageBuddy(page))			goto isolate_fail; 		/*		 * If we already hold the lock, we can skip some rechecking.		 * Note that if we hold the lock now, checked_pageblock was		 * already set in some previous iteration (or strict is true),		 * so it is correct to skip the suitable migration target		 * recheck as well.		 */		if (!locked) {			/*			 * The zone lock must be held to isolate freepages.			 * Unfortunately this is a very coarse lock and can be			 * heavily contended if there are parallel allocations			 * or parallel compactions. For async compaction do not			 * spin on the lock and we acquire the lock as late as			 * possible.			 */			locked = compact_trylock_irqsave(&cc->zone->lock,								&flags, cc);			if (!locked)				break; 			/* Recheck this is a buddy page under lock */			if (!PageBuddy(page))				goto isolate_fail;		} 		/* Found a free page, will break it into order-0 pages */		order = page_order(page);		isolated = __isolate_free_page(page, order);		if (!isolated)			break;		set_page_private(page, order); 		total_isolated += isolated;		cc->nr_freepages += isolated;		list_add_tail(&page->lru, freelist); 

Lines 1–500 of 2130. Load more