History

mm, compaction: be selective about what pageblocks to clear skip hints

Pageblock hints are cleared when compaction restarts or kswapd makes enough progress that it can sleep but it's over-eager in that the bit is cleared for migration sources with no LRU pages and migration targets with no free pages. As pageblock skip hint flushes are relatively rare and out-of-band with respect to kswapd, this patch makes a few more expensive checks to see if it's appropriate to even clear the bit. Every pageblock that is not cleared will avoid 512 pages being scanned unnecessarily on x86-64. The impact is variable with different workloads showing small differences in latency, success rates and scan rates. This is expected as clearing the hints is not that common but doing a small amount of work out-of-band to avoid a large amount of work in-band later is generally a good thing.

Mel Gorman committed 7 years ago

Commit e332f741 parent 4fca9732 files · +108 −18Browse files

Unified · Split

2 files
  1. include/linux/mmzone.h +2 −0
  2. mm/compaction.c +106 −18
include/linux/mmzone.h+2−0
@@ -480,6 +480,8 @@ struct zone { · 479 unmodified lines
480480 unsigned long compact_cached_free_pfn;
481481 /* pfn where async and sync compaction migration scanner should start */
482482 unsigned long compact_cached_migrate_pfn[2];
483+ unsigned long compact_init_migrate_pfn;
484+ unsigned long compact_init_free_pfn;
483485 #endif
484486
485487 #ifdef CONFIG_COMPACTION
mm/compaction.c+106−18
@@ -237,6 +237,70 @@ static bool pageblock_skip_persistent(struct page *page) · 236 unmodified lines
237237 return false;
238238 }
239239
240+static bool
241+__reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,
242+ bool check_target)
243+{
244+ struct page *page = pfn_to_online_page(pfn);
245+ struct page *end_page;
246+ unsigned long block_pfn;
247+
248+ if (!page)
249+ return false;
250+ if (zone != page_zone(page))
251+ return false;
252+ if (pageblock_skip_persistent(page))
253+ return false;
254+
255+ /*
256+ * If skip is already cleared do no further checking once the
257+ * restart points have been set.
258+ */
259+ if (check_source && check_target && !get_pageblock_skip(page))
260+ return true;
261+
262+ /*
263+ * If clearing skip for the target scanner, do not select a
264+ * non-movable pageblock as the starting point.
265+ */
266+ if (!check_source && check_target &&
267+ get_pageblock_migratetype(page) != MIGRATE_MOVABLE)
268+ return false;
269+
270+ /*
271+ * Only clear the hint if a sample indicates there is either a
272+ * free page or an LRU page in the block. One or other condition
273+ * is necessary for the block to be a migration source/target.
274+ */
275+ block_pfn = pageblock_start_pfn(pfn);
276+ pfn = max(block_pfn, zone->zone_start_pfn);
277+ page = pfn_to_page(pfn);
278+ if (zone != page_zone(page))
279+ return false;
280+ pfn = block_pfn + pageblock_nr_pages;
281+ pfn = min(pfn, zone_end_pfn(zone));
282+ end_page = pfn_to_page(pfn);
283+
284+ do {
285+ if (pfn_valid_within(pfn)) {
286+ if (check_source && PageLRU(page)) {
287+ clear_pageblock_skip(page);
288+ return true;
289+ }
290+
291+ if (check_target && PageBuddy(page)) {
292+ clear_pageblock_skip(page);
293+ return true;
294+ }
295+ }
296+
297+ page += (1 << PAGE_ALLOC_COSTLY_ORDER);
298+ pfn += (1 << PAGE_ALLOC_COSTLY_ORDER);
299+ } while (page < end_page);
300+
301+ return false;
302+}
303+
240304 /*
241305 * This function is called to clear all cached information on pageblocks that
242306 * should be skipped for page isolation when the migrate and free page scanner
@@ -244,30 +308,54 @@ static bool pageblock_skip_persistent(struct page *page) · 1 unmodified line
244308 */
245309 static void __reset_isolation_suitable(struct zone *zone)
246310 {
247− unsigned long start_pfn = zone->zone_start_pfn;
248− unsigned long end_pfn = zone_end_pfn(zone);
249− unsigned long pfn;
311+ unsigned long migrate_pfn = zone->zone_start_pfn;
312+ unsigned long free_pfn = zone_end_pfn(zone);
313+ unsigned long reset_migrate = free_pfn;
314+ unsigned long reset_free = migrate_pfn;
315+ bool source_set = false;
316+ bool free_set = false;
250317
251− zone->compact_blockskip_flush = false;
318+ if (!zone->compact_blockskip_flush)
319+ return;
252320
253− /* Walk the zone and mark every pageblock as suitable for isolation */
254− for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
255− struct page *page;
321+ zone->compact_blockskip_flush = false;
256322
323+ /*
324+ * Walk the zone and update pageblock skip information. Source looks
325+ * for PageLRU while target looks for PageBuddy. When the scanner
326+ * is found, both PageBuddy and PageLRU are checked as the pageblock
327+ * is suitable as both source and target.
328+ */
329+ for (; migrate_pfn < free_pfn; migrate_pfn += pageblock_nr_pages,
330+ free_pfn -= pageblock_nr_pages) {
257331 cond_resched();
258332
259− page = pfn_to_online_page(pfn);
260− if (!page)
261− continue;
262− if (zone != page_zone(page))
263− continue;
264− if (pageblock_skip_persistent(page))
265− continue;
333+ /* Update the migrate PFN */
334+ if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&
335+ migrate_pfn < reset_migrate) {
336+ source_set = true;
337+ reset_migrate = migrate_pfn;
338+ zone->compact_init_migrate_pfn = reset_migrate;
339+ zone->compact_cached_migrate_pfn[0] = reset_migrate;
340+ zone->compact_cached_migrate_pfn[1] = reset_migrate;
341+ }
266342
267− clear_pageblock_skip(page);
343+ /* Update the free PFN */
344+ if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&
345+ free_pfn > reset_free) {
346+ free_set = true;
347+ reset_free = free_pfn;
348+ zone->compact_init_free_pfn = reset_free;
349+ zone->compact_cached_free_pfn = reset_free;
350+ }
268351 }
269352
270− reset_cached_positions(zone);
353+ /* Leave no distance if no suitable block was reset */
354+ if (reset_migrate >= reset_free) {
355+ zone->compact_cached_migrate_pfn[0] = migrate_pfn;
356+ zone->compact_cached_migrate_pfn[1] = migrate_pfn;
357+ zone->compact_cached_free_pfn = free_pfn;
358+ }
271359 }
272360
273361 void reset_isolation_suitable(pg_data_t *pgdat)
@@ -1190,7 +1278,7 @@ fast_isolate_freepages(struct compact_control *cc) · 916 unmodified lines
11901278 * If starting the scan, use a deeper search and use the highest
11911279 * PFN found if a suitable one is not found.
11921280 */
1193− if (cc->free_pfn == pageblock_start_pfn(zone_end_pfn(cc->zone) - 1)) {
1281+ if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {
11941282 limit = pageblock_nr_pages >> 1;
11951283 scan_start = true;
11961284 }
@@ -2017,7 +2105,7 @@ static enum compact_result compact_zone(struct compact_control *cc) · 820 unmodified lines
20172105 cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
20182106 }
20192107
2020− if (cc->migrate_pfn == start_pfn)
2108+ if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)
20212109 cc->whole_zone = true;
20222110 }
20232111

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t