If the memory compaction free scanner cannot successfully split a free page (only possible due to per-zone low watermark), terminate the free scanner rather than continuing to scan memory needlessly. If the watermark is insufficient for a free page of order <= cc->order, then terminate the scanner since all future splits will also likely fail. This prevents the compaction freeing scanner from scanning all memory on very large zones (very noticeable for zones > 128GB, for instance) when all splits will likely fail while holding zone->lock. compaction_alloc() iterating a 128GB zone has been benchmarked to take over 400ms on some systems whereas any free page isolated and ready to be split ends up failing in split_free_page() because of the low watermark check and thus the iteration continues. The next time compaction occurs, the freeing scanner will likely start at the end of the zone again since no success was made previously and we get the same lengthy iteration until the zone is brought above the low watermark. All thp page faults can take >400ms in such a state without this fix.
Unified · Split
| @@ -441,25 +441,23 @@ static unsigned long isolate_freepages_block(struct compact_control *cc, · 440 unmodified lines | ||
| 441 | 441 | |
| 442 | 442 | /* Found a free page, break it into order-0 pages */ |
| 443 | 443 | isolated = split_free_page(page); |
| 444 | + if (!isolated) | |
| 445 | + break; | |
| 446 | + | |
| 444 | 447 | total_isolated += isolated; |
| 448 | + cc->nr_freepages += isolated; | |
| 445 | 449 | for (i = 0; i < isolated; i++) { |
| 446 | 450 | list_add(&page->lru, freelist); |
| 447 | 451 | page++; |
| 448 | 452 | } |
| 449 | − | |
| 450 | − /* If a page was split, advance to the end of it */ | |
| 451 | − if (isolated) { | |
| 452 | − cc->nr_freepages += isolated; | |
| 453 | − if (!strict && | |
| 454 | − cc->nr_migratepages <= cc->nr_freepages) { | |
| 455 | − blockpfn += isolated; | |
| 456 | − break; | |
| 457 | − } | |
| 458 | − | |
| 459 | − blockpfn += isolated - 1; | |
| 460 | − cursor += isolated - 1; | |
| 461 | − continue; | |
| 453 | + if (!strict && cc->nr_migratepages <= cc->nr_freepages) { | |
| 454 | + blockpfn += isolated; | |
| 455 | + break; | |
| 462 | 456 | } |
| 457 | + /* Advance to the end of split page */ | |
| 458 | + blockpfn += isolated - 1; | |
| 459 | + cursor += isolated - 1; | |
| 460 | + continue; | |
| 463 | 461 | |
| 464 | 462 | isolate_fail: |
| 465 | 463 | if (strict) |
| @@ -469,6 +467,9 @@ isolate_fail: · 3 unmodified lines | ||
| 469 | 467 | |
| 470 | 468 | } |
| 471 | 469 | |
| 470 | + if (locked) | |
| 471 | + spin_unlock_irqrestore(&cc->zone->lock, flags); | |
| 472 | + | |
| 472 | 473 | /* |
| 473 | 474 | * There is a tiny chance that we have read bogus compound_order(), |
| 474 | 475 | * so be careful to not go outside of the pageblock. |
| @@ -490,9 +491,6 @@ isolate_fail: · 15 unmodified lines | ||
| 490 | 491 | if (strict && blockpfn < end_pfn) |
| 491 | 492 | total_isolated = 0; |
| 492 | 493 | |
| 493 | − if (locked) | |
| 494 | − spin_unlock_irqrestore(&cc->zone->lock, flags); | |
| 495 | − | |
| 496 | 494 | /* Update the pageblock-skip if the whole pageblock was scanned */ |
| 497 | 495 | if (blockpfn == end_pfn) |
| 498 | 496 | update_pageblock_skip(cc, valid_page, total_isolated, false); |
| @@ -1011,6 +1009,7 @@ static void isolate_freepages(struct compact_control *cc) · 512 unmodified lines | ||
| 1011 | 1009 | block_end_pfn = block_start_pfn, |
| 1012 | 1010 | block_start_pfn -= pageblock_nr_pages, |
| 1013 | 1011 | isolate_start_pfn = block_start_pfn) { |
| 1012 | + unsigned long isolated; | |
| 1014 | 1013 | |
| 1015 | 1014 | /* |
| 1016 | 1015 | * This can iterate a massively long zone without finding any |
| @@ -1035,8 +1034,12 @@ static void isolate_freepages(struct compact_control *cc) · 18 unmodified lines | ||
| 1035 | 1034 | continue; |
| 1036 | 1035 | |
| 1037 | 1036 | /* Found a block suitable for isolating free pages from. */ |
| 1038 | − isolate_freepages_block(cc, &isolate_start_pfn, | |
| 1039 | − block_end_pfn, freelist, false); | |
| 1037 | + isolated = isolate_freepages_block(cc, &isolate_start_pfn, | |
| 1038 | + block_end_pfn, freelist, false); | |
| 1039 | + /* If isolation failed early, do not continue needlessly */ | |
| 1040 | + if (!isolated && isolate_start_pfn < block_end_pfn && | |
| 1041 | + cc->nr_migratepages > cc->nr_freepages) | |
| 1042 | + break; | |
| 1040 | 1043 | |
| 1041 | 1044 | /* |
| 1042 | 1045 | * If we isolated enough freepages, or aborted due to async |