History

mm: page cache: store only head pages in i_pages

Transparent Huge Pages are currently stored in i_pages as pointers to consecutive subpages. This patch changes that to storing consecutive pointers to the head page in preparation for storing huge pages more efficiently in i_pages. Large parts of this are "inspired" by Kirill's patch https://lore.kernel.org/lkml/20170126115819.58875-2-kirill.shutemov@linux.intel.com/ Kirill and Huang Ying contributed several fixes. [willy@infradead.org: use compound_nr, squish uninit-var warning] Link: http://lkml.kernel.org/r/20190731210400.7419-1-willy@infradead.org Signed-off-by: Matthew Wilcox <willy@infradead.org> Acked-by: Jan Kara <jack@suse.cz> Reviewed-by: Kirill Shutemov <kirill@shutemov.name> Reviewed-by: Song Liu <songliubraving@fb.com> Tested-by: Song Liu <songliubraving@fb.com> Tested-by: William Kucharski <william.kucharski@oracle.com> Reviewed-by: William Kucharski <william.kucharski@oracle.com> Tested-by: Qian Cai <cai@lca.pw> Tested-by: Mikhail Gavrilov <mikhail.v.gavrilov@gmail.com> Cc: Hugh Dickins <hughd@google.com> Cc: Chris Wilson <chris@chris-wilson.co.uk> Cc: Song Liu <songliubraving@fb.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Matthew Wilcox (Oracle) committed 7 years ago

Commit 41011961 parent 875d91b8 files · +95 −96Browse files

Unified · Split

8 files
  1. include/linux/pagemap.h +10 −0
  2. mm/filemap.c +57 −88
  3. mm/huge_memory.c +20 −2
  4. mm/khugepaged.c +2 −2
  5. mm/memfd.c +2 −0
  6. mm/migrate.c +1 −1
  7. mm/shmem.c +1 −1
  8. mm/swap_state.c +2 −2
include/linux/pagemap.h+10−0
@@ -333,6 +333,16 @@ static inline struct page *grab_cache_page_nowait(struct address_space *mapping, · 332 unmodified lines
333333 mapping_gfp_mask(mapping));
334334 }
335335
336+static inline struct page *find_subpage(struct page *page, pgoff_t offset)
337+{
338+ if (PageHuge(page))
339+ return page;
340+
341+ VM_BUG_ON_PAGE(PageTail(page), page);
342+
343+ return page + (offset & (compound_nr(page) - 1));
344+}
345+
336346 struct page *find_get_entry(struct address_space *mapping, pgoff_t offset);
337347 struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset);
338348 unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
mm/filemap.c+57−88
@@ -281,11 +281,11 @@ EXPORT_SYMBOL(delete_from_page_cache); · 280 unmodified lines
281281 * @pvec: pagevec with pages to delete
282282 *
283283 * The function walks over mapping->i_pages and removes pages passed in @pvec
284− * from the mapping. The function expects @pvec to be sorted by page index.
284+ * from the mapping. The function expects @pvec to be sorted by page index
285+ * and is optimised for it to be dense.
285286 * It tolerates holes in @pvec (mapping entries at those indices are not
286287 * modified). The function expects only THP head pages to be present in the
287− * @pvec and takes care to delete all corresponding tail pages from the
288− * mapping as well.
288+ * @pvec.
289289 *
290290 * The function expects the i_pages lock to be held.
291291 */
@@ -294,40 +294,43 @@ static void page_cache_delete_batch(struct address_space *mapping, · 2 unmodified lines
294294 {
295295 XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index);
296296 int total_pages = 0;
297− int i = 0, tail_pages = 0;
297+ int i = 0;
298298 struct page *page;
299299
300300 mapping_set_update(&xas, mapping);
301301 xas_for_each(&xas, page, ULONG_MAX) {
302− if (i >= pagevec_count(pvec) && !tail_pages)
302+ if (i >= pagevec_count(pvec))
303303 break;
304+
305+ /* A swap/dax/shadow entry got inserted? Skip it. */
304306 if (xa_is_value(page))
305307 continue;
306− if (!tail_pages) {
307− /*
308− * Some page got inserted in our range? Skip it. We
309− * have our pages locked so they are protected from
310− * being removed.
311− */
312− if (page != pvec->pages[i]) {
313− VM_BUG_ON_PAGE(page->index >
314− pvec->pages[i]->index, page);
315− continue;
316− }
317− WARN_ON_ONCE(!PageLocked(page));
318− if (PageTransHuge(page) && !PageHuge(page))
319− tail_pages = HPAGE_PMD_NR - 1;
308+ /*
309+ * A page got inserted in our range? Skip it. We have our
310+ * pages locked so they are protected from being removed.
311+ * If we see a page whose index is higher than ours, it
312+ * means our page has been removed, which shouldn't be
313+ * possible because we're holding the PageLock.
314+ */
315+ if (page != pvec->pages[i]) {
316+ VM_BUG_ON_PAGE(page->index > pvec->pages[i]->index,
317+ page);
318+ continue;
319+ }
320+
321+ WARN_ON_ONCE(!PageLocked(page));
322+
323+ if (page->index == xas.xa_index)
320324 page->mapping = NULL;
321− /*
322− * Leave page->index set: truncation lookup relies
323− * upon it
324− */
325+ /* Leave page->index set: truncation lookup relies on it */
326+
327+ /*
328+ * Move to the next page in the vector if this is a regular
329+ * page or the index is of the last sub-page of this compound
330+ * page.
331+ */
332+ if (page->index + compound_nr(page) - 1 == xas.xa_index)
325333 i++;
326− } else {
327− VM_BUG_ON_PAGE(page->index + HPAGE_PMD_NR - tail_pages
328− != pvec->pages[i]->index, page);
329− tail_pages--;
330− }
331334 xas_store(&xas, NULL);
332335 total_pages++;
333336 }
@@ -1520,7 +1523,7 @@ EXPORT_SYMBOL(page_cache_prev_miss); · 1186 unmodified lines
15201523 struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
15211524 {
15221525 XA_STATE(xas, &mapping->i_pages, offset);
1523− struct page *head, *page;
1526+ struct page *page;
15241527
15251528 rcu_read_lock();
15261529 repeat:
@@ -1535,25 +1538,19 @@ repeat: · 8 unmodified lines
15351538 if (!page || xa_is_value(page))
15361539 goto out;
15371540
1538− head = compound_head(page);
1539− if (!page_cache_get_speculative(head))
1540− goto repeat;
1541−
1542− /* The page was split under us? */
1543− if (compound_head(page) != head) {
1544− put_page(head);
1541+ if (!page_cache_get_speculative(page))
15451542 goto repeat;
1546− }
15471543
15481544 /*
1549− * Has the page moved?
1545+ * Has the page moved or been split?
15501546 * This is part of the lockless pagecache protocol. See
15511547 * include/linux/pagemap.h for details.
15521548 */
15531549 if (unlikely(page != xas_reload(&xas))) {
1554− put_page(head);
1550+ put_page(page);
15551551 goto repeat;
15561552 }
1553+ page = find_subpage(page, offset);
15571554 out:
15581555 rcu_read_unlock();
15591556
@@ -1735,7 +1732,6 @@ unsigned find_get_entries(struct address_space *mapping, · 175 unmodified lines
17351732
17361733 rcu_read_lock();
17371734 xas_for_each(&xas, page, ULONG_MAX) {
1738− struct page *head;
17391735 if (xas_retry(&xas, page))
17401736 continue;
17411737 /*
@@ -1746,17 +1742,13 @@ unsigned find_get_entries(struct address_space *mapping, · 4 unmodified lines
17461742 if (xa_is_value(page))
17471743 goto export;
17481744
1749− head = compound_head(page);
1750− if (!page_cache_get_speculative(head))
1745+ if (!page_cache_get_speculative(page))
17511746 goto retry;
17521747
1753− /* The page was split under us? */
1754− if (compound_head(page) != head)
1755− goto put_page;
1756−
1757− /* Has the page moved? */
1748+ /* Has the page moved or been split? */
17581749 if (unlikely(page != xas_reload(&xas)))
17591750 goto put_page;
1751+ page = find_subpage(page, xas.xa_index);
17601752
17611753 export:
17621754 indices[ret] = xas.xa_index;
@@ -1765,7 +1757,7 @@ export: · 2 unmodified lines
17651757 break;
17661758 continue;
17671759 put_page:
1768− put_page(head);
1760+ put_page(page);
17691761 retry:
17701762 xas_reset(&xas);
17711763 }
@@ -1807,33 +1799,27 @@ unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, · 35 unmodified lines
18071799
18081800 rcu_read_lock();
18091801 xas_for_each(&xas, page, end) {
1810− struct page *head;
18111802 if (xas_retry(&xas, page))
18121803 continue;
18131804 /* Skip over shadow, swap and DAX entries */
18141805 if (xa_is_value(page))
18151806 continue;
18161807
1817− head = compound_head(page);
1818− if (!page_cache_get_speculative(head))
1808+ if (!page_cache_get_speculative(page))
18191809 goto retry;
18201810
1821− /* The page was split under us? */
1822− if (compound_head(page) != head)
1823− goto put_page;
1824−
1825− /* Has the page moved? */
1811+ /* Has the page moved or been split? */
18261812 if (unlikely(page != xas_reload(&xas)))
18271813 goto put_page;
18281814
1829− pages[ret] = page;
1815+ pages[ret] = find_subpage(page, xas.xa_index);
18301816 if (++ret == nr_pages) {
18311817 *start = xas.xa_index + 1;
18321818 goto out;
18331819 }
18341820 continue;
18351821 put_page:
1836− put_page(head);
1822+ put_page(page);
18371823 retry:
18381824 xas_reset(&xas);
18391825 }
@@ -1878,7 +1864,6 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, · 38 unmodified lines
18781864
18791865 rcu_read_lock();
18801866 for (page = xas_load(&xas); page; page = xas_next(&xas)) {
1881− struct page *head;
18821867 if (xas_retry(&xas, page))
18831868 continue;
18841869 /*
@@ -1888,24 +1873,19 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, · 3 unmodified lines
18881873 if (xa_is_value(page))
18891874 break;
18901875
1891− head = compound_head(page);
1892− if (!page_cache_get_speculative(head))
1876+ if (!page_cache_get_speculative(page))
18931877 goto retry;
18941878
1895− /* The page was split under us? */
1896− if (compound_head(page) != head)
1897− goto put_page;
1898−
1899− /* Has the page moved? */
1879+ /* Has the page moved or been split? */
19001880 if (unlikely(page != xas_reload(&xas)))
19011881 goto put_page;
19021882
1903− pages[ret] = page;
1883+ pages[ret] = find_subpage(page, xas.xa_index);
19041884 if (++ret == nr_pages)
19051885 break;
19061886 continue;
19071887 put_page:
1908− put_page(head);
1888+ put_page(page);
19091889 retry:
19101890 xas_reset(&xas);
19111891 }
@@ -1941,7 +1921,6 @@ unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, · 29 unmodified lines
19411921
19421922 rcu_read_lock();
19431923 xas_for_each_marked(&xas, page, end, tag) {
1944− struct page *head;
19451924 if (xas_retry(&xas, page))
19461925 continue;
19471926 /*
@@ -1952,26 +1931,21 @@ unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, · 4 unmodified lines
19521931 if (xa_is_value(page))
19531932 continue;
19541933
1955− head = compound_head(page);
1956− if (!page_cache_get_speculative(head))
1934+ if (!page_cache_get_speculative(page))
19571935 goto retry;
19581936
1959− /* The page was split under us? */
1960− if (compound_head(page) != head)
1961− goto put_page;
1962−
1963− /* Has the page moved? */
1937+ /* Has the page moved or been split? */
19641938 if (unlikely(page != xas_reload(&xas)))
19651939 goto put_page;
19661940
1967− pages[ret] = page;
1941+ pages[ret] = find_subpage(page, xas.xa_index);
19681942 if (++ret == nr_pages) {
19691943 *index = xas.xa_index + 1;
19701944 goto out;
19711945 }
19721946 continue;
19731947 put_page:
1974− put_page(head);
1948+ put_page(page);
19751949 retry:
19761950 xas_reset(&xas);
19771951 }
@@ -2652,7 +2626,7 @@ void filemap_map_pages(struct vm_fault *vmf, · 674 unmodified lines
26522626 pgoff_t last_pgoff = start_pgoff;
26532627 unsigned long max_idx;
26542628 XA_STATE(xas, &mapping->i_pages, start_pgoff);
2655− struct page *head, *page;
2629+ struct page *page;
26562630
26572631 rcu_read_lock();
26582632 xas_for_each(&xas, page, end_pgoff) {
@@ -2661,24 +2635,19 @@ void filemap_map_pages(struct vm_fault *vmf, · 2 unmodified lines
26612635 if (xa_is_value(page))
26622636 goto next;
26632637
2664− head = compound_head(page);
2665−
26662638 /*
26672639 * Check for a locked page first, as a speculative
26682640 * reference may adversely influence page migration.
26692641 */
2670− if (PageLocked(head))
2642+ if (PageLocked(page))
26712643 goto next;
2672− if (!page_cache_get_speculative(head))
2644+ if (!page_cache_get_speculative(page))
26732645 goto next;
26742646
2675− /* The page was split under us? */
2676− if (compound_head(page) != head)
2677− goto skip;
2678−
2679− /* Has the page moved? */
2647+ /* Has the page moved or been split? */
26802648 if (unlikely(page != xas_reload(&xas)))
26812649 goto skip;
2650+ page = find_subpage(page, xas.xa_index);
26822651
26832652 if (!PageUptodate(page) ||
26842653 PageReadahead(page) ||
mm/huge_memory.c+20−2
@@ -2497,6 +2497,8 @@ static void __split_huge_page(struct page *page, struct list_head *list, · 2496 unmodified lines
24972497 struct page *head = compound_head(page);
24982498 pg_data_t *pgdat = page_pgdat(head);
24992499 struct lruvec *lruvec;
2500+ struct address_space *swap_cache = NULL;
2501+ unsigned long offset = 0;
25002502 int i;
25012503
25022504 lruvec = mem_cgroup_page_lruvec(head, pgdat);
@@ -2504,6 +2506,14 @@ static void __split_huge_page(struct page *page, struct list_head *list, · 1 unmodified line
25042506 /* complete memcg works before add pages to LRU */
25052507 mem_cgroup_split_huge_fixup(head);
25062508
2509+ if (PageAnon(head) && PageSwapCache(head)) {
2510+ swp_entry_t entry = { .val = page_private(head) };
2511+
2512+ offset = swp_offset(entry);
2513+ swap_cache = swap_address_space(entry);
2514+ xa_lock(&swap_cache->i_pages);
2515+ }
2516+
25072517 for (i = HPAGE_PMD_NR - 1; i >= 1; i--) {
25082518 __split_huge_page_tail(head, i, lruvec, list);
25092519 /* Some pages can be beyond i_size: drop them from page cache */
@@ -2513,6 +2523,12 @@ static void __split_huge_page(struct page *page, struct list_head *list, · 3 unmodified lines
25132523 if (IS_ENABLED(CONFIG_SHMEM) && PageSwapBacked(head))
25142524 shmem_uncharge(head->mapping->host, 1);
25152525 put_page(head + i);
2526+ } else if (!PageAnon(page)) {
2527+ __xa_store(&head->mapping->i_pages, head[i].index,
2528+ head + i, 0);
2529+ } else if (swap_cache) {
2530+ __xa_store(&swap_cache->i_pages, offset + i,
2531+ head + i, 0);
25162532 }
25172533 }
25182534
@@ -2523,10 +2539,12 @@ static void __split_huge_page(struct page *page, struct list_head *list, · 4 unmodified lines
25232539 /* See comment in __split_huge_page_tail() */
25242540 if (PageAnon(head)) {
25252541 /* Additional pin to swap cache */
2526− if (PageSwapCache(head))
2542+ if (PageSwapCache(head)) {
25272543 page_ref_add(head, 2);
2528− else
2544+ xa_unlock(&swap_cache->i_pages);
2545+ } else {
25292546 page_ref_inc(head);
2547+ }
25302548 } else {
25312549 /* Additional pin to page cache */
25322550 page_ref_add(head, 2);
mm/khugepaged.c+2−2
@@ -1378,7 +1378,7 @@ static void collapse_shmem(struct mm_struct *mm, · 1377 unmodified lines
13781378 result = SCAN_FAIL;
13791379 goto xa_locked;
13801380 }
1381− xas_store(&xas, new_page + (index % HPAGE_PMD_NR));
1381+ xas_store(&xas, new_page);
13821382 nr_none++;
13831383 continue;
13841384 }
@@ -1454,7 +1454,7 @@ static void collapse_shmem(struct mm_struct *mm, · 69 unmodified lines
14541454 list_add_tail(&page->lru, &pagelist);
14551455
14561456 /* Finally, replace with the new page. */
1457− xas_store(&xas, new_page + (index % HPAGE_PMD_NR));
1457+ xas_store(&xas, new_page);
14581458 continue;
14591459 out_unlock:
14601460 unlock_page(page);
mm/memfd.c+2−0
@@ -39,6 +39,7 @@ static void memfd_tag_pins(struct xa_state *xas) · 38 unmodified lines
3939 xas_for_each(xas, page, ULONG_MAX) {
4040 if (xa_is_value(page))
4141 continue;
42+ page = find_subpage(page, xas->xa_index);
4243 if (page_count(page) - page_mapcount(page) > 1)
4344 xas_set_mark(xas, MEMFD_TAG_PINNED);
4445
@@ -88,6 +89,7 @@ static int memfd_wait_for_pins(struct address_space *mapping) · 43 unmodified lines
8889 bool clear = true;
8990 if (xa_is_value(page))
9091 continue;
92+ page = find_subpage(page, xas.xa_index);
9193 if (page_count(page) - page_mapcount(page) != 1) {
9294 /*
9395 * On the last scan, we clean up all those tags
mm/migrate.c+1−1
@@ -460,7 +460,7 @@ int migrate_page_move_mapping(struct address_space *mapping, · 459 unmodified lines
460460
461461 for (i = 1; i < HPAGE_PMD_NR; i++) {
462462 xas_next(&xas);
463− xas_store(&xas, newpage + i);
463+ xas_store(&xas, newpage);
464464 }
465465 }
466466
mm/shmem.c+1−1
@@ -631,7 +631,7 @@ static int shmem_add_to_page_cache(struct page *page, · 630 unmodified lines
631631 if (xas_error(&xas))
632632 goto unlock;
633633 next:
634− xas_store(&xas, page + i);
634+ xas_store(&xas, page);
635635 if (++i < nr) {
636636 xas_next(&xas);
637637 goto next;
mm/swap_state.c+2−2
@@ -133,7 +133,7 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp) · 132 unmodified lines
133133 for (i = 0; i < nr; i++) {
134134 VM_BUG_ON_PAGE(xas.xa_index != idx + i, page);
135135 set_page_private(page + i, entry.val + i);
136− xas_store(&xas, page + i);
136+ xas_store(&xas, page);
137137 xas_next(&xas);
138138 }
139139 address_space->nrpages += nr;
@@ -168,7 +168,7 @@ void __delete_from_swap_cache(struct page *page, swp_entry_t entry) · 28 unmodified lines
168168
169169 for (i = 0; i < nr; i++) {
170170 void *entry = xas_store(&xas, NULL);
171− VM_BUG_ON_PAGE(entry != page + i, entry);
171+ VM_BUG_ON_PAGE(entry != page, entry);
172172 set_page_private(page + i, 0);
173173 xas_next(&xas);
174174 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t