mm,thp: add read-only THP support for (non-shmem) FS

This patch is (hopefully) the first step to enable THP for non-shmem filesystems. This patch enables an application to put part of its text sections to THP via madvise, for example: madvise((void *)0x600000, 0x200000, MADV_HUGEPAGE); We tried to reuse the logic for THP on tmpfs. Currently, write is not supported for non-shmem THP. khugepaged will only process vma with VM_DENYWRITE. sys_mmap() ignores VM_DENYWRITE requests (see ksys_mmap_pgoff). The only way to create vma with VM_DENYWRITE is execve(). This requirement limits non-shmem THP to text sections. The next patch will handle writes, which would only happen when the all the vmas with VM_DENYWRITE are unmapped. An EXPERIMENTAL config, READ_ONLY_THP_FOR_FS, is added to gate this feature. [songliubraving@fb.com: fix build without CONFIG_SHMEM] Link: http://lkml.kernel.org/r/F53407FB-96CC-42E8-9862-105C92CC2B98@fb.com [songliubraving@fb.com: fix double unlock in collapse_file()] Link: http://lkml.kernel.org/r/B960CBFA-8EFC-4DA4-ABC5-1977FFF2CA57@fb.com Link: http://lkml.kernel.org/r/20190801184244.3169074-7-songliubraving@fb.com Signed-off-by: Song Liu <songliubraving@fb.com> Acked-by: Rik van Riel <riel@surriel.com> Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Acked-by: Johannes Weiner <hannes@cmpxchg.org> Cc: Stephen Rothwell <sfr@canb.auug.org.au> Cc: Dan Carpenter <dan.carpenter@oracle.com> Cc: Hillf Danton <hdanton@sina.com> Cc: Hugh Dickins <hughd@google.com> Cc: William Kucharski <william.kucharski@oracle.com> Cc: Oleg Nesterov <oleg@redhat.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Song Liu committed 7 years ago

Commit 99cb0db1 parent 579c5714 files · +128 −48Browse files

Unified · Split

4 files

mm/Kconfig +11 −0
mm/filemap.c +2 −2
mm/khugepaged.c +107 −42
mm/rmap.c +8 −4

mm/Kconfig+11−0

@@ -712,6 +712,17 @@ config GUP_BENCHMARK · 711 unmodified lines
712	config GUP_GET_PTE_LOW_HIGH	712	config GUP_GET_PTE_LOW_HIGH
713	bool	713	bool
714		714
		715	config READ_ONLY_THP_FOR_FS
		716	bool "Read-only THP for filesystems (EXPERIMENTAL)"
		717	depends on TRANSPARENT_HUGE_PAGECACHE && SHMEM
		718
		719	help
		720	Allow khugepaged to put read-only file-backed pages in THP.
		721
		722	This is marked experimental because it is a new feature. Write
		723	support of file THPs will be developed in the next few release
		724	cycles.
		725
715	config ARCH_HAS_PTE_SPECIAL	726	config ARCH_HAS_PTE_SPECIAL
716	bool	727	bool
717		728

mm/filemap.c+2−2

@@ -203,8 +203,8 @@ static void unaccount_page_cache_page(struct address_space *mapping, · 202 unmodified lines
203	__mod_node_page_state(page_pgdat(page), NR_SHMEM, -nr);	203	__mod_node_page_state(page_pgdat(page), NR_SHMEM, -nr);
204	if (PageTransHuge(page))	204	if (PageTransHuge(page))
205	__dec_node_page_state(page, NR_SHMEM_THPS);	205	__dec_node_page_state(page, NR_SHMEM_THPS);
206	} else {	206	} else if (PageTransHuge(page)) {
207	VM_BUG_ON_PAGE(PageTransHuge(page), page);	207	__dec_node_page_state(page, NR_FILE_THPS);
208	}	208	}
209		209
210	/*	210	/*

mm/khugepaged.c+107−42

@@ -48,6 +48,7 @@ enum scan_result { · 47 unmodified lines
48	SCAN_CGROUP_CHARGE_FAIL,	48	SCAN_CGROUP_CHARGE_FAIL,
49	SCAN_EXCEED_SWAP_PTE,	49	SCAN_EXCEED_SWAP_PTE,
50	SCAN_TRUNCATED,	50	SCAN_TRUNCATED,
		51	SCAN_PAGE_HAS_PRIVATE,
51	};	52	};
52		53
53	#define CREATE_TRACE_POINTS	54	#define CREATE_TRACE_POINTS
@@ -404,7 +405,11 @@ static bool hugepage_vma_check(struct vm_area_struct *vma, · 350 unmodified lines
404	(vm_flags & VM_NOHUGEPAGE) \|\|	405	(vm_flags & VM_NOHUGEPAGE) \|\|
405	test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))	406	test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
406	return false;	407	return false;
407	if (shmem_file(vma->vm_file)) {	408
		409	if (shmem_file(vma->vm_file) \|\|
		410	(IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&
		411	vma->vm_file &&
		412	(vm_flags & VM_DENYWRITE))) {
408	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))	413	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
409	return false;	414	return false;
410	return IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff,	415	return IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff,
@@ -456,8 +461,9 @@ int khugepaged_enter_vma_merge(struct vm_area_struct *vma, · 45 unmodified lines
456	unsigned long hstart, hend;	461	unsigned long hstart, hend;
457		462
458	/*	463	/*
459	* khugepaged does not yet work on non-shmem files or special	464	* khugepaged only supports read-only files for non-shmem files.
460	* mappings. And file-private shmem THP is not supported.	465	* khugepaged does not yet work on special mappings. And
		466	* file-private shmem THP is not supported.
461	*/	467	*/
462	if (!hugepage_vma_check(vma, vm_flags))	468	if (!hugepage_vma_check(vma, vm_flags))
463	return 0;	469	return 0;
@@ -1287,12 +1293,12 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) · 823 unmodified lines
1287	}	1293	}
1288		1294
1289	/**	1295	/**
1290	* collapse_file - collapse small tmpfs/shmem pages into huge one.	1296	* collapse_file - collapse filemap/tmpfs/shmem pages into huge one.
1291	*	1297	*
1292	* Basic scheme is simple, details are more complex:	1298	* Basic scheme is simple, details are more complex:
1293	* - allocate and lock a new huge page;	1299	* - allocate and lock a new huge page;
1294	* - scan page cache replacing old pages with the new one	1300	* - scan page cache replacing old pages with the new one
1295	* + swap in pages if necessary;	1301	* + swap/gup in pages if necessary;
1296	* + fill in gaps;	1302	* + fill in gaps;
1297	* + keep old pages around in case rollback is required;	1303	* + keep old pages around in case rollback is required;
1298	* - if replacing succeeds:	1304	* - if replacing succeeds:
@@ -1316,7 +1322,9 @@ static void collapse_file(struct mm_struct *mm, · 17 unmodified lines
1316	LIST_HEAD(pagelist);	1322	LIST_HEAD(pagelist);
1317	XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER);	1323	XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER);
1318	int nr_none = 0, result = SCAN_SUCCEED;	1324	int nr_none = 0, result = SCAN_SUCCEED;
		1325	bool is_shmem = shmem_file(file);
1319		1326
		1327	VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem);
1320	VM_BUG_ON(start & (HPAGE_PMD_NR - 1));	1328	VM_BUG_ON(start & (HPAGE_PMD_NR - 1));
1321		1329
1322	/* Only allocate from the target node */	1330	/* Only allocate from the target node */
@@ -1348,7 +1356,8 @@ static void collapse_file(struct mm_struct *mm, · 25 unmodified lines
1348	} while (1);	1356	} while (1);
1349		1357
1350	__SetPageLocked(new_page);	1358	__SetPageLocked(new_page);
1351	__SetPageSwapBacked(new_page);	1359	if (is_shmem)
		1360	__SetPageSwapBacked(new_page);
1352	new_page->index = start;	1361	new_page->index = start;
1353	new_page->mapping = mapping;	1362	new_page->mapping = mapping;
1354		1363
@@ -1363,41 +1372,75 @@ static void collapse_file(struct mm_struct *mm, · 8 unmodified lines
1363	struct page *page = xas_next(&xas);	1372	struct page *page = xas_next(&xas);
1364		1373
1365	VM_BUG_ON(index != xas.xa_index);	1374	VM_BUG_ON(index != xas.xa_index);
1366	if (!page) {	1375	if (is_shmem) {
1367	/*	1376	if (!page) {
1368	* Stop if extent has been truncated or hole-punched,	1377	/*
1369	* and is now completely empty.	1378	* Stop if extent has been truncated or
1370	*/	1379	* hole-punched, and is now completely
1371	if (index == start) {	1380	* empty.
1372	if (!xas_next_entry(&xas, end - 1)) {	1381	*/
1373	result = SCAN_TRUNCATED;	1382	if (index == start) {
		1383	if (!xas_next_entry(&xas, end - 1)) {
		1384	result = SCAN_TRUNCATED;
		1385	goto xa_locked;
		1386	}
		1387	xas_set(&xas, index);
		1388	}
		1389	if (!shmem_charge(mapping->host, 1)) {
		1390	result = SCAN_FAIL;
1374	goto xa_locked;	1391	goto xa_locked;
1375	}	1392	}
1376	xas_set(&xas, index);	1393	xas_store(&xas, new_page);
		1394	nr_none++;
		1395	continue;
1377	}	1396	}
1378	if (!shmem_charge(mapping->host, 1)) {	1397
1379	result = SCAN_FAIL;	1398	if (xa_is_value(page) \|\| !PageUptodate(page)) {
		1399	xas_unlock_irq(&xas);
		1400	/* swap in or instantiate fallocated page */
		1401	if (shmem_getpage(mapping->host, index, &page,
		1402	SGP_NOHUGE)) {
		1403	result = SCAN_FAIL;
		1404	goto xa_unlocked;
		1405	}
		1406	} else if (trylock_page(page)) {
		1407	get_page(page);
		1408	xas_unlock_irq(&xas);
		1409	} else {
		1410	result = SCAN_PAGE_LOCK;
1380	goto xa_locked;	1411	goto xa_locked;
1381	}	1412	}
1382	xas_store(&xas, new_page);	1413	} else { /* !is_shmem */
1383	nr_none++;	1414	if (!page \|\| xa_is_value(page)) {
1384	continue;	1415	xas_unlock_irq(&xas);
1385	}	1416	page_cache_sync_readahead(mapping, &file->f_ra,
1386		1417	file, index,
1387	if (xa_is_value(page) \|\| !PageUptodate(page)) {	1418	PAGE_SIZE);
1388	xas_unlock_irq(&xas);	1419	/* drain pagevecs to help isolate_lru_page() */
1389	/* swap in or instantiate fallocated page */	1420	lru_add_drain();
1390	if (shmem_getpage(mapping->host, index, &page,	1421	page = find_lock_page(mapping, index);
1391	SGP_NOHUGE)) {	1422	if (unlikely(page == NULL)) {
		1423	result = SCAN_FAIL;
		1424	goto xa_unlocked;
		1425	}
		1426	} else if (!PageUptodate(page)) {
		1427	xas_unlock_irq(&xas);
		1428	wait_on_page_locked(page);
		1429	if (!trylock_page(page)) {
		1430	result = SCAN_PAGE_LOCK;
		1431	goto xa_unlocked;
		1432	}
		1433	get_page(page);
		1434	} else if (PageDirty(page)) {
1392	result = SCAN_FAIL;	1435	result = SCAN_FAIL;
1393	goto xa_unlocked;	1436	goto xa_locked;
		1437	} else if (trylock_page(page)) {
		1438	get_page(page);
		1439	xas_unlock_irq(&xas);
		1440	} else {
		1441	result = SCAN_PAGE_LOCK;
		1442	goto xa_locked;
1394	}	1443	}
1395	} else if (trylock_page(page)) {
1396	get_page(page);
1397	xas_unlock_irq(&xas);
1398	} else {
1399	result = SCAN_PAGE_LOCK;
1400	goto xa_locked;
1401	}	1444	}
1402		1445
1403	/*	1446	/*
@@ -1426,6 +1469,12 @@ static void collapse_file(struct mm_struct *mm, · 22 unmodified lines
1426	goto out_unlock;	1469	goto out_unlock;
1427	}	1470	}
1428		1471
		1472	if (page_has_private(page) &&
		1473	!try_to_release_page(page, GFP_KERNEL)) {
		1474	result = SCAN_PAGE_HAS_PRIVATE;
		1475	goto out_unlock;
		1476	}
		1477
1429	if (page_mapped(page))	1478	if (page_mapped(page))
1430	unmap_mapping_pages(mapping, index, 1, false);	1479	unmap_mapping_pages(mapping, index, 1, false);
1431		1480
@@ -1463,12 +1512,18 @@ out_unlock: · 31 unmodified lines
1463	goto xa_unlocked;	1512	goto xa_unlocked;
1464	}	1513	}
1465		1514
1466	__inc_node_page_state(new_page, NR_SHMEM_THPS);	1515	if (is_shmem)
		1516	__inc_node_page_state(new_page, NR_SHMEM_THPS);
		1517	else
		1518	__inc_node_page_state(new_page, NR_FILE_THPS);
		1519
1467	if (nr_none) {	1520	if (nr_none) {
1468	struct zone *zone = page_zone(new_page);	1521	struct zone *zone = page_zone(new_page);
1469		1522
1470	__mod_node_page_state(zone->zone_pgdat, NR_FILE_PAGES, nr_none);	1523	__mod_node_page_state(zone->zone_pgdat, NR_FILE_PAGES, nr_none);
1471	__mod_node_page_state(zone->zone_pgdat, NR_SHMEM, nr_none);	1524	if (is_shmem)
		1525	__mod_node_page_state(zone->zone_pgdat,
		1526	NR_SHMEM, nr_none);
1472	}	1527	}
1473		1528
1474	xa_locked:	1529	xa_locked:
@@ -1506,10 +1561,15 @@ xa_unlocked: · 31 unmodified lines
1506		1561
1507	SetPageUptodate(new_page);	1562	SetPageUptodate(new_page);
1508	page_ref_add(new_page, HPAGE_PMD_NR - 1);	1563	page_ref_add(new_page, HPAGE_PMD_NR - 1);
1509	set_page_dirty(new_page);
1510	mem_cgroup_commit_charge(new_page, memcg, false, true);	1564	mem_cgroup_commit_charge(new_page, memcg, false, true);
		1565
		1566	if (is_shmem) {
		1567	set_page_dirty(new_page);
		1568	lru_cache_add_anon(new_page);
		1569	} else {
		1570	lru_cache_add_file(new_page);
		1571	}
1511	count_memcg_events(memcg, THP_COLLAPSE_ALLOC, 1);	1572	count_memcg_events(memcg, THP_COLLAPSE_ALLOC, 1);
1512	lru_cache_add_anon(new_page);
1513		1573
1514	/*	1574	/*
1515	* Remove pte page tables, so we can re-fault the page as huge.	1575	* Remove pte page tables, so we can re-fault the page as huge.
@@ -1524,7 +1584,9 @@ xa_unlocked: · 8 unmodified lines
1524	/* Something went wrong: roll back page cache changes */	1584	/* Something went wrong: roll back page cache changes */
1525	xas_lock_irq(&xas);	1585	xas_lock_irq(&xas);
1526	mapping->nrpages -= nr_none;	1586	mapping->nrpages -= nr_none;
1527	shmem_uncharge(mapping->host, nr_none);	1587
		1588	if (is_shmem)
		1589	shmem_uncharge(mapping->host, nr_none);
1528		1590
1529	xas_set(&xas, start);	1591	xas_set(&xas, start);
1530	xas_for_each(&xas, page, end - 1) {	1592	xas_for_each(&xas, page, end - 1) {
@@ -1607,7 +1669,8 @@ static void khugepaged_scan_file(struct mm_struct *mm, · 76 unmodified lines
1607	break;	1669	break;
1608	}	1670	}
1609		1671
1610	if (page_count(page) != 1 + page_mapcount(page)) {	1672	if (page_count(page) !=
		1673	1 + page_mapcount(page) + page_has_private(page)) {
1611	result = SCAN_PAGE_COUNT;	1674	result = SCAN_PAGE_COUNT;
1612	break;	1675	break;
1613	}	1676	}
@@ -1713,11 +1776,13 @@ skip: · 99 unmodified lines
1713	VM_BUG_ON(khugepaged_scan.address < hstart \|\|	1776	VM_BUG_ON(khugepaged_scan.address < hstart \|\|
1714	khugepaged_scan.address + HPAGE_PMD_SIZE >	1777	khugepaged_scan.address + HPAGE_PMD_SIZE >
1715	hend);	1778	hend);
1716	if (shmem_file(vma->vm_file)) {	1779	if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) {
1717	struct file *file;	1780	struct file *file;
1718	pgoff_t pgoff = linear_page_index(vma,	1781	pgoff_t pgoff = linear_page_index(vma,
1719	khugepaged_scan.address);	1782	khugepaged_scan.address);
1720	if (!shmem_huge_enabled(vma))	1783
		1784	if (shmem_file(vma->vm_file)
		1785	&& !shmem_huge_enabled(vma))
1721	goto skip;	1786	goto skip;
1722	file = get_file(vma->vm_file);	1787	file = get_file(vma->vm_file);
1723	up_read(&mm->mmap_sem);	1788	up_read(&mm->mmap_sem);

mm/rmap.c+8−4

 		}
 		if (!atomic_inc_and_test(compound_mapcount_ptr(page)))
 			goto out;
-		VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
+		if (PageSwapBacked(page))
-		__inc_node_page_state(page, NR_SHMEM_PMDMAPPED);
+			__inc_node_page_state(page, NR_SHMEM_PMDMAPPED);
+		else
+			__inc_node_page_state(page, NR_FILE_PMDMAPPED);
 	} else {
 		if (PageTransCompound(page) && page_mapping(page)) {
 			VM_WARN_ON_ONCE(!PageLocked(page));
 		}
 		if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))
 			goto out;
-		VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
+		if (PageSwapBacked(page))
-		__dec_node_page_state(page, NR_SHMEM_PMDMAPPED);
+			__dec_node_page_state(page, NR_SHMEM_PMDMAPPED);
+		else
+			__dec_node_page_state(page, NR_FILE_PMDMAPPED);
 	} else {
 		if (!atomic_add_negative(-1, &page->_mapcount))
 			goto out;

mm,thp: add read-only THP support for (non-shmem) FS

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code