mm: devmap: refactor 1-based refcounting for ZONE_DEVICE pages

An upcoming patch changes and complicates the refcounting and especially the "put page" aspects of it. In order to keep everything clean, refactor the devmap page release routines: * Rename put_devmap_managed_page() to page_is_devmap_managed(), and limit the functionality to "read only": return a bool, with no side effects. * Add a new routine, put_devmap_managed_page(), to handle decrementing the refcount for ZONE_DEVICE pages. * Change callers (just release_pages() and put_page()) to check page_is_devmap_managed() before calling the new put_devmap_managed_page() routine. This is a performance point: put_page() is a hot path, so we need to avoid non- inline function calls where possible. * Rename __put_devmap_managed_page() to free_devmap_managed_page(), and limit the functionality to unconditionally freeing a devmap page. This is originally based on a separate patch by Ira Weiny, which applied to an early version of the put_user_page() experiments. Since then, Jérôme Glisse suggested the refactoring described above.

John Hubbard committed 6 years ago

Commit 07d80261 parent 429589d3 files · +40 −20Browse files

Unified · Split

3 files

include/linux/mm.h +13 −5
mm/memremap.c +1 −14
mm/swap.c +26 −1

include/linux/mm.h+13−5

@@ -947,9 +947,10 @@ static inline bool is_zone_device_page(const struct page *page) · 946 unmodified lines
947	947	#endif
948	948
949	949	#ifdef CONFIG_DEV_PAGEMAP_OPS
950		−void __put_devmap_managed_page(struct page *page);
	950	+void free_devmap_managed_page(struct page *page);
951	951	DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
952		−static inline bool put_devmap_managed_page(struct page *page)
	952	+
	953	+static inline bool page_is_devmap_managed(struct page *page)
953	954	{
954	955	if (!static_branch_unlikely(&devmap_managed_key))
955	956	return false;
@@ -958,7 +959,6 @@ static inline bool put_devmap_managed_page(struct page *page) · 2 unmodified lines
958	959	switch (page->pgmap->type) {
959	960	case MEMORY_DEVICE_PRIVATE:
960	961	case MEMORY_DEVICE_FS_DAX:
961		− __put_devmap_managed_page(page);
962	962	return true;
963	963	default:
964	964	break;
@@ -966,11 +966,17 @@ static inline bool put_devmap_managed_page(struct page *page) · 1 unmodified line
966	966	return false;
967	967	}
968	968
	969	+void put_devmap_managed_page(struct page *page);
	970	+
969	971	#else /* CONFIG_DEV_PAGEMAP_OPS */
970		−static inline bool put_devmap_managed_page(struct page *page)
	972	+static inline bool page_is_devmap_managed(struct page *page)
971	973	{
972	974	return false;
973	975	}
	976	+
	977	+static inline void put_devmap_managed_page(struct page *page)
	978	+{
	979	+}
974	980	#endif /* CONFIG_DEV_PAGEMAP_OPS */
975	981
976	982	static inline bool is_device_private_page(const struct page *page)
@@ -1023,8 +1029,10 @@ static inline void put_page(struct page *page) · 46 unmodified lines
1023	1029	* need to inform the device driver through callback. See
1024	1030	* include/linux/memremap.h and HMM for details.
1025	1031	*/
1026		− if (put_devmap_managed_page(page))
	1032	+ if (page_is_devmap_managed(page)) {
	1033	+ put_devmap_managed_page(page);
1027	1034	return;
	1035	+ }
1028	1036
1029	1037	if (put_page_testzero(page))
1030	1038	__put_page(page);

mm/memremap.c+1−14

@@ -411,20 +411,8 @@ struct dev_pagemap *get_dev_pagemap(unsigned long pfn, · 410 unmodified lines
411	411	EXPORT_SYMBOL_GPL(get_dev_pagemap);
412	412
413	413	#ifdef CONFIG_DEV_PAGEMAP_OPS
414		−void __put_devmap_managed_page(struct page *page)
	414	+void free_devmap_managed_page(struct page *page)
415	415	{
416		− int count = page_ref_dec_return(page);
417		−
418		− /* still busy */
419		− if (count > 1)
420		− return;
421		−
422		− /* only triggered by the dev_pagemap shutdown path */
423		− if (count == 0) {
424		− __put_page(page);
425		− return;
426		− }
427		−
428	416	/* notify page idle for dax */
429	417	if (!is_device_private_page(page)) {
430	418	wake_up_var(&page->_refcount);
@@ -461,5 +449,4 @@ void __put_devmap_managed_page(struct page *page) · 30 unmodified lines
461	449	page->mapping = NULL;
462	450	page->pgmap->ops->page_free(page);
463	451	}
464		−EXPORT_SYMBOL(__put_devmap_managed_page);
465	452	#endif /* CONFIG_DEV_PAGEMAP_OPS */

mm/swap.c+26−1

@@ -813,8 +813,10 @@ void release_pages(struct page **pages, int nr) · 812 unmodified lines
813	813	* processing, and instead, expect a call to
814	814	* put_page_testzero().
815	815	*/
816		− if (put_devmap_managed_page(page))
	816	+ if (page_is_devmap_managed(page)) {
	817	+ put_devmap_managed_page(page);
817	818	continue;
	819	+ }
818	820	}
819	821
820	822	page = compound_head(page);
@@ -1102,3 +1104,26 @@ void __init swap_setup(void) · 281 unmodified lines
1102	1104	* _really_ don't want to cluster much more
1103	1105	*/
1104	1106	}
	1107	+
	1108	+#ifdef CONFIG_DEV_PAGEMAP_OPS
	1109	+void put_devmap_managed_page(struct page *page)
	1110	+{
	1111	+ int count;
	1112	+
	1113	+ if (WARN_ON_ONCE(!page_is_devmap_managed(page)))
	1114	+ return;
	1115	+
	1116	+ count = page_ref_dec_return(page);
	1117	+
	1118	+ /*
	1119	+ * devmap page refcounts are 1-based, rather than 0-based: if
	1120	+ * refcount is 1, then the page is free and the refcount is
	1121	+ * stable because nobody holds a reference on the page.
	1122	+ */
	1123	+ if (count == 1)
	1124	+ free_devmap_managed_page(page);
	1125	+ else if (!count)
	1126	+ __put_page(page);
	1127	+}
	1128	+EXPORT_SYMBOL(put_devmap_managed_page);
	1129	+#endif