An upcoming patch changes and complicates the refcounting and especially the "put page" aspects of it. In order to keep everything clean, refactor the devmap page release routines: * Rename put_devmap_managed_page() to page_is_devmap_managed(), and limit the functionality to "read only": return a bool, with no side effects. * Add a new routine, put_devmap_managed_page(), to handle decrementing the refcount for ZONE_DEVICE pages. * Change callers (just release_pages() and put_page()) to check page_is_devmap_managed() before calling the new put_devmap_managed_page() routine. This is a performance point: put_page() is a hot path, so we need to avoid non- inline function calls where possible. * Rename __put_devmap_managed_page() to free_devmap_managed_page(), and limit the functionality to unconditionally freeing a devmap page. This is originally based on a separate patch by Ira Weiny, which applied to an early version of the put_user_page() experiments. Since then, Jérôme Glisse suggested the refactoring described above.
Showing the changes to include/linux (1 of 3 files). Show all
Unified · Split
| @@ -947,9 +947,10 @@ static inline bool is_zone_device_page(const struct page *page) · 946 unmodified lines | ||
| 947 | 947 | #endif |
| 948 | 948 | |
| 949 | 949 | #ifdef CONFIG_DEV_PAGEMAP_OPS |
| 950 | −void __put_devmap_managed_page(struct page *page); | |
| 950 | +void free_devmap_managed_page(struct page *page); | |
| 951 | 951 | DECLARE_STATIC_KEY_FALSE(devmap_managed_key); |
| 952 | −static inline bool put_devmap_managed_page(struct page *page) | |
| 952 | + | |
| 953 | +static inline bool page_is_devmap_managed(struct page *page) | |
| 953 | 954 | { |
| 954 | 955 | if (!static_branch_unlikely(&devmap_managed_key)) |
| 955 | 956 | return false; |
| @@ -958,7 +959,6 @@ static inline bool put_devmap_managed_page(struct page *page) · 2 unmodified lines | ||
| 958 | 959 | switch (page->pgmap->type) { |
| 959 | 960 | case MEMORY_DEVICE_PRIVATE: |
| 960 | 961 | case MEMORY_DEVICE_FS_DAX: |
| 961 | − __put_devmap_managed_page(page); | |
| 962 | 962 | return true; |
| 963 | 963 | default: |
| 964 | 964 | break; |
| @@ -966,11 +966,17 @@ static inline bool put_devmap_managed_page(struct page *page) · 1 unmodified line | ||
| 966 | 966 | return false; |
| 967 | 967 | } |
| 968 | 968 | |
| 969 | +void put_devmap_managed_page(struct page *page); | |
| 970 | + | |
| 969 | 971 | #else /* CONFIG_DEV_PAGEMAP_OPS */ |
| 970 | −static inline bool put_devmap_managed_page(struct page *page) | |
| 972 | +static inline bool page_is_devmap_managed(struct page *page) | |
| 971 | 973 | { |
| 972 | 974 | return false; |
| 973 | 975 | } |
| 976 | + | |
| 977 | +static inline void put_devmap_managed_page(struct page *page) | |
| 978 | +{ | |
| 979 | +} | |
| 974 | 980 | #endif /* CONFIG_DEV_PAGEMAP_OPS */ |
| 975 | 981 | |
| 976 | 982 | static inline bool is_device_private_page(const struct page *page) |
| @@ -1023,8 +1029,10 @@ static inline void put_page(struct page *page) · 46 unmodified lines | ||
| 1023 | 1029 | * need to inform the device driver through callback. See |
| 1024 | 1030 | * include/linux/memremap.h and HMM for details. |
| 1025 | 1031 | */ |
| 1026 | − if (put_devmap_managed_page(page)) | |
| 1032 | + if (page_is_devmap_managed(page)) { | |
| 1033 | + put_devmap_managed_page(page); | |
| 1027 | 1034 | return; |
| 1035 | + } | |
| 1028 | 1036 | |
| 1029 | 1037 | if (put_page_testzero(page)) |
| 1030 | 1038 | __put_page(page); |