... and instead convert page_add_anon_rmap() to accept flags. Passing flags instead of bools is usually nicer either way, and we want to more often also pass RMAP_EXCLUSIVE in follow up patches when detecting that an anonymous page is exclusive: for example, when restoring an anonymous page from a writable migration entry. This is a preparation for marking an anonymous page inside page_add_anon_rmap() as exclusive when RMAP_EXCLUSIVE is passed.
Unified · Split
| @@ -183,8 +183,6 @@ typedef int __bitwise rmap_t; · 182 unmodified lines | |||
| 183 | */ | 183 | */ |
| 184 | void page_move_anon_rmap(struct page *, struct vm_area_struct *); | 184 | void page_move_anon_rmap(struct page *, struct vm_area_struct *); |
| 185 | void page_add_anon_rmap(struct page *, struct vm_area_struct *, | 185 | void page_add_anon_rmap(struct page *, struct vm_area_struct *, |
| 186 | unsigned long address, bool compound); | ||
| 187 | void do_page_add_anon_rmap(struct page *, struct vm_area_struct *, | ||
| 188 | unsigned long address, rmap_t flags); | 186 | unsigned long address, rmap_t flags); |
| 189 | void page_add_new_anon_rmap(struct page *, struct vm_area_struct *, | 187 | void page_add_new_anon_rmap(struct page *, struct vm_area_struct *, |
| 190 | unsigned long address, bool compound); | 188 | unsigned long address, bool compound); |
| @@ -3072,7 +3072,7 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) · 3071 unmodified lines | |||
| 3072 | pmde = pmd_wrprotect(pmd_mkuffd_wp(pmde)); | 3072 | pmde = pmd_wrprotect(pmd_mkuffd_wp(pmde)); |
| 3073 | 3073 | ||
| 3074 | if (PageAnon(new)) | 3074 | if (PageAnon(new)) |
| 3075 | page_add_anon_rmap(new, vma, mmun_start, true); | 3075 | page_add_anon_rmap(new, vma, mmun_start, RMAP_COMPOUND); |
| 3076 | else | 3076 | else |
| 3077 | page_add_file_rmap(new, vma, true); | 3077 | page_add_file_rmap(new, vma, true); |
| 3078 | set_pmd_at(mm, mmun_start, pvmw->pmd, pmde); | 3078 | set_pmd_at(mm, mmun_start, pvmw->pmd, pmde); |
| @@ -1156,7 +1156,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, · 1155 unmodified lines | |||
| 1156 | */ | 1156 | */ |
| 1157 | if (!is_zero_pfn(page_to_pfn(kpage))) { | 1157 | if (!is_zero_pfn(page_to_pfn(kpage))) { |
| 1158 | get_page(kpage); | 1158 | get_page(kpage); |
| 1159 | page_add_anon_rmap(kpage, vma, addr, false); | 1159 | page_add_anon_rmap(kpage, vma, addr, RMAP_NONE); |
| 1160 | newpte = mk_pte(kpage, vma->vm_page_prot); | 1160 | newpte = mk_pte(kpage, vma->vm_page_prot); |
| 1161 | } else { | 1161 | } else { |
| 1162 | newpte = pte_mkspecial(pfn_pte(page_to_pfn(kpage), | 1162 | newpte = pte_mkspecial(pfn_pte(page_to_pfn(kpage), |
| @@ -725,7 +725,7 @@ static void restore_exclusive_pte(struct vm_area_struct *vma, · 724 unmodified lines | |||
| 725 | * created when the swap entry was made. | 725 | * created when the swap entry was made. |
| 726 | */ | 726 | */ |
| 727 | if (PageAnon(page)) | 727 | if (PageAnon(page)) |
| 728 | page_add_anon_rmap(page, vma, address, false); | 728 | page_add_anon_rmap(page, vma, address, RMAP_NONE); |
| 729 | else | 729 | else |
| 730 | /* | 730 | /* |
| 731 | * Currently device exclusive access only supports anonymous | 731 | * Currently device exclusive access only supports anonymous |
| @@ -3705,7 +3705,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) · 2973 unmodified lines | |||
| 3705 | page_add_new_anon_rmap(page, vma, vmf->address, false); | 3705 | page_add_new_anon_rmap(page, vma, vmf->address, false); |
| 3706 | lru_cache_add_inactive_or_unevictable(page, vma); | 3706 | lru_cache_add_inactive_or_unevictable(page, vma); |
| 3707 | } else { | 3707 | } else { |
| 3708 | do_page_add_anon_rmap(page, vma, vmf->address, rmap_flags); | 3708 | page_add_anon_rmap(page, vma, vmf->address, rmap_flags); |
| 3709 | } | 3709 | } |
| 3710 | 3710 | ||
| 3711 | set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); | 3711 | set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); |
| @@ -240,7 +240,8 @@ static bool remove_migration_pte(struct folio *folio, · 239 unmodified lines | |||
| 240 | #endif | 240 | #endif |
| 241 | { | 241 | { |
| 242 | if (folio_test_anon(folio)) | 242 | if (folio_test_anon(folio)) |
| 243 | page_add_anon_rmap(new, vma, pvmw.address, false); | 243 | page_add_anon_rmap(new, vma, pvmw.address, |
| 244 | RMAP_NONE); | ||
| 244 | else | 245 | else |
| 245 | page_add_file_rmap(new, vma, false); | 246 | page_add_file_rmap(new, vma, false); |
| 246 | set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); | 247 | set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); |
| @@ -1171,7 +1171,7 @@ static void __page_check_anon_rmap(struct page *page, · 1170 unmodified lines | |||
| 1171 | * @page: the page to add the mapping to | 1171 | * @page: the page to add the mapping to |
| 1172 | * @vma: the vm area in which the mapping is added | 1172 | * @vma: the vm area in which the mapping is added |
| 1173 | * @address: the user virtual address mapped | 1173 | * @address: the user virtual address mapped |
| 1174 | * @compound: charge the page as compound or small page | 1174 | * @flags: the rmap flags |
| 1175 | * | 1175 | * |
| 1176 | * The caller needs to hold the pte lock, and the page must be locked in | 1176 | * The caller needs to hold the pte lock, and the page must be locked in |
| 1177 | * the anon_vma case: to serialize mapping,index checking after setting, | 1177 | * the anon_vma case: to serialize mapping,index checking after setting, |
| @@ -1179,18 +1179,6 @@ static void __page_check_anon_rmap(struct page *page, · 1 unmodified line | |||
| 1179 | * (but PageKsm is never downgraded to PageAnon). | 1179 | * (but PageKsm is never downgraded to PageAnon). |
| 1180 | */ | 1180 | */ |
| 1181 | void page_add_anon_rmap(struct page *page, | 1181 | void page_add_anon_rmap(struct page *page, |
| 1182 | struct vm_area_struct *vma, unsigned long address, bool compound) | ||
| 1183 | { | ||
| 1184 | do_page_add_anon_rmap(page, vma, address, | ||
| 1185 | compound ? RMAP_COMPOUND : RMAP_NONE); | ||
| 1186 | } | ||
| 1187 | |||
| 1188 | /* | ||
| 1189 | * Special version of the above for do_swap_page, which often runs | ||
| 1190 | * into pages that are exclusively owned by the current process. | ||
| 1191 | * Everybody else should continue to use page_add_anon_rmap above. | ||
| 1192 | */ | ||
| 1193 | void do_page_add_anon_rmap(struct page *page, | ||
| 1194 | struct vm_area_struct *vma, unsigned long address, rmap_t flags) | 1182 | struct vm_area_struct *vma, unsigned long address, rmap_t flags) |
| 1195 | { | 1183 | { |
| 1196 | bool compound = flags & RMAP_COMPOUND; | 1184 | bool compound = flags & RMAP_COMPOUND; |
| @@ -1800,7 +1800,7 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, · 1799 unmodified lines | |||
| 1800 | inc_mm_counter(vma->vm_mm, MM_ANONPAGES); | 1800 | inc_mm_counter(vma->vm_mm, MM_ANONPAGES); |
| 1801 | get_page(page); | 1801 | get_page(page); |
| 1802 | if (page == swapcache) { | 1802 | if (page == swapcache) { |
| 1803 | page_add_anon_rmap(page, vma, addr, false); | 1803 | page_add_anon_rmap(page, vma, addr, RMAP_NONE); |
| 1804 | } else { /* ksm created a completely new copy */ | 1804 | } else { /* ksm created a completely new copy */ |
| 1805 | page_add_new_anon_rmap(page, vma, addr, false); | 1805 | page_add_new_anon_rmap(page, vma, addr, false); |
| 1806 | lru_cache_add_inactive_or_unevictable(page, vma); | 1806 | lru_cache_add_inactive_or_unevictable(page, vma); |