This patch allows do_fault() to trigger on !pte_none() cases too. This prepares for the pte markers to be handled by do_fault() just like none pte. To achieve this, instead of unconditionally check against pte_none() in finish_fault(), we may hit the case that the orig_pte was some pte marker so what we want to do is to replace the pte marker with some valid pte entry. Then if orig_pte was set we'd want to check the current *pte (under pgtable lock) against orig_pte rather than none pte. Right now there's no solid way to safely reference orig_pte because when pmd is not allocated handle_pte_fault() will not initialize orig_pte, so it's not safe to reference it. There's another solution proposed before this patch to do pte_clear() for vmf->orig_pte for pmd==NULL case, however it turns out it'll break arm32 because arm32 could have assumption that pte_t* pointer will always reside on a real ram32 pgtable, not any kernel stack variable. To solve this, we add a new flag FAULT_FLAG_ORIG_PTE_VALID, and it'll be set along with orig_pte when there is valid orig_pte, or it'll be cleared when orig_pte was not initialized. It'll be updated every time we call handle_pte_fault(), so e.g. if a page fault retry happened it'll be properly updated along with orig_pte. [1] https://lore.kernel.org/lkml/710c48c9-406d-e4c5-a394-10501b951316@samsung.com/ [akpm@linux-foundation.org: coding-style cleanups] [peterx@redhat.com: fix crash reported by Marek] Link: https://lkml.kernel.org/r/Ylb9rXJyPm8/ao8f@xz-m1.local Link: https://lkml.kernel.org/r/20220405014836.14077-1-peterx@redhat.com Signed-off-by: Peter Xu <peterx@redhat.com> Reviewed-by: Alistair Popple <apopple@nvidia.com> Tested-by: Marek Szyprowski <m.szyprowski@samsung.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Axel Rasmussen <axelrasmussen@google.com> Cc: David Hildenbrand <david@redhat.com> Cc: Hugh Dickins <hughd@google.com> Cc: Jerome Glisse <jglisse@redhat.com> Cc: "Kirill A . Shutemov" <kirill@shutemov.name> Cc: Matthew Wilcox <willy@infradead.org> Cc: Mike Kravetz <mike.kravetz@oracle.com> Cc: Mike Rapoport <rppt@linux.vnet.ibm.com> Cc: Nadav Amit <nadav.amit@gmail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Unified · Split
| @@ -822,6 +822,8 @@ typedef struct { · 821 unmodified lines | |||
| 822 | * @FAULT_FLAG_UNSHARE: The fault is an unsharing request to unshare (and mark | 822 | * @FAULT_FLAG_UNSHARE: The fault is an unsharing request to unshare (and mark |
| 823 | * exclusive) a possibly shared anonymous page that is | 823 | * exclusive) a possibly shared anonymous page that is |
| 824 | * mapped R/O. | 824 | * mapped R/O. |
| 825 | * @FAULT_FLAG_ORIG_PTE_VALID: whether the fault has vmf->orig_pte cached. | ||
| 826 | * We should only access orig_pte if this flag set. | ||
| 825 | * | 827 | * |
| 826 | * About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify | 828 | * About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify |
| 827 | * whether we would allow page faults to retry by specifying these two | 829 | * whether we would allow page faults to retry by specifying these two |
| @@ -858,6 +860,7 @@ enum fault_flag { · 30 unmodified lines | |||
| 858 | FAULT_FLAG_INSTRUCTION = 1 << 8, | 860 | FAULT_FLAG_INSTRUCTION = 1 << 8, |
| 859 | FAULT_FLAG_INTERRUPTIBLE = 1 << 9, | 861 | FAULT_FLAG_INTERRUPTIBLE = 1 << 9, |
| 860 | FAULT_FLAG_UNSHARE = 1 << 10, | 862 | FAULT_FLAG_UNSHARE = 1 << 10, |
| 863 | FAULT_FLAG_ORIG_PTE_VALID = 1 << 11, | ||
| 861 | }; | 864 | }; |
| 862 | 865 | ||
| 863 | #endif /* _LINUX_MM_TYPES_H */ | 866 | #endif /* _LINUX_MM_TYPES_H */ |
| @@ -4183,6 +4183,14 @@ void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr) · 4182 unmodified lines | |||
| 4183 | set_pte_at(vma->vm_mm, addr, vmf->pte, entry); | 4183 | set_pte_at(vma->vm_mm, addr, vmf->pte, entry); |
| 4184 | } | 4184 | } |
| 4185 | 4185 | ||
| 4186 | static bool vmf_pte_changed(struct vm_fault *vmf) | ||
| 4187 | { | ||
| 4188 | if (vmf->flags & FAULT_FLAG_ORIG_PTE_VALID) | ||
| 4189 | return !pte_same(*vmf->pte, vmf->orig_pte); | ||
| 4190 | |||
| 4191 | return !pte_none(*vmf->pte); | ||
| 4192 | } | ||
| 4193 | |||
| 4186 | /** | 4194 | /** |
| 4187 | * finish_fault - finish page fault once we have prepared the page to fault | 4195 | * finish_fault - finish page fault once we have prepared the page to fault |
| 4188 | * | 4196 | * |
| @@ -4241,7 +4249,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf) · 52 unmodified lines | |||
| 4241 | vmf->address, &vmf->ptl); | 4249 | vmf->address, &vmf->ptl); |
| 4242 | ret = 0; | 4250 | ret = 0; |
| 4243 | /* Re-check under ptl */ | 4251 | /* Re-check under ptl */ |
| 4244 | if (likely(pte_none(*vmf->pte))) | 4252 | if (likely(!vmf_pte_changed(vmf))) |
| 4245 | do_set_pte(vmf, page, vmf->address); | 4253 | do_set_pte(vmf, page, vmf->address); |
| 4246 | else | 4254 | else |
| 4247 | ret = VM_FAULT_NOPAGE; | 4255 | ret = VM_FAULT_NOPAGE; |
| @@ -4709,6 +4717,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) · 461 unmodified lines | |||
| 4709 | * concurrent faults and from rmap lookups. | 4717 | * concurrent faults and from rmap lookups. |
| 4710 | */ | 4718 | */ |
| 4711 | vmf->pte = NULL; | 4719 | vmf->pte = NULL; |
| 4720 | vmf->flags &= ~FAULT_FLAG_ORIG_PTE_VALID; | ||
| 4712 | } else { | 4721 | } else { |
| 4713 | /* | 4722 | /* |
| 4714 | * If a huge pmd materialized under us just retry later. Use | 4723 | * If a huge pmd materialized under us just retry later. Use |
| @@ -4732,6 +4741,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) · 17 unmodified lines | |||
| 4732 | */ | 4741 | */ |
| 4733 | vmf->pte = pte_offset_map(vmf->pmd, vmf->address); | 4742 | vmf->pte = pte_offset_map(vmf->pmd, vmf->address); |
| 4734 | vmf->orig_pte = *vmf->pte; | 4743 | vmf->orig_pte = *vmf->pte; |
| 4744 | vmf->flags |= FAULT_FLAG_ORIG_PTE_VALID; | ||
| 4735 | 4745 | ||
| 4736 | /* | 4746 | /* |
| 4737 | * some architectures can have larger ptes than wordsize, | 4747 | * some architectures can have larger ptes than wordsize, |