Jann reported a race between munmap() and unmap_mapping_range(), where unmap_mapping_range() will no-op once unmap_vmas() has unlinked the VMA; however munmap() will not yet have invalidated the TLBs. Therefore unmap_mapping_range() will complete while there are still (stale) TLB entries for the specified range. Mitigate this by force flushing TLBs for VM_PFNMAP ranges.
Unified · Split
| @@ -303,6 +303,7 @@ struct mmu_gather { · 302 unmodified lines | |||
| 303 | */ | 303 | */ |
| 304 | unsigned int vma_exec : 1; | 304 | unsigned int vma_exec : 1; |
| 305 | unsigned int vma_huge : 1; | 305 | unsigned int vma_huge : 1; |
| 306 | unsigned int vma_pfn : 1; | ||
| 306 | 307 | ||
| 307 | unsigned int batch_count; | 308 | unsigned int batch_count; |
| 308 | 309 | ||
| @@ -373,7 +374,6 @@ tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { } · 64 unmodified lines | |||
| 373 | #else /* CONFIG_MMU_GATHER_NO_RANGE */ | 374 | #else /* CONFIG_MMU_GATHER_NO_RANGE */ |
| 374 | 375 | ||
| 375 | #ifndef tlb_flush | 376 | #ifndef tlb_flush |
| 376 | |||
| 377 | /* | 377 | /* |
| 378 | * When an architecture does not provide its own tlb_flush() implementation | 378 | * When an architecture does not provide its own tlb_flush() implementation |
| 379 | * but does have a reasonably efficient flush_vma_range() implementation | 379 | * but does have a reasonably efficient flush_vma_range() implementation |
| @@ -393,6 +393,9 @@ static inline void tlb_flush(struct mmu_gather *tlb) · 13 unmodified lines | |||
| 393 | flush_tlb_range(&vma, tlb->start, tlb->end); | 393 | flush_tlb_range(&vma, tlb->start, tlb->end); |
| 394 | } | 394 | } |
| 395 | } | 395 | } |
| 396 | #endif | ||
| 397 | |||
| 398 | #endif /* CONFIG_MMU_GATHER_NO_RANGE */ | ||
| 396 | 399 | ||
| 397 | static inline void | 400 | static inline void |
| 398 | tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) | 401 | tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) |
| @@ -410,17 +413,9 @@ tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) · 11 unmodified lines | |||
| 410 | */ | 413 | */ |
| 411 | tlb->vma_huge = is_vm_hugetlb_page(vma); | 414 | tlb->vma_huge = is_vm_hugetlb_page(vma); |
| 412 | tlb->vma_exec = !!(vma->vm_flags & VM_EXEC); | 415 | tlb->vma_exec = !!(vma->vm_flags & VM_EXEC); |
| 416 | tlb->vma_pfn = !!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)); | ||
| 413 | } | 417 | } |
| 414 | 418 | ||
| 415 | #else | ||
| 416 | |||
| 417 | static inline void | ||
| 418 | tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { } | ||
| 419 | |||
| 420 | #endif | ||
| 421 | |||
| 422 | #endif /* CONFIG_MMU_GATHER_NO_RANGE */ | ||
| 423 | |||
| 424 | static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) | 419 | static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) |
| 425 | { | 420 | { |
| 426 | /* | 421 | /* |
| @@ -507,16 +502,22 @@ static inline void tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct * · 80 unmodified lines | |||
| 507 | 502 | ||
| 508 | static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) | 503 | static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) |
| 509 | { | 504 | { |
| 510 | if (tlb->fullmm || IS_ENABLED(CONFIG_MMU_GATHER_MERGE_VMAS)) | 505 | if (tlb->fullmm) |
| 511 | return; | 506 | return; |
| 512 | 507 | ||
| 513 | /* | 508 | /* |
| 514 | * Do a TLB flush and reset the range at VMA boundaries; this avoids | 509 | * VM_PFNMAP is more fragile because the core mm will not track the |
| 515 | * the ranges growing with the unused space between consecutive VMAs, | 510 | * page mapcount -- there might not be page-frames for these PFNs after |
| 516 | * but also the mmu_gather::vma_* flags from tlb_start_vma() rely on | 511 | * all. Force flush TLBs for such ranges to avoid munmap() vs |
| 517 | * this. | 512 | * unmap_mapping_range() races. |
| 518 | */ | 513 | */ |
| 519 | tlb_flush_mmu_tlbonly(tlb); | 514 | if (tlb->vma_pfn || !IS_ENABLED(CONFIG_MMU_GATHER_MERGE_VMAS)) { |
| 515 | /* | ||
| 516 | * Do a TLB flush and reset the range at VMA boundaries; this avoids | ||
| 517 | * the ranges growing with the unused space between consecutive VMAs. | ||
| 518 | */ | ||
| 519 | tlb_flush_mmu_tlbonly(tlb); | ||
| 520 | } | ||
| 520 | } | 521 | } |
| 521 | 522 | ||
| 522 | /* | 523 | /* |