Jann reported a race between munmap() and unmap_mapping_range(), where unmap_mapping_range() will no-op once unmap_vmas() has unlinked the VMA; however munmap() will not yet have invalidated the TLBs. Therefore unmap_mapping_range() will complete while there are still (stale) TLB entries for the specified range. Mitigate this by force flushing TLBs for VM_PFNMAP ranges.
Unified · Split
| @@ -303,6 +303,7 @@ struct mmu_gather { · 302 unmodified lines | ||
| 303 | 303 | */ |
| 304 | 304 | unsigned int vma_exec : 1; |
| 305 | 305 | unsigned int vma_huge : 1; |
| 306 | + unsigned int vma_pfn : 1; | |
| 306 | 307 | |
| 307 | 308 | unsigned int batch_count; |
| 308 | 309 | |
| @@ -373,7 +374,6 @@ tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { } · 64 unmodified lines | ||
| 373 | 374 | #else /* CONFIG_MMU_GATHER_NO_RANGE */ |
| 374 | 375 | |
| 375 | 376 | #ifndef tlb_flush |
| 376 | − | |
| 377 | 377 | /* |
| 378 | 378 | * When an architecture does not provide its own tlb_flush() implementation |
| 379 | 379 | * but does have a reasonably efficient flush_vma_range() implementation |
| @@ -393,6 +393,9 @@ static inline void tlb_flush(struct mmu_gather *tlb) · 13 unmodified lines | ||
| 393 | 393 | flush_tlb_range(&vma, tlb->start, tlb->end); |
| 394 | 394 | } |
| 395 | 395 | } |
| 396 | +#endif | |
| 397 | + | |
| 398 | +#endif /* CONFIG_MMU_GATHER_NO_RANGE */ | |
| 396 | 399 | |
| 397 | 400 | static inline void |
| 398 | 401 | tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) |
| @@ -410,17 +413,9 @@ tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) · 11 unmodified lines | ||
| 410 | 413 | */ |
| 411 | 414 | tlb->vma_huge = is_vm_hugetlb_page(vma); |
| 412 | 415 | tlb->vma_exec = !!(vma->vm_flags & VM_EXEC); |
| 416 | + tlb->vma_pfn = !!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)); | |
| 413 | 417 | } |
| 414 | 418 | |
| 415 | −#else | |
| 416 | − | |
| 417 | −static inline void | |
| 418 | −tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { } | |
| 419 | − | |
| 420 | −#endif | |
| 421 | − | |
| 422 | −#endif /* CONFIG_MMU_GATHER_NO_RANGE */ | |
| 423 | − | |
| 424 | 419 | static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) |
| 425 | 420 | { |
| 426 | 421 | /* |
| @@ -507,16 +502,22 @@ static inline void tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct * · 80 unmodified lines | ||
| 507 | 502 | |
| 508 | 503 | static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) |
| 509 | 504 | { |
| 510 | − if (tlb->fullmm || IS_ENABLED(CONFIG_MMU_GATHER_MERGE_VMAS)) | |
| 505 | + if (tlb->fullmm) | |
| 511 | 506 | return; |
| 512 | 507 | |
| 513 | 508 | /* |
| 514 | − * Do a TLB flush and reset the range at VMA boundaries; this avoids | |
| 515 | − * the ranges growing with the unused space between consecutive VMAs, | |
| 516 | − * but also the mmu_gather::vma_* flags from tlb_start_vma() rely on | |
| 517 | − * this. | |
| 509 | + * VM_PFNMAP is more fragile because the core mm will not track the | |
| 510 | + * page mapcount -- there might not be page-frames for these PFNs after | |
| 511 | + * all. Force flush TLBs for such ranges to avoid munmap() vs | |
| 512 | + * unmap_mapping_range() races. | |
| 518 | 513 | */ |
| 519 | − tlb_flush_mmu_tlbonly(tlb); | |
| 514 | + if (tlb->vma_pfn || !IS_ENABLED(CONFIG_MMU_GATHER_MERGE_VMAS)) { | |
| 515 | + /* | |
| 516 | + * Do a TLB flush and reset the range at VMA boundaries; this avoids | |
| 517 | + * the ranges growing with the unused space between consecutive VMAs. | |
| 518 | + */ | |
| 519 | + tlb_flush_mmu_tlbonly(tlb); | |
| 520 | + } | |
| 520 | 521 | } |
| 521 | 522 | |
| 522 | 523 | /* |