History

mm: enable maple tree RCU mode by default

Use the maple tree in RCU mode for VMA tracking. The maple tree tracks the stack and is able to update the pivot (lower/upper boundary) in-place to allow the page fault handler to write to the tree while holding just the mmap read lock. This is safe as the writes to the stack have a guard VMA which ensures there will always be a NULL in the direction of the growth and thus will only update a pivot. It is possible, but not recommended, to have VMAs that grow up/down without guard VMAs. syzbot has constructed a testcase which sets up a VMA to grow and consume the empty space. Overwriting the entire NULL entry causes the tree to be altered in a way that is not safe for concurrent readers; the readers may see a node being rewritten or one that does not match the maple state they are using. Enabling RCU mode allows the concurrent readers to see a stable node and will return the expected result. [Liam.Howlett@Oracle.com: we don't need to free the nodes with RCU[ Link: https://lore.kernel.org/linux-mm/000000000000b0a65805f663ace6@google.com/ Link: https://lkml.kernel.org/r/20230227173632.3292573-9-surenb@google.com Fixes: d4af56c5c7c6 ("mm: start tracking VMAs with maple tree") Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com> Signed-off-by: Suren Baghdasaryan <surenb@google.com> Reported-by: syzbot+8d95422d3537159ca390@syzkaller.appspotmail.com Cc: <stable@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

Liam R. Howlett committed 3 years ago

Commit 3dd44321 parent 790e1fa3 files · +7 −2Browse files

Unified · Split

3 files
  1. include/linux/mm_types.h +2 −1
  2. kernel/fork.c +3 −0
  3. mm/mmap.c +2 −1
include/linux/mm_types.h+2−1
@@ -774,7 +774,8 @@ struct mm_struct { · 773 unmodified lines
774 unsigned long cpu_bitmap[];774 unsigned long cpu_bitmap[];
775};775};
776 776
777#define MM_MT_FLAGS (MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN)777#define MM_MT_FLAGS (MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN | \
778 MT_FLAGS_USE_RCU)
778extern struct mm_struct init_mm;779extern struct mm_struct init_mm;
779 780
780/* Pointer magic because the dynamic array size confuses some compilers. */781/* Pointer magic because the dynamic array size confuses some compilers. */
kernel/fork.c+3−0
@@ -617,6 +617,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, · 616 unmodified lines
617 if (retval)617 if (retval)
618 goto out;618 goto out;
619 619
620 mt_clear_in_rcu(vmi.mas.tree);
620 for_each_vma(old_vmi, mpnt) {621 for_each_vma(old_vmi, mpnt) {
621 struct file *file;622 struct file *file;
622 623
@@ -700,6 +701,8 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, · 77 unmodified lines
700 retval = arch_dup_mmap(oldmm, mm);701 retval = arch_dup_mmap(oldmm, mm);
701loop_out:702loop_out:
702 vma_iter_free(&vmi);703 vma_iter_free(&vmi);
704 if (!retval)
705 mt_set_in_rcu(vmi.mas.tree);
703out:706out:
704 mmap_write_unlock(mm);707 mmap_write_unlock(mm);
705 flush_tlb_mm(oldmm);708 flush_tlb_mm(oldmm);
mm/mmap.c+2−1
@@ -2277,7 +2277,7 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, · 2276 unmodified lines
2277 int count = 0;2277 int count = 0;
2278 int error = -ENOMEM;2278 int error = -ENOMEM;
2279 MA_STATE(mas_detach, &mt_detach, 0, 0);2279 MA_STATE(mas_detach, &mt_detach, 0, 0);
2280 mt_init_flags(&mt_detach, MT_FLAGS_LOCK_EXTERN);2280 mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
2281 mt_set_external_lock(&mt_detach, &mm->mmap_lock);2281 mt_set_external_lock(&mt_detach, &mm->mmap_lock);
2282 2282
2283 /*2283 /*
@@ -3037,6 +3037,7 @@ void exit_mmap(struct mm_struct *mm) · 753 unmodified lines
3037 */3037 */
3038 set_bit(MMF_OOM_SKIP, &mm->flags);3038 set_bit(MMF_OOM_SKIP, &mm->flags);
3039 mmap_write_lock(mm);3039 mmap_write_lock(mm);
3040 mt_clear_in_rcu(&mm->mm_mt);
3040 free_pgtables(&tlb, &mm->mm_mt, vma, FIRST_USER_ADDRESS,3041 free_pgtables(&tlb, &mm->mm_mt, vma, FIRST_USER_ADDRESS,
3041 USER_PGTABLES_CEILING);3042 USER_PGTABLES_CEILING);
3042 tlb_finish_mmu(&tlb);3043 tlb_finish_mmu(&tlb);