mm->rss_stat counters have per-task delta: task->rss_stat. Before changing task->mm pointer the kernel must flush this delta with sync_mm_rss(). do_exit() already calls sync_mm_rss() to flush the rss-counters before committing the rss statistics into task->signal->maxrss, taskstats, audit and other stuff. Unfortunately the kernel does this before calling mm_release(), which can call put_user() for processing task->clear_child_tid. So at this point we can trigger page-faults and task->rss_stat becomes non-zero again. As a result mm->rss_stat becomes inconsistent and check_mm() will print something like this: | BUG: Bad rss-counter state mm:ffff88020813c380 idx:1 val:-1 | BUG: Bad rss-counter state mm:ffff88020813c380 idx:2 val:1 This patch moves sync_mm_rss() into mm_release(), and moves mm_release() out of do_exit() and calls it earlier. After mm_release() there should be no pagefaults. [akpm@linux-foundation.org: tweak comment] Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org> Reported-by: Markus Trippelsdorf <markus@trippelsdorf.de> Cc: Hugh Dickins <hughd@google.com> Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Cc: Oleg Nesterov <oleg@redhat.com> Cc: <stable@vger.kernel.org> [3.4.x] Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Unified · Split
| @@ -819,7 +819,6 @@ static int exec_mmap(struct mm_struct *mm) · 818 unmodified lines | ||
| 819 | 819 | /* Notify parent that we're no longer interested in the old VM */ |
| 820 | 820 | tsk = current; |
| 821 | 821 | old_mm = current->mm; |
| 822 | − sync_mm_rss(old_mm); | |
| 823 | 822 | mm_release(tsk, old_mm); |
| 824 | 823 | |
| 825 | 824 | if (old_mm) { |
| @@ -423,6 +423,7 @@ void daemonize(const char *name, ...) · 422 unmodified lines | ||
| 423 | 423 | * user space pages. We don't need them, and if we didn't close them |
| 424 | 424 | * they would be locked into memory. |
| 425 | 425 | */ |
| 426 | + mm_release(current, current->mm); | |
| 426 | 427 | exit_mm(current); |
| 427 | 428 | /* |
| 428 | 429 | * We don't want to get frozen, in case system-wide hibernation |
| @@ -640,7 +641,6 @@ static void exit_mm(struct task_struct * tsk) · 211 unmodified lines | ||
| 640 | 641 | struct mm_struct *mm = tsk->mm; |
| 641 | 642 | struct core_state *core_state; |
| 642 | 643 | |
| 643 | − mm_release(tsk, mm); | |
| 644 | 644 | if (!mm) |
| 645 | 645 | return; |
| 646 | 646 | /* |
| @@ -960,9 +960,13 @@ void do_exit(long code) · 313 unmodified lines | ||
| 960 | 960 | preempt_count()); |
| 961 | 961 | |
| 962 | 962 | acct_update_integrals(tsk); |
| 963 | − /* sync mm's RSS info before statistics gathering */ | |
| 964 | − if (tsk->mm) | |
| 965 | − sync_mm_rss(tsk->mm); | |
| 963 | + | |
| 964 | + /* Set exit_code before complete_vfork_done() in mm_release() */ | |
| 965 | + tsk->exit_code = code; | |
| 966 | + | |
| 967 | + /* Release mm and sync mm's RSS info before statistics gathering */ | |
| 968 | + mm_release(tsk, tsk->mm); | |
| 969 | + | |
| 966 | 970 | group_dead = atomic_dec_and_test(&tsk->signal->live); |
| 967 | 971 | if (group_dead) { |
| 968 | 972 | hrtimer_cancel(&tsk->signal->real_timer); |
| @@ -975,7 +979,6 @@ void do_exit(long code) · 6 unmodified lines | ||
| 975 | 979 | tty_audit_exit(); |
| 976 | 980 | audit_free(tsk); |
| 977 | 981 | |
| 978 | − tsk->exit_code = code; | |
| 979 | 982 | taskstats_exit(tsk, group_dead); |
| 980 | 983 | |
| 981 | 984 | exit_mm(tsk); |
| @@ -619,6 +619,14 @@ void mmput(struct mm_struct *mm) · 618 unmodified lines | ||
| 619 | 619 | module_put(mm->binfmt->module); |
| 620 | 620 | mmdrop(mm); |
| 621 | 621 | } |
| 622 | + | |
| 623 | + /* | |
| 624 | + * Final rss-counter synchronization. After this point there must be | |
| 625 | + * no pagefaults into this mm from the current context. Otherwise | |
| 626 | + * mm->rss_stat will be inconsistent. | |
| 627 | + */ | |
| 628 | + if (mm) | |
| 629 | + sync_mm_rss(mm); | |
| 622 | 630 | } |
| 623 | 631 | EXPORT_SYMBOL_GPL(mmput); |
| 624 | 632 | |