2038 lines · 50.6 KB · C
RawDownload
		return;	/*	 * The current owner is exiting/execing and there are no other	 * candidates.  Do not leave the mm pointing to a possibly	 * freed task structure.	 */	if (atomic_read(&mm->mm_users) <= 1) {		WRITE_ONCE(mm->owner, NULL);		return;	} 	read_lock(&tasklist_lock);	/*	 * Search in the children	 */	list_for_each_entry(g, &p->children, sibling) {		if (try_to_set_owner(g, mm))			goto ret;	}	/*	 * Search in the siblings	 */	list_for_each_entry(g, &p->real_parent->children, sibling) {		if (try_to_set_owner(g, mm))			goto ret;	}	/*	 * Search through everything else, we should not get here often.	 */	for_each_process(g) {		if (atomic_read(&mm->mm_users) <= 1)			break;		if (g->flags & PF_KTHREAD)			continue;		if (try_to_set_owner(g, mm))			goto ret;	}	read_unlock(&tasklist_lock);	/*	 * We found no owner yet mm_users > 1: this implies that we are	 * most likely racing with swapoff (try_to_unuse()) or /proc or	 * ptrace or page migration (get_task_mm()).  Mark owner as NULL.	 */	WRITE_ONCE(mm->owner, NULL); ret:	return; }#endif /* CONFIG_MEMCG */ #if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING)/* * Subtract the memory footprint of the current task from * mm. */static void exit_mm_sched_cache(struct mm_struct *mm){	unsigned long fp, sub; 	if (!current->total_numa_faults)		return;	/*	 * No lock protection due to performance considerations.	 * Make sure mm->sc_stat.footprint does not become	 * negative.	 */	fp = READ_ONCE(mm->sc_stat.footprint);	sub = min(fp, current->total_numa_faults);	WRITE_ONCE(mm->sc_stat.footprint, fp - sub);}#elsestatic inline void exit_mm_sched_cache(struct mm_struct *mm){}#endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */ /* * Turn us into a lazy TLB process if we * aren't already.. */static void exit_mm(void){	struct mm_struct *mm = current->mm; 	exit_mm_release(current, mm);	if (!mm)		return; 	exit_mm_sched_cache(mm); 	mmap_read_lock(mm);	mmgrab_lazy_tlb(mm);	BUG_ON(mm != current->active_mm);	/* more a memory barrier than a real lock */	task_lock(current);	/*	 * When a thread stops operating on an address space, the loop	 * in membarrier_private_expedited() may not observe that	 * tsk->mm, and the loop in membarrier_global_expedited() may	 * not observe a MEMBARRIER_STATE_GLOBAL_EXPEDITED	 * rq->membarrier_state, so those would not issue an IPI.	 * Membarrier requires a memory barrier after accessing	 * user-space memory, before clearing tsk->mm or the	 * rq->membarrier_state.	 */	smp_mb__after_spinlock();	local_irq_disable();	current->mm = NULL;	membarrier_update_current_mm(NULL);	enter_lazy_tlb(mm, current);	local_irq_enable();	task_unlock(current);	mmap_read_unlock(mm);	mm_update_next_owner(mm);	mmput(mm);	if (test_thread_flag(TIF_MEMDIE))		exit_oom_victim();} static struct task_struct *find_alive_thread(struct task_struct *p){	struct task_struct *t; 	for_each_thread(p, t) {		if (!(t->flags & PF_EXITING))			return t;	}	return NULL;} static struct task_struct *find_child_reaper(struct task_struct *father,						struct list_head *dead)	__releases(&tasklist_lock)	__acquires(&tasklist_lock){	struct pid_namespace *pid_ns = task_active_pid_ns(father);	struct task_struct *reaper = pid_ns->child_reaper;	struct task_struct *p, *n; 	if (likely(reaper != father))		return reaper; 	reaper = find_alive_thread(father);	if (reaper) {		ASSERT_EXCLUSIVE_WRITER(pid_ns->child_reaper);		WRITE_ONCE(pid_ns->child_reaper, reaper);		return reaper;	} 	write_unlock_irq(&tasklist_lock); 	list_for_each_entry_safe(p, n, dead, ptrace_entry) {		list_del_init(&p->ptrace_entry);		release_task(p);	} 	zap_pid_ns_processes(pid_ns);	write_lock_irq(&tasklist_lock); 	return father;} /* * When we die, we re-parent all our children, and try to: * 1. give them to another thread in our thread group, if such a member exists * 2. give it to the first ancestor process which prctl'd itself as a *    child_subreaper for its children (like a service manager) * 3. give it to the init process (PID 1) in our pid namespace */static struct task_struct *find_new_reaper(struct task_struct *father,					   struct task_struct *child_reaper){	struct task_struct *thread, *reaper; 	thread = find_alive_thread(father);	if (thread)		return thread; 	if (father->signal->has_child_subreaper) {		unsigned int ns_level = task_pid(father)->level;		/*		 * Find the first ->is_child_subreaper ancestor in our pid_ns.		 * We can't check reaper != child_reaper to ensure we do not		 * cross the namespaces, the exiting parent could be injected		 * by setns() + fork().		 * We check pid->level, this is slightly more efficient than		 * task_active_pid_ns(reaper) != task_active_pid_ns(father).		 */		for (reaper = father->real_parent;		     task_pid(reaper)->level == ns_level;		     reaper = reaper->real_parent) {			if (reaper == &init_task)				break;			if (!reaper->signal->is_child_subreaper)				continue;			thread = find_alive_thread(reaper);			if (thread)				return thread;		}	} 	return child_reaper;} /** Any that need to be release_task'd are put on the @dead list. */static void reparent_leader(struct task_struct *father, struct task_struct *p,				struct list_head *dead){	if (unlikely(p->exit_state == EXIT_DEAD))		return; 	/* We don't want people slaying init. */	p->exit_signal = SIGCHLD; 	/* If it has exited notify the new parent about this child's death. */	if (!p->ptrace &&	    p->exit_state == EXIT_ZOMBIE && thread_group_empty(p)) {		if (do_notify_parent(p, p->exit_signal)) {			p->exit_state = EXIT_DEAD;			list_add(&p->ptrace_entry, dead);		}	} 	kill_orphaned_pgrp(p, father);} /* * Make init inherit all the child processes */static void forget_original_parent(struct task_struct *father,					struct list_head *dead){	struct task_struct *p, *t, *reaper; 	if (unlikely(!list_empty(&father->ptraced)))		exit_ptrace(father, dead); 	/* Can drop and reacquire tasklist_lock */	reaper = find_child_reaper(father, dead);	if (list_empty(&father->children))		return; 	reaper = find_new_reaper(father, reaper);	list_for_each_entry(p, &father->children, sibling) {		for_each_thread(p, t) {			RCU_INIT_POINTER(t->real_parent, reaper);			BUG_ON((!t->ptrace) != (rcu_access_pointer(t->parent) == father));			if (likely(!t->ptrace))				t->parent = t->real_parent;			if (t->pdeath_signal)				group_send_sig_info(t->pdeath_signal,						    SEND_SIG_NOINFO, t,						    PIDTYPE_TGID);		}		/*		 * If this is a threaded reparent there is no need to		 * notify anyone anything has happened.		 */		if (!same_thread_group(reaper, father))			reparent_leader(father, p, dead);	}	list_splice_tail_init(&father->children, &reaper->children);} /* * Send signals to all our closest relatives so that they know * to properly mourn us.. */static void exit_notify(struct task_struct *tsk, int group_dead){	bool autoreap;	struct task_struct *p, *n;	LIST_HEAD(dead); 	write_lock_irq(&tasklist_lock);	forget_original_parent(tsk, &dead); 	if (group_dead)		kill_orphaned_pgrp(tsk->group_leader, NULL); 	tsk->exit_state = EXIT_ZOMBIE; 	if (unlikely(tsk->ptrace)) {		int sig = thread_group_empty(tsk) && !ptrace_reparented(tsk)			  ? tsk->exit_signal : SIGCHLD;		autoreap = do_notify_parent(tsk, sig);	} else if (thread_group_leader(tsk)) {		autoreap = thread_group_empty(tsk) &&			   do_notify_parent(tsk, tsk->exit_signal);	} else {		autoreap = true;		/* untraced sub-thread */		do_notify_pidfd(tsk);	} 	if (autoreap) {		tsk->exit_state = EXIT_DEAD;		list_add(&tsk->ptrace_entry, &dead);	} 	/* mt-exec, de_thread() is waiting for group leader */	if (unlikely(tsk->signal->notify_count < 0))		wake_up_process(tsk->signal->group_exec_task);	write_unlock_irq(&tasklist_lock); 	list_for_each_entry_safe(p, n, &dead, ptrace_entry) {		list_del_init(&p->ptrace_entry);		release_task(p);	}} #ifdef CONFIG_DEBUG_STACK_USAGE#ifdef CONFIG_STACK_GROWSUPunsigned long stack_not_used(struct task_struct *p){	unsigned long *n = end_of_stack(p); 	do {	/* Skip over canary */		n--;	} while (!*n); 	return (unsigned long)end_of_stack(p) - (unsigned long)n;}#else /* !CONFIG_STACK_GROWSUP */unsigned long stack_not_used(struct task_struct *p){	unsigned long *n = end_of_stack(p); 	do {	/* Skip over canary */		n++;	} while (!*n); 	return (unsigned long)n - (unsigned long)end_of_stack(p);}#endif /* CONFIG_STACK_GROWSUP */ /* Count the maximum pages reached in kernel stacks */static inline void kstack_histogram(unsigned long used_stack){#ifdef CONFIG_VM_EVENT_COUNTERS	if (used_stack <= 1024)		count_vm_event(KSTACK_1K);#if THREAD_SIZE > 1024	else if (used_stack <= 2048)		count_vm_event(KSTACK_2K);#endif#if THREAD_SIZE > 2048	else if (used_stack <= 4096)		count_vm_event(KSTACK_4K);#endif#if THREAD_SIZE > 4096	else if (used_stack <= 8192)		count_vm_event(KSTACK_8K);#endif#if THREAD_SIZE > 8192	else if (used_stack <= 16384)		count_vm_event(KSTACK_16K);#endif#if THREAD_SIZE > 16384	else if (used_stack <= 32768)		count_vm_event(KSTACK_32K);#endif#if THREAD_SIZE > 32768	else if (used_stack <= 65536)		count_vm_event(KSTACK_64K);#endif#if THREAD_SIZE > 65536	else		count_vm_event(KSTACK_REST);#endif#endif /* CONFIG_VM_EVENT_COUNTERS */} static void check_stack_usage(void){	static DEFINE_SPINLOCK(low_water_lock);	static int lowest_to_date = THREAD_SIZE;	unsigned long free; 	free = stack_not_used(current);	kstack_histogram(THREAD_SIZE - free); 	if (free >= lowest_to_date)		return; 	spin_lock(&low_water_lock);	if (free < lowest_to_date) {		pr_info("%s (%d) used greatest stack depth: %lu bytes left\n",			current->comm, task_pid_nr(current), free);		lowest_to_date = free;	}	spin_unlock(&low_water_lock);}#else /* !CONFIG_DEBUG_STACK_USAGE */static inline void check_stack_usage(void) {}#endif /* CONFIG_DEBUG_STACK_USAGE */ static void synchronize_group_exit(struct task_struct *tsk, long code){	struct sighand_struct *sighand = tsk->sighand;	struct signal_struct *signal = tsk->signal;	struct core_state *core_state; 	spin_lock_irq(&sighand->siglock);	signal->quick_threads--;	if ((signal->quick_threads == 0) &&	    !(signal->flags & SIGNAL_GROUP_EXIT)) {		signal->flags = SIGNAL_GROUP_EXIT;		signal->group_exit_code = code;		signal->group_stop_count = 0;	}	/*	 * Serialize with any possible pending coredump.	 * We must hold siglock around checking core_state	 * and setting PF_POSTCOREDUMP.  The core-inducing thread	 * will increment ->nr_threads for each thread in the	 * group without PF_POSTCOREDUMP set.	 */	tsk->flags |= PF_POSTCOREDUMP;	core_state = signal->core_state;	spin_unlock_irq(&sighand->siglock); 	if (unlikely(core_state))		coredump_task_exit(tsk, core_state);} void __noreturn do_exit(long code){	struct task_struct *tsk = current;	struct kthread *kthread;	int group_dead; 	WARN_ON(irqs_disabled());	WARN_ON(tsk->plug); 	kthread = tsk_is_kthread(tsk);	if (unlikely(kthread))		kthread_do_exit(kthread, code); 	kcov_task_exit(tsk);	kmsan_task_exit(tsk); 	synchronize_group_exit(tsk, code);	ptrace_event(PTRACE_EVENT_EXIT, code);	user_events_exit(tsk); 	io_uring_files_cancel();	sched_mm_cid_exit(tsk);	exit_signals(tsk);  /* sets PF_EXITING */ 	seccomp_filter_release(tsk); 	acct_update_integrals(tsk);	group_dead = atomic_dec_and_test(&tsk->signal->live);	if (group_dead) {		/*		 * If the last thread of global init has exited, panic		 * immediately to get a useable coredump.		 */		if (unlikely(is_global_init(tsk)))			panic("Attempted to kill init! exitcode=0x%08x\n",				tsk->signal->group_exit_code ?: (int)code); #ifdef CONFIG_POSIX_TIMERS		hrtimer_cancel(&tsk->signal->real_timer);		exit_itimers(tsk);#endif		if (tsk->mm)			setmax_mm_hiwater_rss(&tsk->signal->maxrss, tsk->mm);	}	acct_collect(code, group_dead);	if (group_dead)		tty_audit_exit();	audit_free(tsk); 	tsk->exit_code = code;	taskstats_exit(tsk, group_dead);	trace_sched_process_exit(tsk, group_dead); 	/*	 * Since sampling can touch ->mm, make sure to stop everything before we	 * tear it down.	 *	 * Also flushes inherited counters to the parent - before the parent	 * gets woken up by child-exit notifications.	 */	perf_event_exit_task(tsk);	/*	 * PF_EXITING (above) ensures unwind_deferred_request() will no	 * longer add new unwinds. While exit_mm() (below) will destroy the	 * abaility to do unwinds. So flush any pending unwinds here.	 */	unwind_deferred_task_exit(tsk); 	exit_mm(); 	if (group_dead)		acct_process();

Lines 501–1000 of 2038. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t