3409 lines · 83.7 KB · C
RawDownload
				struct pid_namespace *ns = ns_of_pid(pid); 				ASSERT_EXCLUSIVE_WRITER(ns->child_reaper);				WRITE_ONCE(ns->child_reaper, p);				p->signal->flags |= SIGNAL_UNKILLABLE;			}			p->signal->shared_pending.signal = delayed.signal;			p->signal->tty = tty_kref_get(current->signal->tty);			/*			 * Inherit has_child_subreaper flag under the same			 * tasklist_lock with adding child to the process tree			 * for propagate_has_child_subreaper optimization.			 */			p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper ||							 p->real_parent->signal->is_child_subreaper;			if (clone_flags & CLONE_AUTOREAP)				p->signal->autoreap = 1;			list_add_tail(&p->sibling, &p->real_parent->children);			list_add_tail_rcu(&p->tasks, &init_task.tasks);			attach_pid(p, PIDTYPE_TGID);			attach_pid(p, PIDTYPE_PGID);			attach_pid(p, PIDTYPE_SID);			__this_cpu_inc(process_counts);		} else {			current->signal->nr_threads++;			current->signal->quick_threads++;			atomic_inc(&current->signal->live);			refcount_inc(&current->signal->sigcnt);			task_join_group_stop(p);			list_add_tail_rcu(&p->thread_node,					  &p->signal->thread_head);		}		attach_pid(p, PIDTYPE_PID);		nr_threads++;	}	total_forks++;	hlist_del_init(&delayed.node);	spin_unlock(&current->sighand->siglock);	syscall_tracepoint_update(p);	write_unlock_irq(&tasklist_lock); 	if (pidfile)		fd_install(pidfd, pidfile); 	proc_fork_connector(p);	/*	 * sched_ext needs @p to be associated with its cgroup in its post_fork	 * hook. cgroup_post_fork() should come before sched_post_fork().	 */	cgroup_post_fork(p, args);	sched_post_fork(p);	perf_event_fork(p); 	trace_task_newtask(p, clone_flags);	uprobe_copy_process(p, clone_flags);	user_events_fork(p, clone_flags); 	copy_oom_score_adj(clone_flags, p); 	return p; bad_fork_core_free:	sched_core_free(p);	spin_unlock(&current->sighand->siglock);	write_unlock_irq(&tasklist_lock);bad_fork_cancel_cgroup:	cgroup_cancel_fork(p, args);bad_fork_put_pidfd:	if (clone_flags & CLONE_PIDFD) {		fput(pidfile);		put_unused_fd(pidfd);	}bad_fork_free_pid:	if (pid != &init_struct_pid)		free_pid(pid);bad_fork_cleanup_thread:	exit_thread(p);bad_fork_cleanup_io:	if (p->io_context)		exit_io_context(p);bad_fork_cleanup_namespaces:	exit_nsproxy_namespaces(p);bad_fork_cleanup_mm:	if (p->mm) {		mm_clear_owner(p->mm, p);		mmput(p->mm);	}bad_fork_cleanup_signal:	if (!(clone_flags & CLONE_THREAD))		free_signal_struct(p->signal);bad_fork_cleanup_sighand:	__cleanup_sighand(p->sighand);bad_fork_cleanup_fs:	exit_fs(p); /* blocking */bad_fork_cleanup_files:	exit_files(p); /* blocking */bad_fork_cleanup_semundo:	exit_sem(p);bad_fork_cleanup_security:	security_task_free(p);bad_fork_cleanup_audit:	audit_free(p);bad_fork_cleanup_perf:	perf_event_free_task(p);bad_fork_sched_cancel_fork:	sched_cancel_fork(p);bad_fork_cleanup_policy:	lockdep_free_task(p);#ifdef CONFIG_NUMA	mpol_put(p->mempolicy);#endifbad_fork_cleanup_delayacct:	io_uring_free(p);	delayacct_tsk_free(p);bad_fork_cleanup_count:	dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1);	exit_cred_namespaces(p);	exit_creds(p);bad_fork_free:	WRITE_ONCE(p->__state, TASK_DEAD);	exit_task_stack_account(p);	put_task_stack(p);	delayed_free_task(p);fork_out:	spin_lock_irq(&current->sighand->siglock);	hlist_del_init(&delayed.node);	spin_unlock_irq(&current->sighand->siglock);	return ERR_PTR(retval);} static inline void init_idle_pids(struct task_struct *idle){	enum pid_type type; 	for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {		INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */		init_task_pid(idle, type, &init_struct_pid);	}} static int idle_dummy(void *dummy){	/* This function is never called */	return 0;} struct task_struct * __init fork_idle(int cpu){	struct task_struct *task;	struct kernel_clone_args args = {		.flags		= CLONE_VM,		.fn		= &idle_dummy,		.fn_arg		= NULL,		.kthread	= 1,		.idle		= 1,	}; 	task = copy_process(&init_struct_pid, 0, cpu_to_node(cpu), &args);	if (!IS_ERR(task)) {		init_idle_pids(task);		init_idle(task, cpu);	} 	return task;} /* * This is like kernel_clone(), but shaved down and tailored to just * creating io_uring workers. It returns a created task, or an error pointer. * The returned task is inactive, and the caller must fire it up through * wake_up_new_task(p). All signals are blocked in the created task. */struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node){	unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|			      CLONE_IO|CLONE_VM|CLONE_UNTRACED;	struct kernel_clone_args args = {		.flags		= flags,		.fn		= fn,		.fn_arg		= arg,		.io_thread	= 1,		.user_worker	= 1,	}; 	return copy_process(NULL, 0, node, &args);} /* *  Ok, this is the main fork-routine. * * It copies the process, and if successful kick-starts * it and waits for it to finish using the VM if required. */pid_t kernel_clone(struct kernel_clone_args *args){	u64 clone_flags = args->flags;	struct completion vfork;	struct pid *pid;	struct task_struct *p;	int trace = 0;	pid_t nr; 	/*	 * Creating an empty mount namespace implies creating a new mount	 * namespace.  Set this before copy_process() so that the	 * CLONE_NEWNS|CLONE_FS mutual exclusion check works correctly.	 */	if (clone_flags & CLONE_EMPTY_MNTNS) {		clone_flags |= CLONE_NEWNS;		args->flags = clone_flags;	} 	/*	 * For legacy clone() calls, CLONE_PIDFD uses the parent_tid argument	 * to return the pidfd. Hence, CLONE_PIDFD and CLONE_PARENT_SETTID are	 * mutually exclusive. With clone3() CLONE_PIDFD has grown a separate	 * field in struct clone_args and it still doesn't make sense to have	 * them both point at the same memory location. Performing this check	 * here has the advantage that we don't need to have a separate helper	 * to check for legacy clone().	 */	if ((clone_flags & CLONE_PIDFD) &&	    (clone_flags & CLONE_PARENT_SETTID) &&	    (args->pidfd == args->parent_tid))		return -EINVAL; 	if (!valid_signal(args->exit_signal))		return -EINVAL; 	/*	 * Determine whether and which event to report to ptracer.  When	 * called from kernel_thread or CLONE_UNTRACED is explicitly	 * requested, no event is reported; otherwise, report if the event	 * for the type of forking is enabled.	 */	if (!(clone_flags & CLONE_UNTRACED)) {		if (clone_flags & CLONE_VFORK)			trace = PTRACE_EVENT_VFORK;		else if (args->exit_signal != SIGCHLD)			trace = PTRACE_EVENT_CLONE;		else			trace = PTRACE_EVENT_FORK; 		if (likely(!ptrace_event_enabled(current, trace)))			trace = 0;	} 	p = copy_process(NULL, trace, NUMA_NO_NODE, args);	add_latent_entropy(); 	if (IS_ERR(p))		return PTR_ERR(p); 	/*	 * Do this prior waking up the new thread - the thread pointer	 * might get invalid after that point, if the thread exits quickly.	 */	trace_sched_process_fork(current, p); 	pid = get_task_pid(p, PIDTYPE_PID);	nr = pid_vnr(pid); 	if (clone_flags & CLONE_PARENT_SETTID)		put_user(nr, args->parent_tid); 	if (clone_flags & CLONE_VFORK) {		p->vfork_done = &vfork;		init_completion(&vfork);		get_task_struct(p);	} 	if (IS_ENABLED(CONFIG_LRU_GEN_WALKS_MMU) && !(clone_flags & CLONE_VM)) {		/* lock the task to synchronize with memcg migration */		task_lock(p);		lru_gen_add_mm(p->mm);		task_unlock(p);	} 	wake_up_new_task(p); 	/* forking complete and child started to run, tell ptracer */	if (unlikely(trace))		ptrace_event_pid(trace, pid); 	if (clone_flags & CLONE_VFORK) {		if (!wait_for_vfork_done(p, &vfork))			ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid);	} 	put_pid(pid);	return nr;} /* * Create a kernel thread. */pid_t kernel_thread(int (*fn)(void *), void *arg, const char *name,		    unsigned long flags){	struct kernel_clone_args args = {		.flags		= ((flags | CLONE_VM | CLONE_UNTRACED) & ~CSIGNAL),		.exit_signal	= (flags & CSIGNAL),		.fn		= fn,		.fn_arg		= arg,		.name		= name,		.kthread	= 1,	}; 	return kernel_clone(&args);} /* * Create a user mode thread. */pid_t user_mode_thread(int (*fn)(void *), void *arg, unsigned long flags){	struct kernel_clone_args args = {		.flags		= ((flags | CLONE_VM | CLONE_UNTRACED) & ~CSIGNAL),		.exit_signal	= (flags & CSIGNAL),		.fn		= fn,		.fn_arg		= arg,	}; 	return kernel_clone(&args);} #ifdef __ARCH_WANT_SYS_FORKSYSCALL_DEFINE0(fork){#ifdef CONFIG_MMU	struct kernel_clone_args args = {		.exit_signal = SIGCHLD,	}; 	return kernel_clone(&args);#else	/* can not support in nommu mode */	return -EINVAL;#endif}#endif #ifdef __ARCH_WANT_SYS_VFORKSYSCALL_DEFINE0(vfork){	struct kernel_clone_args args = {		.flags		= CLONE_VFORK | CLONE_VM,		.exit_signal	= SIGCHLD,	}; 	return kernel_clone(&args);}#endif #ifdef __ARCH_WANT_SYS_CLONE#ifdef CONFIG_CLONE_BACKWARDSSYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,		 int __user *, parent_tidptr,		 unsigned long, tls,		 int __user *, child_tidptr)#elif defined(CONFIG_CLONE_BACKWARDS2)SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,		 int __user *, parent_tidptr,		 int __user *, child_tidptr,		 unsigned long, tls)#elif defined(CONFIG_CLONE_BACKWARDS3)SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,		int, stack_size,		int __user *, parent_tidptr,		int __user *, child_tidptr,		unsigned long, tls)#elseSYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,		 int __user *, parent_tidptr,		 int __user *, child_tidptr,		 unsigned long, tls)#endif{	struct kernel_clone_args args = {		.flags		= (lower_32_bits(clone_flags) & ~CSIGNAL),		.pidfd		= parent_tidptr,		.child_tid	= child_tidptr,		.parent_tid	= parent_tidptr,		.exit_signal	= (lower_32_bits(clone_flags) & CSIGNAL),		.stack		= newsp,		.tls		= tls,	}; 	return kernel_clone(&args);}#endif static noinline int copy_clone_args_from_user(struct kernel_clone_args *kargs,					      struct clone_args __user *uargs,					      size_t usize){	int err;	struct clone_args args;	pid_t *kset_tid = kargs->set_tid; 	BUILD_BUG_ON(offsetofend(struct clone_args, tls) !=		     CLONE_ARGS_SIZE_VER0);	BUILD_BUG_ON(offsetofend(struct clone_args, set_tid_size) !=		     CLONE_ARGS_SIZE_VER1);	BUILD_BUG_ON(offsetofend(struct clone_args, cgroup) !=		     CLONE_ARGS_SIZE_VER2);	BUILD_BUG_ON(sizeof(struct clone_args) != CLONE_ARGS_SIZE_VER2); 	if (unlikely(usize > PAGE_SIZE))		return -E2BIG;	if (unlikely(usize < CLONE_ARGS_SIZE_VER0))		return -EINVAL; 	err = copy_struct_from_user(&args, sizeof(args), uargs, usize);	if (err)		return err; 	if (unlikely(args.set_tid_size > MAX_PID_NS_LEVEL))		return -EINVAL; 	if (unlikely(!args.set_tid && args.set_tid_size > 0))		return -EINVAL; 	if (unlikely(args.set_tid && args.set_tid_size == 0))		return -EINVAL; 	/*	 * Verify that higher 32bits of exit_signal are unset	 */	if (unlikely(args.exit_signal & ~((u64)CSIGNAL)))		return -EINVAL; 	if ((args.flags & CLONE_INTO_CGROUP) &&	    (args.cgroup > INT_MAX || usize < CLONE_ARGS_SIZE_VER2))		return -EINVAL; 	*kargs = (struct kernel_clone_args){		.flags		= args.flags,		.pidfd		= u64_to_user_ptr(args.pidfd),		.child_tid	= u64_to_user_ptr(args.child_tid),		.parent_tid	= u64_to_user_ptr(args.parent_tid),		.exit_signal	= args.exit_signal,		.stack		= args.stack,		.stack_size	= args.stack_size,		.tls		= args.tls,		.set_tid_size	= args.set_tid_size,		.cgroup		= args.cgroup,	}; 	if (args.set_tid &&		copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid),			(kargs->set_tid_size * sizeof(pid_t))))		return -EFAULT; 	kargs->set_tid = kset_tid; 	return 0;} /** * clone3_stack_valid - check and prepare stack * @kargs: kernel clone args * * Verify that the stack arguments userspace gave us are sane. * In addition, set the stack direction for userspace since it's easy for us to * determine. */static inline bool clone3_stack_valid(struct kernel_clone_args *kargs){	if (kargs->stack == 0) {		if (kargs->stack_size > 0)			return false;	} else {		if (kargs->stack_size == 0)			return false; 		if (!access_ok((void __user *)kargs->stack, kargs->stack_size))			return false; #if !defined(CONFIG_STACK_GROWSUP)		kargs->stack += kargs->stack_size;#endif	} 	return true;} static bool clone3_args_valid(struct kernel_clone_args *kargs){	/* Verify that no unknown flags are passed along. */	if (kargs->flags &	    ~(CLONE_LEGACY_FLAGS | CLONE_CLEAR_SIGHAND |	      CLONE_INTO_CGROUP | CLONE_AUTOREAP | CLONE_NNP |	      CLONE_PIDFD_AUTOKILL | CLONE_EMPTY_MNTNS))		return false; 	/*	 * - make the CLONE_DETACHED bit reusable for clone3	 * - make the CSIGNAL bits reusable for clone3	 */

Lines 2501–3000 of 3409. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t