1525 lines · 41.5 KB · C
RawDownload
// SPDX-License-Identifier: GPL-2.0-only/* Kernel thread helper functions. *   Copyright (C) 2004 IBM Corporation, Rusty Russell. *   Copyright (C) 2009 Red Hat, Inc. * * Creation is done via kthreadd, so that we get a clean environment * even if we're invoked from userspace (think modprobe, hotplug cpu, * etc.). */#include <uapi/linux/sched/types.h>#include <linux/mm.h>#include <linux/mmu_context.h>#include <linux/sched.h>#include <linux/sched/mm.h>#include <linux/sched/task.h>#include <linux/kthread.h>#include <linux/completion.h>#include <linux/err.h>#include <linux/cgroup.h>#include <linux/cpuset.h>#include <linux/unistd.h>#include <linux/file.h>#include <linux/export.h>#include <linux/mutex.h>#include <linux/slab.h>#include <linux/freezer.h>#include <linux/ptrace.h>#include <linux/uaccess.h>#include <linux/numa.h>#include <linux/sched/isolation.h>#include <trace/events/sched.h>  static DEFINE_SPINLOCK(kthread_create_lock);static LIST_HEAD(kthread_create_list);struct task_struct *kthreadd_task; struct kthread_create_info{	/* Information passed to kthread() from kthreadd. */	int (*threadfn)(void *data);	void *data;	int node; 	/* Result passed back to kthread_create() from kthreadd. */	struct task_struct *result;	struct completion *done; 	struct list_head list;}; struct kthread {	unsigned long flags;	unsigned int cpu;	int result;	int (*threadfn)(void *);	void *data;	struct completion parked;	struct completion exited;#ifdef CONFIG_BLK_CGROUP	struct cgroup_subsys_state *blkcg_css;#endif	/* To store the full name if task comm is truncated. */	char *full_name;}; enum KTHREAD_BITS {	KTHREAD_IS_PER_CPU = 0,	KTHREAD_SHOULD_STOP,	KTHREAD_SHOULD_PARK,}; static inline struct kthread *to_kthread(struct task_struct *k){	WARN_ON(!(k->flags & PF_KTHREAD));	return k->worker_private;} /* * Variant of to_kthread() that doesn't assume @p is a kthread. * * Per construction; when: * *   (p->flags & PF_KTHREAD) && p->worker_private * * the task is both a kthread and struct kthread is persistent. However * PF_KTHREAD on it's own is not, kernel_thread() can exec() (See umh.c and * begin_new_exec()). */static inline struct kthread *__to_kthread(struct task_struct *p){	void *kthread = p->worker_private;	if (kthread && !(p->flags & PF_KTHREAD))		kthread = NULL;	return kthread;} void get_kthread_comm(char *buf, size_t buf_size, struct task_struct *tsk){	struct kthread *kthread = to_kthread(tsk); 	if (!kthread || !kthread->full_name) {		__get_task_comm(buf, buf_size, tsk);		return;	} 	strscpy_pad(buf, kthread->full_name, buf_size);} bool set_kthread_struct(struct task_struct *p){	struct kthread *kthread; 	if (WARN_ON_ONCE(to_kthread(p)))		return false; 	kthread = kzalloc(sizeof(*kthread), GFP_KERNEL);	if (!kthread)		return false; 	init_completion(&kthread->exited);	init_completion(&kthread->parked);	p->vfork_done = &kthread->exited; 	p->worker_private = kthread;	return true;} void free_kthread_struct(struct task_struct *k){	struct kthread *kthread; 	/*	 * Can be NULL if kmalloc() in set_kthread_struct() failed.	 */	kthread = to_kthread(k);	if (!kthread)		return; #ifdef CONFIG_BLK_CGROUP	WARN_ON_ONCE(kthread->blkcg_css);#endif	k->worker_private = NULL;	kfree(kthread->full_name);	kfree(kthread);} /** * kthread_should_stop - should this kthread return now? * * When someone calls kthread_stop() on your kthread, it will be woken * and this will return true.  You should then return, and your return * value will be passed through to kthread_stop(). */bool kthread_should_stop(void){	return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags);}EXPORT_SYMBOL(kthread_should_stop); bool __kthread_should_park(struct task_struct *k){	return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(k)->flags);}EXPORT_SYMBOL_GPL(__kthread_should_park); /** * kthread_should_park - should this kthread park now? * * When someone calls kthread_park() on your kthread, it will be woken * and this will return true.  You should then do the necessary * cleanup and call kthread_parkme() * * Similar to kthread_should_stop(), but this keeps the thread alive * and in a park position. kthread_unpark() "restarts" the thread and * calls the thread function again. */bool kthread_should_park(void){	return __kthread_should_park(current);}EXPORT_SYMBOL_GPL(kthread_should_park); /** * kthread_freezable_should_stop - should this freezable kthread return now? * @was_frozen: optional out parameter, indicates whether %current was frozen * * kthread_should_stop() for freezable kthreads, which will enter * refrigerator if necessary.  This function is safe from kthread_stop() / * freezer deadlock and freezable kthreads should use this function instead * of calling try_to_freeze() directly. */bool kthread_freezable_should_stop(bool *was_frozen){	bool frozen = false; 	might_sleep(); 	if (unlikely(freezing(current)))		frozen = __refrigerator(true); 	if (was_frozen)		*was_frozen = frozen; 	return kthread_should_stop();}EXPORT_SYMBOL_GPL(kthread_freezable_should_stop); /** * kthread_func - return the function specified on kthread creation * @task: kthread task in question * * Returns NULL if the task is not a kthread. */void *kthread_func(struct task_struct *task){	struct kthread *kthread = __to_kthread(task);	if (kthread)		return kthread->threadfn;	return NULL;}EXPORT_SYMBOL_GPL(kthread_func); /** * kthread_data - return data value specified on kthread creation * @task: kthread task in question * * Return the data value specified when kthread @task was created. * The caller is responsible for ensuring the validity of @task when * calling this function. */void *kthread_data(struct task_struct *task){	return to_kthread(task)->data;}EXPORT_SYMBOL_GPL(kthread_data); /** * kthread_probe_data - speculative version of kthread_data() * @task: possible kthread task in question * * @task could be a kthread task.  Return the data value specified when it * was created if accessible.  If @task isn't a kthread task or its data is * inaccessible for any reason, %NULL is returned.  This function requires * that @task itself is safe to dereference. */void *kthread_probe_data(struct task_struct *task){	struct kthread *kthread = __to_kthread(task);	void *data = NULL; 	if (kthread)		copy_from_kernel_nofault(&data, &kthread->data, sizeof(data));	return data;} static void __kthread_parkme(struct kthread *self){	for (;;) {		/*		 * TASK_PARKED is a special state; we must serialize against		 * possible pending wakeups to avoid store-store collisions on		 * task->state.		 *		 * Such a collision might possibly result in the task state		 * changin from TASK_PARKED and us failing the		 * wait_task_inactive() in kthread_park().		 */		set_special_state(TASK_PARKED);		if (!test_bit(KTHREAD_SHOULD_PARK, &self->flags))			break; 		/*		 * Thread is going to call schedule(), do not preempt it,		 * or the caller of kthread_park() may spend more time in		 * wait_task_inactive().		 */		preempt_disable();		complete(&self->parked);		schedule_preempt_disabled();		preempt_enable();	}	__set_current_state(TASK_RUNNING);} void kthread_parkme(void){	__kthread_parkme(to_kthread(current));}EXPORT_SYMBOL_GPL(kthread_parkme); /** * kthread_exit - Cause the current kthread return @result to kthread_stop(). * @result: The integer value to return to kthread_stop(). * * While kthread_exit can be called directly, it exists so that * functions which do some additional work in non-modular code such as * module_put_and_kthread_exit can be implemented. * * Does not return. */void __noreturn kthread_exit(long result){	struct kthread *kthread = to_kthread(current);	kthread->result = result;	do_exit(0);} /** * kthread_complete_and_exit - Exit the current kthread. * @comp: Completion to complete * @code: The integer value to return to kthread_stop(). * * If present complete @comp and the reuturn code to kthread_stop(). * * A kernel thread whose module may be removed after the completion of * @comp can use this function exit safely. * * Does not return. */void __noreturn kthread_complete_and_exit(struct completion *comp, long code){	if (comp)		complete(comp); 	kthread_exit(code);}EXPORT_SYMBOL(kthread_complete_and_exit); static int kthread(void *_create){	static const struct sched_param param = { .sched_priority = 0 };	/* Copy data: it's on kthread's stack */	struct kthread_create_info *create = _create;	int (*threadfn)(void *data) = create->threadfn;	void *data = create->data;	struct completion *done;	struct kthread *self;	int ret; 	self = to_kthread(current); 	/* Release the structure when caller killed by a fatal signal. */	done = xchg(&create->done, NULL);	if (!done) {		kfree(create);		kthread_exit(-EINTR);	} 	self->threadfn = threadfn;	self->data = data; 	/*	 * The new thread inherited kthreadd's priority and CPU mask. Reset	 * back to default in case they have been changed.	 */	sched_setscheduler_nocheck(current, SCHED_NORMAL, &param);	set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_KTHREAD)); 	/* OK, tell user we're spawned, wait for stop or wakeup */	__set_current_state(TASK_UNINTERRUPTIBLE);	create->result = current;	/*	 * Thread is going to call schedule(), do not preempt it,	 * or the creator may spend more time in wait_task_inactive().	 */	preempt_disable();	complete(done);	schedule_preempt_disabled();	preempt_enable(); 	ret = -EINTR;	if (!test_bit(KTHREAD_SHOULD_STOP, &self->flags)) {		cgroup_kthread_ready();		__kthread_parkme(self);		ret = threadfn(data);	}	kthread_exit(ret);} /* called from kernel_clone() to get node information for about to be created task */int tsk_fork_get_node(struct task_struct *tsk){#ifdef CONFIG_NUMA	if (tsk == kthreadd_task)		return tsk->pref_node_fork;#endif	return NUMA_NO_NODE;} static void create_kthread(struct kthread_create_info *create){	int pid; #ifdef CONFIG_NUMA	current->pref_node_fork = create->node;#endif	/* We want our own signal handler (we take no signals by default). */	pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD);	if (pid < 0) {		/* Release the structure when caller killed by a fatal signal. */		struct completion *done = xchg(&create->done, NULL); 		if (!done) {			kfree(create);			return;		}		create->result = ERR_PTR(pid);		complete(done);	}} static __printf(4, 0)struct task_struct *__kthread_create_on_node(int (*threadfn)(void *data),						    void *data, int node,						    const char namefmt[],						    va_list args){	DECLARE_COMPLETION_ONSTACK(done);	struct task_struct *task;	struct kthread_create_info *create = kmalloc(sizeof(*create),						     GFP_KERNEL); 	if (!create)		return ERR_PTR(-ENOMEM);	create->threadfn = threadfn;	create->data = data;	create->node = node;	create->done = &done; 	spin_lock(&kthread_create_lock);	list_add_tail(&create->list, &kthread_create_list);	spin_unlock(&kthread_create_lock); 	wake_up_process(kthreadd_task);	/*	 * Wait for completion in killable state, for I might be chosen by	 * the OOM killer while kthreadd is trying to allocate memory for	 * new kernel thread.	 */	if (unlikely(wait_for_completion_killable(&done))) {		/*		 * If I was killed by a fatal signal before kthreadd (or new		 * kernel thread) calls complete(), leave the cleanup of this		 * structure to that thread.		 */		if (xchg(&create->done, NULL))			return ERR_PTR(-EINTR);		/*		 * kthreadd (or new kernel thread) will call complete()		 * shortly.		 */		wait_for_completion(&done);	}	task = create->result;	if (!IS_ERR(task)) {		char name[TASK_COMM_LEN];		va_list aq;		int len; 		/*		 * task is already visible to other tasks, so updating		 * COMM must be protected.		 */		va_copy(aq, args);		len = vsnprintf(name, sizeof(name), namefmt, aq);		va_end(aq);		if (len >= TASK_COMM_LEN) {			struct kthread *kthread = to_kthread(task); 			/* leave it truncated when out of memory. */			kthread->full_name = kvasprintf(GFP_KERNEL, namefmt, args);		}		set_task_comm(task, name);	}	kfree(create);	return task;} /** * kthread_create_on_node - create a kthread. * @threadfn: the function to run until signal_pending(current). * @data: data ptr for @threadfn. * @node: task and thread structures for the thread are allocated on this node * @namefmt: printf-style name for the thread. * * Description: This helper function creates and names a kernel * thread.  The thread will be stopped: use wake_up_process() to start * it.  See also kthread_run().  The new thread has SCHED_NORMAL policy and * is affine to all CPUs. * * If thread is going to be bound on a particular cpu, give its node * in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE. * When woken, the thread will run @threadfn() with @data as its * argument. @threadfn() can either return directly if it is a * standalone thread for which no one will call kthread_stop(), or * return when 'kthread_should_stop()' is true (which means * kthread_stop() has been called).  The return value should be zero * or a negative error number; it will be passed to kthread_stop(). *

Lines 1–500 of 1525.