3409 lines · 83.7 KB · C
RawDownload
} void exit_task_stack_account(struct task_struct *tsk){	account_kernel_stack(tsk, -1); 	if (IS_ENABLED(CONFIG_VMAP_STACK)) {		struct vm_struct *vm_area;		int i; 		vm_area = task_stack_vm_area(tsk);		for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)			memcg_kmem_uncharge_page(vm_area->pages[i], 0);	}} static void release_task_stack(struct task_struct *tsk){	if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD))		return;  /* Better to leak the stack than to free prematurely */ 	free_thread_stack(tsk);} #ifdef CONFIG_THREAD_INFO_IN_TASKvoid put_task_stack(struct task_struct *tsk){	if (refcount_dec_and_test(&tsk->stack_refcount))		release_task_stack(tsk);}#endif void free_task(struct task_struct *tsk){#ifdef CONFIG_SECCOMP	WARN_ON_ONCE(tsk->seccomp.filter);#endif	release_user_cpus_ptr(tsk);	scs_release(tsk); #ifndef CONFIG_THREAD_INFO_IN_TASK	/*	 * The task is finally done with both the stack and thread_info,	 * so free both.	 */	release_task_stack(tsk);#else	/*	 * If the task had a separate stack allocation, it should be gone	 * by now.	 */	WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0);#endif	rt_mutex_debug_task_free(tsk);	ftrace_graph_exit_task(tsk);	arch_release_task_struct(tsk);	if (tsk->flags & PF_KTHREAD)		free_kthread_struct(tsk);	bpf_task_storage_free(tsk);	put_task_exec_state(rcu_access_pointer(tsk->exec_state));	free_task_struct(tsk);}EXPORT_SYMBOL(free_task); void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm){	struct file *exe_file; 	exe_file = get_mm_exe_file(oldmm);	RCU_INIT_POINTER(mm->exe_file, exe_file);	/*	 * We depend on the oldmm having properly denied write access to the	 * exe_file already.	 */	if (exe_file && exe_file_deny_write_access(exe_file))		pr_warn_once("exe_file_deny_write_access() failed in %s\n", __func__);} #ifdef CONFIG_MMUstatic inline int mm_alloc_pgd(struct mm_struct *mm){	mm->pgd = pgd_alloc(mm);	if (unlikely(!mm->pgd))		return -ENOMEM;	return 0;} static inline void mm_free_pgd(struct mm_struct *mm){	pgd_free(mm, mm->pgd);}#else#define mm_alloc_pgd(mm)	(0)#define mm_free_pgd(mm)#endif /* CONFIG_MMU */ #ifdef CONFIG_MM_IDstatic DEFINE_IDA(mm_ida); static inline int mm_alloc_id(struct mm_struct *mm){	int ret; 	ret = ida_alloc_range(&mm_ida, MM_ID_MIN, MM_ID_MAX, GFP_KERNEL);	if (ret < 0)		return ret;	mm->mm_id = ret;	return 0;} static inline void mm_free_id(struct mm_struct *mm){	const mm_id_t id = mm->mm_id; 	mm->mm_id = MM_ID_DUMMY;	if (id == MM_ID_DUMMY)		return;	if (WARN_ON_ONCE(id < MM_ID_MIN || id > MM_ID_MAX))		return;	ida_free(&mm_ida, id);}#else /* !CONFIG_MM_ID */static inline int mm_alloc_id(struct mm_struct *mm) { return 0; }static inline void mm_free_id(struct mm_struct *mm) {}#endif /* CONFIG_MM_ID */ static void check_mm(struct mm_struct *mm){	int i; 	BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS,			 "Please make sure 'struct resident_page_types[]' is updated as well"); 	for (i = 0; i < NR_MM_COUNTERS; i++) {		long x = percpu_counter_sum(&mm->rss_stat[i]); 		if (unlikely(x)) {			pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld Comm:%s Pid:%d\n",				 mm, resident_page_types[i], x,				 current->comm,				 task_pid_nr(current));		}	} 	if (mm_pgtables_bytes(mm))		pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",				mm_pgtables_bytes(mm)); #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)	VM_BUG_ON_MM(mm->pmd_huge_pte, mm);#endif} #define allocate_mm()	(kmem_cache_alloc(mm_cachep, GFP_KERNEL))#define free_mm(mm)	(kmem_cache_free(mm_cachep, (mm))) static void do_check_lazy_tlb(void *arg){	struct mm_struct *mm = arg; 	WARN_ON_ONCE(current->active_mm == mm);} static void do_shoot_lazy_tlb(void *arg){	struct mm_struct *mm = arg; 	if (current->active_mm == mm) {		WARN_ON_ONCE(current->mm);		current->active_mm = &init_mm;		switch_mm(mm, &init_mm, current);	}} static void cleanup_lazy_tlbs(struct mm_struct *mm){	if (!IS_ENABLED(CONFIG_MMU_LAZY_TLB_SHOOTDOWN)) {		/*		 * In this case, lazy tlb mms are refounted and would not reach		 * __mmdrop until all CPUs have switched away and mmdrop()ed.		 */		return;	} 	/*	 * Lazy mm shootdown does not refcount "lazy tlb mm" usage, rather it	 * requires lazy mm users to switch to another mm when the refcount	 * drops to zero, before the mm is freed. This requires IPIs here to	 * switch kernel threads to init_mm.	 *	 * archs that use IPIs to flush TLBs can piggy-back that lazy tlb mm	 * switch with the final userspace teardown TLB flush which leaves the	 * mm lazy on this CPU but no others, reducing the need for additional	 * IPIs here. There are cases where a final IPI is still required here,	 * such as the final mmdrop being performed on a different CPU than the	 * one exiting, or kernel threads using the mm when userspace exits.	 *	 * IPI overheads have not found to be expensive, but they could be	 * reduced in a number of possible ways, for example (roughly	 * increasing order of complexity):	 * - The last lazy reference created by exit_mm() could instead switch	 *   to init_mm, however it's probable this will run on the same CPU	 *   immediately afterwards, so this may not reduce IPIs much.	 * - A batch of mms requiring IPIs could be gathered and freed at once.	 * - CPUs store active_mm where it can be remotely checked without a	 *   lock, to filter out false-positives in the cpumask.	 * - After mm_users or mm_count reaches zero, switching away from the	 *   mm could clear mm_cpumask to reduce some IPIs, perhaps together	 *   with some batching or delaying of the final IPIs.	 * - A delayed freeing and RCU-like quiescing sequence based on mm	 *   switching to avoid IPIs completely.	 */	on_each_cpu_mask(mm_cpumask(mm), do_shoot_lazy_tlb, (void *)mm, 1);	if (IS_ENABLED(CONFIG_DEBUG_VM_SHOOT_LAZIES))		on_each_cpu(do_check_lazy_tlb, (void *)mm, 1);} /* * Called when the last reference to the mm * is dropped: either by a lazy thread or by * mmput. Free the page directory and the mm. */void __mmdrop(struct mm_struct *mm){	BUG_ON(mm == &init_mm);	WARN_ON_ONCE(mm == current->mm); 	/* Ensure no CPUs are using this as their lazy tlb mm */	cleanup_lazy_tlbs(mm); 	WARN_ON_ONCE(mm == current->active_mm);	mm_destroy_sched(mm);	mm_free_pgd(mm);	mm_free_id(mm);	destroy_context(mm);	mmu_notifier_subscriptions_destroy(mm);	check_mm(mm);	mm_pasid_drop(mm);	mm_destroy_cid(mm);	percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS); 	free_mm(mm);}EXPORT_SYMBOL_GPL(__mmdrop); static void mmdrop_async_fn(struct work_struct *work){	struct mm_struct *mm; 	mm = container_of(work, struct mm_struct, async_put_work);	__mmdrop(mm);} static void mmdrop_async(struct mm_struct *mm){	if (unlikely(atomic_dec_and_test(&mm->mm_count))) {		INIT_WORK(&mm->async_put_work, mmdrop_async_fn);		schedule_work(&mm->async_put_work);	}} static inline void free_signal_struct(struct signal_struct *sig){	taskstats_tgid_free(sig);	sched_autogroup_exit(sig);	/*	 * __mmdrop is not safe to call from softirq context on x86 due to	 * pgd_dtor so postpone it to the async context	 */	if (sig->oom_mm)		mmdrop_async(sig->oom_mm);	kmem_cache_free(signal_cachep, sig);} static inline void put_signal_struct(struct signal_struct *sig){	if (refcount_dec_and_test(&sig->sigcnt))		free_signal_struct(sig);} void __put_task_struct(struct task_struct *tsk){	WARN_ON(!tsk->exit_state);	WARN_ON(refcount_read(&tsk->usage));	WARN_ON(tsk == current); 	unwind_task_free(tsk);	io_uring_free(tsk);	cgroup_task_free(tsk);	task_numa_free(tsk, true);	security_task_free(tsk);	exit_creds(tsk);	delayacct_tsk_free(tsk);	put_signal_struct(tsk->signal);	sched_core_free(tsk);	free_task(tsk);}EXPORT_SYMBOL_GPL(__put_task_struct); void __put_task_struct_rcu_cb(struct rcu_head *rhp){	struct task_struct *task = container_of(rhp, struct task_struct, rcu); 	__put_task_struct(task);}EXPORT_SYMBOL_GPL(__put_task_struct_rcu_cb); void __init __weak arch_task_cache_init(void) { } /* * set_max_threads */static void __init set_max_threads(unsigned int max_threads_suggested){	u64 threads;	unsigned long nr_pages = memblock_estimated_nr_free_pages(); 	/*	 * The number of threads shall be limited such that the thread	 * structures may only consume a small part of the available memory.	 */	if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64)		threads = MAX_THREADS;	else		threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE,				    (u64) THREAD_SIZE * 8UL); 	if (threads > max_threads_suggested)		threads = max_threads_suggested; 	max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);} #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT/* Initialized by the architecture: */int arch_task_struct_size __read_mostly;#endif static void __init task_struct_whitelist(unsigned long *offset, unsigned long *size){	/* Fetch thread_struct whitelist for the architecture. */	arch_thread_struct_whitelist(offset, size); 	/*	 * Handle zero-sized whitelist or empty thread_struct, otherwise	 * adjust offset to position of thread_struct in task_struct.	 */	if (unlikely(*size == 0))		*offset = 0;	else		*offset += offsetof(struct task_struct, thread);} void __init fork_init(void){	int i;#ifndef ARCH_MIN_TASKALIGN#define ARCH_MIN_TASKALIGN	0#endif	int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN);	unsigned long useroffset, usersize; 	/* create a slab on which task_structs can be allocated */	task_struct_whitelist(&useroffset, &usersize);	task_struct_cachep = kmem_cache_create_usercopy("task_struct",			arch_task_struct_size, align,			SLAB_PANIC|SLAB_ACCOUNT,			useroffset, usersize, NULL); 	/* do the arch specific task caches init */	arch_task_cache_init(); 	set_max_threads(MAX_THREADS); 	init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;	init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;	init_task.signal->rlim[RLIMIT_SIGPENDING] =		init_task.signal->rlim[RLIMIT_NPROC]; 	for (i = 0; i < UCOUNT_COUNTS; i++)		init_user_ns.ucount_max[i] = max_threads/2; 	set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_NPROC,      RLIM_INFINITY);	set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MSGQUEUE,   RLIM_INFINITY);	set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_SIGPENDING, RLIM_INFINITY);	set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MEMLOCK,    RLIM_INFINITY); #ifdef CONFIG_VMAP_STACK	cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache",			  NULL, free_vm_stack_cache);#endif 	scs_init(); 	lockdep_init_task(&init_task);	uprobes_init();} int __weak arch_dup_task_struct(struct task_struct *dst,					       struct task_struct *src){	*dst = *src;	return 0;} void set_task_stack_end_magic(struct task_struct *tsk){	unsigned long *stackend; 	stackend = end_of_stack(tsk);	*stackend = STACK_END_MAGIC;	/* for overflow detection */} static struct task_struct *dup_task_struct(struct task_struct *orig, int node){	struct task_struct *tsk;	int err; 	if (node == NUMA_NO_NODE)		node = tsk_fork_get_node(orig);	tsk = alloc_task_struct_node(node);	if (!tsk)		return NULL; 	err = arch_dup_task_struct(tsk, orig);	if (err)		goto free_tsk; 	err = alloc_thread_stack_node(tsk, node);	if (err)		goto free_tsk; #ifdef CONFIG_THREAD_INFO_IN_TASK	refcount_set(&tsk->stack_refcount, 1);#endif	account_kernel_stack(tsk, 1); 	err = scs_prepare(tsk, node);	if (err)		goto free_stack; #ifdef CONFIG_SECCOMP	/*	 * We must handle setting up seccomp filters once we're under	 * the sighand lock in case orig has changed between now and	 * then. Until then, filter must be NULL to avoid messing up	 * the usage counts on the error path calling free_task.	 */	tsk->seccomp.filter = NULL;#endif 	RCU_INIT_POINTER(tsk->exec_state, NULL); 	setup_thread_stack(tsk, orig);	clear_user_return_notifier(tsk);	clear_tsk_need_resched(tsk);	set_task_stack_end_magic(tsk);	clear_syscall_work_syscall_user_dispatch(tsk); #ifdef CONFIG_STACKPROTECTOR	tsk->stack_canary = get_random_canary();#endif	if (orig->cpus_ptr == &orig->cpus_mask)		tsk->cpus_ptr = &tsk->cpus_mask;	dup_user_cpus_ptr(tsk, orig, node); 	/*	 * One for the user space visible state that goes away when reaped.	 * One for the scheduler.	 */	refcount_set(&tsk->rcu_users, 2);	/* One for the rcu users */	refcount_set(&tsk->usage, 1);#ifdef CONFIG_BLK_DEV_IO_TRACE	tsk->btrace_seq = 0;#endif	tsk->splice_pipe = NULL;	tsk->task_frag.page = NULL;	tsk->wake_q.next = NULL;	tsk->worker_private = NULL; 	kcov_task_init(tsk);	kmsan_task_create(tsk);	kmap_local_fork(tsk); #ifdef CONFIG_FAULT_INJECTION	tsk->fail_nth = 0;#endif #ifdef CONFIG_BLK_CGROUP	tsk->throttle_disk = NULL;	tsk->use_memdelay = 0;#endif #ifdef CONFIG_ARCH_HAS_CPU_PASID	tsk->pasid_activated = 0;#endif #ifdef CONFIG_MEMCG	tsk->active_memcg = NULL;

Lines 501–1000 of 3409. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t