3409 lines · 83.7 KB · C
RawDownload
#endif #ifdef CONFIG_X86_BUS_LOCK_DETECT	tsk->reported_split_lock = 0;#endif #ifdef CONFIG_SCHED_MM_CID	tsk->mm_cid.cid = MM_CID_UNSET;	tsk->mm_cid.active = 0;	INIT_HLIST_NODE(&tsk->mm_cid.node);#endif #ifdef CONFIG_BPF_SYSCALL	RCU_INIT_POINTER(tsk->bpf_storage, NULL);	tsk->bpf_ctx = NULL;#endif	return tsk; free_stack:	exit_task_stack_account(tsk);	free_thread_stack(tsk);free_tsk:	free_task_struct(tsk);	return NULL;} __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); static unsigned long coredump_filter = MMF_DUMP_FILTER_DEFAULT; static int __init coredump_filter_setup(char *s){	if (kstrtoul(s, 0, &coredump_filter))		return 0;	coredump_filter <<= MMF_DUMP_FILTER_SHIFT;	coredump_filter &= MMF_DUMP_FILTER_MASK;	return 1;} __setup("coredump_filter=", coredump_filter_setup); #include <linux/init_task.h> static void mm_init_aio(struct mm_struct *mm){#ifdef CONFIG_AIO	spin_lock_init(&mm->ioctx_lock);	mm->ioctx_table = NULL;#endif} static __always_inline void mm_clear_owner(struct mm_struct *mm,					   struct task_struct *p){#ifdef CONFIG_MEMCG	if (mm->owner == p)		WRITE_ONCE(mm->owner, NULL);#endif} static void mm_init_owner(struct mm_struct *mm, struct task_struct *p){#ifdef CONFIG_MEMCG	mm->owner = p;#endif} static void mm_init_uprobes_state(struct mm_struct *mm){#ifdef CONFIG_UPROBES	mm->uprobes_state.xol_area = NULL;	arch_uprobe_init_state(mm);#endif} static void mmap_init_lock(struct mm_struct *mm){	init_rwsem(&mm->mmap_lock);	mm_lock_seqcount_init(mm);#ifdef CONFIG_PER_VMA_LOCK	rcuwait_init(&mm->vma_writer_wait);#endif} static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p){	mt_init_flags(&mm->mm_mt, MM_MT_FLAGS);	mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock);	atomic_set(&mm->mm_users, 1);	atomic_set(&mm->mm_count, 1);	seqcount_init(&mm->write_protect_seq);	mmap_init_lock(mm);	INIT_LIST_HEAD(&mm->mmlist);	mm_pgtables_bytes_init(mm);	mm->map_count = 0;	mm->locked_vm = 0;	atomic64_set(&mm->pinned_vm, 0);	memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));	spin_lock_init(&mm->page_table_lock);	spin_lock_init(&mm->arg_lock);	mm_init_cpumask(mm);	mm_init_aio(mm);	mm_init_owner(mm, p);	mm_pasid_init(mm);	RCU_INIT_POINTER(mm->exe_file, NULL);	mmu_notifier_subscriptions_init(mm);	init_tlb_flush_pending(mm);#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)	mm->pmd_huge_pte = NULL;#endif	mm_init_uprobes_state(mm);	hugetlb_count_init(mm);	futex_mm_init(mm); 	mm_flags_clear_all(mm);	if (current->mm) {		unsigned long flags = __mm_flags_get_word(current->mm); 		__mm_flags_overwrite_word(mm, mmf_init_legacy_flags(flags));		mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;	} else {		__mm_flags_overwrite_word(mm, coredump_filter);		mm->def_flags = 0;	} 	if (mm_alloc_pgd(mm))		goto fail_mm_init; 	if (mm_alloc_id(mm))		goto fail_noid; 	if (init_new_context(p, mm))		goto fail_nocontext; 	if (mm_alloc_cid(mm, p))		goto fail_cid; 	if (mm_alloc_sched(mm))		goto fail_sched; 	if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT,				     NR_MM_COUNTERS))		goto fail_pcpu; 	lru_gen_init_mm(mm);	return mm; fail_pcpu:	mm_destroy_sched(mm);fail_sched:	mm_destroy_cid(mm);fail_cid:	destroy_context(mm);fail_nocontext:	mm_free_id(mm);fail_noid:	mm_free_pgd(mm);fail_mm_init:	free_mm(mm);	return NULL;} /* * Allocate and initialize an mm_struct. */struct mm_struct *mm_alloc(void){	struct mm_struct *mm; 	mm = allocate_mm();	if (!mm)		return NULL; 	memset(mm, 0, sizeof(*mm));	return mm_init(mm, current);}EXPORT_SYMBOL_IF_KUNIT(mm_alloc); static inline void __mmput(struct mm_struct *mm){	VM_BUG_ON(atomic_read(&mm->mm_users)); 	uprobe_clear_state(mm);	exit_aio(mm);	ksm_exit(mm);	khugepaged_exit(mm); /* must run before exit_mmap */	exit_mmap(mm);	mm_put_huge_zero_folio(mm);	set_mm_exe_file(mm, NULL);	if (!list_empty(&mm->mmlist)) {		spin_lock(&mmlist_lock);		list_del(&mm->mmlist);		spin_unlock(&mmlist_lock);	}	if (mm->binfmt)		module_put(mm->binfmt->module);	lru_gen_del_mm(mm);	futex_hash_free(mm);	mmdrop(mm);} /* * Decrement the use count and release all resources for an mm. */void mmput(struct mm_struct *mm){	might_sleep(); 	if (atomic_dec_and_test(&mm->mm_users))		__mmput(mm);}EXPORT_SYMBOL_GPL(mmput); #if defined(CONFIG_MMU) || defined(CONFIG_FUTEX_PRIVATE_HASH)static void mmput_async_fn(struct work_struct *work){	struct mm_struct *mm = container_of(work, struct mm_struct,					    async_put_work); 	__mmput(mm);} void mmput_async(struct mm_struct *mm){	if (atomic_dec_and_test(&mm->mm_users)) {		INIT_WORK(&mm->async_put_work, mmput_async_fn);		schedule_work(&mm->async_put_work);	}}EXPORT_SYMBOL_GPL(mmput_async);#endif /** * set_mm_exe_file - change a reference to the mm's executable file * @mm: The mm to change. * @new_exe_file: The new file to use. * * This changes mm's executable file (shown as symlink /proc/[pid]/exe). * * Main users are mmput() and sys_execve(). Callers prevent concurrent * invocations: in mmput() nobody alive left, in execve it happens before * the new mm is made visible to anyone. * * Can only fail if new_exe_file != NULL. */int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file){	struct file *old_exe_file; 	/*	 * It is safe to dereference the exe_file without RCU as	 * this function is only called if nobody else can access	 * this mm -- see comment above for justification.	 */	old_exe_file = rcu_dereference_raw(mm->exe_file); 	if (new_exe_file) {		/*		 * We expect the caller (i.e., sys_execve) to already denied		 * write access, so this is unlikely to fail.		 */		if (unlikely(exe_file_deny_write_access(new_exe_file)))			return -EACCES;		get_file(new_exe_file);	}	rcu_assign_pointer(mm->exe_file, new_exe_file);	if (old_exe_file) {		exe_file_allow_write_access(old_exe_file);		fput(old_exe_file);	}	return 0;} /** * replace_mm_exe_file - replace a reference to the mm's executable file * @mm: The mm to change. * @new_exe_file: The new file to use. * * This changes mm's executable file (shown as symlink /proc/[pid]/exe). * * Main user is sys_prctl(PR_SET_MM_MAP/EXE_FILE). */int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file){	struct vm_area_struct *vma;	struct file *old_exe_file;	int ret = 0; 	/* Forbid mm->exe_file change if old file still mapped. */	old_exe_file = get_mm_exe_file(mm);	if (old_exe_file) {		VMA_ITERATOR(vmi, mm, 0);		mmap_read_lock(mm);		for_each_vma(vmi, vma) {			if (!vma->vm_file)				continue;			if (path_equal(&vma->vm_file->f_path,				       &old_exe_file->f_path)) {				ret = -EBUSY;				break;			}		}		mmap_read_unlock(mm);		fput(old_exe_file);		if (ret)			return ret;	} 	ret = exe_file_deny_write_access(new_exe_file);	if (ret)		return -EACCES;	get_file(new_exe_file); 	/* set the new file */	mmap_write_lock(mm);	old_exe_file = rcu_dereference_raw(mm->exe_file);	rcu_assign_pointer(mm->exe_file, new_exe_file);	mmap_write_unlock(mm); 	if (old_exe_file) {		exe_file_allow_write_access(old_exe_file);		fput(old_exe_file);	}	return 0;} /** * get_mm_exe_file - acquire a reference to the mm's executable file * @mm: The mm of interest. * * Returns %NULL if mm has no associated executable file. * User must release file via fput(). */struct file *get_mm_exe_file(struct mm_struct *mm){	struct file *exe_file; 	rcu_read_lock();	exe_file = get_file_rcu(&mm->exe_file);	rcu_read_unlock();	return exe_file;} /** * get_task_exe_file - acquire a reference to the task's executable file * @task: The task. * * Returns %NULL if task's mm (if any) has no associated executable file or * this is a kernel thread with borrowed mm (see the comment above get_task_mm). * User must release file via fput(). */struct file *get_task_exe_file(struct task_struct *task){	struct file *exe_file = NULL;	struct mm_struct *mm; 	if (task->flags & PF_KTHREAD)		return NULL; 	task_lock(task);	mm = task->mm;	if (mm)		exe_file = get_mm_exe_file(mm);	task_unlock(task);	return exe_file;} /** * get_task_mm - acquire a reference to the task's mm * @task: The task. * * Returns %NULL if the task has no mm.  Checks PF_KTHREAD (meaning * this kernel workthread has transiently adopted a user mm with kthread_use_mm, * to do its AIO) is not set and if so returns a reference to it, after * bumping up the use count.  User must release the mm via mmput() * after use.  Typically used by /proc and ptrace. */struct mm_struct *get_task_mm(struct task_struct *task){	struct mm_struct *mm; 	if (task->flags & PF_KTHREAD)		return NULL; 	task_lock(task);	mm = task->mm;	if (mm)		mmget(mm);	task_unlock(task);	return mm;}EXPORT_SYMBOL_GPL(get_task_mm); static bool may_access_mm(struct mm_struct *mm, struct task_struct *task, unsigned int mode){	if (mm == current->mm)		return true;	if (ptrace_may_access(task, mode))		return true;	if ((mode & PTRACE_MODE_READ) && perfmon_capable())		return true;	return false;} struct mm_struct *mm_access(struct task_struct *task, unsigned int mode){	struct mm_struct *mm;	int err; 	err =  down_read_killable(&task->signal->exec_update_lock);	if (err)		return ERR_PTR(err); 	mm = get_task_mm(task);	if (!mm) {		mm = ERR_PTR(-ESRCH);	} else if (!may_access_mm(mm, task, mode)) {		mmput(mm);		mm = ERR_PTR(-EACCES);	}	up_read(&task->signal->exec_update_lock); 	return mm;} static void complete_vfork_done(struct task_struct *tsk){	struct completion *vfork; 	task_lock(tsk);	vfork = tsk->vfork_done;	if (likely(vfork)) {		tsk->vfork_done = NULL;		complete(vfork);	}	task_unlock(tsk);} static int wait_for_vfork_done(struct task_struct *child,				struct completion *vfork){	unsigned int state = TASK_KILLABLE|TASK_FREEZABLE;	int killed; 	cgroup_enter_frozen();	killed = wait_for_completion_state(vfork, state);	cgroup_leave_frozen(false); 	if (killed) {		task_lock(child);		child->vfork_done = NULL;		task_unlock(child);	} 	put_task_struct(child);	return killed;} /* Please note the differences between mmput and mm_release. * mmput is called whenever we stop holding onto a mm_struct, * error success whatever. * * mm_release is called after a mm_struct has been removed * from the current process. * * This difference is important for error handling, when we * only half set up a mm_struct for a new process and need to restore * the old one.  Because we mmput the new mm_struct before * restoring the old one. . . * Eric Biederman 10 January 1998 */static void mm_release(struct task_struct *tsk, struct mm_struct *mm){	uprobe_free_utask(tsk); 	/* Get rid of any cached register state */	deactivate_mm(tsk, mm); 	/*	 * Signal userspace if we're not exiting with a core dump	 * because we want to leave the value intact for debugging	 * purposes.	 */	if (tsk->clear_child_tid) {		if (atomic_read(&mm->mm_users) > 1) {			/*			 * We don't check the error code - if userspace has			 * not set up a proper pointer then tough luck.			 */			put_user(0, tsk->clear_child_tid);			do_futex(tsk->clear_child_tid, FUTEX_WAKE,					1, NULL, NULL, 0, 0);		}		tsk->clear_child_tid = NULL;	} 	/*	 * All done, finally we can wake up parent and return this mm to him.	 * Also kthread_stop() uses this completion for synchronization.	 */

Lines 1001–1500 of 3409. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t