/* * linux/kernel/fork.c * * Copyright (C) 1991, 1992 Linus Torvalds */ /* * 'fork.c' contains the help-routines for the 'fork' system call * (see also entry.S and others). * Fork is rather simple, once you get the hang of it, but the memory * management can be a bitch. See 'mm/memory.c': 'copy_page_range()' */ #include <linux/slab.h>#include <linux/sched/autogroup.h>#include <linux/sched/mm.h>#include <linux/sched/coredump.h>#include <linux/sched/user.h>#include <linux/sched/numa_balancing.h>#include <linux/sched/stat.h>#include <linux/sched/task.h>#include <linux/sched/task_stack.h>#include <linux/sched/cputime.h>#include <linux/rtmutex.h>#include <linux/init.h>#include <linux/unistd.h>#include <linux/module.h>#include <linux/vmalloc.h>#include <linux/completion.h>#include <linux/personality.h>#include <linux/mempolicy.h>#include <linux/sem.h>#include <linux/file.h>#include <linux/fdtable.h>#include <linux/iocontext.h>#include <linux/key.h>#include <linux/binfmts.h>#include <linux/mman.h>#include <linux/mmu_notifier.h>#include <linux/hmm.h>#include <linux/fs.h>#include <linux/mm.h>#include <linux/vmacache.h>#include <linux/nsproxy.h>#include <linux/capability.h>#include <linux/cpu.h>#include <linux/cgroup.h>#include <linux/security.h>#include <linux/hugetlb.h>#include <linux/seccomp.h>#include <linux/swap.h>#include <linux/syscalls.h>#include <linux/jiffies.h>#include <linux/futex.h>#include <linux/compat.h>#include <linux/kthread.h>#include <linux/task_io_accounting_ops.h>#include <linux/rcupdate.h>#include <linux/ptrace.h>#include <linux/mount.h>#include <linux/audit.h>#include <linux/memcontrol.h>#include <linux/ftrace.h>#include <linux/proc_fs.h>#include <linux/profile.h>#include <linux/rmap.h>#include <linux/ksm.h>#include <linux/acct.h>#include <linux/userfaultfd_k.h>#include <linux/tsacct_kern.h>#include <linux/cn_proc.h>#include <linux/freezer.h>#include <linux/delayacct.h>#include <linux/taskstats_kern.h>#include <linux/random.h>#include <linux/tty.h>#include <linux/blkdev.h>#include <linux/fs_struct.h>#include <linux/magic.h>#include <linux/perf_event.h>#include <linux/posix-timers.h>#include <linux/user-return-notifier.h>#include <linux/oom.h>#include <linux/khugepaged.h>#include <linux/signalfd.h>#include <linux/uprobes.h>#include <linux/aio.h>#include <linux/compiler.h>#include <linux/sysctl.h>#include <linux/kcov.h>#include <linux/livepatch.h>#include <linux/thread_info.h> #include <asm/pgtable.h>#include <asm/pgalloc.h>#include <linux/uaccess.h>#include <asm/mmu_context.h>#include <asm/cacheflush.h>#include <asm/tlbflush.h> #include <trace/events/sched.h> #define CREATE_TRACE_POINTS#include <trace/events/task.h> /* * Minimum number of threads to boot the kernel */#define MIN_THREADS 20 /* * Maximum number of threads */#define MAX_THREADS FUTEX_TID_MASK /* * Protected counters by write_lock_irq(&tasklist_lock) */unsigned long total_forks; /* Handle normal Linux uptimes. */int nr_threads; /* The idle threads do not count.. */ int max_threads; /* tunable limit on nr_threads */ DEFINE_PER_CPU(unsigned long, process_counts) = 0; __cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ #ifdef CONFIG_PROVE_RCUint lockdep_tasklist_lock_is_held(void){ return lockdep_is_held(&tasklist_lock);}EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);#endif /* #ifdef CONFIG_PROVE_RCU */ int nr_processes(void){ int cpu; int total = 0; for_each_possible_cpu(cpu) total += per_cpu(process_counts, cpu); return total;} void __weak arch_release_task_struct(struct task_struct *tsk){} #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATORstatic struct kmem_cache *task_struct_cachep; static inline struct task_struct *alloc_task_struct_node(int node){ return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);} static inline void free_task_struct(struct task_struct *tsk){ kmem_cache_free(task_struct_cachep, tsk);}#endif void __weak arch_release_thread_stack(unsigned long *stack){} #ifndef CONFIG_ARCH_THREAD_STACK_ALLOCATOR /* * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a * kmemcache based allocator. */# if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) #ifdef CONFIG_VMAP_STACK/* * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB * flush. Try to minimize the number of calls by caching stacks. */#define NR_CACHED_STACKS 2static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); static int free_vm_stack_cache(unsigned int cpu){ struct vm_struct **cached_vm_stacks = per_cpu_ptr(cached_stacks, cpu); int i; for (i = 0; i < NR_CACHED_STACKS; i++) { struct vm_struct *vm_stack = cached_vm_stacks[i]; if (!vm_stack) continue; vfree(vm_stack->addr); cached_vm_stacks[i] = NULL; } return 0;}#endif static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node){#ifdef CONFIG_VMAP_STACK void *stack; int i; for (i = 0; i < NR_CACHED_STACKS; i++) { struct vm_struct *s; s = this_cpu_xchg(cached_stacks[i], NULL); if (!s) continue; #ifdef CONFIG_DEBUG_KMEMLEAK /* Clear stale pointers from reused stack. */ memset(s->addr, 0, THREAD_SIZE);#endif tsk->stack_vm_area = s; return s->addr; } stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN, VMALLOC_START, VMALLOC_END, THREADINFO_GFP, PAGE_KERNEL, 0, node, __builtin_return_address(0)); /* * We can't call find_vm_area() in interrupt context, and * free_thread_stack() can be called in interrupt context, * so cache the vm_struct. */ if (stack) tsk->stack_vm_area = find_vm_area(stack); return stack;#else struct page *page = alloc_pages_node(node, THREADINFO_GFP, THREAD_SIZE_ORDER); return page ? page_address(page) : NULL;#endif} static inline void free_thread_stack(struct task_struct *tsk){#ifdef CONFIG_VMAP_STACK if (task_stack_vm_area(tsk)) { int i; for (i = 0; i < NR_CACHED_STACKS; i++) { if (this_cpu_cmpxchg(cached_stacks[i], NULL, tsk->stack_vm_area) != NULL) continue; return; } vfree_atomic(tsk->stack); return; }#endif __free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);}# elsestatic struct kmem_cache *thread_stack_cache; static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node){ return kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);} static void free_thread_stack(struct task_struct *tsk){ kmem_cache_free(thread_stack_cache, tsk->stack);} void thread_stack_cache_init(void){ thread_stack_cache = kmem_cache_create("thread_stack", THREAD_SIZE, THREAD_SIZE, 0, NULL); BUG_ON(thread_stack_cache == NULL);}# endif#endif /* SLAB cache for signal_struct structures (tsk->signal) */static struct kmem_cache *signal_cachep; /* SLAB cache for sighand_struct structures (tsk->sighand) */struct kmem_cache *sighand_cachep; /* SLAB cache for files_struct structures (tsk->files) */struct kmem_cache *files_cachep; /* SLAB cache for fs_struct structures (tsk->fs) */struct kmem_cache *fs_cachep; /* SLAB cache for vm_area_struct structures */struct kmem_cache *vm_area_cachep; /* SLAB cache for mm_struct structures (tsk->mm) */static struct kmem_cache *mm_cachep; static void account_kernel_stack(struct task_struct *tsk, int account){ void *stack = task_stack_page(tsk); struct vm_struct *vm = task_stack_vm_area(tsk); BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0); if (vm) { int i; BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE); for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) { mod_zone_page_state(page_zone(vm->pages[i]), NR_KERNEL_STACK_KB, PAGE_SIZE / 1024 * account); } /* All stack pages belong to the same memcg. */ mod_memcg_page_state(vm->pages[0], MEMCG_KERNEL_STACK_KB, account * (THREAD_SIZE / 1024)); } else { /* * All stack pages are in the same zone and belong to the * same memcg. */ struct page *first_page = virt_to_page(stack); mod_zone_page_state(page_zone(first_page), NR_KERNEL_STACK_KB, THREAD_SIZE / 1024 * account); mod_memcg_page_state(first_page, MEMCG_KERNEL_STACK_KB, account * (THREAD_SIZE / 1024)); }} static void release_task_stack(struct task_struct *tsk){ if (WARN_ON(tsk->state != TASK_DEAD)) return; /* Better to leak the stack than to free prematurely */ account_kernel_stack(tsk, -1); arch_release_thread_stack(tsk->stack); free_thread_stack(tsk); tsk->stack = NULL;#ifdef CONFIG_VMAP_STACK tsk->stack_vm_area = NULL;#endif} #ifdef CONFIG_THREAD_INFO_IN_TASKvoid put_task_stack(struct task_struct *tsk){ if (atomic_dec_and_test(&tsk->stack_refcount)) release_task_stack(tsk);}#endif void free_task(struct task_struct *tsk){#ifndef CONFIG_THREAD_INFO_IN_TASK /* * The task is finally done with both the stack and thread_info, * so free both. */ release_task_stack(tsk);#else /* * If the task had a separate stack allocation, it should be gone * by now. */ WARN_ON_ONCE(atomic_read(&tsk->stack_refcount) != 0);#endif rt_mutex_debug_task_free(tsk); ftrace_graph_exit_task(tsk); put_seccomp_filter(tsk); arch_release_task_struct(tsk); if (tsk->flags & PF_KTHREAD) free_kthread_struct(tsk); free_task_struct(tsk);}EXPORT_SYMBOL(free_task); static inline void free_signal_struct(struct signal_struct *sig){ taskstats_tgid_free(sig); sched_autogroup_exit(sig); /* * __mmdrop is not safe to call from softirq context on x86 due to * pgd_dtor so postpone it to the async context */ if (sig->oom_mm) mmdrop_async(sig->oom_mm); kmem_cache_free(signal_cachep, sig);} static inline void put_signal_struct(struct signal_struct *sig){ if (atomic_dec_and_test(&sig->sigcnt)) free_signal_struct(sig);} void __put_task_struct(struct task_struct *tsk){ WARN_ON(!tsk->exit_state); WARN_ON(atomic_read(&tsk->usage)); WARN_ON(tsk == current); cgroup_free(tsk); task_numa_free(tsk); security_task_free(tsk); exit_creds(tsk); delayacct_tsk_free(tsk); put_signal_struct(tsk->signal); if (!profile_handoff_task(tsk)) free_task(tsk);}EXPORT_SYMBOL_GPL(__put_task_struct); void __init __weak arch_task_cache_init(void) { } /* * set_max_threads */static void set_max_threads(unsigned int max_threads_suggested){ u64 threads; /* * The number of threads shall be limited such that the thread * structures may only consume a small part of the available memory. */ if (fls64(totalram_pages) + fls64(PAGE_SIZE) > 64) threads = MAX_THREADS; else threads = div64_u64((u64) totalram_pages * (u64) PAGE_SIZE, (u64) THREAD_SIZE * 8UL); if (threads > max_threads_suggested) threads = max_threads_suggested; max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);} #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT/* Initialized by the architecture: */int arch_task_struct_size __read_mostly;#endif void __init fork_init(void){ int i;#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR#ifndef ARCH_MIN_TASKALIGN#define ARCH_MIN_TASKALIGN 0#endif int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN); /* create a slab on which task_structs can be allocated */ task_struct_cachep = kmem_cache_create("task_struct", arch_task_struct_size, align, SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT, NULL);#endif /* do the arch specific task caches init */ arch_task_cache_init(); set_max_threads(MAX_THREADS); init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; init_task.signal->rlim[RLIMIT_SIGPENDING] = init_task.signal->rlim[RLIMIT_NPROC]; for (i = 0; i < UCOUNT_COUNTS; i++) { init_user_ns.ucount_max[i] = max_threads/2; } #ifdef CONFIG_VMAP_STACK cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache", NULL, free_vm_stack_cache);#endif lockdep_init_task(&init_task);} int __weak arch_dup_task_struct(struct task_struct *dst, struct task_struct *src){ *dst = *src;Lines 1–500 of 2481. Load more
/* * linux/kernel/fork.c * * Copyright (C) 1991, 1992 Linus Torvalds */ /* * 'fork.c' contains the help-routines for the 'fork' system call * (see also entry.S and others). * Fork is rather simple, once you get the hang of it, but the memory * management can be a bitch. See 'mm/memory.c': 'copy_page_range()' */ #include <linux/slab.h> #include <linux/sched/autogroup.h> #include <linux/sched/mm.h> #include <linux/sched/coredump.h> #include <linux/sched/user.h> #include <linux/sched/numa_balancing.h> #include <linux/sched/stat.h> #include <linux/sched/task.h> #include <linux/sched/task_stack.h> #include <linux/sched/cputime.h> #include <linux/rtmutex.h> #include <linux/init.h> #include <linux/unistd.h> #include <linux/module.h> #include <linux/vmalloc.h> #include <linux/completion.h> #include <linux/personality.h> #include <linux/mempolicy.h> #include <linux/sem.h> #include <linux/file.h> #include <linux/fdtable.h> #include <linux/iocontext.h> #include <linux/key.h> #include <linux/binfmts.h> #include <linux/mman.h> #include <linux/mmu_notifier.h> #include <linux/hmm.h> #include <linux/fs.h> #include <linux/mm.h> #include <linux/vmacache.h> #include <linux/nsproxy.h> #include <linux/capability.h> #include <linux/cpu.h> #include <linux/cgroup.h> #include <linux/security.h> #include <linux/hugetlb.h> #include <linux/seccomp.h> #include <linux/swap.h> #include <linux/syscalls.h> #include <linux/jiffies.h> #include <linux/futex.h> #include <linux/compat.h> #include <linux/kthread.h> #include <linux/task_io_accounting_ops.h> #include <linux/rcupdate.h> #include <linux/ptrace.h> #include <linux/mount.h> #include <linux/audit.h> #include <linux/memcontrol.h> #include <linux/ftrace.h> #include <linux/proc_fs.h> #include <linux/profile.h> #include <linux/rmap.h> #include <linux/ksm.h> #include <linux/acct.h> #include <linux/userfaultfd_k.h> #include <linux/tsacct_kern.h> #include <linux/cn_proc.h> #include <linux/freezer.h> #include <linux/delayacct.h> #include <linux/taskstats_kern.h> #include <linux/random.h> #include <linux/tty.h> #include <linux/blkdev.h> #include <linux/fs_struct.h> #include <linux/magic.h> #include <linux/perf_event.h> #include <linux/posix-timers.h> #include <linux/user-return-notifier.h> #include <linux/oom.h> #include <linux/khugepaged.h> #include <linux/signalfd.h> #include <linux/uprobes.h> #include <linux/aio.h> #include <linux/compiler.h> #include <linux/sysctl.h> #include <linux/kcov.h> #include <linux/livepatch.h> #include <linux/thread_info.h> #include <asm/pgtable.h> #include <asm/pgalloc.h> #include <linux/uaccess.h> #include <asm/mmu_context.h> #include <asm/cacheflush.h> #include <asm/tlbflush.h> #include <trace/events/sched.h> #define CREATE_TRACE_POINTS #include <trace/events/task.h> /* * Minimum number of threads to boot the kernel */ #define MIN_THREADS 20 /* * Maximum number of threads */ #define MAX_THREADS FUTEX_TID_MASK /* * Protected counters by write_lock_irq(&tasklist_lock) */ unsigned long total_forks; /* Handle normal Linux uptimes. */ int nr_threads; /* The idle threads do not count.. */ int max_threads; /* tunable limit on nr_threads */ DEFINE_PER_CPU(unsigned long, process_counts) = 0; __cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ #ifdef CONFIG_PROVE_RCU int lockdep_tasklist_lock_is_held(void) { return lockdep_is_held(&tasklist_lock); } EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held); #endif /* #ifdef CONFIG_PROVE_RCU */ int nr_processes(void) { int cpu; int total = 0; for_each_possible_cpu(cpu) total += per_cpu(process_counts, cpu); return total; } void __weak arch_release_task_struct(struct task_struct *tsk) { } #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR static struct kmem_cache *task_struct_cachep; static inline struct task_struct *alloc_task_struct_node(int node) { return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node); } static inline void free_task_struct(struct task_struct *tsk) { kmem_cache_free(task_struct_cachep, tsk); } #endif void __weak arch_release_thread_stack(unsigned long *stack) { } #ifndef CONFIG_ARCH_THREAD_STACK_ALLOCATOR /* * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a * kmemcache based allocator. */ # if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) #ifdef CONFIG_VMAP_STACK /* * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB * flush. Try to minimize the number of calls by caching stacks. */ #define NR_CACHED_STACKS 2 static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); static int free_vm_stack_cache(unsigned int cpu) { struct vm_struct **cached_vm_stacks = per_cpu_ptr(cached_stacks, cpu); int i; for (i = 0; i < NR_CACHED_STACKS; i++) { struct vm_struct *vm_stack = cached_vm_stacks[i]; if (!vm_stack) continue; vfree(vm_stack->addr); cached_vm_stacks[i] = NULL; } return 0; } #endif static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node) { #ifdef CONFIG_VMAP_STACK void *stack; int i; for (i = 0; i < NR_CACHED_STACKS; i++) { struct vm_struct *s; s = this_cpu_xchg(cached_stacks[i], NULL); if (!s) continue; #ifdef CONFIG_DEBUG_KMEMLEAK /* Clear stale pointers from reused stack. */ memset(s->addr, 0, THREAD_SIZE); #endif tsk->stack_vm_area = s; return s->addr; } stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN, VMALLOC_START, VMALLOC_END, THREADINFO_GFP, PAGE_KERNEL, 0, node, __builtin_return_address(0)); /* * We can't call find_vm_area() in interrupt context, and * free_thread_stack() can be called in interrupt context, * so cache the vm_struct. */ if (stack) tsk->stack_vm_area = find_vm_area(stack); return stack; #else struct page *page = alloc_pages_node(node, THREADINFO_GFP, THREAD_SIZE_ORDER); return page ? page_address(page) : NULL; #endif } static inline void free_thread_stack(struct task_struct *tsk) { #ifdef CONFIG_VMAP_STACK if (task_stack_vm_area(tsk)) { int i; for (i = 0; i < NR_CACHED_STACKS; i++) { if (this_cpu_cmpxchg(cached_stacks[i], NULL, tsk->stack_vm_area) != NULL) continue; return; } vfree_atomic(tsk->stack); return; } #endif __free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER); } # else static struct kmem_cache *thread_stack_cache; static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node) { return kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node); } static void free_thread_stack(struct task_struct *tsk) { kmem_cache_free(thread_stack_cache, tsk->stack); } void thread_stack_cache_init(void) { thread_stack_cache = kmem_cache_create("thread_stack", THREAD_SIZE, THREAD_SIZE, 0, NULL); BUG_ON(thread_stack_cache == NULL); } # endif #endif /* SLAB cache for signal_struct structures (tsk->signal) */ static struct kmem_cache *signal_cachep; /* SLAB cache for sighand_struct structures (tsk->sighand) */ struct kmem_cache *sighand_cachep; /* SLAB cache for files_struct structures (tsk->files) */ struct kmem_cache *files_cachep; /* SLAB cache for fs_struct structures (tsk->fs) */ struct kmem_cache *fs_cachep; /* SLAB cache for vm_area_struct structures */ struct kmem_cache *vm_area_cachep; /* SLAB cache for mm_struct structures (tsk->mm) */ static struct kmem_cache *mm_cachep; static void account_kernel_stack(struct task_struct *tsk, int account) { void *stack = task_stack_page(tsk); struct vm_struct *vm = task_stack_vm_area(tsk); BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0); if (vm) { int i; BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE); for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) { mod_zone_page_state(page_zone(vm->pages[i]), NR_KERNEL_STACK_KB, PAGE_SIZE / 1024 * account); } /* All stack pages belong to the same memcg. */ mod_memcg_page_state(vm->pages[0], MEMCG_KERNEL_STACK_KB, account * (THREAD_SIZE / 1024)); } else { /* * All stack pages are in the same zone and belong to the * same memcg. */ struct page *first_page = virt_to_page(stack); mod_zone_page_state(page_zone(first_page), NR_KERNEL_STACK_KB, THREAD_SIZE / 1024 * account); mod_memcg_page_state(first_page, MEMCG_KERNEL_STACK_KB, account * (THREAD_SIZE / 1024)); } } static void release_task_stack(struct task_struct *tsk) { if (WARN_ON(tsk->state != TASK_DEAD)) return; /* Better to leak the stack than to free prematurely */ account_kernel_stack(tsk, -1); arch_release_thread_stack(tsk->stack); free_thread_stack(tsk); tsk->stack = NULL; #ifdef CONFIG_VMAP_STACK tsk->stack_vm_area = NULL; #endif } #ifdef CONFIG_THREAD_INFO_IN_TASK void put_task_stack(struct task_struct *tsk) { if (atomic_dec_and_test(&tsk->stack_refcount)) release_task_stack(tsk); } #endif void free_task(struct task_struct *tsk) { #ifndef CONFIG_THREAD_INFO_IN_TASK /* * The task is finally done with both the stack and thread_info, * so free both. */ release_task_stack(tsk); #else /* * If the task had a separate stack allocation, it should be gone * by now. */ WARN_ON_ONCE(atomic_read(&tsk->stack_refcount) != 0); #endif rt_mutex_debug_task_free(tsk); ftrace_graph_exit_task(tsk); put_seccomp_filter(tsk); arch_release_task_struct(tsk); if (tsk->flags & PF_KTHREAD) free_kthread_struct(tsk); free_task_struct(tsk); } EXPORT_SYMBOL(free_task); static inline void free_signal_struct(struct signal_struct *sig) { taskstats_tgid_free(sig); sched_autogroup_exit(sig); /* * __mmdrop is not safe to call from softirq context on x86 due to * pgd_dtor so postpone it to the async context */ if (sig->oom_mm) mmdrop_async(sig->oom_mm); kmem_cache_free(signal_cachep, sig); } static inline void put_signal_struct(struct signal_struct *sig) { if (atomic_dec_and_test(&sig->sigcnt)) free_signal_struct(sig); } void __put_task_struct(struct task_struct *tsk) { WARN_ON(!tsk->exit_state); WARN_ON(atomic_read(&tsk->usage)); WARN_ON(tsk == current); cgroup_free(tsk); task_numa_free(tsk); security_task_free(tsk); exit_creds(tsk); delayacct_tsk_free(tsk); put_signal_struct(tsk->signal); if (!profile_handoff_task(tsk)) free_task(tsk); } EXPORT_SYMBOL_GPL(__put_task_struct); void __init __weak arch_task_cache_init(void) { } /* * set_max_threads */ static void set_max_threads(unsigned int max_threads_suggested) { u64 threads; /* * The number of threads shall be limited such that the thread * structures may only consume a small part of the available memory. */ if (fls64(totalram_pages) + fls64(PAGE_SIZE) > 64) threads = MAX_THREADS; else threads = div64_u64((u64) totalram_pages * (u64) PAGE_SIZE, (u64) THREAD_SIZE * 8UL); if (threads > max_threads_suggested) threads = max_threads_suggested; max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS); } #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT /* Initialized by the architecture: */ int arch_task_struct_size __read_mostly; #endif void __init fork_init(void) { int i; #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR #ifndef ARCH_MIN_TASKALIGN #define ARCH_MIN_TASKALIGN 0 #endif int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN); /* create a slab on which task_structs can be allocated */ task_struct_cachep = kmem_cache_create("task_struct", arch_task_struct_size, align, SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT, NULL); #endif /* do the arch specific task caches init */ arch_task_cache_init(); set_max_threads(MAX_THREADS); init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; init_task.signal->rlim[RLIMIT_SIGPENDING] = init_task.signal->rlim[RLIMIT_NPROC]; for (i = 0; i < UCOUNT_COUNTS; i++) { init_user_ns.ucount_max[i] = max_threads/2; } #ifdef CONFIG_VMAP_STACK cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache", NULL, free_vm_stack_cache); #endif lockdep_init_task(&init_task); } int __weak arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { *dst = *src;