// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2020 Facebook */ #include <linux/init.h>#include <linux/namei.h>#include <linux/pid_namespace.h>#include <linux/fs.h>#include <linux/filter.h>#include <linux/bpf_mem_alloc.h>#include <linux/btf_ids.h>#include <linux/mm_types.h>#include <linux/mmap_lock.h>#include <linux/sched/mm.h>#include "mmap_unlock_work.h" static const char * const iter_task_type_names[] = { "ALL", "TID", "PID",}; struct bpf_iter_seq_task_common { struct pid_namespace *ns; enum bpf_iter_task_type type; u32 pid; u32 pid_visiting;}; struct bpf_iter_seq_task_info { /* The first field must be struct bpf_iter_seq_task_common. * this is assumed by {init, fini}_seq_pidns() callback functions. */ struct bpf_iter_seq_task_common common; u32 tid;}; static struct task_struct *task_group_seq_get_next(struct bpf_iter_seq_task_common *common, u32 *tid, bool skip_if_dup_files){ struct task_struct *task; struct pid *pid; u32 next_tid; if (!*tid) { /* The first time, the iterator calls this function. */ pid = find_pid_ns(common->pid, common->ns); task = get_pid_task(pid, PIDTYPE_TGID); if (!task) return NULL; *tid = common->pid; common->pid_visiting = common->pid; return task; } /* If the control returns to user space and comes back to the * kernel again, *tid and common->pid_visiting should be the * same for task_seq_start() to pick up the correct task. */ if (*tid == common->pid_visiting) { pid = find_pid_ns(common->pid_visiting, common->ns); task = get_pid_task(pid, PIDTYPE_PID); return task; } task = find_task_by_pid_ns(common->pid_visiting, common->ns); if (!task) return NULL; retry: task = __next_thread(task); if (!task) return NULL; next_tid = __task_pid_nr_ns(task, PIDTYPE_PID, common->ns); if (!next_tid) goto retry; if (skip_if_dup_files && task->files == task->group_leader->files) goto retry; *tid = common->pid_visiting = next_tid; get_task_struct(task); return task;} static struct task_struct *task_seq_get_next(struct bpf_iter_seq_task_common *common, u32 *tid, bool skip_if_dup_files){ struct task_struct *task = NULL; struct pid *pid; if (common->type == BPF_TASK_ITER_TID) { if (*tid && *tid != common->pid) return NULL; rcu_read_lock(); pid = find_pid_ns(common->pid, common->ns); if (pid) { task = get_pid_task(pid, PIDTYPE_PID); *tid = common->pid; } rcu_read_unlock(); return task; } if (common->type == BPF_TASK_ITER_TGID) { rcu_read_lock(); task = task_group_seq_get_next(common, tid, skip_if_dup_files); rcu_read_unlock(); return task; } rcu_read_lock();retry: pid = find_ge_pid(*tid, common->ns); if (pid) { *tid = pid_nr_ns(pid, common->ns); task = get_pid_task(pid, PIDTYPE_PID); if (!task) { ++*tid; goto retry; } else if (skip_if_dup_files && !thread_group_leader(task) && task->files == task->group_leader->files) { put_task_struct(task); task = NULL; ++*tid; goto retry; } } rcu_read_unlock(); return task;} static void *task_seq_start(struct seq_file *seq, loff_t *pos){ struct bpf_iter_seq_task_info *info = seq->private; struct task_struct *task; task = task_seq_get_next(&info->common, &info->tid, false); if (!task) return NULL; if (*pos == 0) ++*pos; return task;} static void *task_seq_next(struct seq_file *seq, void *v, loff_t *pos){ struct bpf_iter_seq_task_info *info = seq->private; struct task_struct *task; ++*pos; ++info->tid; put_task_struct((struct task_struct *)v); task = task_seq_get_next(&info->common, &info->tid, false); if (!task) return NULL; return task;} struct bpf_iter__task { __bpf_md_ptr(struct bpf_iter_meta *, meta); __bpf_md_ptr(struct task_struct *, task);}; DEFINE_BPF_ITER_FUNC(task, struct bpf_iter_meta *meta, struct task_struct *task) static int __task_seq_show(struct seq_file *seq, struct task_struct *task, bool in_stop){ struct bpf_iter_meta meta; struct bpf_iter__task ctx; struct bpf_prog *prog; meta.seq = seq; prog = bpf_iter_get_info(&meta, in_stop); if (!prog) return 0; ctx.meta = &meta; ctx.task = task; return bpf_iter_run_prog(prog, &ctx);} static int task_seq_show(struct seq_file *seq, void *v){ return __task_seq_show(seq, v, false);} static void task_seq_stop(struct seq_file *seq, void *v){ if (!v) (void)__task_seq_show(seq, v, true); else put_task_struct((struct task_struct *)v);} static int bpf_iter_attach_task(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux){ unsigned int flags; struct pid *pid; pid_t tgid; if ((!!linfo->task.tid + !!linfo->task.pid + !!linfo->task.pid_fd) > 1) return -EINVAL; aux->task.type = BPF_TASK_ITER_ALL; if (linfo->task.tid != 0) { aux->task.type = BPF_TASK_ITER_TID; aux->task.pid = linfo->task.tid; } if (linfo->task.pid != 0) { aux->task.type = BPF_TASK_ITER_TGID; aux->task.pid = linfo->task.pid; } if (linfo->task.pid_fd != 0) { aux->task.type = BPF_TASK_ITER_TGID; pid = pidfd_get_pid(linfo->task.pid_fd, &flags); if (IS_ERR(pid)) return PTR_ERR(pid); tgid = pid_nr_ns(pid, task_active_pid_ns(current)); aux->task.pid = tgid; put_pid(pid); } return 0;} static const struct seq_operations task_seq_ops = { .start = task_seq_start, .next = task_seq_next, .stop = task_seq_stop, .show = task_seq_show,}; struct bpf_iter_seq_task_file_info { /* The first field must be struct bpf_iter_seq_task_common. * this is assumed by {init, fini}_seq_pidns() callback functions. */ struct bpf_iter_seq_task_common common; struct task_struct *task; u32 tid; u32 fd;}; static struct file *task_file_seq_get_next(struct bpf_iter_seq_task_file_info *info){ u32 saved_tid = info->tid; struct task_struct *curr_task; unsigned int curr_fd = info->fd; struct file *f; /* If this function returns a non-NULL file object, * it held a reference to the task/file. * Otherwise, it does not hold any reference. */again: if (info->task) { curr_task = info->task; curr_fd = info->fd; } else { curr_task = task_seq_get_next(&info->common, &info->tid, true); if (!curr_task) { info->task = NULL; return NULL; } /* set info->task */ info->task = curr_task; if (saved_tid == info->tid) curr_fd = info->fd; else curr_fd = 0; } f = fget_task_next(curr_task, &curr_fd); if (f) { /* set info->fd */ info->fd = curr_fd; return f; } /* the current task is done, go to the next task */ put_task_struct(curr_task); if (info->common.type == BPF_TASK_ITER_TID) { info->task = NULL; return NULL; } info->task = NULL; info->fd = 0; saved_tid = ++(info->tid); goto again;} static void *task_file_seq_start(struct seq_file *seq, loff_t *pos){ struct bpf_iter_seq_task_file_info *info = seq->private; struct file *file; info->task = NULL; file = task_file_seq_get_next(info); if (file && *pos == 0) ++*pos; return file;} static void *task_file_seq_next(struct seq_file *seq, void *v, loff_t *pos){ struct bpf_iter_seq_task_file_info *info = seq->private; ++*pos; ++info->fd; fput((struct file *)v); return task_file_seq_get_next(info);} struct bpf_iter__task_file { __bpf_md_ptr(struct bpf_iter_meta *, meta); __bpf_md_ptr(struct task_struct *, task); u32 fd __aligned(8); __bpf_md_ptr(struct file *, file);}; DEFINE_BPF_ITER_FUNC(task_file, struct bpf_iter_meta *meta, struct task_struct *task, u32 fd, struct file *file) static int __task_file_seq_show(struct seq_file *seq, struct file *file, bool in_stop){ struct bpf_iter_seq_task_file_info *info = seq->private; struct bpf_iter__task_file ctx; struct bpf_iter_meta meta; struct bpf_prog *prog; meta.seq = seq; prog = bpf_iter_get_info(&meta, in_stop); if (!prog) return 0; ctx.meta = &meta; ctx.task = info->task; ctx.fd = info->fd; ctx.file = file; return bpf_iter_run_prog(prog, &ctx);} static int task_file_seq_show(struct seq_file *seq, void *v){ return __task_file_seq_show(seq, v, false);} static void task_file_seq_stop(struct seq_file *seq, void *v){ struct bpf_iter_seq_task_file_info *info = seq->private; if (!v) { (void)__task_file_seq_show(seq, v, true); } else { fput((struct file *)v); put_task_struct(info->task); info->task = NULL; }} static int init_seq_pidns(void *priv_data, struct bpf_iter_aux_info *aux){ struct bpf_iter_seq_task_common *common = priv_data; common->ns = get_pid_ns(task_active_pid_ns(current)); common->type = aux->task.type; common->pid = aux->task.pid; return 0;} static void fini_seq_pidns(void *priv_data){ struct bpf_iter_seq_task_common *common = priv_data; put_pid_ns(common->ns);} static const struct seq_operations task_file_seq_ops = { .start = task_file_seq_start, .next = task_file_seq_next, .stop = task_file_seq_stop, .show = task_file_seq_show,}; struct bpf_iter_seq_task_vma_info { /* The first field must be struct bpf_iter_seq_task_common. * this is assumed by {init, fini}_seq_pidns() callback functions. */ struct bpf_iter_seq_task_common common; struct task_struct *task; struct mm_struct *mm; struct vm_area_struct *vma; u32 tid; unsigned long prev_vm_start; unsigned long prev_vm_end;}; enum bpf_task_vma_iter_find_op { task_vma_iter_first_vma, /* use find_vma() with addr 0 */ task_vma_iter_next_vma, /* use vma_next() with curr_vma */ task_vma_iter_find_vma, /* use find_vma() to find next vma */}; static struct vm_area_struct *task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info){ enum bpf_task_vma_iter_find_op op; struct vm_area_struct *curr_vma; struct task_struct *curr_task; struct mm_struct *curr_mm; u32 saved_tid = info->tid; /* If this function returns a non-NULL vma, it holds a reference to * the task_struct, holds a refcount on mm->mm_users, and holds * read lock on vma->mm->mmap_lock. * If this function returns NULL, it does not hold any reference or * lock. */ if (info->task) { curr_task = info->task; curr_vma = info->vma; curr_mm = info->mm; /* In case of lock contention, drop mmap_lock to unblock * the writer. * * After relock, call find(mm, prev_vm_end - 1) to find * new vma to process. * * +------+------+-----------+ * | VMA1 | VMA2 | VMA3 | * +------+------+-----------+ * | | | | * 4k 8k 16k 400k * * For example, curr_vma == VMA2. Before unlock, we set * * prev_vm_start = 8k * prev_vm_end = 16k * * There are a few cases: * * 1) VMA2 is freed, but VMA3 exists. * * find_vma() will return VMA3, just process VMA3. * * 2) VMA2 still exists. * * find_vma() will return VMA2, process VMA2->next. * * 3) no more vma in this mm. * * Process the next task. * * 4) find_vma() returns a different vma, VMA2'. * * 4.1) If VMA2 covers same range as VMA2', skip VMA2', * because we already covered the range; * 4.2) VMA2 and VMA2' covers different ranges, process * VMA2'. */ if (mmap_lock_is_contended(curr_mm)) { info->prev_vm_start = curr_vma->vm_start; info->prev_vm_end = curr_vma->vm_end; op = task_vma_iter_find_vma; mmap_read_unlock(curr_mm); if (mmap_read_lock_killable(curr_mm)) { mmput(curr_mm); goto finish; } } else { op = task_vma_iter_next_vma; } } else {again: curr_task = task_seq_get_next(&info->common, &info->tid, true); if (!curr_task) { info->tid++;Lines 1–500 of 1191. Load more
// SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2020 Facebook */ #include <linux/init.h> #include <linux/namei.h> #include <linux/pid_namespace.h> #include <linux/fs.h> #include <linux/filter.h> #include <linux/bpf_mem_alloc.h> #include <linux/btf_ids.h> #include <linux/mm_types.h> #include <linux/mmap_lock.h> #include <linux/sched/mm.h> #include "mmap_unlock_work.h" static const char * const iter_task_type_names[] = { "ALL", "TID", "PID", }; struct bpf_iter_seq_task_common { struct pid_namespace *ns; enum bpf_iter_task_type type; u32 pid; u32 pid_visiting; }; struct bpf_iter_seq_task_info { /* The first field must be struct bpf_iter_seq_task_common. * this is assumed by {init, fini}_seq_pidns() callback functions. */ struct bpf_iter_seq_task_common common; u32 tid; }; static struct task_struct *task_group_seq_get_next(struct bpf_iter_seq_task_common *common, u32 *tid, bool skip_if_dup_files) { struct task_struct *task; struct pid *pid; u32 next_tid; if (!*tid) { /* The first time, the iterator calls this function. */ pid = find_pid_ns(common->pid, common->ns); task = get_pid_task(pid, PIDTYPE_TGID); if (!task) return NULL; *tid = common->pid; common->pid_visiting = common->pid; return task; } /* If the control returns to user space and comes back to the * kernel again, *tid and common->pid_visiting should be the * same for task_seq_start() to pick up the correct task. */ if (*tid == common->pid_visiting) { pid = find_pid_ns(common->pid_visiting, common->ns); task = get_pid_task(pid, PIDTYPE_PID); return task; } task = find_task_by_pid_ns(common->pid_visiting, common->ns); if (!task) return NULL; retry: task = __next_thread(task); if (!task) return NULL; next_tid = __task_pid_nr_ns(task, PIDTYPE_PID, common->ns); if (!next_tid) goto retry; if (skip_if_dup_files && task->files == task->group_leader->files) goto retry; *tid = common->pid_visiting = next_tid; get_task_struct(task); return task; } static struct task_struct *task_seq_get_next(struct bpf_iter_seq_task_common *common, u32 *tid, bool skip_if_dup_files) { struct task_struct *task = NULL; struct pid *pid; if (common->type == BPF_TASK_ITER_TID) { if (*tid && *tid != common->pid) return NULL; rcu_read_lock(); pid = find_pid_ns(common->pid, common->ns); if (pid) { task = get_pid_task(pid, PIDTYPE_PID); *tid = common->pid; } rcu_read_unlock(); return task; } if (common->type == BPF_TASK_ITER_TGID) { rcu_read_lock(); task = task_group_seq_get_next(common, tid, skip_if_dup_files); rcu_read_unlock(); return task; } rcu_read_lock(); retry: pid = find_ge_pid(*tid, common->ns); if (pid) { *tid = pid_nr_ns(pid, common->ns); task = get_pid_task(pid, PIDTYPE_PID); if (!task) { ++*tid; goto retry; } else if (skip_if_dup_files && !thread_group_leader(task) && task->files == task->group_leader->files) { put_task_struct(task); task = NULL; ++*tid; goto retry; } } rcu_read_unlock(); return task; } static void *task_seq_start(struct seq_file *seq, loff_t *pos) { struct bpf_iter_seq_task_info *info = seq->private; struct task_struct *task; task = task_seq_get_next(&info->common, &info->tid, false); if (!task) return NULL; if (*pos == 0) ++*pos; return task; } static void *task_seq_next(struct seq_file *seq, void *v, loff_t *pos) { struct bpf_iter_seq_task_info *info = seq->private; struct task_struct *task; ++*pos; ++info->tid; put_task_struct((struct task_struct *)v); task = task_seq_get_next(&info->common, &info->tid, false); if (!task) return NULL; return task; } struct bpf_iter__task { __bpf_md_ptr(struct bpf_iter_meta *, meta); __bpf_md_ptr(struct task_struct *, task); }; DEFINE_BPF_ITER_FUNC(task, struct bpf_iter_meta *meta, struct task_struct *task) static int __task_seq_show(struct seq_file *seq, struct task_struct *task, bool in_stop) { struct bpf_iter_meta meta; struct bpf_iter__task ctx; struct bpf_prog *prog; meta.seq = seq; prog = bpf_iter_get_info(&meta, in_stop); if (!prog) return 0; ctx.meta = &meta; ctx.task = task; return bpf_iter_run_prog(prog, &ctx); } static int task_seq_show(struct seq_file *seq, void *v) { return __task_seq_show(seq, v, false); } static void task_seq_stop(struct seq_file *seq, void *v) { if (!v) (void)__task_seq_show(seq, v, true); else put_task_struct((struct task_struct *)v); } static int bpf_iter_attach_task(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux) { unsigned int flags; struct pid *pid; pid_t tgid; if ((!!linfo->task.tid + !!linfo->task.pid + !!linfo->task.pid_fd) > 1) return -EINVAL; aux->task.type = BPF_TASK_ITER_ALL; if (linfo->task.tid != 0) { aux->task.type = BPF_TASK_ITER_TID; aux->task.pid = linfo->task.tid; } if (linfo->task.pid != 0) { aux->task.type = BPF_TASK_ITER_TGID; aux->task.pid = linfo->task.pid; } if (linfo->task.pid_fd != 0) { aux->task.type = BPF_TASK_ITER_TGID; pid = pidfd_get_pid(linfo->task.pid_fd, &flags); if (IS_ERR(pid)) return PTR_ERR(pid); tgid = pid_nr_ns(pid, task_active_pid_ns(current)); aux->task.pid = tgid; put_pid(pid); } return 0; } static const struct seq_operations task_seq_ops = { .start = task_seq_start, .next = task_seq_next, .stop = task_seq_stop, .show = task_seq_show, }; struct bpf_iter_seq_task_file_info { /* The first field must be struct bpf_iter_seq_task_common. * this is assumed by {init, fini}_seq_pidns() callback functions. */ struct bpf_iter_seq_task_common common; struct task_struct *task; u32 tid; u32 fd; }; static struct file * task_file_seq_get_next(struct bpf_iter_seq_task_file_info *info) { u32 saved_tid = info->tid; struct task_struct *curr_task; unsigned int curr_fd = info->fd; struct file *f; /* If this function returns a non-NULL file object, * it held a reference to the task/file. * Otherwise, it does not hold any reference. */ again: if (info->task) { curr_task = info->task; curr_fd = info->fd; } else { curr_task = task_seq_get_next(&info->common, &info->tid, true); if (!curr_task) { info->task = NULL; return NULL; } /* set info->task */ info->task = curr_task; if (saved_tid == info->tid) curr_fd = info->fd; else curr_fd = 0; } f = fget_task_next(curr_task, &curr_fd); if (f) { /* set info->fd */ info->fd = curr_fd; return f; } /* the current task is done, go to the next task */ put_task_struct(curr_task); if (info->common.type == BPF_TASK_ITER_TID) { info->task = NULL; return NULL; } info->task = NULL; info->fd = 0; saved_tid = ++(info->tid); goto again; } static void *task_file_seq_start(struct seq_file *seq, loff_t *pos) { struct bpf_iter_seq_task_file_info *info = seq->private; struct file *file; info->task = NULL; file = task_file_seq_get_next(info); if (file && *pos == 0) ++*pos; return file; } static void *task_file_seq_next(struct seq_file *seq, void *v, loff_t *pos) { struct bpf_iter_seq_task_file_info *info = seq->private; ++*pos; ++info->fd; fput((struct file *)v); return task_file_seq_get_next(info); } struct bpf_iter__task_file { __bpf_md_ptr(struct bpf_iter_meta *, meta); __bpf_md_ptr(struct task_struct *, task); u32 fd __aligned(8); __bpf_md_ptr(struct file *, file); }; DEFINE_BPF_ITER_FUNC(task_file, struct bpf_iter_meta *meta, struct task_struct *task, u32 fd, struct file *file) static int __task_file_seq_show(struct seq_file *seq, struct file *file, bool in_stop) { struct bpf_iter_seq_task_file_info *info = seq->private; struct bpf_iter__task_file ctx; struct bpf_iter_meta meta; struct bpf_prog *prog; meta.seq = seq; prog = bpf_iter_get_info(&meta, in_stop); if (!prog) return 0; ctx.meta = &meta; ctx.task = info->task; ctx.fd = info->fd; ctx.file = file; return bpf_iter_run_prog(prog, &ctx); } static int task_file_seq_show(struct seq_file *seq, void *v) { return __task_file_seq_show(seq, v, false); } static void task_file_seq_stop(struct seq_file *seq, void *v) { struct bpf_iter_seq_task_file_info *info = seq->private; if (!v) { (void)__task_file_seq_show(seq, v, true); } else { fput((struct file *)v); put_task_struct(info->task); info->task = NULL; } } static int init_seq_pidns(void *priv_data, struct bpf_iter_aux_info *aux) { struct bpf_iter_seq_task_common *common = priv_data; common->ns = get_pid_ns(task_active_pid_ns(current)); common->type = aux->task.type; common->pid = aux->task.pid; return 0; } static void fini_seq_pidns(void *priv_data) { struct bpf_iter_seq_task_common *common = priv_data; put_pid_ns(common->ns); } static const struct seq_operations task_file_seq_ops = { .start = task_file_seq_start, .next = task_file_seq_next, .stop = task_file_seq_stop, .show = task_file_seq_show, }; struct bpf_iter_seq_task_vma_info { /* The first field must be struct bpf_iter_seq_task_common. * this is assumed by {init, fini}_seq_pidns() callback functions. */ struct bpf_iter_seq_task_common common; struct task_struct *task; struct mm_struct *mm; struct vm_area_struct *vma; u32 tid; unsigned long prev_vm_start; unsigned long prev_vm_end; }; enum bpf_task_vma_iter_find_op { task_vma_iter_first_vma, /* use find_vma() with addr 0 */ task_vma_iter_next_vma, /* use vma_next() with curr_vma */ task_vma_iter_find_vma, /* use find_vma() to find next vma */ }; static struct vm_area_struct * task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info) { enum bpf_task_vma_iter_find_op op; struct vm_area_struct *curr_vma; struct task_struct *curr_task; struct mm_struct *curr_mm; u32 saved_tid = info->tid; /* If this function returns a non-NULL vma, it holds a reference to * the task_struct, holds a refcount on mm->mm_users, and holds * read lock on vma->mm->mmap_lock. * If this function returns NULL, it does not hold any reference or * lock. */ if (info->task) { curr_task = info->task; curr_vma = info->vma; curr_mm = info->mm; /* In case of lock contention, drop mmap_lock to unblock * the writer. * * After relock, call find(mm, prev_vm_end - 1) to find * new vma to process. * * +------+------+-----------+ * | VMA1 | VMA2 | VMA3 | * +------+------+-----------+ * | | | | * 4k 8k 16k 400k * * For example, curr_vma == VMA2. Before unlock, we set * * prev_vm_start = 8k * prev_vm_end = 16k * * There are a few cases: * * 1) VMA2 is freed, but VMA3 exists. * * find_vma() will return VMA3, just process VMA3. * * 2) VMA2 still exists. * * find_vma() will return VMA2, process VMA2->next. * * 3) no more vma in this mm. * * Process the next task. * * 4) find_vma() returns a different vma, VMA2'. * * 4.1) If VMA2 covers same range as VMA2', skip VMA2', * because we already covered the range; * 4.2) VMA2 and VMA2' covers different ranges, process * VMA2'. */ if (mmap_lock_is_contended(curr_mm)) { info->prev_vm_start = curr_vma->vm_start; info->prev_vm_end = curr_vma->vm_end; op = task_vma_iter_find_vma; mmap_read_unlock(curr_mm); if (mmap_read_lock_killable(curr_mm)) { mmput(curr_mm); goto finish; } } else { op = task_vma_iter_next_vma; } } else { again: curr_task = task_seq_get_next(&info->common, &info->tid, true); if (!curr_task) { info->tid++;