// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */#include <linux/bpf.h>#include <linux/btf.h>#include <linux/cacheflush.h>#include <linux/err.h>#include <linux/irq_work.h>#include "linux/filter.h"#include <linux/llist.h>#include <linux/btf_ids.h>#include <linux/vmalloc.h>#include <linux/pagemap.h>#include <asm/tlbflush.h>#include "range_tree.h" /* * bpf_arena is a sparsely populated shared memory region between bpf program and * user space process. * * For example on x86-64 the values could be: * user_vm_start 7f7d26200000 // picked by mmap() * kern_vm_start ffffc90001e69000 // picked by get_vm_area() * For user space all pointers within the arena are normal 8-byte addresses. * In this example 7f7d26200000 is the address of the first page (pgoff=0). * The bpf program will access it as: kern_vm_start + lower_32bit_of_user_ptr * (u32)7f7d26200000 -> 26200000 * hence * ffffc90001e69000 + 26200000 == ffffc90028069000 is "pgoff=0" within 4Gb * kernel memory region. * * BPF JITs generate the following code to access arena: * mov eax, eax // eax has lower 32-bit of user pointer * mov word ptr [rax + r12 + off], bx * where r12 == kern_vm_start and off is s16. * Hence allocate 4Gb + GUARD_SZ/2 on each side. * * Initially kernel vm_area and user vma are not populated. * User space can fault-in any address which will insert the page * into kernel and user vma. * bpf program can allocate a page via bpf_arena_alloc_pages() kfunc * which will insert it into kernel vm_area. * The later fault-in from user space will populate that page into user vma. */ /* number of bytes addressable by LDX/STX insn with 16-bit 'off' field */#define GUARD_SZ round_up(1ull << sizeof_field(struct bpf_insn, off) * 8, PAGE_SIZE << 1)#define KERN_VM_SZ (SZ_4G + GUARD_SZ) static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable); struct bpf_arena { struct bpf_map map; u64 user_vm_start; u64 user_vm_end; struct vm_struct *kern_vm; struct page *scratch_page; struct range_tree rt; /* protects rt and nr_pages */ rqspinlock_t spinlock; /* number of pages currently populated in the arena */ u64 nr_pages; struct list_head vma_list; /* protects vma_list */ struct mutex lock; u64 zap_gen; struct mutex zap_mutex; struct irq_work free_irq; struct work_struct free_work; struct llist_head free_spans;}; static void arena_free_worker(struct work_struct *work);static void arena_free_irq(struct irq_work *iw); struct arena_free_span { struct llist_node node; unsigned long uaddr; u32 page_cnt;}; u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena){ return arena ? (u64) (long) arena->kern_vm->addr + GUARD_SZ / 2 : 0;} u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena){ return arena ? arena->user_vm_start : 0;} /** * bpf_arena_map_kern_vm_start - kern_vm_start lookup by struct bpf_map * * @map: a BPF_MAP_TYPE_ARENA map * * Return @map's kern_vm_start. */u64 bpf_arena_map_kern_vm_start(struct bpf_map *map){ return bpf_arena_get_kern_vm_start(container_of(map, struct bpf_arena, map));} /** * bpf_prog_arena - return the bpf_map of the arena referenced by @prog * @prog: a loaded BPF program * * The verifier enforces at most one arena per program and stores it in * prog->aux->arena. Return that arena's underlying bpf_map, or NULL if * @prog does not reference an arena. */struct bpf_map *bpf_prog_arena(struct bpf_prog *prog){ struct bpf_arena *arena = prog->aux->arena; return arena ? &arena->map : NULL;} static long arena_map_peek_elem(struct bpf_map *map, void *value){ return -EOPNOTSUPP;} static long arena_map_push_elem(struct bpf_map *map, void *value, u64 flags){ return -EOPNOTSUPP;} static long arena_map_pop_elem(struct bpf_map *map, void *value){ return -EOPNOTSUPP;} static long arena_map_delete_elem(struct bpf_map *map, void *value){ return -EOPNOTSUPP;} static int arena_map_get_next_key(struct bpf_map *map, void *key, void *next_key){ return -EOPNOTSUPP;} static long compute_pgoff(struct bpf_arena *arena, long uaddr){ return (u32)(uaddr - (u32)arena->user_vm_start) >> PAGE_SHIFT;} struct apply_range_data { struct bpf_arena *arena; struct page **pages; int i;}; struct clear_range_data { struct bpf_arena *arena; struct llist_head *free_pages;}; static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data){ struct apply_range_data *d = data; struct page *page; pte_t pteval; if (!data) return 0; page = d->pages[d->i]; /* paranoia, similar to vmap_pages_pte_range() */ if (WARN_ON_ONCE(!pfn_valid(page_to_pfn(page)))) return -EINVAL; pteval = mk_pte(page, PAGE_KERNEL);#ifdef ptep_try_set /* * Kernel-fault recovery may have installed the scratch page here, and * some architectures (arm64) prohibit valid->valid PTE transitions. * Install atomically into a none slot. If scratch is present, clear it * and flush_tlb_before_set() (break-before-make) before retrying. */ while (!ptep_try_set(pte, pteval)) { pte_t old = ptep_get(pte); if (pte_none(old)) continue; if (WARN_ON_ONCE(pte_page(old) != d->arena->scratch_page)) return -EBUSY; ptep_get_and_clear(&init_mm, addr, pte); flush_tlb_before_set(addr); }#else /* * Without ptep_try_set() there is no atomic installer, but such arches * also do not wire up bpf_arena_handle_page_fault(), so no scratch page * is ever installed and the slot is always none here. */ if (unlikely(!pte_none(ptep_get(pte)))) return -EBUSY; set_pte_at(&init_mm, addr, pte, pteval);#endif d->i++; WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages + 1); return 0;} static void flush_vmap_cache(unsigned long start, unsigned long size){ flush_cache_vmap(start, start + size);} static int apply_range_clear_cb(pte_t *pte, unsigned long addr, void *data){ struct clear_range_data *d = data; pte_t old_pte; struct page *page; /* * Pairs with ptep_try_set() in the kernel-fault scratch installer. * Both sides must be atomic. */ old_pte = ptep_get_and_clear(&init_mm, addr, pte); if (pte_none(old_pte) || !pte_present(old_pte)) return 0; page = pte_page(old_pte); if (WARN_ON_ONCE(!page)) return -EINVAL; /* * Skip the per-arena scratch page. A kernel fault on an unallocated uaddr * scratches its PTE. A later bpf_arena_free_pages() over that range walks * here. Without the skip, scratch_page would be freed. */ if (page == d->arena->scratch_page) return 0; __llist_add(&page->pcp_llist, d->free_pages); WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages - 1); return 0;} static int apply_range_set_scratch_cb(pte_t *pte, unsigned long addr, void *data){ struct page *scratch_page = data; if (!pte_none(ptep_get(pte))) return 0; /* * Best-effort install. ptep_try_set() returns false only if another * installer (real allocation or concurrent fault) won the cmpxchg. * Their PTE is already valid, so the access retry succeeds. * * No flush_tlb_kernel_range() needed. Stale "not mapped" entries just * cause one extra re-fault through this same path. */ ptep_try_set(pte, mk_pte(scratch_page, PAGE_KERNEL)); return 0;} static int populate_pgtable_except_pte(struct bpf_arena *arena){ /* Populate intermediates for the recovery range (4 GiB + upper half-guard). */ return apply_to_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena), SZ_4G + GUARD_SZ / 2, apply_range_set_cb, NULL);} static struct bpf_map *arena_map_alloc(union bpf_attr *attr){ struct vm_struct *kern_vm; int numa_node = bpf_map_attr_numa_node(attr); struct bpf_arena *arena; u64 vm_range; int err = -ENOMEM; if (!bpf_jit_supports_arena()) return ERR_PTR(-EOPNOTSUPP); if (attr->key_size || attr->value_size || attr->max_entries == 0 || /* BPF_F_MMAPABLE must be set */ !(attr->map_flags & BPF_F_MMAPABLE) || /* No unsupported flags present */ (attr->map_flags & ~(BPF_F_SEGV_ON_FAULT | BPF_F_MMAPABLE | BPF_F_NO_USER_CONV))) return ERR_PTR(-EINVAL); if (attr->map_extra & ~PAGE_MASK) /* If non-zero the map_extra is an expected user VMA start address */ return ERR_PTR(-EINVAL); vm_range = (u64)attr->max_entries * PAGE_SIZE; if (vm_range > SZ_4G) return ERR_PTR(-E2BIG); if ((attr->map_extra >> 32) != ((attr->map_extra + vm_range - 1) >> 32)) /* user vma must not cross 32-bit boundary */ return ERR_PTR(-ERANGE); kern_vm = get_vm_area(KERN_VM_SZ, VM_SPARSE | VM_USERMAP); if (!kern_vm) return ERR_PTR(-ENOMEM); arena = bpf_map_area_alloc(sizeof(*arena), numa_node); if (!arena) goto err; arena->kern_vm = kern_vm; arena->user_vm_start = attr->map_extra; if (arena->user_vm_start) arena->user_vm_end = arena->user_vm_start + vm_range; INIT_LIST_HEAD(&arena->vma_list); init_llist_head(&arena->free_spans); init_irq_work(&arena->free_irq, arena_free_irq); INIT_WORK(&arena->free_work, arena_free_worker); bpf_map_init_from_attr(&arena->map, attr); err = bpf_map_alloc_pages(&arena->map, NUMA_NO_NODE, 1, &arena->scratch_page); if (err) goto err_free_arena; range_tree_init(&arena->rt); err = range_tree_set(&arena->rt, 0, attr->max_entries); if (err) goto err_free_scratch; mutex_init(&arena->lock); mutex_init(&arena->zap_mutex); raw_res_spin_lock_init(&arena->spinlock); err = populate_pgtable_except_pte(arena); if (err) goto err_destroy_rt; return &arena->map; err_destroy_rt: range_tree_destroy(&arena->rt);err_free_scratch: __free_page(arena->scratch_page);err_free_arena: bpf_map_area_free(arena);err: free_vm_area(kern_vm); return ERR_PTR(err);} static int existing_page_cb(pte_t *ptep, unsigned long addr, void *data){ struct bpf_arena *arena = data; struct page *page; pte_t pte; pte = ptep_get(ptep); if (!pte_present(pte)) /* sanity check */ return 0; page = pte_page(pte); /* * Skip the scratch page. The walk is page-table-driven, not range-tree-driven, * so it can visit scratch PTEs at uaddrs the BPF program never allocated. */ if (page == arena->scratch_page) return 0; /* * We do not update pte here: * 1. Nobody should be accessing bpf_arena's range outside of a kernel bug * 2. TLB flushing is batched or deferred. Even if we clear pte, * the TLB entries can stick around and continue to permit access to * the freed page. So it all relies on 1. */ __free_page(page); return 0;} static void arena_map_free(struct bpf_map *map){ struct bpf_arena *arena = container_of(map, struct bpf_arena, map); /* * Check that user vma-s are not around when bpf map is freed. * mmap() holds vm_file which holds bpf_map refcnt. * munmap() must have happened on vma followed by arena_vm_close() * which would clear arena->vma_list. */ if (WARN_ON_ONCE(!list_empty(&arena->vma_list))) return; /* Ensure no pending deferred frees */ irq_work_sync(&arena->free_irq); flush_work(&arena->free_work); /* * free_vm_area() calls remove_vm_area() that calls free_unmap_vmap_area(). * It unmaps everything from vmalloc area and clears pgtables. * Call apply_to_existing_page_range() first to find populated ptes and * free those pages. */ apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena), SZ_4G + GUARD_SZ / 2, existing_page_cb, arena); free_vm_area(arena->kern_vm); range_tree_destroy(&arena->rt); __free_page(arena->scratch_page); bpf_map_area_free(arena);} static void *arena_map_lookup_elem(struct bpf_map *map, void *key){ return ERR_PTR(-EINVAL);} static long arena_map_update_elem(struct bpf_map *map, void *key, void *value, u64 flags){ return -EOPNOTSUPP;} static int arena_map_check_btf(struct bpf_map *map, const struct btf *btf, const struct btf_type *key_type, const struct btf_type *value_type){ return 0;} static u64 arena_map_mem_usage(const struct bpf_map *map){ struct bpf_arena *arena = container_of(map, struct bpf_arena, map); return (u64)READ_ONCE(arena->nr_pages) << PAGE_SHIFT;} struct vma_list { struct vm_area_struct *vma; struct list_head head; refcount_t mmap_count; u64 zap_gen;}; static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma){ struct vma_list *vml; vml = kmalloc_obj(*vml); if (!vml) return -ENOMEM; refcount_set(&vml->mmap_count, 1); vma->vm_private_data = vml; vml->vma = vma; vml->zap_gen = 0; list_add(&vml->head, &arena->vma_list); return 0;} static void arena_vm_open(struct vm_area_struct *vma){ struct vma_list *vml = vma->vm_private_data; refcount_inc(&vml->mmap_count);} static int arena_vm_may_split(struct vm_area_struct *vma, unsigned long addr){ return -EINVAL;} static int arena_vm_mremap(struct vm_area_struct *vma){ return -EINVAL;} static void arena_vm_close(struct vm_area_struct *vma){ struct bpf_map *map = vma->vm_file->private_data; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); struct vma_list *vml = vma->vm_private_data; if (!refcount_dec_and_test(&vml->mmap_count)) return; guard(mutex)(&arena->lock); /* update link list under lock */ list_del(&vml->head); vma->vm_private_data = NULL; kfree(vml);} static vm_fault_t arena_vm_fault(struct vm_fault *vmf){ struct bpf_map *map = vmf->vma->vm_file->private_data; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); struct mem_cgroup *new_memcg, *old_memcg; struct page *page; long kbase, kaddr; unsigned long flags; int ret; kbase = bpf_arena_get_kern_vm_start(arena); kaddr = kbase + (u32)(vmf->address); if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) /* * A failed lock means a possible deadlock was detected. Don't * return VM_FAULT_RETRY: this handler never took mmap_lock, but * the fault path would re-take it on retry and deadlock. Fail. */ return VM_FAULT_SIGBUS; page = vmalloc_to_page((void *)kaddr);Lines 1–500 of 1204. Load more
// SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ #include <linux/bpf.h> #include <linux/btf.h> #include <linux/cacheflush.h> #include <linux/err.h> #include <linux/irq_work.h> #include "linux/filter.h" #include <linux/llist.h> #include <linux/btf_ids.h> #include <linux/vmalloc.h> #include <linux/pagemap.h> #include <asm/tlbflush.h> #include "range_tree.h" /* * bpf_arena is a sparsely populated shared memory region between bpf program and * user space process. * * For example on x86-64 the values could be: * user_vm_start 7f7d26200000 // picked by mmap() * kern_vm_start ffffc90001e69000 // picked by get_vm_area() * For user space all pointers within the arena are normal 8-byte addresses. * In this example 7f7d26200000 is the address of the first page (pgoff=0). * The bpf program will access it as: kern_vm_start + lower_32bit_of_user_ptr * (u32)7f7d26200000 -> 26200000 * hence * ffffc90001e69000 + 26200000 == ffffc90028069000 is "pgoff=0" within 4Gb * kernel memory region. * * BPF JITs generate the following code to access arena: * mov eax, eax // eax has lower 32-bit of user pointer * mov word ptr [rax + r12 + off], bx * where r12 == kern_vm_start and off is s16. * Hence allocate 4Gb + GUARD_SZ/2 on each side. * * Initially kernel vm_area and user vma are not populated. * User space can fault-in any address which will insert the page * into kernel and user vma. * bpf program can allocate a page via bpf_arena_alloc_pages() kfunc * which will insert it into kernel vm_area. * The later fault-in from user space will populate that page into user vma. */ /* number of bytes addressable by LDX/STX insn with 16-bit 'off' field */ #define GUARD_SZ round_up(1ull << sizeof_field(struct bpf_insn, off) * 8, PAGE_SIZE << 1) #define KERN_VM_SZ (SZ_4G + GUARD_SZ) static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable); struct bpf_arena { struct bpf_map map; u64 user_vm_start; u64 user_vm_end; struct vm_struct *kern_vm; struct page *scratch_page; struct range_tree rt; /* protects rt and nr_pages */ rqspinlock_t spinlock; /* number of pages currently populated in the arena */ u64 nr_pages; struct list_head vma_list; /* protects vma_list */ struct mutex lock; u64 zap_gen; struct mutex zap_mutex; struct irq_work free_irq; struct work_struct free_work; struct llist_head free_spans; }; static void arena_free_worker(struct work_struct *work); static void arena_free_irq(struct irq_work *iw); struct arena_free_span { struct llist_node node; unsigned long uaddr; u32 page_cnt; }; u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena) { return arena ? (u64) (long) arena->kern_vm->addr + GUARD_SZ / 2 : 0; } u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena) { return arena ? arena->user_vm_start : 0; } /** * bpf_arena_map_kern_vm_start - kern_vm_start lookup by struct bpf_map * * @map: a BPF_MAP_TYPE_ARENA map * * Return @map's kern_vm_start. */ u64 bpf_arena_map_kern_vm_start(struct bpf_map *map) { return bpf_arena_get_kern_vm_start(container_of(map, struct bpf_arena, map)); } /** * bpf_prog_arena - return the bpf_map of the arena referenced by @prog * @prog: a loaded BPF program * * The verifier enforces at most one arena per program and stores it in * prog->aux->arena. Return that arena's underlying bpf_map, or NULL if * @prog does not reference an arena. */ struct bpf_map *bpf_prog_arena(struct bpf_prog *prog) { struct bpf_arena *arena = prog->aux->arena; return arena ? &arena->map : NULL; } static long arena_map_peek_elem(struct bpf_map *map, void *value) { return -EOPNOTSUPP; } static long arena_map_push_elem(struct bpf_map *map, void *value, u64 flags) { return -EOPNOTSUPP; } static long arena_map_pop_elem(struct bpf_map *map, void *value) { return -EOPNOTSUPP; } static long arena_map_delete_elem(struct bpf_map *map, void *value) { return -EOPNOTSUPP; } static int arena_map_get_next_key(struct bpf_map *map, void *key, void *next_key) { return -EOPNOTSUPP; } static long compute_pgoff(struct bpf_arena *arena, long uaddr) { return (u32)(uaddr - (u32)arena->user_vm_start) >> PAGE_SHIFT; } struct apply_range_data { struct bpf_arena *arena; struct page **pages; int i; }; struct clear_range_data { struct bpf_arena *arena; struct llist_head *free_pages; }; static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) { struct apply_range_data *d = data; struct page *page; pte_t pteval; if (!data) return 0; page = d->pages[d->i]; /* paranoia, similar to vmap_pages_pte_range() */ if (WARN_ON_ONCE(!pfn_valid(page_to_pfn(page)))) return -EINVAL; pteval = mk_pte(page, PAGE_KERNEL); #ifdef ptep_try_set /* * Kernel-fault recovery may have installed the scratch page here, and * some architectures (arm64) prohibit valid->valid PTE transitions. * Install atomically into a none slot. If scratch is present, clear it * and flush_tlb_before_set() (break-before-make) before retrying. */ while (!ptep_try_set(pte, pteval)) { pte_t old = ptep_get(pte); if (pte_none(old)) continue; if (WARN_ON_ONCE(pte_page(old) != d->arena->scratch_page)) return -EBUSY; ptep_get_and_clear(&init_mm, addr, pte); flush_tlb_before_set(addr); } #else /* * Without ptep_try_set() there is no atomic installer, but such arches * also do not wire up bpf_arena_handle_page_fault(), so no scratch page * is ever installed and the slot is always none here. */ if (unlikely(!pte_none(ptep_get(pte)))) return -EBUSY; set_pte_at(&init_mm, addr, pte, pteval); #endif d->i++; WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages + 1); return 0; } static void flush_vmap_cache(unsigned long start, unsigned long size) { flush_cache_vmap(start, start + size); } static int apply_range_clear_cb(pte_t *pte, unsigned long addr, void *data) { struct clear_range_data *d = data; pte_t old_pte; struct page *page; /* * Pairs with ptep_try_set() in the kernel-fault scratch installer. * Both sides must be atomic. */ old_pte = ptep_get_and_clear(&init_mm, addr, pte); if (pte_none(old_pte) || !pte_present(old_pte)) return 0; page = pte_page(old_pte); if (WARN_ON_ONCE(!page)) return -EINVAL; /* * Skip the per-arena scratch page. A kernel fault on an unallocated uaddr * scratches its PTE. A later bpf_arena_free_pages() over that range walks * here. Without the skip, scratch_page would be freed. */ if (page == d->arena->scratch_page) return 0; __llist_add(&page->pcp_llist, d->free_pages); WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages - 1); return 0; } static int apply_range_set_scratch_cb(pte_t *pte, unsigned long addr, void *data) { struct page *scratch_page = data; if (!pte_none(ptep_get(pte))) return 0; /* * Best-effort install. ptep_try_set() returns false only if another * installer (real allocation or concurrent fault) won the cmpxchg. * Their PTE is already valid, so the access retry succeeds. * * No flush_tlb_kernel_range() needed. Stale "not mapped" entries just * cause one extra re-fault through this same path. */ ptep_try_set(pte, mk_pte(scratch_page, PAGE_KERNEL)); return 0; } static int populate_pgtable_except_pte(struct bpf_arena *arena) { /* Populate intermediates for the recovery range (4 GiB + upper half-guard). */ return apply_to_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena), SZ_4G + GUARD_SZ / 2, apply_range_set_cb, NULL); } static struct bpf_map *arena_map_alloc(union bpf_attr *attr) { struct vm_struct *kern_vm; int numa_node = bpf_map_attr_numa_node(attr); struct bpf_arena *arena; u64 vm_range; int err = -ENOMEM; if (!bpf_jit_supports_arena()) return ERR_PTR(-EOPNOTSUPP); if (attr->key_size || attr->value_size || attr->max_entries == 0 || /* BPF_F_MMAPABLE must be set */ !(attr->map_flags & BPF_F_MMAPABLE) || /* No unsupported flags present */ (attr->map_flags & ~(BPF_F_SEGV_ON_FAULT | BPF_F_MMAPABLE | BPF_F_NO_USER_CONV))) return ERR_PTR(-EINVAL); if (attr->map_extra & ~PAGE_MASK) /* If non-zero the map_extra is an expected user VMA start address */ return ERR_PTR(-EINVAL); vm_range = (u64)attr->max_entries * PAGE_SIZE; if (vm_range > SZ_4G) return ERR_PTR(-E2BIG); if ((attr->map_extra >> 32) != ((attr->map_extra + vm_range - 1) >> 32)) /* user vma must not cross 32-bit boundary */ return ERR_PTR(-ERANGE); kern_vm = get_vm_area(KERN_VM_SZ, VM_SPARSE | VM_USERMAP); if (!kern_vm) return ERR_PTR(-ENOMEM); arena = bpf_map_area_alloc(sizeof(*arena), numa_node); if (!arena) goto err; arena->kern_vm = kern_vm; arena->user_vm_start = attr->map_extra; if (arena->user_vm_start) arena->user_vm_end = arena->user_vm_start + vm_range; INIT_LIST_HEAD(&arena->vma_list); init_llist_head(&arena->free_spans); init_irq_work(&arena->free_irq, arena_free_irq); INIT_WORK(&arena->free_work, arena_free_worker); bpf_map_init_from_attr(&arena->map, attr); err = bpf_map_alloc_pages(&arena->map, NUMA_NO_NODE, 1, &arena->scratch_page); if (err) goto err_free_arena; range_tree_init(&arena->rt); err = range_tree_set(&arena->rt, 0, attr->max_entries); if (err) goto err_free_scratch; mutex_init(&arena->lock); mutex_init(&arena->zap_mutex); raw_res_spin_lock_init(&arena->spinlock); err = populate_pgtable_except_pte(arena); if (err) goto err_destroy_rt; return &arena->map; err_destroy_rt: range_tree_destroy(&arena->rt); err_free_scratch: __free_page(arena->scratch_page); err_free_arena: bpf_map_area_free(arena); err: free_vm_area(kern_vm); return ERR_PTR(err); } static int existing_page_cb(pte_t *ptep, unsigned long addr, void *data) { struct bpf_arena *arena = data; struct page *page; pte_t pte; pte = ptep_get(ptep); if (!pte_present(pte)) /* sanity check */ return 0; page = pte_page(pte); /* * Skip the scratch page. The walk is page-table-driven, not range-tree-driven, * so it can visit scratch PTEs at uaddrs the BPF program never allocated. */ if (page == arena->scratch_page) return 0; /* * We do not update pte here: * 1. Nobody should be accessing bpf_arena's range outside of a kernel bug * 2. TLB flushing is batched or deferred. Even if we clear pte, * the TLB entries can stick around and continue to permit access to * the freed page. So it all relies on 1. */ __free_page(page); return 0; } static void arena_map_free(struct bpf_map *map) { struct bpf_arena *arena = container_of(map, struct bpf_arena, map); /* * Check that user vma-s are not around when bpf map is freed. * mmap() holds vm_file which holds bpf_map refcnt. * munmap() must have happened on vma followed by arena_vm_close() * which would clear arena->vma_list. */ if (WARN_ON_ONCE(!list_empty(&arena->vma_list))) return; /* Ensure no pending deferred frees */ irq_work_sync(&arena->free_irq); flush_work(&arena->free_work); /* * free_vm_area() calls remove_vm_area() that calls free_unmap_vmap_area(). * It unmaps everything from vmalloc area and clears pgtables. * Call apply_to_existing_page_range() first to find populated ptes and * free those pages. */ apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena), SZ_4G + GUARD_SZ / 2, existing_page_cb, arena); free_vm_area(arena->kern_vm); range_tree_destroy(&arena->rt); __free_page(arena->scratch_page); bpf_map_area_free(arena); } static void *arena_map_lookup_elem(struct bpf_map *map, void *key) { return ERR_PTR(-EINVAL); } static long arena_map_update_elem(struct bpf_map *map, void *key, void *value, u64 flags) { return -EOPNOTSUPP; } static int arena_map_check_btf(struct bpf_map *map, const struct btf *btf, const struct btf_type *key_type, const struct btf_type *value_type) { return 0; } static u64 arena_map_mem_usage(const struct bpf_map *map) { struct bpf_arena *arena = container_of(map, struct bpf_arena, map); return (u64)READ_ONCE(arena->nr_pages) << PAGE_SHIFT; } struct vma_list { struct vm_area_struct *vma; struct list_head head; refcount_t mmap_count; u64 zap_gen; }; static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma) { struct vma_list *vml; vml = kmalloc_obj(*vml); if (!vml) return -ENOMEM; refcount_set(&vml->mmap_count, 1); vma->vm_private_data = vml; vml->vma = vma; vml->zap_gen = 0; list_add(&vml->head, &arena->vma_list); return 0; } static void arena_vm_open(struct vm_area_struct *vma) { struct vma_list *vml = vma->vm_private_data; refcount_inc(&vml->mmap_count); } static int arena_vm_may_split(struct vm_area_struct *vma, unsigned long addr) { return -EINVAL; } static int arena_vm_mremap(struct vm_area_struct *vma) { return -EINVAL; } static void arena_vm_close(struct vm_area_struct *vma) { struct bpf_map *map = vma->vm_file->private_data; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); struct vma_list *vml = vma->vm_private_data; if (!refcount_dec_and_test(&vml->mmap_count)) return; guard(mutex)(&arena->lock); /* update link list under lock */ list_del(&vml->head); vma->vm_private_data = NULL; kfree(vml); } static vm_fault_t arena_vm_fault(struct vm_fault *vmf) { struct bpf_map *map = vmf->vma->vm_file->private_data; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); struct mem_cgroup *new_memcg, *old_memcg; struct page *page; long kbase, kaddr; unsigned long flags; int ret; kbase = bpf_arena_get_kern_vm_start(arena); kaddr = kbase + (u32)(vmf->address); if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) /* * A failed lock means a possible deadlock was detected. Don't * return VM_FAULT_RETRY: this handler never took mmap_lock, but * the fault path would re-take it on retry and deadlock. Fail. */ return VM_FAULT_SIGBUS; page = vmalloc_to_page((void *)kaddr);