.map_direct_value_addr = array_map_direct_value_addr,	.map_direct_value_meta = array_map_direct_value_meta,	.map_mmap = array_map_mmap,	.map_seq_show_elem = array_map_seq_show_elem,	.map_check_btf = array_map_check_btf,	.map_lookup_batch = generic_map_lookup_batch,	.map_update_batch = generic_map_update_batch,}; const struct bpf_map_ops percpu_array_map_ops = {	.map_alloc_check = array_map_alloc_check,	.map_alloc = array_map_alloc,	.map_free = array_map_free,	.map_get_next_key = array_map_get_next_key,	.map_lookup_elem = percpu_array_map_lookup_elem,	.map_update_elem = array_map_update_elem,	.map_delete_elem = array_map_delete_elem,	.map_seq_show_elem = percpu_array_map_seq_show_elem,	.map_check_btf = array_map_check_btf,}; static int fd_array_map_alloc_check(union bpf_attr *attr){	/* only file descriptors can be stored in this type of map */	if (attr->value_size != sizeof(u32))		return -EINVAL;	/* Program read-only/write-only not supported for special maps yet. */	if (attr->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG))		return -EINVAL;	return array_map_alloc_check(attr);} static void fd_array_map_free(struct bpf_map *map){	struct bpf_array *array = container_of(map, struct bpf_array, map);	int i; 	synchronize_rcu(); 	/* make sure it's empty */	for (i = 0; i < array->map.max_entries; i++)		BUG_ON(array->ptrs[i] != NULL); 	bpf_map_area_free(array);} static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key){	return ERR_PTR(-EOPNOTSUPP);} /* only called from syscall */int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value){	void **elem, *ptr;	int ret =  0; 	if (!map->ops->map_fd_sys_lookup_elem)		return -ENOTSUPP; 	rcu_read_lock();	elem = array_map_lookup_elem(map, key);	if (elem && (ptr = READ_ONCE(*elem)))		*value = map->ops->map_fd_sys_lookup_elem(ptr);	else		ret = -ENOENT;	rcu_read_unlock(); 	return ret;} /* only called from syscall */int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,				 void *key, void *value, u64 map_flags){	struct bpf_array *array = container_of(map, struct bpf_array, map);	void *new_ptr, *old_ptr;	u32 index = *(u32 *)key, ufd; 	if (map_flags != BPF_ANY)		return -EINVAL; 	if (index >= array->map.max_entries)		return -E2BIG; 	ufd = *(u32 *)value;	new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd);	if (IS_ERR(new_ptr))		return PTR_ERR(new_ptr); 	if (map->ops->map_poke_run) {		mutex_lock(&array->aux->poke_mutex);		old_ptr = xchg(array->ptrs + index, new_ptr);		map->ops->map_poke_run(map, index, old_ptr, new_ptr);		mutex_unlock(&array->aux->poke_mutex);	} else {		old_ptr = xchg(array->ptrs + index, new_ptr);	} 	if (old_ptr)		map->ops->map_fd_put_ptr(old_ptr);	return 0;} static int fd_array_map_delete_elem(struct bpf_map *map, void *key){	struct bpf_array *array = container_of(map, struct bpf_array, map);	void *old_ptr;	u32 index = *(u32 *)key; 	if (index >= array->map.max_entries)		return -E2BIG; 	if (map->ops->map_poke_run) {		mutex_lock(&array->aux->poke_mutex);		old_ptr = xchg(array->ptrs + index, NULL);		map->ops->map_poke_run(map, index, old_ptr, NULL);		mutex_unlock(&array->aux->poke_mutex);	} else {		old_ptr = xchg(array->ptrs + index, NULL);	} 	if (old_ptr) {		map->ops->map_fd_put_ptr(old_ptr);		return 0;	} else {		return -ENOENT;	}} static void *prog_fd_array_get_ptr(struct bpf_map *map,				   struct file *map_file, int fd){	struct bpf_array *array = container_of(map, struct bpf_array, map);	struct bpf_prog *prog = bpf_prog_get(fd); 	if (IS_ERR(prog))		return prog; 	if (!bpf_prog_array_compatible(array, prog)) {		bpf_prog_put(prog);		return ERR_PTR(-EINVAL);	} 	return prog;} static void prog_fd_array_put_ptr(void *ptr){	bpf_prog_put(ptr);} static u32 prog_fd_array_sys_lookup_elem(void *ptr){	return ((struct bpf_prog *)ptr)->aux->id;} /* decrement refcnt of all bpf_progs that are stored in this map */static void bpf_fd_array_map_clear(struct bpf_map *map){	struct bpf_array *array = container_of(map, struct bpf_array, map);	int i; 	for (i = 0; i < array->map.max_entries; i++)		fd_array_map_delete_elem(map, &i);} static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key,					 struct seq_file *m){	void **elem, *ptr;	u32 prog_id; 	rcu_read_lock(); 	elem = array_map_lookup_elem(map, key);	if (elem) {		ptr = READ_ONCE(*elem);		if (ptr) {			seq_printf(m, "%u: ", *(u32 *)key);			prog_id = prog_fd_array_sys_lookup_elem(ptr);			btf_type_seq_show(map->btf, map->btf_value_type_id,					  &prog_id, m);			seq_puts(m, "\n");		}	} 	rcu_read_unlock();} struct prog_poke_elem {	struct list_head list;	struct bpf_prog_aux *aux;}; static int prog_array_map_poke_track(struct bpf_map *map,				     struct bpf_prog_aux *prog_aux){	struct prog_poke_elem *elem;	struct bpf_array_aux *aux;	int ret = 0; 	aux = container_of(map, struct bpf_array, map)->aux;	mutex_lock(&aux->poke_mutex);	list_for_each_entry(elem, &aux->poke_progs, list) {		if (elem->aux == prog_aux)			goto out;	} 	elem = kmalloc(sizeof(*elem), GFP_KERNEL);	if (!elem) {		ret = -ENOMEM;		goto out;	} 	INIT_LIST_HEAD(&elem->list);	/* We must track the program's aux info at this point in time	 * since the program pointer itself may not be stable yet, see	 * also comment in prog_array_map_poke_run().	 */	elem->aux = prog_aux; 	list_add_tail(&elem->list, &aux->poke_progs);out:	mutex_unlock(&aux->poke_mutex);	return ret;} static void prog_array_map_poke_untrack(struct bpf_map *map,					struct bpf_prog_aux *prog_aux){	struct prog_poke_elem *elem, *tmp;	struct bpf_array_aux *aux; 	aux = container_of(map, struct bpf_array, map)->aux;	mutex_lock(&aux->poke_mutex);	list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) {		if (elem->aux == prog_aux) {			list_del_init(&elem->list);			kfree(elem);			break;		}	}	mutex_unlock(&aux->poke_mutex);} static void prog_array_map_poke_run(struct bpf_map *map, u32 key,				    struct bpf_prog *old,				    struct bpf_prog *new){	struct prog_poke_elem *elem;	struct bpf_array_aux *aux; 	aux = container_of(map, struct bpf_array, map)->aux;	WARN_ON_ONCE(!mutex_is_locked(&aux->poke_mutex)); 	list_for_each_entry(elem, &aux->poke_progs, list) {		struct bpf_jit_poke_descriptor *poke;		int i, ret; 		for (i = 0; i < elem->aux->size_poke_tab; i++) {			poke = &elem->aux->poke_tab[i]; 			/* Few things to be aware of:			 *			 * 1) We can only ever access aux in this context, but			 *    not aux->prog since it might not be stable yet and			 *    there could be danger of use after free otherwise.			 * 2) Initially when we start tracking aux, the program			 *    is not JITed yet and also does not have a kallsyms			 *    entry. We skip these as poke->ip_stable is not			 *    active yet. The JIT will do the final fixup before			 *    setting it stable. The various poke->ip_stable are			 *    successively activated, so tail call updates can			 *    arrive from here while JIT is still finishing its			 *    final fixup for non-activated poke entries.			 * 3) On program teardown, the program's kallsym entry gets			 *    removed out of RCU callback, but we can only untrack			 *    from sleepable context, therefore bpf_arch_text_poke()			 *    might not see that this is in BPF text section and			 *    bails out with -EINVAL. As these are unreachable since			 *    RCU grace period already passed, we simply skip them.			 * 4) Also programs reaching refcount of zero while patching			 *    is in progress is okay since we're protected under			 *    poke_mutex and untrack the programs before the JIT			 *    buffer is freed. When we're still in the middle of			 *    patching and suddenly kallsyms entry of the program			 *    gets evicted, we just skip the rest which is fine due			 *    to point 3).			 * 5) Any other error happening below from bpf_arch_text_poke()			 *    is a unexpected bug.			 */			if (!READ_ONCE(poke->ip_stable))				continue;			if (poke->reason != BPF_POKE_REASON_TAIL_CALL)				continue;			if (poke->tail_call.map != map ||			    poke->tail_call.key != key)				continue; 			ret = bpf_arch_text_poke(poke->ip, BPF_MOD_JUMP,						 old ? (u8 *)old->bpf_func +						 poke->adj_off : NULL,						 new ? (u8 *)new->bpf_func +						 poke->adj_off : NULL);			BUG_ON(ret < 0 && ret != -EINVAL);		}	}} static void prog_array_map_clear_deferred(struct work_struct *work){	struct bpf_map *map = container_of(work, struct bpf_array_aux,					   work)->map;	bpf_fd_array_map_clear(map);	bpf_map_put(map);} static void prog_array_map_clear(struct bpf_map *map){	struct bpf_array_aux *aux = container_of(map, struct bpf_array,						 map)->aux;	bpf_map_inc(map);	schedule_work(&aux->work);} static struct bpf_map *prog_array_map_alloc(union bpf_attr *attr){	struct bpf_array_aux *aux;	struct bpf_map *map; 	aux = kzalloc(sizeof(*aux), GFP_KERNEL);	if (!aux)		return ERR_PTR(-ENOMEM); 	INIT_WORK(&aux->work, prog_array_map_clear_deferred);	INIT_LIST_HEAD(&aux->poke_progs);	mutex_init(&aux->poke_mutex); 	map = array_map_alloc(attr);	if (IS_ERR(map)) {		kfree(aux);		return map;	} 	container_of(map, struct bpf_array, map)->aux = aux;	aux->map = map; 	return map;} static void prog_array_map_free(struct bpf_map *map){	struct prog_poke_elem *elem, *tmp;	struct bpf_array_aux *aux; 	aux = container_of(map, struct bpf_array, map)->aux;	list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) {		list_del_init(&elem->list);		kfree(elem);	}	kfree(aux);	fd_array_map_free(map);} const struct bpf_map_ops prog_array_map_ops = {	.map_alloc_check = fd_array_map_alloc_check,	.map_alloc = prog_array_map_alloc,	.map_free = prog_array_map_free,	.map_poke_track = prog_array_map_poke_track,	.map_poke_untrack = prog_array_map_poke_untrack,	.map_poke_run = prog_array_map_poke_run,	.map_get_next_key = array_map_get_next_key,	.map_lookup_elem = fd_array_map_lookup_elem,	.map_delete_elem = fd_array_map_delete_elem,	.map_fd_get_ptr = prog_fd_array_get_ptr,	.map_fd_put_ptr = prog_fd_array_put_ptr,	.map_fd_sys_lookup_elem = prog_fd_array_sys_lookup_elem,	.map_release_uref = prog_array_map_clear,	.map_seq_show_elem = prog_array_map_seq_show_elem,}; static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,						   struct file *map_file){	struct bpf_event_entry *ee; 	ee = kzalloc(sizeof(*ee), GFP_ATOMIC);	if (ee) {		ee->event = perf_file->private_data;		ee->perf_file = perf_file;		ee->map_file = map_file;	} 	return ee;} static void __bpf_event_entry_free(struct rcu_head *rcu){	struct bpf_event_entry *ee; 	ee = container_of(rcu, struct bpf_event_entry, rcu);	fput(ee->perf_file);	kfree(ee);} static void bpf_event_entry_free_rcu(struct bpf_event_entry *ee){	call_rcu(&ee->rcu, __bpf_event_entry_free);} static void *perf_event_fd_array_get_ptr(struct bpf_map *map,					 struct file *map_file, int fd){	struct bpf_event_entry *ee;	struct perf_event *event;	struct file *perf_file;	u64 value; 	perf_file = perf_event_get(fd);	if (IS_ERR(perf_file))		return perf_file; 	ee = ERR_PTR(-EOPNOTSUPP);	event = perf_file->private_data;	if (perf_event_read_local(event, &value, NULL, NULL) == -EOPNOTSUPP)		goto err_out; 	ee = bpf_event_entry_gen(perf_file, map_file);	if (ee)		return ee;	ee = ERR_PTR(-ENOMEM);err_out:	fput(perf_file);	return ee;} static void perf_event_fd_array_put_ptr(void *ptr){	bpf_event_entry_free_rcu(ptr);} static void perf_event_fd_array_release(struct bpf_map *map,					struct file *map_file){	struct bpf_array *array = container_of(map, struct bpf_array, map);	struct bpf_event_entry *ee;	int i; 	rcu_read_lock();	for (i = 0; i < array->map.max_entries; i++) {		ee = READ_ONCE(array->ptrs[i]);		if (ee && ee->map_file == map_file)			fd_array_map_delete_elem(map, &i);	}	rcu_read_unlock();} const struct bpf_map_ops perf_event_array_map_ops = {	.map_alloc_check = fd_array_map_alloc_check,	.map_alloc = array_map_alloc,	.map_free = fd_array_map_free,	.map_get_next_key = array_map_get_next_key,	.map_lookup_elem = fd_array_map_lookup_elem,	.map_delete_elem = fd_array_map_delete_elem,	.map_fd_get_ptr = perf_event_fd_array_get_ptr,	.map_fd_put_ptr = perf_event_fd_array_put_ptr,	.map_release = perf_event_fd_array_release,	.map_check_btf = map_check_no_btf,}; #ifdef CONFIG_CGROUPSstatic void *cgroup_fd_array_get_ptr(struct bpf_map *map,				     struct file *map_file /* not used */,				     int fd){	return cgroup_get_from_fd(fd);} static void cgroup_fd_array_put_ptr(void *ptr){	/* cgroup_put free cgrp after a rcu grace period */	cgroup_put(ptr);} static void cgroup_fd_array_free(struct bpf_map *map){	bpf_fd_array_map_clear(map);	fd_array_map_free(map);} const struct bpf_map_ops cgroup_array_map_ops = {	.map_alloc_check = fd_array_map_alloc_check,	.map_alloc = array_map_alloc,	.map_free = cgroup_fd_array_free,	.map_get_next_key = array_map_get_next_key,	.map_lookup_elem = fd_array_map_lookup_elem,	.map_delete_elem = fd_array_map_delete_elem,	.map_fd_get_ptr = cgroup_fd_array_get_ptr,	.map_fd_put_ptr = cgroup_fd_array_put_ptr,

Lines 501–1000 of 1087. Previous lines Load more