750 lines · 19.4 KB · C
RawDownload
	if (IS_ERR(prog))		return prog; 	if (!bpf_prog_array_compatible(array, prog)) {		bpf_prog_put(prog);		return ERR_PTR(-EINVAL);	} 	return prog;} static void prog_fd_array_put_ptr(void *ptr){	bpf_prog_put(ptr);} static u32 prog_fd_array_sys_lookup_elem(void *ptr){	return ((struct bpf_prog *)ptr)->aux->id;} /* decrement refcnt of all bpf_progs that are stored in this map */static void bpf_fd_array_map_clear(struct bpf_map *map){	struct bpf_array *array = container_of(map, struct bpf_array, map);	int i; 	for (i = 0; i < array->map.max_entries; i++)		fd_array_map_delete_elem(map, &i);} const struct bpf_map_ops prog_array_map_ops = {	.map_alloc_check = fd_array_map_alloc_check,	.map_alloc = array_map_alloc,	.map_free = fd_array_map_free,	.map_get_next_key = array_map_get_next_key,	.map_lookup_elem = fd_array_map_lookup_elem,	.map_delete_elem = fd_array_map_delete_elem,	.map_fd_get_ptr = prog_fd_array_get_ptr,	.map_fd_put_ptr = prog_fd_array_put_ptr,	.map_fd_sys_lookup_elem = prog_fd_array_sys_lookup_elem,	.map_release_uref = bpf_fd_array_map_clear,	.map_check_btf = map_check_no_btf,}; static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,						   struct file *map_file){	struct bpf_event_entry *ee; 	ee = kzalloc(sizeof(*ee), GFP_ATOMIC);	if (ee) {		ee->event = perf_file->private_data;		ee->perf_file = perf_file;		ee->map_file = map_file;	} 	return ee;} static void __bpf_event_entry_free(struct rcu_head *rcu){	struct bpf_event_entry *ee; 	ee = container_of(rcu, struct bpf_event_entry, rcu);	fput(ee->perf_file);	kfree(ee);} static void bpf_event_entry_free_rcu(struct bpf_event_entry *ee){	call_rcu(&ee->rcu, __bpf_event_entry_free);} static void *perf_event_fd_array_get_ptr(struct bpf_map *map,					 struct file *map_file, int fd){	struct bpf_event_entry *ee;	struct perf_event *event;	struct file *perf_file;	u64 value; 	perf_file = perf_event_get(fd);	if (IS_ERR(perf_file))		return perf_file; 	ee = ERR_PTR(-EOPNOTSUPP);	event = perf_file->private_data;	if (perf_event_read_local(event, &value, NULL, NULL) == -EOPNOTSUPP)		goto err_out; 	ee = bpf_event_entry_gen(perf_file, map_file);	if (ee)		return ee;	ee = ERR_PTR(-ENOMEM);err_out:	fput(perf_file);	return ee;} static void perf_event_fd_array_put_ptr(void *ptr){	bpf_event_entry_free_rcu(ptr);} static void perf_event_fd_array_release(struct bpf_map *map,					struct file *map_file){	struct bpf_array *array = container_of(map, struct bpf_array, map);	struct bpf_event_entry *ee;	int i; 	rcu_read_lock();	for (i = 0; i < array->map.max_entries; i++) {		ee = READ_ONCE(array->ptrs[i]);		if (ee && ee->map_file == map_file)			fd_array_map_delete_elem(map, &i);	}	rcu_read_unlock();} const struct bpf_map_ops perf_event_array_map_ops = {	.map_alloc_check = fd_array_map_alloc_check,	.map_alloc = array_map_alloc,	.map_free = fd_array_map_free,	.map_get_next_key = array_map_get_next_key,	.map_lookup_elem = fd_array_map_lookup_elem,	.map_delete_elem = fd_array_map_delete_elem,	.map_fd_get_ptr = perf_event_fd_array_get_ptr,	.map_fd_put_ptr = perf_event_fd_array_put_ptr,	.map_release = perf_event_fd_array_release,	.map_check_btf = map_check_no_btf,}; #ifdef CONFIG_CGROUPSstatic void *cgroup_fd_array_get_ptr(struct bpf_map *map,				     struct file *map_file /* not used */,				     int fd){	return cgroup_get_from_fd(fd);} static void cgroup_fd_array_put_ptr(void *ptr){	/* cgroup_put free cgrp after a rcu grace period */	cgroup_put(ptr);} static void cgroup_fd_array_free(struct bpf_map *map){	bpf_fd_array_map_clear(map);	fd_array_map_free(map);} const struct bpf_map_ops cgroup_array_map_ops = {	.map_alloc_check = fd_array_map_alloc_check,	.map_alloc = array_map_alloc,	.map_free = cgroup_fd_array_free,	.map_get_next_key = array_map_get_next_key,	.map_lookup_elem = fd_array_map_lookup_elem,	.map_delete_elem = fd_array_map_delete_elem,	.map_fd_get_ptr = cgroup_fd_array_get_ptr,	.map_fd_put_ptr = cgroup_fd_array_put_ptr,	.map_check_btf = map_check_no_btf,};#endif static struct bpf_map *array_of_map_alloc(union bpf_attr *attr){	struct bpf_map *map, *inner_map_meta; 	inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd);	if (IS_ERR(inner_map_meta))		return inner_map_meta; 	map = array_map_alloc(attr);	if (IS_ERR(map)) {		bpf_map_meta_free(inner_map_meta);		return map;	} 	map->inner_map_meta = inner_map_meta; 	return map;} static void array_of_map_free(struct bpf_map *map){	/* map->inner_map_meta is only accessed by syscall which	 * is protected by fdget/fdput.	 */	bpf_map_meta_free(map->inner_map_meta);	bpf_fd_array_map_clear(map);	fd_array_map_free(map);} static void *array_of_map_lookup_elem(struct bpf_map *map, void *key){	struct bpf_map **inner_map = array_map_lookup_elem(map, key); 	if (!inner_map)		return NULL; 	return READ_ONCE(*inner_map);} static u32 array_of_map_gen_lookup(struct bpf_map *map,				   struct bpf_insn *insn_buf){	struct bpf_array *array = container_of(map, struct bpf_array, map);	u32 elem_size = round_up(map->value_size, 8);	struct bpf_insn *insn = insn_buf;	const int ret = BPF_REG_0;	const int map_ptr = BPF_REG_1;	const int index = BPF_REG_2; 	*insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));	*insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);	if (map->unpriv_array) {		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6);		*insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);	} else {		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5);	}	if (is_power_of_2(elem_size))		*insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size));	else		*insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size);	*insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr);	*insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0);	*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);	*insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);	*insn++ = BPF_MOV64_IMM(ret, 0); 	return insn - insn_buf;} const struct bpf_map_ops array_of_maps_map_ops = {	.map_alloc_check = fd_array_map_alloc_check,	.map_alloc = array_of_map_alloc,	.map_free = array_of_map_free,	.map_get_next_key = array_map_get_next_key,	.map_lookup_elem = array_of_map_lookup_elem,	.map_delete_elem = fd_array_map_delete_elem,	.map_fd_get_ptr = bpf_map_fd_get_ptr,	.map_fd_put_ptr = bpf_map_fd_put_ptr,	.map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem,	.map_gen_lookup = array_of_map_gen_lookup,	.map_check_btf = map_check_no_btf,};

Lines 501–750 of 750. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t