As we need to introduce a third type of attachment for trampolines, the flattened signature of arch_prepare_bpf_trampoline gets even more complicated. Refactor the prog and count argument to arch_prepare_bpf_trampoline to use bpf_tramp_progs to simplify the addition and accounting for new attachment types.
Showing the changes to kernel (2 of 4 files). Show all
Unified · Split
| @@ -320,6 +320,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 319 unmodified lines | ||
| 320 | 320 | struct bpf_struct_ops_value *uvalue, *kvalue; |
| 321 | 321 | const struct btf_member *member; |
| 322 | 322 | const struct btf_type *t = st_ops->type; |
| 323 | + struct bpf_tramp_progs *tprogs = NULL; | |
| 323 | 324 | void *udata, *kdata; |
| 324 | 325 | int prog_fd, err = 0; |
| 325 | 326 | void *image; |
| @@ -343,6 +344,10 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 17 unmodified lines | ||
| 343 | 344 | if (uvalue->state || refcount_read(&uvalue->refcnt)) |
| 344 | 345 | return -EINVAL; |
| 345 | 346 | |
| 347 | + tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL); | |
| 348 | + if (!tprogs) | |
| 349 | + return -ENOMEM; | |
| 350 | + | |
| 346 | 351 | uvalue = (struct bpf_struct_ops_value *)st_map->uvalue; |
| 347 | 352 | kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue; |
| 348 | 353 | |
| @@ -425,10 +430,12 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 76 unmodified lines | ||
| 425 | 430 | goto reset_unlock; |
| 426 | 431 | } |
| 427 | 432 | |
| 433 | + tprogs[BPF_TRAMP_FENTRY].progs[0] = prog; | |
| 434 | + tprogs[BPF_TRAMP_FENTRY].nr_progs = 1; | |
| 428 | 435 | err = arch_prepare_bpf_trampoline(image, |
| 429 | 436 | st_map->image + PAGE_SIZE, |
| 430 | 437 | &st_ops->func_models[i], 0, |
| 431 | − &prog, 1, NULL, 0, NULL); | |
| 438 | + tprogs, NULL); | |
| 432 | 439 | if (err < 0) |
| 433 | 440 | goto reset_unlock; |
| 434 | 441 | |
| @@ -469,6 +476,7 @@ reset_unlock: · 34 unmodified lines | ||
| 469 | 476 | memset(uvalue, 0, map->value_size); |
| 470 | 477 | memset(kvalue, 0, map->value_size); |
| 471 | 478 | unlock: |
| 479 | + kfree(tprogs); | |
| 472 | 480 | mutex_unlock(&st_map->lock); |
| 473 | 481 | return err; |
| 474 | 482 | } |
| @@ -190,40 +190,49 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) · 189 unmodified lines | ||
| 190 | 190 | return ret; |
| 191 | 191 | } |
| 192 | 192 | |
| 193 | −/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 | |
| 194 | − * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2 | |
| 195 | − */ | |
| 196 | −#define BPF_MAX_TRAMP_PROGS 40 | |
| 193 | +static struct bpf_tramp_progs * | |
| 194 | +bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total) | |
| 195 | +{ | |
| 196 | + const struct bpf_prog_aux *aux; | |
| 197 | + struct bpf_tramp_progs *tprogs; | |
| 198 | + struct bpf_prog **progs; | |
| 199 | + int kind; | |
| 200 | + | |
| 201 | + *total = 0; | |
| 202 | + tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL); | |
| 203 | + if (!tprogs) | |
| 204 | + return ERR_PTR(-ENOMEM); | |
| 205 | + | |
| 206 | + for (kind = 0; kind < BPF_TRAMP_MAX; kind++) { | |
| 207 | + tprogs[kind].nr_progs = tr->progs_cnt[kind]; | |
| 208 | + *total += tr->progs_cnt[kind]; | |
| 209 | + progs = tprogs[kind].progs; | |
| 210 | + | |
| 211 | + hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist) | |
| 212 | + *progs++ = aux->prog; | |
| 213 | + } | |
| 214 | + return tprogs; | |
| 215 | +} | |
| 197 | 216 | |
| 198 | 217 | static int bpf_trampoline_update(struct bpf_trampoline *tr) |
| 199 | 218 | { |
| 200 | 219 | void *old_image = tr->image + ((tr->selector + 1) & 1) * BPF_IMAGE_SIZE/2; |
| 201 | 220 | void *new_image = tr->image + (tr->selector & 1) * BPF_IMAGE_SIZE/2; |
| 202 | − struct bpf_prog *progs_to_run[BPF_MAX_TRAMP_PROGS]; | |
| 203 | − int fentry_cnt = tr->progs_cnt[BPF_TRAMP_FENTRY]; | |
| 204 | − int fexit_cnt = tr->progs_cnt[BPF_TRAMP_FEXIT]; | |
| 205 | − struct bpf_prog **progs, **fentry, **fexit; | |
| 221 | + struct bpf_tramp_progs *tprogs; | |
| 206 | 222 | u32 flags = BPF_TRAMP_F_RESTORE_REGS; |
| 207 | − struct bpf_prog_aux *aux; | |
| 208 | − int err; | |
| 223 | + int err, total; | |
| 209 | 224 | |
| 210 | − if (fentry_cnt + fexit_cnt == 0) { | |
| 225 | + tprogs = bpf_trampoline_get_progs(tr, &total); | |
| 226 | + if (IS_ERR(tprogs)) | |
| 227 | + return PTR_ERR(tprogs); | |
| 228 | + | |
| 229 | + if (total == 0) { | |
| 211 | 230 | err = unregister_fentry(tr, old_image); |
| 212 | 231 | tr->selector = 0; |
| 213 | 232 | goto out; |
| 214 | 233 | } |
| 215 | 234 | |
| 216 | − /* populate fentry progs */ | |
| 217 | − fentry = progs = progs_to_run; | |
| 218 | − hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FENTRY], tramp_hlist) | |
| 219 | − *progs++ = aux->prog; | |
| 220 | − | |
| 221 | − /* populate fexit progs */ | |
| 222 | − fexit = progs; | |
| 223 | − hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FEXIT], tramp_hlist) | |
| 224 | − *progs++ = aux->prog; | |
| 225 | − | |
| 226 | − if (fexit_cnt) | |
| 235 | + if (tprogs[BPF_TRAMP_FEXIT].nr_progs) | |
| 227 | 236 | flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME; |
| 228 | 237 | |
| 229 | 238 | /* Though the second half of trampoline page is unused a task could be |
| @@ -232,12 +241,11 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 2 unmodified lines | ||
| 232 | 241 | * preempted task. Hence wait for tasks to voluntarily schedule or go |
| 233 | 242 | * to userspace. |
| 234 | 243 | */ |
| 244 | + | |
| 235 | 245 | synchronize_rcu_tasks(); |
| 236 | 246 | |
| 237 | 247 | err = arch_prepare_bpf_trampoline(new_image, new_image + BPF_IMAGE_SIZE / 2, |
| 238 | − &tr->func.model, flags, | |
| 239 | − fentry, fentry_cnt, | |
| 240 | − fexit, fexit_cnt, | |
| 248 | + &tr->func.model, flags, tprogs, | |
| 241 | 249 | tr->func.addr); |
| 242 | 250 | if (err < 0) |
| 243 | 251 | goto out; |
| @@ -252,6 +260,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 8 unmodified lines | ||
| 252 | 260 | goto out; |
| 253 | 261 | tr->selector++; |
| 254 | 262 | out: |
| 263 | + kfree(tprogs); | |
| 255 | 264 | return err; |
| 256 | 265 | } |
| 257 | 266 | |
| @@ -409,8 +418,7 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) · 151 unmodified lines | ||
| 409 | 418 | int __weak |
| 410 | 419 | arch_prepare_bpf_trampoline(void *image, void *image_end, |
| 411 | 420 | const struct btf_func_model *m, u32 flags, |
| 412 | − struct bpf_prog **fentry_progs, int fentry_cnt, | |
| 413 | − struct bpf_prog **fexit_progs, int fexit_cnt, | |
| 421 | + struct bpf_tramp_progs *tprogs, | |
| 414 | 422 | void *orig_call) |
| 415 | 423 | { |
| 416 | 424 | return -ENOTSUPP; |