As we need to introduce a third type of attachment for trampolines, the flattened signature of arch_prepare_bpf_trampoline gets even more complicated. Refactor the prog and count argument to arch_prepare_bpf_trampoline to use bpf_tramp_progs to simplify the addition and accounting for new attachment types.
Showing the changes to kernel (2 of 4 files). Show all
Unified · Split
| @@ -320,6 +320,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 319 unmodified lines | |||
| 320 | struct bpf_struct_ops_value *uvalue, *kvalue; | 320 | struct bpf_struct_ops_value *uvalue, *kvalue; |
| 321 | const struct btf_member *member; | 321 | const struct btf_member *member; |
| 322 | const struct btf_type *t = st_ops->type; | 322 | const struct btf_type *t = st_ops->type; |
| 323 | struct bpf_tramp_progs *tprogs = NULL; | ||
| 323 | void *udata, *kdata; | 324 | void *udata, *kdata; |
| 324 | int prog_fd, err = 0; | 325 | int prog_fd, err = 0; |
| 325 | void *image; | 326 | void *image; |
| @@ -343,6 +344,10 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 17 unmodified lines | |||
| 343 | if (uvalue->state || refcount_read(&uvalue->refcnt)) | 344 | if (uvalue->state || refcount_read(&uvalue->refcnt)) |
| 344 | return -EINVAL; | 345 | return -EINVAL; |
| 345 | 346 | ||
| 347 | tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL); | ||
| 348 | if (!tprogs) | ||
| 349 | return -ENOMEM; | ||
| 350 | |||
| 346 | uvalue = (struct bpf_struct_ops_value *)st_map->uvalue; | 351 | uvalue = (struct bpf_struct_ops_value *)st_map->uvalue; |
| 347 | kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue; | 352 | kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue; |
| 348 | 353 | ||
| @@ -425,10 +430,12 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 76 unmodified lines | |||
| 425 | goto reset_unlock; | 430 | goto reset_unlock; |
| 426 | } | 431 | } |
| 427 | 432 | ||
| 433 | tprogs[BPF_TRAMP_FENTRY].progs[0] = prog; | ||
| 434 | tprogs[BPF_TRAMP_FENTRY].nr_progs = 1; | ||
| 428 | err = arch_prepare_bpf_trampoline(image, | 435 | err = arch_prepare_bpf_trampoline(image, |
| 429 | st_map->image + PAGE_SIZE, | 436 | st_map->image + PAGE_SIZE, |
| 430 | &st_ops->func_models[i], 0, | 437 | &st_ops->func_models[i], 0, |
| 431 | &prog, 1, NULL, 0, NULL); | 438 | tprogs, NULL); |
| 432 | if (err < 0) | 439 | if (err < 0) |
| 433 | goto reset_unlock; | 440 | goto reset_unlock; |
| 434 | 441 | ||
| @@ -469,6 +476,7 @@ reset_unlock: · 34 unmodified lines | |||
| 469 | memset(uvalue, 0, map->value_size); | 476 | memset(uvalue, 0, map->value_size); |
| 470 | memset(kvalue, 0, map->value_size); | 477 | memset(kvalue, 0, map->value_size); |
| 471 | unlock: | 478 | unlock: |
| 479 | kfree(tprogs); | ||
| 472 | mutex_unlock(&st_map->lock); | 480 | mutex_unlock(&st_map->lock); |
| 473 | return err; | 481 | return err; |
| 474 | } | 482 | } |
| @@ -190,40 +190,49 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) · 189 unmodified lines | |||
| 190 | return ret; | 190 | return ret; |
| 191 | } | 191 | } |
| 192 | 192 | ||
| 193 | /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 | 193 | static struct bpf_tramp_progs * |
| 194 | * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2 | 194 | bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total) |
| 195 | */ | 195 | { |
| 196 | #define BPF_MAX_TRAMP_PROGS 40 | 196 | const struct bpf_prog_aux *aux; |
| 197 | struct bpf_tramp_progs *tprogs; | ||
| 198 | struct bpf_prog **progs; | ||
| 199 | int kind; | ||
| 200 | |||
| 201 | *total = 0; | ||
| 202 | tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL); | ||
| 203 | if (!tprogs) | ||
| 204 | return ERR_PTR(-ENOMEM); | ||
| 205 | |||
| 206 | for (kind = 0; kind < BPF_TRAMP_MAX; kind++) { | ||
| 207 | tprogs[kind].nr_progs = tr->progs_cnt[kind]; | ||
| 208 | *total += tr->progs_cnt[kind]; | ||
| 209 | progs = tprogs[kind].progs; | ||
| 210 | |||
| 211 | hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist) | ||
| 212 | *progs++ = aux->prog; | ||
| 213 | } | ||
| 214 | return tprogs; | ||
| 215 | } | ||
| 197 | 216 | ||
| 198 | static int bpf_trampoline_update(struct bpf_trampoline *tr) | 217 | static int bpf_trampoline_update(struct bpf_trampoline *tr) |
| 199 | { | 218 | { |
| 200 | void *old_image = tr->image + ((tr->selector + 1) & 1) * BPF_IMAGE_SIZE/2; | 219 | void *old_image = tr->image + ((tr->selector + 1) & 1) * BPF_IMAGE_SIZE/2; |
| 201 | void *new_image = tr->image + (tr->selector & 1) * BPF_IMAGE_SIZE/2; | 220 | void *new_image = tr->image + (tr->selector & 1) * BPF_IMAGE_SIZE/2; |
| 202 | struct bpf_prog *progs_to_run[BPF_MAX_TRAMP_PROGS]; | 221 | struct bpf_tramp_progs *tprogs; |
| 203 | int fentry_cnt = tr->progs_cnt[BPF_TRAMP_FENTRY]; | ||
| 204 | int fexit_cnt = tr->progs_cnt[BPF_TRAMP_FEXIT]; | ||
| 205 | struct bpf_prog **progs, **fentry, **fexit; | ||
| 206 | u32 flags = BPF_TRAMP_F_RESTORE_REGS; | 222 | u32 flags = BPF_TRAMP_F_RESTORE_REGS; |
| 207 | struct bpf_prog_aux *aux; | 223 | int err, total; |
| 208 | int err; | ||
| 209 | 224 | ||
| 210 | if (fentry_cnt + fexit_cnt == 0) { | 225 | tprogs = bpf_trampoline_get_progs(tr, &total); |
| 226 | if (IS_ERR(tprogs)) | ||
| 227 | return PTR_ERR(tprogs); | ||
| 228 | |||
| 229 | if (total == 0) { | ||
| 211 | err = unregister_fentry(tr, old_image); | 230 | err = unregister_fentry(tr, old_image); |
| 212 | tr->selector = 0; | 231 | tr->selector = 0; |
| 213 | goto out; | 232 | goto out; |
| 214 | } | 233 | } |
| 215 | 234 | ||
| 216 | /* populate fentry progs */ | 235 | if (tprogs[BPF_TRAMP_FEXIT].nr_progs) |
| 217 | fentry = progs = progs_to_run; | ||
| 218 | hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FENTRY], tramp_hlist) | ||
| 219 | *progs++ = aux->prog; | ||
| 220 | |||
| 221 | /* populate fexit progs */ | ||
| 222 | fexit = progs; | ||
| 223 | hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FEXIT], tramp_hlist) | ||
| 224 | *progs++ = aux->prog; | ||
| 225 | |||
| 226 | if (fexit_cnt) | ||
| 227 | flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME; | 236 | flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME; |
| 228 | 237 | ||
| 229 | /* Though the second half of trampoline page is unused a task could be | 238 | /* Though the second half of trampoline page is unused a task could be |
| @@ -232,12 +241,11 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 2 unmodified lines | |||
| 232 | * preempted task. Hence wait for tasks to voluntarily schedule or go | 241 | * preempted task. Hence wait for tasks to voluntarily schedule or go |
| 233 | * to userspace. | 242 | * to userspace. |
| 234 | */ | 243 | */ |
| 244 | |||
| 235 | synchronize_rcu_tasks(); | 245 | synchronize_rcu_tasks(); |
| 236 | 246 | ||
| 237 | err = arch_prepare_bpf_trampoline(new_image, new_image + BPF_IMAGE_SIZE / 2, | 247 | err = arch_prepare_bpf_trampoline(new_image, new_image + BPF_IMAGE_SIZE / 2, |
| 238 | &tr->func.model, flags, | 248 | &tr->func.model, flags, tprogs, |
| 239 | fentry, fentry_cnt, | ||
| 240 | fexit, fexit_cnt, | ||
| 241 | tr->func.addr); | 249 | tr->func.addr); |
| 242 | if (err < 0) | 250 | if (err < 0) |
| 243 | goto out; | 251 | goto out; |
| @@ -252,6 +260,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 8 unmodified lines | |||
| 252 | goto out; | 260 | goto out; |
| 253 | tr->selector++; | 261 | tr->selector++; |
| 254 | out: | 262 | out: |
| 263 | kfree(tprogs); | ||
| 255 | return err; | 264 | return err; |
| 256 | } | 265 | } |
| 257 | 266 | ||
| @@ -409,8 +418,7 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) · 151 unmodified lines | |||
| 409 | int __weak | 418 | int __weak |
| 410 | arch_prepare_bpf_trampoline(void *image, void *image_end, | 419 | arch_prepare_bpf_trampoline(void *image, void *image_end, |
| 411 | const struct btf_func_model *m, u32 flags, | 420 | const struct btf_func_model *m, u32 flags, |
| 412 | struct bpf_prog **fentry_progs, int fentry_cnt, | 421 | struct bpf_tramp_progs *tprogs, |
| 413 | struct bpf_prog **fexit_progs, int fexit_cnt, | ||
| 414 | void *orig_call) | 422 | void *orig_call) |
| 415 | { | 423 | { |
| 416 | return -ENOTSUPP; | 424 | return -ENOTSUPP; |