History

bpf: Refactor trampoline update code

As we need to introduce a third type of attachment for trampolines, the flattened signature of arch_prepare_bpf_trampoline gets even more complicated. Refactor the prog and count argument to arch_prepare_bpf_trampoline to use bpf_tramp_progs to simplify the addition and accounting for new attachment types.

KP Singh committed 6 years ago

Commit 88fd9e51 parent cc6fa774 files · +71 −45Browse files

Showing the changes to kernel (2 of 4 files). Show all

Unified · Split

2 files
  1. kernel/bpf/bpf_struct_ops.c +9 −1
  2. kernel/bpf/trampoline.c +35 −27
kernel/bpf/bpf_struct_ops.c+9−1
@@ -320,6 +320,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 319 unmodified lines
320 struct bpf_struct_ops_value *uvalue, *kvalue;320 struct bpf_struct_ops_value *uvalue, *kvalue;
321 const struct btf_member *member;321 const struct btf_member *member;
322 const struct btf_type *t = st_ops->type;322 const struct btf_type *t = st_ops->type;
323 struct bpf_tramp_progs *tprogs = NULL;
323 void *udata, *kdata;324 void *udata, *kdata;
324 int prog_fd, err = 0;325 int prog_fd, err = 0;
325 void *image;326 void *image;
@@ -343,6 +344,10 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 17 unmodified lines
343 if (uvalue->state || refcount_read(&uvalue->refcnt))344 if (uvalue->state || refcount_read(&uvalue->refcnt))
344 return -EINVAL;345 return -EINVAL;
345 346
347 tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL);
348 if (!tprogs)
349 return -ENOMEM;
350
346 uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;351 uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
347 kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;352 kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;
348 353
@@ -425,10 +430,12 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 76 unmodified lines
425 goto reset_unlock;430 goto reset_unlock;
426 }431 }
427 432
433 tprogs[BPF_TRAMP_FENTRY].progs[0] = prog;
434 tprogs[BPF_TRAMP_FENTRY].nr_progs = 1;
428 err = arch_prepare_bpf_trampoline(image,435 err = arch_prepare_bpf_trampoline(image,
429 st_map->image + PAGE_SIZE,436 st_map->image + PAGE_SIZE,
430 &st_ops->func_models[i], 0,437 &st_ops->func_models[i], 0,
431 &prog, 1, NULL, 0, NULL);438 tprogs, NULL);
432 if (err < 0)439 if (err < 0)
433 goto reset_unlock;440 goto reset_unlock;
434 441
@@ -469,6 +476,7 @@ reset_unlock: · 34 unmodified lines
469 memset(uvalue, 0, map->value_size);476 memset(uvalue, 0, map->value_size);
470 memset(kvalue, 0, map->value_size);477 memset(kvalue, 0, map->value_size);
471unlock:478unlock:
479 kfree(tprogs);
472 mutex_unlock(&st_map->lock);480 mutex_unlock(&st_map->lock);
473 return err;481 return err;
474}482}
kernel/bpf/trampoline.c+35−27
@@ -190,40 +190,49 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) · 189 unmodified lines
190 return ret;190 return ret;
191}191}
192 192
193/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50193static struct bpf_tramp_progs *
194 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2194bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total)
195 */195{
196#define BPF_MAX_TRAMP_PROGS 40196 const struct bpf_prog_aux *aux;
197 struct bpf_tramp_progs *tprogs;
198 struct bpf_prog **progs;
199 int kind;
200
201 *total = 0;
202 tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL);
203 if (!tprogs)
204 return ERR_PTR(-ENOMEM);
205
206 for (kind = 0; kind < BPF_TRAMP_MAX; kind++) {
207 tprogs[kind].nr_progs = tr->progs_cnt[kind];
208 *total += tr->progs_cnt[kind];
209 progs = tprogs[kind].progs;
210
211 hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist)
212 *progs++ = aux->prog;
213 }
214 return tprogs;
215}
197 216
198static int bpf_trampoline_update(struct bpf_trampoline *tr)217static int bpf_trampoline_update(struct bpf_trampoline *tr)
199{218{
200 void *old_image = tr->image + ((tr->selector + 1) & 1) * BPF_IMAGE_SIZE/2;219 void *old_image = tr->image + ((tr->selector + 1) & 1) * BPF_IMAGE_SIZE/2;
201 void *new_image = tr->image + (tr->selector & 1) * BPF_IMAGE_SIZE/2;220 void *new_image = tr->image + (tr->selector & 1) * BPF_IMAGE_SIZE/2;
202 struct bpf_prog *progs_to_run[BPF_MAX_TRAMP_PROGS];221 struct bpf_tramp_progs *tprogs;
203 int fentry_cnt = tr->progs_cnt[BPF_TRAMP_FENTRY];
204 int fexit_cnt = tr->progs_cnt[BPF_TRAMP_FEXIT];
205 struct bpf_prog **progs, **fentry, **fexit;
206 u32 flags = BPF_TRAMP_F_RESTORE_REGS;222 u32 flags = BPF_TRAMP_F_RESTORE_REGS;
207 struct bpf_prog_aux *aux;223 int err, total;
208 int err;
209 224
210 if (fentry_cnt + fexit_cnt == 0) {225 tprogs = bpf_trampoline_get_progs(tr, &total);
226 if (IS_ERR(tprogs))
227 return PTR_ERR(tprogs);
228
229 if (total == 0) {
211 err = unregister_fentry(tr, old_image);230 err = unregister_fentry(tr, old_image);
212 tr->selector = 0;231 tr->selector = 0;
213 goto out;232 goto out;
214 }233 }
215 234
216 /* populate fentry progs */235 if (tprogs[BPF_TRAMP_FEXIT].nr_progs)
217 fentry = progs = progs_to_run;
218 hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FENTRY], tramp_hlist)
219 *progs++ = aux->prog;
220
221 /* populate fexit progs */
222 fexit = progs;
223 hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FEXIT], tramp_hlist)
224 *progs++ = aux->prog;
225
226 if (fexit_cnt)
227 flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;236 flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;
228 237
229 /* Though the second half of trampoline page is unused a task could be238 /* Though the second half of trampoline page is unused a task could be
@@ -232,12 +241,11 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 2 unmodified lines
232 * preempted task. Hence wait for tasks to voluntarily schedule or go241 * preempted task. Hence wait for tasks to voluntarily schedule or go
233 * to userspace.242 * to userspace.
234 */243 */
244
235 synchronize_rcu_tasks();245 synchronize_rcu_tasks();
236 246
237 err = arch_prepare_bpf_trampoline(new_image, new_image + BPF_IMAGE_SIZE / 2,247 err = arch_prepare_bpf_trampoline(new_image, new_image + BPF_IMAGE_SIZE / 2,
238 &tr->func.model, flags,248 &tr->func.model, flags, tprogs,
239 fentry, fentry_cnt,
240 fexit, fexit_cnt,
241 tr->func.addr);249 tr->func.addr);
242 if (err < 0)250 if (err < 0)
243 goto out;251 goto out;
@@ -252,6 +260,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 8 unmodified lines
252 goto out;260 goto out;
253 tr->selector++;261 tr->selector++;
254out:262out:
263 kfree(tprogs);
255 return err;264 return err;
256}265}
257 266
@@ -409,8 +418,7 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) · 151 unmodified lines
409int __weak418int __weak
410arch_prepare_bpf_trampoline(void *image, void *image_end,419arch_prepare_bpf_trampoline(void *image, void *image_end,
411 const struct btf_func_model *m, u32 flags,420 const struct btf_func_model *m, u32 flags,
412 struct bpf_prog **fentry_progs, int fentry_cnt,421 struct bpf_tramp_progs *tprogs,
413 struct bpf_prog **fexit_progs, int fexit_cnt,
414 void *orig_call)422 void *orig_call)
415{423{
416 return -ENOTSUPP;424 return -ENOTSUPP;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t