History

bpf: notify offload JITs about optimizations

Let offload JITs know when instructions are replaced and optimized out, so they can update their state appropriately. The optimizations are best effort, if JIT returns an error from any callback verifier will stop notifying it as state may now be out of sync, but the verifier continues making progress.

Jakub Kicinski committed 7 years ago

Commit 08ca90a1 parent 9e4c24e4 files · +53 −0Browse files

Unified · Split

4 files
  1. include/linux/bpf.h +7 −0
  2. include/linux/bpf_verifier.h +5 −0
  3. kernel/bpf/offload.c +35 −0
  4. kernel/bpf/verifier.c +6 −0
include/linux/bpf.h+7−0
@@ -268,9 +268,15 @@ struct bpf_verifier_ops { · 267 unmodified lines
268268 };
269269
270270 struct bpf_prog_offload_ops {
271+ /* verifier basic callbacks */
271272 int (*insn_hook)(struct bpf_verifier_env *env,
272273 int insn_idx, int prev_insn_idx);
273274 int (*finalize)(struct bpf_verifier_env *env);
275+ /* verifier optimization callbacks (called after .finalize) */
276+ int (*replace_insn)(struct bpf_verifier_env *env, u32 off,
277+ struct bpf_insn *insn);
278+ int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);
279+ /* program management callbacks */
274280 int (*prepare)(struct bpf_prog *prog);
275281 int (*translate)(struct bpf_prog *prog);
276282 void (*destroy)(struct bpf_prog *prog);
@@ -283,6 +289,7 @@ struct bpf_prog_offload { · 6 unmodified lines
283289 void *dev_priv;
284290 struct list_head offloads;
285291 bool dev_state;
292+ bool opt_failed;
286293 void *jited_image;
287294 u32 jited_len;
288295 };
include/linux/bpf_verifier.h+5−0
@@ -266,5 +266,10 @@ int bpf_prog_offload_verifier_prep(struct bpf_prog *prog); · 265 unmodified lines
266266 int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env,
267267 int insn_idx, int prev_insn_idx);
268268 int bpf_prog_offload_finalize(struct bpf_verifier_env *env);
269+void
270+bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,
271+ struct bpf_insn *insn);
272+void
273+bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt);
269274
270275 #endif /* _LINUX_BPF_VERIFIER_H */
kernel/bpf/offload.c+35−0
@@ -173,6 +173,41 @@ int bpf_prog_offload_finalize(struct bpf_verifier_env *env) · 172 unmodified lines
173173 return ret;
174174 }
175175
176+void
177+bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,
178+ struct bpf_insn *insn)
179+{
180+ const struct bpf_prog_offload_ops *ops;
181+ struct bpf_prog_offload *offload;
182+ int ret = -EOPNOTSUPP;
183+
184+ down_read(&bpf_devs_lock);
185+ offload = env->prog->aux->offload;
186+ if (offload) {
187+ ops = offload->offdev->ops;
188+ if (!offload->opt_failed && ops->replace_insn)
189+ ret = ops->replace_insn(env, off, insn);
190+ offload->opt_failed |= ret;
191+ }
192+ up_read(&bpf_devs_lock);
193+}
194+
195+void
196+bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
197+{
198+ struct bpf_prog_offload *offload;
199+ int ret = -EOPNOTSUPP;
200+
201+ down_read(&bpf_devs_lock);
202+ offload = env->prog->aux->offload;
203+ if (offload) {
204+ if (!offload->opt_failed && offload->offdev->ops->remove_insns)
205+ ret = offload->offdev->ops->remove_insns(env, off, cnt);
206+ offload->opt_failed |= ret;
207+ }
208+ up_read(&bpf_devs_lock);
209+}
210+
176211 static void __bpf_prog_offload_destroy(struct bpf_prog *prog)
177212 {
178213 struct bpf_prog_offload *offload = prog->aux->offload;
kernel/bpf/verifier.c+6−0
@@ -6558,6 +6558,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) · 6557 unmodified lines
65586558 unsigned int orig_prog_len = env->prog->len;
65596559 int err;
65606560
6561+ if (bpf_prog_is_dev_bound(env->prog->aux))
6562+ bpf_prog_offload_remove_insns(env, off, cnt);
6563+
65616564 err = bpf_remove_insns(env->prog, off, cnt);
65626565 if (err)
65636566 return err;
@@ -6632,6 +6635,9 @@ static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) · 68 unmodified lines
66326635 else
66336636 continue;
66346637
6638+ if (bpf_prog_is_dev_bound(env->prog->aux))
6639+ bpf_prog_offload_replace_insn(env, i, &ja);
6640+
66356641 memcpy(insn, &ja, sizeof(ja));
66366642 }
66376643 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t