History

bpf: notify offload JITs about optimizations

Let offload JITs know when instructions are replaced and optimized out, so they can update their state appropriately. The optimizations are best effort, if JIT returns an error from any callback verifier will stop notifying it as state may now be out of sync, but the verifier continues making progress.

Jakub Kicinski committed 7 years ago

Commit 08ca90a1 parent 9e4c24e4 files · +53 −0Browse files

Unified · Split

4 files
  1. include/linux/bpf.h +7 −0
  2. include/linux/bpf_verifier.h +5 −0
  3. kernel/bpf/offload.c +35 −0
  4. kernel/bpf/verifier.c +6 −0
include/linux/bpf.h+7−0
@@ -268,9 +268,15 @@ struct bpf_verifier_ops { · 267 unmodified lines
268};268};
269 269
270struct bpf_prog_offload_ops {270struct bpf_prog_offload_ops {
271 /* verifier basic callbacks */
271 int (*insn_hook)(struct bpf_verifier_env *env,272 int (*insn_hook)(struct bpf_verifier_env *env,
272 int insn_idx, int prev_insn_idx);273 int insn_idx, int prev_insn_idx);
273 int (*finalize)(struct bpf_verifier_env *env);274 int (*finalize)(struct bpf_verifier_env *env);
275 /* verifier optimization callbacks (called after .finalize) */
276 int (*replace_insn)(struct bpf_verifier_env *env, u32 off,
277 struct bpf_insn *insn);
278 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);
279 /* program management callbacks */
274 int (*prepare)(struct bpf_prog *prog);280 int (*prepare)(struct bpf_prog *prog);
275 int (*translate)(struct bpf_prog *prog);281 int (*translate)(struct bpf_prog *prog);
276 void (*destroy)(struct bpf_prog *prog);282 void (*destroy)(struct bpf_prog *prog);
@@ -283,6 +289,7 @@ struct bpf_prog_offload { · 6 unmodified lines
283 void *dev_priv;289 void *dev_priv;
284 struct list_head offloads;290 struct list_head offloads;
285 bool dev_state;291 bool dev_state;
292 bool opt_failed;
286 void *jited_image;293 void *jited_image;
287 u32 jited_len;294 u32 jited_len;
288};295};
include/linux/bpf_verifier.h+5−0
@@ -266,5 +266,10 @@ int bpf_prog_offload_verifier_prep(struct bpf_prog *prog); · 265 unmodified lines
266int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env,266int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env,
267 int insn_idx, int prev_insn_idx);267 int insn_idx, int prev_insn_idx);
268int bpf_prog_offload_finalize(struct bpf_verifier_env *env);268int bpf_prog_offload_finalize(struct bpf_verifier_env *env);
269void
270bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,
271 struct bpf_insn *insn);
272void
273bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt);
269 274
270#endif /* _LINUX_BPF_VERIFIER_H */275#endif /* _LINUX_BPF_VERIFIER_H */
kernel/bpf/offload.c+35−0
@@ -173,6 +173,41 @@ int bpf_prog_offload_finalize(struct bpf_verifier_env *env) · 172 unmodified lines
173 return ret;173 return ret;
174}174}
175 175
176void
177bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,
178 struct bpf_insn *insn)
179{
180 const struct bpf_prog_offload_ops *ops;
181 struct bpf_prog_offload *offload;
182 int ret = -EOPNOTSUPP;
183
184 down_read(&bpf_devs_lock);
185 offload = env->prog->aux->offload;
186 if (offload) {
187 ops = offload->offdev->ops;
188 if (!offload->opt_failed && ops->replace_insn)
189 ret = ops->replace_insn(env, off, insn);
190 offload->opt_failed |= ret;
191 }
192 up_read(&bpf_devs_lock);
193}
194
195void
196bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
197{
198 struct bpf_prog_offload *offload;
199 int ret = -EOPNOTSUPP;
200
201 down_read(&bpf_devs_lock);
202 offload = env->prog->aux->offload;
203 if (offload) {
204 if (!offload->opt_failed && offload->offdev->ops->remove_insns)
205 ret = offload->offdev->ops->remove_insns(env, off, cnt);
206 offload->opt_failed |= ret;
207 }
208 up_read(&bpf_devs_lock);
209}
210
176static void __bpf_prog_offload_destroy(struct bpf_prog *prog)211static void __bpf_prog_offload_destroy(struct bpf_prog *prog)
177{212{
178 struct bpf_prog_offload *offload = prog->aux->offload;213 struct bpf_prog_offload *offload = prog->aux->offload;
kernel/bpf/verifier.c+6−0
@@ -6558,6 +6558,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) · 6557 unmodified lines
6558 unsigned int orig_prog_len = env->prog->len;6558 unsigned int orig_prog_len = env->prog->len;
6559 int err;6559 int err;
6560 6560
6561 if (bpf_prog_is_dev_bound(env->prog->aux))
6562 bpf_prog_offload_remove_insns(env, off, cnt);
6563
6561 err = bpf_remove_insns(env->prog, off, cnt);6564 err = bpf_remove_insns(env->prog, off, cnt);
6562 if (err)6565 if (err)
6563 return err;6566 return err;
@@ -6632,6 +6635,9 @@ static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) · 68 unmodified lines
6632 else6635 else
6633 continue;6636 continue;
6634 6637
6638 if (bpf_prog_is_dev_bound(env->prog->aux))
6639 bpf_prog_offload_replace_insn(env, i, &ja);
6640
6635 memcpy(insn, &ja, sizeof(ja));6641 memcpy(insn, &ja, sizeof(ja));
6636 }6642 }
6637}6643}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t