Let offload JITs know when instructions are replaced and optimized out, so they can update their state appropriately. The optimizations are best effort, if JIT returns an error from any callback verifier will stop notifying it as state may now be out of sync, but the verifier continues making progress.
Unified · Split
| @@ -268,9 +268,15 @@ struct bpf_verifier_ops { · 267 unmodified lines | |||
| 268 | }; | 268 | }; |
| 269 | 269 | ||
| 270 | struct bpf_prog_offload_ops { | 270 | struct bpf_prog_offload_ops { |
| 271 | /* verifier basic callbacks */ | ||
| 271 | int (*insn_hook)(struct bpf_verifier_env *env, | 272 | int (*insn_hook)(struct bpf_verifier_env *env, |
| 272 | int insn_idx, int prev_insn_idx); | 273 | int insn_idx, int prev_insn_idx); |
| 273 | int (*finalize)(struct bpf_verifier_env *env); | 274 | int (*finalize)(struct bpf_verifier_env *env); |
| 275 | /* verifier optimization callbacks (called after .finalize) */ | ||
| 276 | int (*replace_insn)(struct bpf_verifier_env *env, u32 off, | ||
| 277 | struct bpf_insn *insn); | ||
| 278 | int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); | ||
| 279 | /* program management callbacks */ | ||
| 274 | int (*prepare)(struct bpf_prog *prog); | 280 | int (*prepare)(struct bpf_prog *prog); |
| 275 | int (*translate)(struct bpf_prog *prog); | 281 | int (*translate)(struct bpf_prog *prog); |
| 276 | void (*destroy)(struct bpf_prog *prog); | 282 | void (*destroy)(struct bpf_prog *prog); |
| @@ -283,6 +289,7 @@ struct bpf_prog_offload { · 6 unmodified lines | |||
| 283 | void *dev_priv; | 289 | void *dev_priv; |
| 284 | struct list_head offloads; | 290 | struct list_head offloads; |
| 285 | bool dev_state; | 291 | bool dev_state; |
| 292 | bool opt_failed; | ||
| 286 | void *jited_image; | 293 | void *jited_image; |
| 287 | u32 jited_len; | 294 | u32 jited_len; |
| 288 | }; | 295 | }; |
| @@ -266,5 +266,10 @@ int bpf_prog_offload_verifier_prep(struct bpf_prog *prog); · 265 unmodified lines | |||
| 266 | int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env, | 266 | int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env, |
| 267 | int insn_idx, int prev_insn_idx); | 267 | int insn_idx, int prev_insn_idx); |
| 268 | int bpf_prog_offload_finalize(struct bpf_verifier_env *env); | 268 | int bpf_prog_offload_finalize(struct bpf_verifier_env *env); |
| 269 | void | ||
| 270 | bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off, | ||
| 271 | struct bpf_insn *insn); | ||
| 272 | void | ||
| 273 | bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt); | ||
| 269 | 274 | ||
| 270 | #endif /* _LINUX_BPF_VERIFIER_H */ | 275 | #endif /* _LINUX_BPF_VERIFIER_H */ |
| @@ -173,6 +173,41 @@ int bpf_prog_offload_finalize(struct bpf_verifier_env *env) · 172 unmodified lines | |||
| 173 | return ret; | 173 | return ret; |
| 174 | } | 174 | } |
| 175 | 175 | ||
| 176 | void | ||
| 177 | bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off, | ||
| 178 | struct bpf_insn *insn) | ||
| 179 | { | ||
| 180 | const struct bpf_prog_offload_ops *ops; | ||
| 181 | struct bpf_prog_offload *offload; | ||
| 182 | int ret = -EOPNOTSUPP; | ||
| 183 | |||
| 184 | down_read(&bpf_devs_lock); | ||
| 185 | offload = env->prog->aux->offload; | ||
| 186 | if (offload) { | ||
| 187 | ops = offload->offdev->ops; | ||
| 188 | if (!offload->opt_failed && ops->replace_insn) | ||
| 189 | ret = ops->replace_insn(env, off, insn); | ||
| 190 | offload->opt_failed |= ret; | ||
| 191 | } | ||
| 192 | up_read(&bpf_devs_lock); | ||
| 193 | } | ||
| 194 | |||
| 195 | void | ||
| 196 | bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) | ||
| 197 | { | ||
| 198 | struct bpf_prog_offload *offload; | ||
| 199 | int ret = -EOPNOTSUPP; | ||
| 200 | |||
| 201 | down_read(&bpf_devs_lock); | ||
| 202 | offload = env->prog->aux->offload; | ||
| 203 | if (offload) { | ||
| 204 | if (!offload->opt_failed && offload->offdev->ops->remove_insns) | ||
| 205 | ret = offload->offdev->ops->remove_insns(env, off, cnt); | ||
| 206 | offload->opt_failed |= ret; | ||
| 207 | } | ||
| 208 | up_read(&bpf_devs_lock); | ||
| 209 | } | ||
| 210 | |||
| 176 | static void __bpf_prog_offload_destroy(struct bpf_prog *prog) | 211 | static void __bpf_prog_offload_destroy(struct bpf_prog *prog) |
| 177 | { | 212 | { |
| 178 | struct bpf_prog_offload *offload = prog->aux->offload; | 213 | struct bpf_prog_offload *offload = prog->aux->offload; |
| @@ -6558,6 +6558,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) · 6557 unmodified lines | |||
| 6558 | unsigned int orig_prog_len = env->prog->len; | 6558 | unsigned int orig_prog_len = env->prog->len; |
| 6559 | int err; | 6559 | int err; |
| 6560 | 6560 | ||
| 6561 | if (bpf_prog_is_dev_bound(env->prog->aux)) | ||
| 6562 | bpf_prog_offload_remove_insns(env, off, cnt); | ||
| 6563 | |||
| 6561 | err = bpf_remove_insns(env->prog, off, cnt); | 6564 | err = bpf_remove_insns(env->prog, off, cnt); |
| 6562 | if (err) | 6565 | if (err) |
| 6563 | return err; | 6566 | return err; |
| @@ -6632,6 +6635,9 @@ static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) · 68 unmodified lines | |||
| 6632 | else | 6635 | else |
| 6633 | continue; | 6636 | continue; |
| 6634 | 6637 | ||
| 6638 | if (bpf_prog_is_dev_bound(env->prog->aux)) | ||
| 6639 | bpf_prog_offload_replace_insn(env, i, &ja); | ||
| 6640 | |||
| 6635 | memcpy(insn, &ja, sizeof(ja)); | 6641 | memcpy(insn, &ja, sizeof(ja)); |
| 6636 | } | 6642 | } |
| 6637 | } | 6643 | } |