| @@ -6432,6 +6432,150 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of · 6431 unmodified lines |
| 6432 | return new_prog; | 6432 | return new_prog; |
| 6433 | } | 6433 | } |
| 6434 | | 6434 | |
| | 6435 | static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, |
| | 6436 | u32 off, u32 cnt) |
| | 6437 | { |
| | 6438 | int i, j; |
| | 6439 | |
| | 6440 | /* find first prog starting at or after off (first to remove) */ |
| | 6441 | for (i = 0; i < env->subprog_cnt; i++) |
| | 6442 | if (env->subprog_info[i].start >= off) |
| | 6443 | break; |
| | 6444 | /* find first prog starting at or after off + cnt (first to stay) */ |
| | 6445 | for (j = i; j < env->subprog_cnt; j++) |
| | 6446 | if (env->subprog_info[j].start >= off + cnt) |
| | 6447 | break; |
| | 6448 | /* if j doesn't start exactly at off + cnt, we are just removing |
| | 6449 | * the front of previous prog |
| | 6450 | */ |
| | 6451 | if (env->subprog_info[j].start != off + cnt) |
| | 6452 | j--; |
| | 6453 | |
| | 6454 | if (j > i) { |
| | 6455 | struct bpf_prog_aux *aux = env->prog->aux; |
| | 6456 | int move; |
| | 6457 | |
| | 6458 | /* move fake 'exit' subprog as well */ |
| | 6459 | move = env->subprog_cnt + 1 - j; |
| | 6460 | |
| | 6461 | memmove(env->subprog_info + i, |
| | 6462 | env->subprog_info + j, |
| | 6463 | sizeof(*env->subprog_info) * move); |
| | 6464 | env->subprog_cnt -= j - i; |
| | 6465 | |
| | 6466 | /* remove func_info */ |
| | 6467 | if (aux->func_info) { |
| | 6468 | move = aux->func_info_cnt - j; |
| | 6469 | |
| | 6470 | memmove(aux->func_info + i, |
| | 6471 | aux->func_info + j, |
| | 6472 | sizeof(*aux->func_info) * move); |
| | 6473 | aux->func_info_cnt -= j - i; |
| | 6474 | /* func_info->insn_off is set after all code rewrites, |
| | 6475 | * in adjust_btf_func() - no need to adjust |
| | 6476 | */ |
| | 6477 | } |
| | 6478 | } else { |
| | 6479 | /* convert i from "first prog to remove" to "first to adjust" */ |
| | 6480 | if (env->subprog_info[i].start == off) |
| | 6481 | i++; |
| | 6482 | } |
| | 6483 | |
| | 6484 | /* update fake 'exit' subprog as well */ |
| | 6485 | for (; i <= env->subprog_cnt; i++) |
| | 6486 | env->subprog_info[i].start -= cnt; |
| | 6487 | |
| | 6488 | return 0; |
| | 6489 | } |
| | 6490 | |
| | 6491 | static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, |
| | 6492 | u32 cnt) |
| | 6493 | { |
| | 6494 | struct bpf_prog *prog = env->prog; |
| | 6495 | u32 i, l_off, l_cnt, nr_linfo; |
| | 6496 | struct bpf_line_info *linfo; |
| | 6497 | |
| | 6498 | nr_linfo = prog->aux->nr_linfo; |
| | 6499 | if (!nr_linfo) |
| | 6500 | return 0; |
| | 6501 | |
| | 6502 | linfo = prog->aux->linfo; |
| | 6503 | |
| | 6504 | /* find first line info to remove, count lines to be removed */ |
| | 6505 | for (i = 0; i < nr_linfo; i++) |
| | 6506 | if (linfo[i].insn_off >= off) |
| | 6507 | break; |
| | 6508 | |
| | 6509 | l_off = i; |
| | 6510 | l_cnt = 0; |
| | 6511 | for (; i < nr_linfo; i++) |
| | 6512 | if (linfo[i].insn_off < off + cnt) |
| | 6513 | l_cnt++; |
| | 6514 | else |
| | 6515 | break; |
| | 6516 | |
| | 6517 | /* First live insn doesn't match first live linfo, it needs to "inherit" |
| | 6518 | * last removed linfo. prog is already modified, so prog->len == off |
| | 6519 | * means no live instructions after (tail of the program was removed). |
| | 6520 | */ |
| | 6521 | if (prog->len != off && l_cnt && |
| | 6522 | (i == nr_linfo || linfo[i].insn_off != off + cnt)) { |
| | 6523 | l_cnt--; |
| | 6524 | linfo[--i].insn_off = off + cnt; |
| | 6525 | } |
| | 6526 | |
| | 6527 | /* remove the line info which refer to the removed instructions */ |
| | 6528 | if (l_cnt) { |
| | 6529 | memmove(linfo + l_off, linfo + i, |
| | 6530 | sizeof(*linfo) * (nr_linfo - i)); |
| | 6531 | |
| | 6532 | prog->aux->nr_linfo -= l_cnt; |
| | 6533 | nr_linfo = prog->aux->nr_linfo; |
| | 6534 | } |
| | 6535 | |
| | 6536 | /* pull all linfo[i].insn_off >= off + cnt in by cnt */ |
| | 6537 | for (i = l_off; i < nr_linfo; i++) |
| | 6538 | linfo[i].insn_off -= cnt; |
| | 6539 | |
| | 6540 | /* fix up all subprogs (incl. 'exit') which start >= off */ |
| | 6541 | for (i = 0; i <= env->subprog_cnt; i++) |
| | 6542 | if (env->subprog_info[i].linfo_idx > l_off) { |
| | 6543 | /* program may have started in the removed region but |
| | 6544 | * may not be fully removed |
| | 6545 | */ |
| | 6546 | if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) |
| | 6547 | env->subprog_info[i].linfo_idx -= l_cnt; |
| | 6548 | else |
| | 6549 | env->subprog_info[i].linfo_idx = l_off; |
| | 6550 | } |
| | 6551 | |
| | 6552 | return 0; |
| | 6553 | } |
| | 6554 | |
| | 6555 | static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) |
| | 6556 | { |
| | 6557 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| | 6558 | unsigned int orig_prog_len = env->prog->len; |
| | 6559 | int err; |
| | 6560 | |
| | 6561 | err = bpf_remove_insns(env->prog, off, cnt); |
| | 6562 | if (err) |
| | 6563 | return err; |
| | 6564 | |
| | 6565 | err = adjust_subprog_starts_after_remove(env, off, cnt); |
| | 6566 | if (err) |
| | 6567 | return err; |
| | 6568 | |
| | 6569 | err = bpf_adj_linfo_after_remove(env, off, cnt); |
| | 6570 | if (err) |
| | 6571 | return err; |
| | 6572 | |
| | 6573 | memmove(aux_data + off, aux_data + off + cnt, |
| | 6574 | sizeof(*aux_data) * (orig_prog_len - off - cnt)); |
| | 6575 | |
| | 6576 | return 0; |
| | 6577 | } |
| | 6578 | |
| 6435 | /* The verifier does more data flow analysis than llvm and will not | 6579 | /* The verifier does more data flow analysis than llvm and will not |
| 6436 | * explore branches that are dead at run time. Malicious programs can | 6580 | * explore branches that are dead at run time. Malicious programs can |
| 6437 | * have dead code too. Therefore replace all dead at-run-time code | 6581 | * have dead code too. Therefore replace all dead at-run-time code |
| @@ -6492,6 +6636,30 @@ static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) · 54 unmodified lines |
| 6492 | } | 6636 | } |
| 6493 | } | 6637 | } |
| 6494 | | 6638 | |
| | 6639 | static int opt_remove_dead_code(struct bpf_verifier_env *env) |
| | 6640 | { |
| | 6641 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| | 6642 | int insn_cnt = env->prog->len; |
| | 6643 | int i, err; |
| | 6644 | |
| | 6645 | for (i = 0; i < insn_cnt; i++) { |
| | 6646 | int j; |
| | 6647 | |
| | 6648 | j = 0; |
| | 6649 | while (i + j < insn_cnt && !aux_data[i + j].seen) |
| | 6650 | j++; |
| | 6651 | if (!j) |
| | 6652 | continue; |
| | 6653 | |
| | 6654 | err = verifier_remove_insns(env, i, j); |
| | 6655 | if (err) |
| | 6656 | return err; |
| | 6657 | insn_cnt = env->prog->len; |
| | 6658 | } |
| | 6659 | |
| | 6660 | return 0; |
| | 6661 | } |
| | 6662 | |
| 6495 | /* convert load instructions that access fields of a context type into a | 6663 | /* convert load instructions that access fields of a context type into a |
| 6496 | * sequence of instructions that access fields of the underlying structure: | 6664 | * sequence of instructions that access fields of the underlying structure: |
| 6497 | * struct __sk_buff -> struct sk_buff | 6665 | * struct __sk_buff -> struct sk_buff |
| @@ -7282,11 +7450,13 @@ skip_full_check: · 784 unmodified lines |
| 7282 | if (is_priv) { | 7450 | if (is_priv) { |
| 7283 | if (ret == 0) | 7451 | if (ret == 0) |
| 7284 | opt_hard_wire_dead_code_branches(env); | 7452 | opt_hard_wire_dead_code_branches(env); |
| | 7453 | if (ret == 0) |
| | 7454 | ret = opt_remove_dead_code(env); |
| | 7455 | } else { |
| | 7456 | if (ret == 0) |
| | 7457 | sanitize_dead_code(env); |
| 7285 | } | 7458 | } |
| 7286 | | 7459 | |
| 7287 | if (ret == 0) | | |
| 7288 | sanitize_dead_code(env); | | |
| 7289 | | | |
| 7290 | if (ret == 0) | 7460 | if (ret == 0) |
| 7291 | /* program is valid, convert *(u32*)(ctx + off) accesses */ | 7461 | /* program is valid, convert *(u32*)(ctx + off) accesses */ |
| 7292 | ret = convert_ctx_accesses(env); | 7462 | ret = convert_ctx_accesses(env); |