| @@ -6432,6 +6432,150 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of · 6431 unmodified lines |
| 6432 | 6432 | return new_prog; |
| 6433 | 6433 | } |
| 6434 | 6434 | |
| 6435 | +static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, |
| 6436 | + u32 off, u32 cnt) |
| 6437 | +{ |
| 6438 | + int i, j; |
| 6439 | + |
| 6440 | + /* find first prog starting at or after off (first to remove) */ |
| 6441 | + for (i = 0; i < env->subprog_cnt; i++) |
| 6442 | + if (env->subprog_info[i].start >= off) |
| 6443 | + break; |
| 6444 | + /* find first prog starting at or after off + cnt (first to stay) */ |
| 6445 | + for (j = i; j < env->subprog_cnt; j++) |
| 6446 | + if (env->subprog_info[j].start >= off + cnt) |
| 6447 | + break; |
| 6448 | + /* if j doesn't start exactly at off + cnt, we are just removing |
| 6449 | + * the front of previous prog |
| 6450 | + */ |
| 6451 | + if (env->subprog_info[j].start != off + cnt) |
| 6452 | + j--; |
| 6453 | + |
| 6454 | + if (j > i) { |
| 6455 | + struct bpf_prog_aux *aux = env->prog->aux; |
| 6456 | + int move; |
| 6457 | + |
| 6458 | + /* move fake 'exit' subprog as well */ |
| 6459 | + move = env->subprog_cnt + 1 - j; |
| 6460 | + |
| 6461 | + memmove(env->subprog_info + i, |
| 6462 | + env->subprog_info + j, |
| 6463 | + sizeof(*env->subprog_info) * move); |
| 6464 | + env->subprog_cnt -= j - i; |
| 6465 | + |
| 6466 | + /* remove func_info */ |
| 6467 | + if (aux->func_info) { |
| 6468 | + move = aux->func_info_cnt - j; |
| 6469 | + |
| 6470 | + memmove(aux->func_info + i, |
| 6471 | + aux->func_info + j, |
| 6472 | + sizeof(*aux->func_info) * move); |
| 6473 | + aux->func_info_cnt -= j - i; |
| 6474 | + /* func_info->insn_off is set after all code rewrites, |
| 6475 | + * in adjust_btf_func() - no need to adjust |
| 6476 | + */ |
| 6477 | + } |
| 6478 | + } else { |
| 6479 | + /* convert i from "first prog to remove" to "first to adjust" */ |
| 6480 | + if (env->subprog_info[i].start == off) |
| 6481 | + i++; |
| 6482 | + } |
| 6483 | + |
| 6484 | + /* update fake 'exit' subprog as well */ |
| 6485 | + for (; i <= env->subprog_cnt; i++) |
| 6486 | + env->subprog_info[i].start -= cnt; |
| 6487 | + |
| 6488 | + return 0; |
| 6489 | +} |
| 6490 | + |
| 6491 | +static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, |
| 6492 | + u32 cnt) |
| 6493 | +{ |
| 6494 | + struct bpf_prog *prog = env->prog; |
| 6495 | + u32 i, l_off, l_cnt, nr_linfo; |
| 6496 | + struct bpf_line_info *linfo; |
| 6497 | + |
| 6498 | + nr_linfo = prog->aux->nr_linfo; |
| 6499 | + if (!nr_linfo) |
| 6500 | + return 0; |
| 6501 | + |
| 6502 | + linfo = prog->aux->linfo; |
| 6503 | + |
| 6504 | + /* find first line info to remove, count lines to be removed */ |
| 6505 | + for (i = 0; i < nr_linfo; i++) |
| 6506 | + if (linfo[i].insn_off >= off) |
| 6507 | + break; |
| 6508 | + |
| 6509 | + l_off = i; |
| 6510 | + l_cnt = 0; |
| 6511 | + for (; i < nr_linfo; i++) |
| 6512 | + if (linfo[i].insn_off < off + cnt) |
| 6513 | + l_cnt++; |
| 6514 | + else |
| 6515 | + break; |
| 6516 | + |
| 6517 | + /* First live insn doesn't match first live linfo, it needs to "inherit" |
| 6518 | + * last removed linfo. prog is already modified, so prog->len == off |
| 6519 | + * means no live instructions after (tail of the program was removed). |
| 6520 | + */ |
| 6521 | + if (prog->len != off && l_cnt && |
| 6522 | + (i == nr_linfo || linfo[i].insn_off != off + cnt)) { |
| 6523 | + l_cnt--; |
| 6524 | + linfo[--i].insn_off = off + cnt; |
| 6525 | + } |
| 6526 | + |
| 6527 | + /* remove the line info which refer to the removed instructions */ |
| 6528 | + if (l_cnt) { |
| 6529 | + memmove(linfo + l_off, linfo + i, |
| 6530 | + sizeof(*linfo) * (nr_linfo - i)); |
| 6531 | + |
| 6532 | + prog->aux->nr_linfo -= l_cnt; |
| 6533 | + nr_linfo = prog->aux->nr_linfo; |
| 6534 | + } |
| 6535 | + |
| 6536 | + /* pull all linfo[i].insn_off >= off + cnt in by cnt */ |
| 6537 | + for (i = l_off; i < nr_linfo; i++) |
| 6538 | + linfo[i].insn_off -= cnt; |
| 6539 | + |
| 6540 | + /* fix up all subprogs (incl. 'exit') which start >= off */ |
| 6541 | + for (i = 0; i <= env->subprog_cnt; i++) |
| 6542 | + if (env->subprog_info[i].linfo_idx > l_off) { |
| 6543 | + /* program may have started in the removed region but |
| 6544 | + * may not be fully removed |
| 6545 | + */ |
| 6546 | + if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) |
| 6547 | + env->subprog_info[i].linfo_idx -= l_cnt; |
| 6548 | + else |
| 6549 | + env->subprog_info[i].linfo_idx = l_off; |
| 6550 | + } |
| 6551 | + |
| 6552 | + return 0; |
| 6553 | +} |
| 6554 | + |
| 6555 | +static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) |
| 6556 | +{ |
| 6557 | + struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| 6558 | + unsigned int orig_prog_len = env->prog->len; |
| 6559 | + int err; |
| 6560 | + |
| 6561 | + err = bpf_remove_insns(env->prog, off, cnt); |
| 6562 | + if (err) |
| 6563 | + return err; |
| 6564 | + |
| 6565 | + err = adjust_subprog_starts_after_remove(env, off, cnt); |
| 6566 | + if (err) |
| 6567 | + return err; |
| 6568 | + |
| 6569 | + err = bpf_adj_linfo_after_remove(env, off, cnt); |
| 6570 | + if (err) |
| 6571 | + return err; |
| 6572 | + |
| 6573 | + memmove(aux_data + off, aux_data + off + cnt, |
| 6574 | + sizeof(*aux_data) * (orig_prog_len - off - cnt)); |
| 6575 | + |
| 6576 | + return 0; |
| 6577 | +} |
| 6578 | + |
| 6435 | 6579 | /* The verifier does more data flow analysis than llvm and will not |
| 6436 | 6580 | * explore branches that are dead at run time. Malicious programs can |
| 6437 | 6581 | * have dead code too. Therefore replace all dead at-run-time code |
| @@ -6492,6 +6636,30 @@ static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) · 54 unmodified lines |
| 6492 | 6636 | } |
| 6493 | 6637 | } |
| 6494 | 6638 | |
| 6639 | +static int opt_remove_dead_code(struct bpf_verifier_env *env) |
| 6640 | +{ |
| 6641 | + struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| 6642 | + int insn_cnt = env->prog->len; |
| 6643 | + int i, err; |
| 6644 | + |
| 6645 | + for (i = 0; i < insn_cnt; i++) { |
| 6646 | + int j; |
| 6647 | + |
| 6648 | + j = 0; |
| 6649 | + while (i + j < insn_cnt && !aux_data[i + j].seen) |
| 6650 | + j++; |
| 6651 | + if (!j) |
| 6652 | + continue; |
| 6653 | + |
| 6654 | + err = verifier_remove_insns(env, i, j); |
| 6655 | + if (err) |
| 6656 | + return err; |
| 6657 | + insn_cnt = env->prog->len; |
| 6658 | + } |
| 6659 | + |
| 6660 | + return 0; |
| 6661 | +} |
| 6662 | + |
| 6495 | 6663 | /* convert load instructions that access fields of a context type into a |
| 6496 | 6664 | * sequence of instructions that access fields of the underlying structure: |
| 6497 | 6665 | * struct __sk_buff -> struct sk_buff |
| @@ -7282,11 +7450,13 @@ skip_full_check: · 784 unmodified lines |
| 7282 | 7450 | if (is_priv) { |
| 7283 | 7451 | if (ret == 0) |
| 7284 | 7452 | opt_hard_wire_dead_code_branches(env); |
| 7453 | + if (ret == 0) |
| 7454 | + ret = opt_remove_dead_code(env); |
| 7455 | + } else { |
| 7456 | + if (ret == 0) |
| 7457 | + sanitize_dead_code(env); |
| 7285 | 7458 | } |
| 7286 | 7459 | |
| 7287 | | − if (ret == 0) |
| 7288 | | − sanitize_dead_code(env); |
| 7289 | | − |
| 7290 | 7460 | if (ret == 0) |
| 7291 | 7461 | /* program is valid, convert *(u32*)(ctx + off) accesses */ |
| 7292 | 7462 | ret = convert_ctx_accesses(env); |