Luke Nelson says: ==================== This patch series introduces a set of optimizations to the BPF JIT on RV64. The optimizations are related to the verifier zero-extension optimization and BPF_JMP BPF_K. We tested the optimizations on a QEMU riscv64 virt machine, using lib/test_bpf and test_verifier, and formally verified their correctness using Serval. ====================
Unified · Split
| @@ -515,7 +515,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, · 514 unmodified lines | ||
| 515 | 515 | case BPF_ALU | BPF_LSH | BPF_X: |
| 516 | 516 | case BPF_ALU64 | BPF_LSH | BPF_X: |
| 517 | 517 | emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); |
| 518 | − if (!is64) | |
| 518 | + if (!is64 && !aux->verifier_zext) | |
| 519 | 519 | emit_zext_32(rd, ctx); |
| 520 | 520 | break; |
| 521 | 521 | case BPF_ALU | BPF_RSH | BPF_X: |
| @@ -542,13 +542,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, · 20 unmodified lines | ||
| 542 | 542 | |
| 543 | 543 | /* dst = BSWAP##imm(dst) */ |
| 544 | 544 | case BPF_ALU | BPF_END | BPF_FROM_LE: |
| 545 | − { | |
| 546 | − int shift = 64 - imm; | |
| 547 | − | |
| 548 | − emit(rv_slli(rd, rd, shift), ctx); | |
| 549 | − emit(rv_srli(rd, rd, shift), ctx); | |
| 545 | + switch (imm) { | |
| 546 | + case 16: | |
| 547 | + emit(rv_slli(rd, rd, 48), ctx); | |
| 548 | + emit(rv_srli(rd, rd, 48), ctx); | |
| 549 | + break; | |
| 550 | + case 32: | |
| 551 | + if (!aux->verifier_zext) | |
| 552 | + emit_zext_32(rd, ctx); | |
| 553 | + break; | |
| 554 | + case 64: | |
| 555 | + /* Do nothing */ | |
| 556 | + break; | |
| 557 | + } | |
| 550 | 558 | break; |
| 551 | − } | |
| 559 | + | |
| 552 | 560 | case BPF_ALU | BPF_END | BPF_FROM_BE: |
| 553 | 561 | emit(rv_addi(RV_REG_T2, RV_REG_ZERO, 0), ctx); |
| 554 | 562 | |
| @@ -692,19 +700,19 @@ out_be: · 137 unmodified lines | ||
| 692 | 700 | case BPF_ALU | BPF_LSH | BPF_K: |
| 693 | 701 | case BPF_ALU64 | BPF_LSH | BPF_K: |
| 694 | 702 | emit(is64 ? rv_slli(rd, rd, imm) : rv_slliw(rd, rd, imm), ctx); |
| 695 | − if (!is64) | |
| 703 | + if (!is64 && !aux->verifier_zext) | |
| 696 | 704 | emit_zext_32(rd, ctx); |
| 697 | 705 | break; |
| 698 | 706 | case BPF_ALU | BPF_RSH | BPF_K: |
| 699 | 707 | case BPF_ALU64 | BPF_RSH | BPF_K: |
| 700 | 708 | emit(is64 ? rv_srli(rd, rd, imm) : rv_srliw(rd, rd, imm), ctx); |
| 701 | − if (!is64) | |
| 709 | + if (!is64 && !aux->verifier_zext) | |
| 702 | 710 | emit_zext_32(rd, ctx); |
| 703 | 711 | break; |
| 704 | 712 | case BPF_ALU | BPF_ARSH | BPF_K: |
| 705 | 713 | case BPF_ALU64 | BPF_ARSH | BPF_K: |
| 706 | 714 | emit(is64 ? rv_srai(rd, rd, imm) : rv_sraiw(rd, rd, imm), ctx); |
| 707 | − if (!is64) | |
| 715 | + if (!is64 && !aux->verifier_zext) | |
| 708 | 716 | emit_zext_32(rd, ctx); |
| 709 | 717 | break; |
| 710 | 718 | |
| @@ -784,11 +792,15 @@ out_be: · 73 unmodified lines | ||
| 784 | 792 | case BPF_JMP32 | BPF_JSGE | BPF_K: |
| 785 | 793 | case BPF_JMP | BPF_JSLE | BPF_K: |
| 786 | 794 | case BPF_JMP32 | BPF_JSLE | BPF_K: |
| 787 | − case BPF_JMP | BPF_JSET | BPF_K: | |
| 788 | − case BPF_JMP32 | BPF_JSET | BPF_K: | |
| 789 | 795 | rvoff = rv_offset(i, off, ctx); |
| 790 | 796 | s = ctx->ninsns; |
| 791 | − emit_imm(RV_REG_T1, imm, ctx); | |
| 797 | + if (imm) { | |
| 798 | + emit_imm(RV_REG_T1, imm, ctx); | |
| 799 | + rs = RV_REG_T1; | |
| 800 | + } else { | |
| 801 | + /* If imm is 0, simply use zero register. */ | |
| 802 | + rs = RV_REG_ZERO; | |
| 803 | + } | |
| 792 | 804 | if (!is64) { |
| 793 | 805 | if (is_signed_bpf_cond(BPF_OP(code))) |
| 794 | 806 | emit_sext_32_rd(&rd, ctx); |
| @@ -799,16 +811,28 @@ out_be: · 4 unmodified lines | ||
| 799 | 811 | |
| 800 | 812 | /* Adjust for extra insns */ |
| 801 | 813 | rvoff -= (e - s) << 2; |
| 814 | + emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); | |
| 815 | + break; | |
| 802 | 816 | |
| 803 | − if (BPF_OP(code) == BPF_JSET) { | |
| 804 | − /* Adjust for and */ | |
| 805 | − rvoff -= 4; | |
| 806 | − emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx); | |
| 807 | − emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, | |
| 808 | − ctx); | |
| 817 | + case BPF_JMP | BPF_JSET | BPF_K: | |
| 818 | + case BPF_JMP32 | BPF_JSET | BPF_K: | |
| 819 | + rvoff = rv_offset(i, off, ctx); | |
| 820 | + s = ctx->ninsns; | |
| 821 | + if (is_12b_int(imm)) { | |
| 822 | + emit(rv_andi(RV_REG_T1, rd, imm), ctx); | |
| 809 | 823 | } else { |
| 810 | − emit_branch(BPF_OP(code), rd, RV_REG_T1, rvoff, ctx); | |
| 824 | + emit_imm(RV_REG_T1, imm, ctx); | |
| 825 | + emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx); | |
| 811 | 826 | } |
| 827 | + /* For jset32, we should clear the upper 32 bits of t1, but | |
| 828 | + * sign-extension is sufficient here and saves one instruction, | |
| 829 | + * as t1 is used only in comparison against zero. | |
| 830 | + */ | |
| 831 | + if (!is64 && imm < 0) | |
| 832 | + emit(rv_addiw(RV_REG_T1, RV_REG_T1, 0), ctx); | |
| 833 | + e = ctx->ninsns; | |
| 834 | + rvoff -= (e - s) << 2; | |
| 835 | + emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); | |
| 812 | 836 | break; |
| 813 | 837 | |
| 814 | 838 | /* function call */ |