History

Merge branch 'bpf-rv64-jit'

Luke Nelson says: ==================== This patch series introduces a set of optimizations to the BPF JIT on RV64. The optimizations are related to the verifier zero-extension optimization and BPF_JMP BPF_K. We tested the optimizations on a QEMU riscv64 virt machine, using lib/test_bpf and test_verifier, and formally verified their correctness using Serval. ====================

Daniel Borkmann committed 6 years ago

Commit a085a1e2 parents 5447e8e 073ca6a1 file · +44 −20Browse files

Unified · Split

arch/riscv/net/bpf_jit_comp64.c+44−20
@@ -515,7 +515,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, · 514 unmodified lines
515 case BPF_ALU | BPF_LSH | BPF_X:515 case BPF_ALU | BPF_LSH | BPF_X:
516 case BPF_ALU64 | BPF_LSH | BPF_X:516 case BPF_ALU64 | BPF_LSH | BPF_X:
517 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx);517 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx);
518 if (!is64)518 if (!is64 && !aux->verifier_zext)
519 emit_zext_32(rd, ctx);519 emit_zext_32(rd, ctx);
520 break;520 break;
521 case BPF_ALU | BPF_RSH | BPF_X:521 case BPF_ALU | BPF_RSH | BPF_X:
@@ -542,13 +542,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, · 20 unmodified lines
542 542
543 /* dst = BSWAP##imm(dst) */543 /* dst = BSWAP##imm(dst) */
544 case BPF_ALU | BPF_END | BPF_FROM_LE:544 case BPF_ALU | BPF_END | BPF_FROM_LE:
545 {545 switch (imm) {
546 int shift = 64 - imm;546 case 16:
547 547 emit(rv_slli(rd, rd, 48), ctx);
548 emit(rv_slli(rd, rd, shift), ctx);548 emit(rv_srli(rd, rd, 48), ctx);
549 emit(rv_srli(rd, rd, shift), ctx);549 break;
550 case 32:
551 if (!aux->verifier_zext)
552 emit_zext_32(rd, ctx);
553 break;
554 case 64:
555 /* Do nothing */
556 break;
557 }
550 break;558 break;
551 }559
552 case BPF_ALU | BPF_END | BPF_FROM_BE:560 case BPF_ALU | BPF_END | BPF_FROM_BE:
553 emit(rv_addi(RV_REG_T2, RV_REG_ZERO, 0), ctx);561 emit(rv_addi(RV_REG_T2, RV_REG_ZERO, 0), ctx);
554 562
@@ -692,19 +700,19 @@ out_be: · 137 unmodified lines
692 case BPF_ALU | BPF_LSH | BPF_K:700 case BPF_ALU | BPF_LSH | BPF_K:
693 case BPF_ALU64 | BPF_LSH | BPF_K:701 case BPF_ALU64 | BPF_LSH | BPF_K:
694 emit(is64 ? rv_slli(rd, rd, imm) : rv_slliw(rd, rd, imm), ctx);702 emit(is64 ? rv_slli(rd, rd, imm) : rv_slliw(rd, rd, imm), ctx);
695 if (!is64)703 if (!is64 && !aux->verifier_zext)
696 emit_zext_32(rd, ctx);704 emit_zext_32(rd, ctx);
697 break;705 break;
698 case BPF_ALU | BPF_RSH | BPF_K:706 case BPF_ALU | BPF_RSH | BPF_K:
699 case BPF_ALU64 | BPF_RSH | BPF_K:707 case BPF_ALU64 | BPF_RSH | BPF_K:
700 emit(is64 ? rv_srli(rd, rd, imm) : rv_srliw(rd, rd, imm), ctx);708 emit(is64 ? rv_srli(rd, rd, imm) : rv_srliw(rd, rd, imm), ctx);
701 if (!is64)709 if (!is64 && !aux->verifier_zext)
702 emit_zext_32(rd, ctx);710 emit_zext_32(rd, ctx);
703 break;711 break;
704 case BPF_ALU | BPF_ARSH | BPF_K:712 case BPF_ALU | BPF_ARSH | BPF_K:
705 case BPF_ALU64 | BPF_ARSH | BPF_K:713 case BPF_ALU64 | BPF_ARSH | BPF_K:
706 emit(is64 ? rv_srai(rd, rd, imm) : rv_sraiw(rd, rd, imm), ctx);714 emit(is64 ? rv_srai(rd, rd, imm) : rv_sraiw(rd, rd, imm), ctx);
707 if (!is64)715 if (!is64 && !aux->verifier_zext)
708 emit_zext_32(rd, ctx);716 emit_zext_32(rd, ctx);
709 break;717 break;
710 718
@@ -784,11 +792,15 @@ out_be: · 73 unmodified lines
784 case BPF_JMP32 | BPF_JSGE | BPF_K:792 case BPF_JMP32 | BPF_JSGE | BPF_K:
785 case BPF_JMP | BPF_JSLE | BPF_K:793 case BPF_JMP | BPF_JSLE | BPF_K:
786 case BPF_JMP32 | BPF_JSLE | BPF_K:794 case BPF_JMP32 | BPF_JSLE | BPF_K:
787 case BPF_JMP | BPF_JSET | BPF_K:
788 case BPF_JMP32 | BPF_JSET | BPF_K:
789 rvoff = rv_offset(i, off, ctx);795 rvoff = rv_offset(i, off, ctx);
790 s = ctx->ninsns;796 s = ctx->ninsns;
791 emit_imm(RV_REG_T1, imm, ctx);797 if (imm) {
798 emit_imm(RV_REG_T1, imm, ctx);
799 rs = RV_REG_T1;
800 } else {
801 /* If imm is 0, simply use zero register. */
802 rs = RV_REG_ZERO;
803 }
792 if (!is64) {804 if (!is64) {
793 if (is_signed_bpf_cond(BPF_OP(code)))805 if (is_signed_bpf_cond(BPF_OP(code)))
794 emit_sext_32_rd(&rd, ctx);806 emit_sext_32_rd(&rd, ctx);
@@ -799,16 +811,28 @@ out_be: · 4 unmodified lines
799 811
800 /* Adjust for extra insns */812 /* Adjust for extra insns */
801 rvoff -= (e - s) << 2;813 rvoff -= (e - s) << 2;
814 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);
815 break;
802 816
803 if (BPF_OP(code) == BPF_JSET) {817 case BPF_JMP | BPF_JSET | BPF_K:
804 /* Adjust for and */818 case BPF_JMP32 | BPF_JSET | BPF_K:
805 rvoff -= 4;819 rvoff = rv_offset(i, off, ctx);
806 emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx);820 s = ctx->ninsns;
807 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff,821 if (is_12b_int(imm)) {
808 ctx);822 emit(rv_andi(RV_REG_T1, rd, imm), ctx);
809 } else {823 } else {
810 emit_branch(BPF_OP(code), rd, RV_REG_T1, rvoff, ctx);824 emit_imm(RV_REG_T1, imm, ctx);
825 emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx);
811 }826 }
827 /* For jset32, we should clear the upper 32 bits of t1, but
828 * sign-extension is sufficient here and saves one instruction,
829 * as t1 is used only in comparison against zero.
830 */
831 if (!is64 && imm < 0)
832 emit(rv_addiw(RV_REG_T1, RV_REG_T1, 0), ctx);
833 e = ctx->ninsns;
834 rvoff -= (e - s) << 2;
835 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx);
812 break;836 break;
813 837
814 /* function call */838 /* function call */