eBPF ISA specification requires high 32-bit cleared when low 32-bit sub-register is written. This applies to destination register of ALU32 etc. JIT back-ends must guarantee this semantic when doing code-gen. x86_64 and AArch64 ISA has the same semantics, so the corresponding JIT back-end doesn't need to do extra work. However, 32-bit arches (arm, x86, nfp etc.) and some other 64-bit arches (PowerPC, SPARC etc) need to do explicit zero extension to meet this requirement, otherwise code like the following will fail. u64_value = (u64) u32_value ... other uses of u64_value This is because compiler could exploit the semantic described above and save those zero extensions for extending u32_value to u64_value, these JIT back-ends are expected to guarantee this through inserting extra zero extensions which however could be a significant increase on the code size. Some benchmarks show there could be ~40% sub-register writes out of total insns, meaning at least ~40% extra code-gen. One observation is these extra zero extensions are not always necessary. Take above code snippet for example, it is possible u32_value will never be casted into a u64, the value of high 32-bit of u32_value then could be ignored and extra zero extension could be eliminated. This patch implements this idea, insns defining sub-registers will be marked when the high 32-bit of the defined sub-register matters. For those unmarked insns, it is safe to eliminate high 32-bit clearnace for them. Algo: - Split read flags into READ32 and READ64. - Record index of insn that does sub-register write. Keep the index inside reg state and update it during verifier insn walking. - A full register read on a sub-register marks its definition insn as needing zero extension on dst register. A new sub-register write overrides the old one. - When propagating read64 during path pruning, also mark any insn defining a sub-register that is read in the pruned path as full-register.
Showing the changes to include/linux (1 of 2 files). Show all
Unified · Split
| @@ -36,9 +36,11 @@ · 35 unmodified lines | |||
| 36 | */ | 36 | */ |
| 37 | enum bpf_reg_liveness { | 37 | enum bpf_reg_liveness { |
| 38 | REG_LIVE_NONE = 0, /* reg hasn't been read or written this branch */ | 38 | REG_LIVE_NONE = 0, /* reg hasn't been read or written this branch */ |
| 39 | REG_LIVE_READ, /* reg was read, so we're sensitive to initial value */ | 39 | REG_LIVE_READ32 = 0x1, /* reg was read, so we're sensitive to initial value */ |
| 40 | REG_LIVE_WRITTEN, /* reg was written first, screening off later reads */ | 40 | REG_LIVE_READ64 = 0x2, /* likewise, but full 64-bit content matters */ |
| 41 | REG_LIVE_DONE = 4, /* liveness won't be updating this register anymore */ | 41 | REG_LIVE_READ = REG_LIVE_READ32 | REG_LIVE_READ64, |
| 42 | REG_LIVE_WRITTEN = 0x4, /* reg was written first, screening off later reads */ | ||
| 43 | REG_LIVE_DONE = 0x8, /* liveness won't be updating this register anymore */ | ||
| 42 | }; | 44 | }; |
| 43 | 45 | ||
| 44 | struct bpf_reg_state { | 46 | struct bpf_reg_state { |
| @@ -131,6 +133,11 @@ struct bpf_reg_state { · 86 unmodified lines | |||
| 131 | * pointing to bpf_func_state. | 133 | * pointing to bpf_func_state. |
| 132 | */ | 134 | */ |
| 133 | u32 frameno; | 135 | u32 frameno; |
| 136 | /* Tracks subreg definition. The stored value is the insn_idx of the | ||
| 137 | * writing insn. This is safe because subreg_def is used before any insn | ||
| 138 | * patching which only happens after main verification finished. | ||
| 139 | */ | ||
| 140 | s32 subreg_def; | ||
| 134 | enum bpf_reg_liveness live; | 141 | enum bpf_reg_liveness live; |
| 135 | }; | 142 | }; |
| 136 | 143 | ||
| @@ -233,6 +240,7 @@ struct bpf_insn_aux_data { · 96 unmodified lines | |||
| 233 | int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ | 240 | int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ |
| 234 | int sanitize_stack_off; /* stack slot to be cleared */ | 241 | int sanitize_stack_off; /* stack slot to be cleared */ |
| 235 | bool seen; /* this insn was processed by the verifier */ | 242 | bool seen; /* this insn was processed by the verifier */ |
| 243 | bool zext_dst; /* this insn zero extends dst reg */ | ||
| 236 | u8 alu_state; /* used in combination with alu_limit */ | 244 | u8 alu_state; /* used in combination with alu_limit */ |
| 237 | bool prune_point; | 245 | bool prune_point; |
| 238 | unsigned int orig_idx; /* original instruction index */ | 246 | unsigned int orig_idx; /* original instruction index */ |