Some code pathes, especially the low level entry code, must be protected against instrumentation for various reasons: - Low level entry code can be a fragile beast, especially on x86. - With NO_HZ_FULL RCU state needs to be established before using it. Having a dedicated section for such code allows to validate with tooling that no unsafe functions are invoked. Add the .noinstr.text section and the noinstr attribute to mark functions. noinstr implies notrace. Kprobes will gain a section check later. Provide also a set of markers: instrumentation_begin()/end() These are used to mark code inside a noinstr function which calls into regular instrumentable text section as safe. The instrumentation markers are only active when CONFIG_DEBUG_ENTRY is enabled as the end marker emits a NOP to prevent the compiler from merging the annotation points. This means the objtool verification requires a kernel compiled with this option.
Showing the changes to include/linux (2 of 6 files). Show all
Unified · Split
| @@ -120,12 +120,65 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 119 unmodified lines | |||
| 120 | /* Annotate a C jump table to allow objtool to follow the code flow */ | 120 | /* Annotate a C jump table to allow objtool to follow the code flow */ |
| 121 | #define __annotate_jump_table __section(.rodata..c_jump_table) | 121 | #define __annotate_jump_table __section(.rodata..c_jump_table) |
| 122 | 122 | ||
| 123 | #ifdef CONFIG_DEBUG_ENTRY | ||
| 124 | /* Begin/end of an instrumentation safe region */ | ||
| 125 | #define instrumentation_begin() ({ \ | ||
| 126 | asm volatile("%c0:\n\t" \ | ||
| 127 | ".pushsection .discard.instr_begin\n\t" \ | ||
| 128 | ".long %c0b - .\n\t" \ | ||
| 129 | ".popsection\n\t" : : "i" (__COUNTER__)); \ | ||
| 130 | }) | ||
| 131 | |||
| 132 | /* | ||
| 133 | * Because instrumentation_{begin,end}() can nest, objtool validation considers | ||
| 134 | * _begin() a +1 and _end() a -1 and computes a sum over the instructions. | ||
| 135 | * When the value is greater than 0, we consider instrumentation allowed. | ||
| 136 | * | ||
| 137 | * There is a problem with code like: | ||
| 138 | * | ||
| 139 | * noinstr void foo() | ||
| 140 | * { | ||
| 141 | * instrumentation_begin(); | ||
| 142 | * ... | ||
| 143 | * if (cond) { | ||
| 144 | * instrumentation_begin(); | ||
| 145 | * ... | ||
| 146 | * instrumentation_end(); | ||
| 147 | * } | ||
| 148 | * bar(); | ||
| 149 | * instrumentation_end(); | ||
| 150 | * } | ||
| 151 | * | ||
| 152 | * If instrumentation_end() would be an empty label, like all the other | ||
| 153 | * annotations, the inner _end(), which is at the end of a conditional block, | ||
| 154 | * would land on the instruction after the block. | ||
| 155 | * | ||
| 156 | * If we then consider the sum of the !cond path, we'll see that the call to | ||
| 157 | * bar() is with a 0-value, even though, we meant it to happen with a positive | ||
| 158 | * value. | ||
| 159 | * | ||
| 160 | * To avoid this, have _end() be a NOP instruction, this ensures it will be | ||
| 161 | * part of the condition block and does not escape. | ||
| 162 | */ | ||
| 163 | #define instrumentation_end() ({ \ | ||
| 164 | asm volatile("%c0: nop\n\t" \ | ||
| 165 | ".pushsection .discard.instr_end\n\t" \ | ||
| 166 | ".long %c0b - .\n\t" \ | ||
| 167 | ".popsection\n\t" : : "i" (__COUNTER__)); \ | ||
| 168 | }) | ||
| 169 | #endif /* CONFIG_DEBUG_ENTRY */ | ||
| 170 | |||
| 123 | #else | 171 | #else |
| 124 | #define annotate_reachable() | 172 | #define annotate_reachable() |
| 125 | #define annotate_unreachable() | 173 | #define annotate_unreachable() |
| 126 | #define __annotate_jump_table | 174 | #define __annotate_jump_table |
| 127 | #endif | 175 | #endif |
| 128 | 176 | ||
| 177 | #ifndef instrumentation_begin | ||
| 178 | #define instrumentation_begin() do { } while(0) | ||
| 179 | #define instrumentation_end() do { } while(0) | ||
| 180 | #endif | ||
| 181 | |||
| 129 | #ifndef ASM_UNREACHABLE | 182 | #ifndef ASM_UNREACHABLE |
| 130 | # define ASM_UNREACHABLE | 183 | # define ASM_UNREACHABLE |
| 131 | #endif | 184 | #endif |
| @@ -118,6 +118,10 @@ struct ftrace_likely_data { · 117 unmodified lines | |||
| 118 | #define notrace __attribute__((__no_instrument_function__)) | 118 | #define notrace __attribute__((__no_instrument_function__)) |
| 119 | #endif | 119 | #endif |
| 120 | 120 | ||
| 121 | /* Section for code which can't be instrumented at all */ | ||
| 122 | #define noinstr \ | ||
| 123 | noinline notrace __attribute((__section__(".noinstr.text"))) | ||
| 124 | |||
| 121 | /* | 125 | /* |
| 122 | * it doesn't make sense on ARM (currently the only user of __naked) | 126 | * it doesn't make sense on ARM (currently the only user of __naked) |
| 123 | * to trace naked functions because then mcount is called without | 127 | * to trace naked functions because then mcount is called without |