Merge reason: After much naming discussion, there seems to be consensus now - queue it up for v3.4.
Unified · Split
| @@ -0,0 +1,286 @@ | |||
| 1 | Static Keys | ||
| 2 | ----------- | ||
| 3 | |||
| 4 | By: Jason Baron <jbaron@redhat.com> | ||
| 5 | |||
| 6 | 0) Abstract | ||
| 7 | |||
| 8 | Static keys allows the inclusion of seldom used features in | ||
| 9 | performance-sensitive fast-path kernel code, via a GCC feature and a code | ||
| 10 | patching technique. A quick example: | ||
| 11 | |||
| 12 | struct static_key key = STATIC_KEY_INIT_FALSE; | ||
| 13 | |||
| 14 | ... | ||
| 15 | |||
| 16 | if (static_key_false(&key)) | ||
| 17 | do unlikely code | ||
| 18 | else | ||
| 19 | do likely code | ||
| 20 | |||
| 21 | ... | ||
| 22 | static_key_slow_inc(); | ||
| 23 | ... | ||
| 24 | static_key_slow_inc(); | ||
| 25 | ... | ||
| 26 | |||
| 27 | The static_key_false() branch will be generated into the code with as little | ||
| 28 | impact to the likely code path as possible. | ||
| 29 | |||
| 30 | |||
| 31 | 1) Motivation | ||
| 32 | |||
| 33 | |||
| 34 | Currently, tracepoints are implemented using a conditional branch. The | ||
| 35 | conditional check requires checking a global variable for each tracepoint. | ||
| 36 | Although the overhead of this check is small, it increases when the memory | ||
| 37 | cache comes under pressure (memory cache lines for these global variables may | ||
| 38 | be shared with other memory accesses). As we increase the number of tracepoints | ||
| 39 | in the kernel this overhead may become more of an issue. In addition, | ||
| 40 | tracepoints are often dormant (disabled) and provide no direct kernel | ||
| 41 | functionality. Thus, it is highly desirable to reduce their impact as much as | ||
| 42 | possible. Although tracepoints are the original motivation for this work, other | ||
| 43 | kernel code paths should be able to make use of the static keys facility. | ||
| 44 | |||
| 45 | |||
| 46 | 2) Solution | ||
| 47 | |||
| 48 | |||
| 49 | gcc (v4.5) adds a new 'asm goto' statement that allows branching to a label: | ||
| 50 | |||
| 51 | http://gcc.gnu.org/ml/gcc-patches/2009-07/msg01556.html | ||
| 52 | |||
| 53 | Using the 'asm goto', we can create branches that are either taken or not taken | ||
| 54 | by default, without the need to check memory. Then, at run-time, we can patch | ||
| 55 | the branch site to change the branch direction. | ||
| 56 | |||
| 57 | For example, if we have a simple branch that is disabled by default: | ||
| 58 | |||
| 59 | if (static_key_false(&key)) | ||
| 60 | printk("I am the true branch\n"); | ||
| 61 | |||
| 62 | Thus, by default the 'printk' will not be emitted. And the code generated will | ||
| 63 | consist of a single atomic 'no-op' instruction (5 bytes on x86), in the | ||
| 64 | straight-line code path. When the branch is 'flipped', we will patch the | ||
| 65 | 'no-op' in the straight-line codepath with a 'jump' instruction to the | ||
| 66 | out-of-line true branch. Thus, changing branch direction is expensive but | ||
| 67 | branch selection is basically 'free'. That is the basic tradeoff of this | ||
| 68 | optimization. | ||
| 69 | |||
| 70 | This lowlevel patching mechanism is called 'jump label patching', and it gives | ||
| 71 | the basis for the static keys facility. | ||
| 72 | |||
| 73 | 3) Static key label API, usage and examples: | ||
| 74 | |||
| 75 | |||
| 76 | In order to make use of this optimization you must first define a key: | ||
| 77 | |||
| 78 | struct static_key key; | ||
| 79 | |||
| 80 | Which is initialized as: | ||
| 81 | |||
| 82 | struct static_key key = STATIC_KEY_INIT_TRUE; | ||
| 83 | |||
| 84 | or: | ||
| 85 | |||
| 86 | struct static_key key = STATIC_KEY_INIT_FALSE; | ||
| 87 | |||
| 88 | If the key is not initialized, it is default false. The 'struct static_key', | ||
| 89 | must be a 'global'. That is, it can't be allocated on the stack or dynamically | ||
| 90 | allocated at run-time. | ||
| 91 | |||
| 92 | The key is then used in code as: | ||
| 93 | |||
| 94 | if (static_key_false(&key)) | ||
| 95 | do unlikely code | ||
| 96 | else | ||
| 97 | do likely code | ||
| 98 | |||
| 99 | Or: | ||
| 100 | |||
| 101 | if (static_key_true(&key)) | ||
| 102 | do likely code | ||
| 103 | else | ||
| 104 | do unlikely code | ||
| 105 | |||
| 106 | A key that is initialized via 'STATIC_KEY_INIT_FALSE', must be used in a | ||
| 107 | 'static_key_false()' construct. Likewise, a key initialized via | ||
| 108 | 'STATIC_KEY_INIT_TRUE' must be used in a 'static_key_true()' construct. A | ||
| 109 | single key can be used in many branches, but all the branches must match the | ||
| 110 | way that the key has been initialized. | ||
| 111 | |||
| 112 | The branch(es) can then be switched via: | ||
| 113 | |||
| 114 | static_key_slow_inc(&key); | ||
| 115 | ... | ||
| 116 | static_key_slow_dec(&key); | ||
| 117 | |||
| 118 | Thus, 'static_key_slow_inc()' means 'make the branch true', and | ||
| 119 | 'static_key_slow_dec()' means 'make the the branch false' with appropriate | ||
| 120 | reference counting. For example, if the key is initialized true, a | ||
| 121 | static_key_slow_dec(), will switch the branch to false. And a subsequent | ||
| 122 | static_key_slow_inc(), will change the branch back to true. Likewise, if the | ||
| 123 | key is initialized false, a 'static_key_slow_inc()', will change the branch to | ||
| 124 | true. And then a 'static_key_slow_dec()', will again make the branch false. | ||
| 125 | |||
| 126 | An example usage in the kernel is the implementation of tracepoints: | ||
| 127 | |||
| 128 | static inline void trace_##name(proto) \ | ||
| 129 | { \ | ||
| 130 | if (static_key_false(&__tracepoint_##name.key)) \ | ||
| 131 | __DO_TRACE(&__tracepoint_##name, \ | ||
| 132 | TP_PROTO(data_proto), \ | ||
| 133 | TP_ARGS(data_args), \ | ||
| 134 | TP_CONDITION(cond)); \ | ||
| 135 | } | ||
| 136 | |||
| 137 | Tracepoints are disabled by default, and can be placed in performance critical | ||
| 138 | pieces of the kernel. Thus, by using a static key, the tracepoints can have | ||
| 139 | absolutely minimal impact when not in use. | ||
| 140 | |||
| 141 | |||
| 142 | 4) Architecture level code patching interface, 'jump labels' | ||
| 143 | |||
| 144 | |||
| 145 | There are a few functions and macros that architectures must implement in order | ||
| 146 | to take advantage of this optimization. If there is no architecture support, we | ||
| 147 | simply fall back to a traditional, load, test, and jump sequence. | ||
| 148 | |||
| 149 | * select HAVE_ARCH_JUMP_LABEL, see: arch/x86/Kconfig | ||
| 150 | |||
| 151 | * #define JUMP_LABEL_NOP_SIZE, see: arch/x86/include/asm/jump_label.h | ||
| 152 | |||
| 153 | * __always_inline bool arch_static_branch(struct static_key *key), see: | ||
| 154 | arch/x86/include/asm/jump_label.h | ||
| 155 | |||
| 156 | * void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type), | ||
| 157 | see: arch/x86/kernel/jump_label.c | ||
| 158 | |||
| 159 | * __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry, enum jump_label_type type), | ||
| 160 | see: arch/x86/kernel/jump_label.c | ||
| 161 | |||
| 162 | |||
| 163 | * struct jump_entry, see: arch/x86/include/asm/jump_label.h | ||
| 164 | |||
| 165 | |||
| 166 | 5) Static keys / jump label analysis, results (x86_64): | ||
| 167 | |||
| 168 | |||
| 169 | As an example, let's add the following branch to 'getppid()', such that the | ||
| 170 | system call now looks like: | ||
| 171 | |||
| 172 | SYSCALL_DEFINE0(getppid) | ||
| 173 | { | ||
| 174 | int pid; | ||
| 175 | |||
| 176 | + if (static_key_false(&key)) | ||
| 177 | + printk("I am the true branch\n"); | ||
| 178 | |||
| 179 | rcu_read_lock(); | ||
| 180 | pid = task_tgid_vnr(rcu_dereference(current->real_parent)); | ||
| 181 | rcu_read_unlock(); | ||
| 182 | |||
| 183 | return pid; | ||
| 184 | } | ||
| 185 | |||
| 186 | The resulting instructions with jump labels generated by GCC is: | ||
| 187 | |||
| 188 | ffffffff81044290 <sys_getppid>: | ||
| 189 | ffffffff81044290: 55 push %rbp | ||
| 190 | ffffffff81044291: 48 89 e5 mov %rsp,%rbp | ||
| 191 | ffffffff81044294: e9 00 00 00 00 jmpq ffffffff81044299 <sys_getppid+0x9> | ||
| 192 | ffffffff81044299: 65 48 8b 04 25 c0 b6 mov %gs:0xb6c0,%rax | ||
| 193 | ffffffff810442a0: 00 00 | ||
| 194 | ffffffff810442a2: 48 8b 80 80 02 00 00 mov 0x280(%rax),%rax | ||
| 195 | ffffffff810442a9: 48 8b 80 b0 02 00 00 mov 0x2b0(%rax),%rax | ||
| 196 | ffffffff810442b0: 48 8b b8 e8 02 00 00 mov 0x2e8(%rax),%rdi | ||
| 197 | ffffffff810442b7: e8 f4 d9 00 00 callq ffffffff81051cb0 <pid_vnr> | ||
| 198 | ffffffff810442bc: 5d pop %rbp | ||
| 199 | ffffffff810442bd: 48 98 cltq | ||
| 200 | ffffffff810442bf: c3 retq | ||
| 201 | ffffffff810442c0: 48 c7 c7 e3 54 98 81 mov $0xffffffff819854e3,%rdi | ||
| 202 | ffffffff810442c7: 31 c0 xor %eax,%eax | ||
| 203 | ffffffff810442c9: e8 71 13 6d 00 callq ffffffff8171563f <printk> | ||
| 204 | ffffffff810442ce: eb c9 jmp ffffffff81044299 <sys_getppid+0x9> | ||
| 205 | |||
| 206 | Without the jump label optimization it looks like: | ||
| 207 | |||
| 208 | ffffffff810441f0 <sys_getppid>: | ||
| 209 | ffffffff810441f0: 8b 05 8a 52 d8 00 mov 0xd8528a(%rip),%eax # ffffffff81dc9480 <key> | ||
| 210 | ffffffff810441f6: 55 push %rbp | ||
| 211 | ffffffff810441f7: 48 89 e5 mov %rsp,%rbp | ||
| 212 | ffffffff810441fa: 85 c0 test %eax,%eax | ||
| 213 | ffffffff810441fc: 75 27 jne ffffffff81044225 <sys_getppid+0x35> | ||
| 214 | ffffffff810441fe: 65 48 8b 04 25 c0 b6 mov %gs:0xb6c0,%rax | ||
| 215 | ffffffff81044205: 00 00 | ||
| 216 | ffffffff81044207: 48 8b 80 80 02 00 00 mov 0x280(%rax),%rax | ||
| 217 | ffffffff8104420e: 48 8b 80 b0 02 00 00 mov 0x2b0(%rax),%rax | ||
| 218 | ffffffff81044215: 48 8b b8 e8 02 00 00 mov 0x2e8(%rax),%rdi | ||
| 219 | ffffffff8104421c: e8 2f da 00 00 callq ffffffff81051c50 <pid_vnr> | ||
| 220 | ffffffff81044221: 5d pop %rbp | ||
| 221 | ffffffff81044222: 48 98 cltq | ||
| 222 | ffffffff81044224: c3 retq | ||
| 223 | ffffffff81044225: 48 c7 c7 13 53 98 81 mov $0xffffffff81985313,%rdi | ||
| 224 | ffffffff8104422c: 31 c0 xor %eax,%eax | ||
| 225 | ffffffff8104422e: e8 60 0f 6d 00 callq ffffffff81715193 <printk> | ||
| 226 | ffffffff81044233: eb c9 jmp ffffffff810441fe <sys_getppid+0xe> | ||
| 227 | ffffffff81044235: 66 66 2e 0f 1f 84 00 data32 nopw %cs:0x0(%rax,%rax,1) | ||
| 228 | ffffffff8104423c: 00 00 00 00 | ||
| 229 | |||
| 230 | Thus, the disable jump label case adds a 'mov', 'test' and 'jne' instruction | ||
| 231 | vs. the jump label case just has a 'no-op' or 'jmp 0'. (The jmp 0, is patched | ||
| 232 | to a 5 byte atomic no-op instruction at boot-time.) Thus, the disabled jump | ||
| 233 | label case adds: | ||
| 234 | |||
| 235 | 6 (mov) + 2 (test) + 2 (jne) = 10 - 5 (5 byte jump 0) = 5 addition bytes. | ||
| 236 | |||
| 237 | If we then include the padding bytes, the jump label code saves, 16 total bytes | ||
| 238 | of instruction memory for this small fucntion. In this case the non-jump label | ||
| 239 | function is 80 bytes long. Thus, we have have saved 20% of the instruction | ||
| 240 | footprint. We can in fact improve this even further, since the 5-byte no-op | ||
| 241 | really can be a 2-byte no-op since we can reach the branch with a 2-byte jmp. | ||
| 242 | However, we have not yet implemented optimal no-op sizes (they are currently | ||
| 243 | hard-coded). | ||
| 244 | |||
| 245 | Since there are a number of static key API uses in the scheduler paths, | ||
| 246 | 'pipe-test' (also known as 'perf bench sched pipe') can be used to show the | ||
| 247 | performance improvement. Testing done on 3.3.0-rc2: | ||
| 248 | |||
| 249 | jump label disabled: | ||
| 250 | |||
| 251 | Performance counter stats for 'bash -c /tmp/pipe-test' (50 runs): | ||
| 252 | |||
| 253 | 855.700314 task-clock # 0.534 CPUs utilized ( +- 0.11% ) | ||
| 254 | 200,003 context-switches # 0.234 M/sec ( +- 0.00% ) | ||
| 255 | 0 CPU-migrations # 0.000 M/sec ( +- 39.58% ) | ||
| 256 | 487 page-faults # 0.001 M/sec ( +- 0.02% ) | ||
| 257 | 1,474,374,262 cycles # 1.723 GHz ( +- 0.17% ) | ||
| 258 | <not supported> stalled-cycles-frontend | ||
| 259 | <not supported> stalled-cycles-backend | ||
| 260 | 1,178,049,567 instructions # 0.80 insns per cycle ( +- 0.06% ) | ||
| 261 | 208,368,926 branches # 243.507 M/sec ( +- 0.06% ) | ||
| 262 | 5,569,188 branch-misses # 2.67% of all branches ( +- 0.54% ) | ||
| 263 | |||
| 264 | 1.601607384 seconds time elapsed ( +- 0.07% ) | ||
| 265 | |||
| 266 | jump label enabled: | ||
| 267 | |||
| 268 | Performance counter stats for 'bash -c /tmp/pipe-test' (50 runs): | ||
| 269 | |||
| 270 | 841.043185 task-clock # 0.533 CPUs utilized ( +- 0.12% ) | ||
| 271 | 200,004 context-switches # 0.238 M/sec ( +- 0.00% ) | ||
| 272 | 0 CPU-migrations # 0.000 M/sec ( +- 40.87% ) | ||
| 273 | 487 page-faults # 0.001 M/sec ( +- 0.05% ) | ||
| 274 | 1,432,559,428 cycles # 1.703 GHz ( +- 0.18% ) | ||
| 275 | <not supported> stalled-cycles-frontend | ||
| 276 | <not supported> stalled-cycles-backend | ||
| 277 | 1,175,363,994 instructions # 0.82 insns per cycle ( +- 0.04% ) | ||
| 278 | 206,859,359 branches # 245.956 M/sec ( +- 0.04% ) | ||
| 279 | 4,884,119 branch-misses # 2.36% of all branches ( +- 0.85% ) | ||
| 280 | |||
| 281 | 1.579384366 seconds time elapsed | ||
| 282 | |||
| 283 | The percentage of saved branches is .7%, and we've saved 12% on | ||
| 284 | 'branch-misses'. This is where we would expect to get the most savings, since | ||
| 285 | this optimization is about reducing the number of branches. In addition, we've | ||
| 286 | saved .2% on instructions, and 2.8% on cycles and 1.4% on elapsed time. | ||
| @@ -47,18 +47,29 @@ config KPROBES · 46 unmodified lines | |||
| 47 | If in doubt, say "N". | 47 | If in doubt, say "N". |
| 48 | 48 | ||
| 49 | config JUMP_LABEL | 49 | config JUMP_LABEL |
| 50 | bool "Optimize trace point call sites" | 50 | bool "Optimize very unlikely/likely branches" |
| 51 | depends on HAVE_ARCH_JUMP_LABEL | 51 | depends on HAVE_ARCH_JUMP_LABEL |
| 52 | help | 52 | help |
| 53 | This option enables a transparent branch optimization that | ||
| 54 | makes certain almost-always-true or almost-always-false branch | ||
| 55 | conditions even cheaper to execute within the kernel. | ||
| 56 | |||
| 57 | Certain performance-sensitive kernel code, such as trace points, | ||
| 58 | scheduler functionality, networking code and KVM have such | ||
| 59 | branches and include support for this optimization technique. | ||
| 60 | |||
| 53 | If it is detected that the compiler has support for "asm goto", | 61 | If it is detected that the compiler has support for "asm goto", |
| 54 | the kernel will compile trace point locations with just a | 62 | the kernel will compile such branches with just a nop |
| 55 | nop instruction. When trace points are enabled, the nop will | 63 | instruction. When the condition flag is toggled to true, the |
| 56 | be converted to a jump to the trace function. This technique | 64 | nop will be converted to a jump instruction to execute the |
| 57 | lowers overhead and stress on the branch prediction of the | 65 | conditional block of instructions. |
| 58 | processor. | 66 | |
| 59 | 67 | This technique lowers overhead and stress on the branch prediction | |
| 60 | On i386, options added to the compiler flags may increase | 68 | of the processor and generally makes the kernel faster. The update |
| 61 | the size of the kernel slightly. | 69 | of the condition is slower, but those are always very rare. |
| 70 | |||
| 71 | ( On 32-bit x86, the necessary options added to the compiler | ||
| 72 | flags may increase the size of the kernel slightly. ) | ||
| 62 | 73 | ||
| 63 | config OPTPROBES | 74 | config OPTPROBES |
| 64 | def_bool y | 75 | def_bool y |
| @@ -281,9 +281,9 @@ paravirt_init_missing_ticks_accounting(int cpu) · 280 unmodified lines | |||
| 281 | pv_time_ops.init_missing_ticks_accounting(cpu); | 281 | pv_time_ops.init_missing_ticks_accounting(cpu); |
| 282 | } | 282 | } |
| 283 | 283 | ||
| 284 | struct jump_label_key; | 284 | struct static_key; |
| 285 | extern struct jump_label_key paravirt_steal_enabled; | 285 | extern struct static_key paravirt_steal_enabled; |
| 286 | extern struct jump_label_key paravirt_steal_rq_enabled; | 286 | extern struct static_key paravirt_steal_rq_enabled; |
| 287 | 287 | ||
| 288 | static inline int | 288 | static inline int |
| 289 | paravirt_do_steal_accounting(unsigned long *new_itm) | 289 | paravirt_do_steal_accounting(unsigned long *new_itm) |
| @@ -634,8 +634,8 @@ struct pv_irq_ops pv_irq_ops = { · 633 unmodified lines | |||
| 634 | * pv_time_ops | 634 | * pv_time_ops |
| 635 | * time operations | 635 | * time operations |
| 636 | */ | 636 | */ |
| 637 | struct jump_label_key paravirt_steal_enabled; | 637 | struct static_key paravirt_steal_enabled; |
| 638 | struct jump_label_key paravirt_steal_rq_enabled; | 638 | struct static_key paravirt_steal_rq_enabled; |
| 639 | 639 | ||
| 640 | static int | 640 | static int |
| 641 | ia64_native_do_steal_accounting(unsigned long *new_itm) | 641 | ia64_native_do_steal_accounting(unsigned long *new_itm) |
| @@ -20,7 +20,7 @@ · 19 unmodified lines | |||
| 20 | #define WORD_INSN ".word" | 20 | #define WORD_INSN ".word" |
| 21 | #endif | 21 | #endif |
| 22 | 22 | ||
| 23 | static __always_inline bool arch_static_branch(struct jump_label_key *key) | 23 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 24 | { | 24 | { |
| 25 | asm goto("1:\tnop\n\t" | 25 | asm goto("1:\tnop\n\t" |
| 26 | "nop\n\t" | 26 | "nop\n\t" |
| @@ -17,7 +17,7 @@ · 16 unmodified lines | |||
| 17 | #define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG) | 17 | #define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG) |
| 18 | #define JUMP_LABEL_NOP_SIZE 4 | 18 | #define JUMP_LABEL_NOP_SIZE 4 |
| 19 | 19 | ||
| 20 | static __always_inline bool arch_static_branch(struct jump_label_key *key) | 20 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 21 | { | 21 | { |
| 22 | asm goto("1:\n\t" | 22 | asm goto("1:\n\t" |
| 23 | "nop\n\t" | 23 | "nop\n\t" |
| @@ -13,7 +13,7 @@ · 12 unmodified lines | |||
| 13 | #define ASM_ALIGN ".balign 4" | 13 | #define ASM_ALIGN ".balign 4" |
| 14 | #endif | 14 | #endif |
| 15 | 15 | ||
| 16 | static __always_inline bool arch_static_branch(struct jump_label_key *key) | 16 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 17 | { | 17 | { |
| 18 | asm goto("0: brcl 0,0\n" | 18 | asm goto("0: brcl 0,0\n" |
| 19 | ".pushsection __jump_table, \"aw\"\n" | 19 | ".pushsection __jump_table, \"aw\"\n" |
| @@ -7,7 +7,7 @@ · 6 unmodified lines | |||
| 7 | 7 | ||
| 8 | #define JUMP_LABEL_NOP_SIZE 4 | 8 | #define JUMP_LABEL_NOP_SIZE 4 |
| 9 | 9 | ||
| 10 | static __always_inline bool arch_static_branch(struct jump_label_key *key) | 10 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 11 | { | 11 | { |
| 12 | asm goto("1:\n\t" | 12 | asm goto("1:\n\t" |
| 13 | "nop\n\t" | 13 | "nop\n\t" |
| @@ -9,12 +9,12 @@ · 8 unmodified lines | |||
| 9 | 9 | ||
| 10 | #define JUMP_LABEL_NOP_SIZE 5 | 10 | #define JUMP_LABEL_NOP_SIZE 5 |
| 11 | 11 | ||
| 12 | #define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t" | 12 | #define STATIC_KEY_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t" |
| 13 | 13 | ||
| 14 | static __always_inline bool arch_static_branch(struct jump_label_key *key) | 14 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 15 | { | 15 | { |
| 16 | asm goto("1:" | 16 | asm goto("1:" |
| 17 | JUMP_LABEL_INITIAL_NOP | 17 | STATIC_KEY_INITIAL_NOP |
| 18 | ".pushsection __jump_table, \"aw\" \n\t" | 18 | ".pushsection __jump_table, \"aw\" \n\t" |
| 19 | _ASM_ALIGN "\n\t" | 19 | _ASM_ALIGN "\n\t" |
| 20 | _ASM_PTR "1b, %l[l_yes], %c0 \n\t" | 20 | _ASM_PTR "1b, %l[l_yes], %c0 \n\t" |
| @@ -230,9 +230,9 @@ static inline unsigned long long paravirt_sched_clock(void) · 229 unmodified lines | |||
| 230 | return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock); | 230 | return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock); |
| 231 | } | 231 | } |
| 232 | 232 | ||
| 233 | struct jump_label_key; | 233 | struct static_key; |
| 234 | extern struct jump_label_key paravirt_steal_enabled; | 234 | extern struct static_key paravirt_steal_enabled; |
| 235 | extern struct jump_label_key paravirt_steal_rq_enabled; | 235 | extern struct static_key paravirt_steal_rq_enabled; |
| 236 | 236 | ||
| 237 | static inline u64 paravirt_steal_clock(int cpu) | 237 | static inline u64 paravirt_steal_clock(int cpu) |
| 238 | { | 238 | { |
| @@ -438,9 +438,9 @@ void __init kvm_guest_init(void) · 437 unmodified lines | |||
| 438 | static __init int activate_jump_labels(void) | 438 | static __init int activate_jump_labels(void) |
| 439 | { | 439 | { |
| 440 | if (has_steal_clock) { | 440 | if (has_steal_clock) { |
| 441 | jump_label_inc(¶virt_steal_enabled); | 441 | static_key_slow_inc(¶virt_steal_enabled); |
| 442 | if (steal_acc) | 442 | if (steal_acc) |
| 443 | jump_label_inc(¶virt_steal_rq_enabled); | 443 | static_key_slow_inc(¶virt_steal_rq_enabled); |
| 444 | } | 444 | } |
| 445 | 445 | ||
| 446 | return 0; | 446 | return 0; |
| @@ -202,8 +202,8 @@ static void native_flush_tlb_single(unsigned long addr) · 201 unmodified lines | |||
| 202 | __native_flush_tlb_single(addr); | 202 | __native_flush_tlb_single(addr); |
| 203 | } | 203 | } |
| 204 | 204 | ||
| 205 | struct jump_label_key paravirt_steal_enabled; | 205 | struct static_key paravirt_steal_enabled; |
| 206 | struct jump_label_key paravirt_steal_rq_enabled; | 206 | struct static_key paravirt_steal_rq_enabled; |
| 207 | 207 | ||
| 208 | static u64 native_steal_clock(int cpu) | 208 | static u64 native_steal_clock(int cpu) |
| 209 | { | 209 | { |
| @@ -234,7 +234,7 @@ static void audit_vcpu_spte(struct kvm_vcpu *vcpu) · 233 unmodified lines | |||
| 234 | } | 234 | } |
| 235 | 235 | ||
| 236 | static bool mmu_audit; | 236 | static bool mmu_audit; |
| 237 | static struct jump_label_key mmu_audit_key; | 237 | static struct static_key mmu_audit_key; |
| 238 | 238 | ||
| 239 | static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) | 239 | static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) |
| 240 | { | 240 | { |
| @@ -250,7 +250,7 @@ static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) · 9 unmodified lines | |||
| 250 | 250 | ||
| 251 | static inline void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) | 251 | static inline void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) |
| 252 | { | 252 | { |
| 253 | if (static_branch((&mmu_audit_key))) | 253 | if (static_key_false((&mmu_audit_key))) |
| 254 | __kvm_mmu_audit(vcpu, point); | 254 | __kvm_mmu_audit(vcpu, point); |
| 255 | } | 255 | } |
| 256 | 256 | ||
| @@ -259,7 +259,7 @@ static void mmu_audit_enable(void) · 2 unmodified lines | |||
| 259 | if (mmu_audit) | 259 | if (mmu_audit) |
| 260 | return; | 260 | return; |
| 261 | 261 | ||
| 262 | jump_label_inc(&mmu_audit_key); | 262 | static_key_slow_inc(&mmu_audit_key); |
| 263 | mmu_audit = true; | 263 | mmu_audit = true; |
| 264 | } | 264 | } |
| 265 | 265 | ||
| @@ -268,7 +268,7 @@ static void mmu_audit_disable(void) · 2 unmodified lines | |||
| 268 | if (!mmu_audit) | 268 | if (!mmu_audit) |
| 269 | return; | 269 | return; |
| 270 | 270 | ||
| 271 | jump_label_dec(&mmu_audit_key); | 271 | static_key_slow_dec(&mmu_audit_key); |
| 272 | mmu_audit = false; | 272 | mmu_audit = false; |
| 273 | } | 273 | } |
| 274 | 274 | ||
| @@ -9,15 +9,15 @@ · 8 unmodified lines | |||
| 9 | * | 9 | * |
| 10 | * Jump labels provide an interface to generate dynamic branches using | 10 | * Jump labels provide an interface to generate dynamic branches using |
| 11 | * self-modifying code. Assuming toolchain and architecture support the result | 11 | * self-modifying code. Assuming toolchain and architecture support the result |
| 12 | * of a "if (static_branch(&key))" statement is a unconditional branch (which | 12 | * of a "if (static_key_false(&key))" statement is a unconditional branch (which |
| 13 | * defaults to false - and the true block is placed out of line). | 13 | * defaults to false - and the true block is placed out of line). |
| 14 | * | 14 | * |
| 15 | * However at runtime we can change the 'static' branch target using | 15 | * However at runtime we can change the branch target using |
| 16 | * jump_label_{inc,dec}(). These function as a 'reference' count on the key | 16 | * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key |
| 17 | * object and for as long as there are references all branches referring to | 17 | * object and for as long as there are references all branches referring to |
| 18 | * that particular key will point to the (out of line) true block. | 18 | * that particular key will point to the (out of line) true block. |
| 19 | * | 19 | * |
| 20 | * Since this relies on modifying code the jump_label_{inc,dec}() functions | 20 | * Since this relies on modifying code the static_key_slow_{inc,dec}() functions |
| 21 | * must be considered absolute slow paths (machine wide synchronization etc.). | 21 | * must be considered absolute slow paths (machine wide synchronization etc.). |
| 22 | * OTOH, since the affected branches are unconditional their runtime overhead | 22 | * OTOH, since the affected branches are unconditional their runtime overhead |
| 23 | * will be absolutely minimal, esp. in the default (off) case where the total | 23 | * will be absolutely minimal, esp. in the default (off) case where the total |
| @@ -26,12 +26,26 @@ · 2 unmodified lines | |||
| 26 | * | 26 | * |
| 27 | * When the control is directly exposed to userspace it is prudent to delay the | 27 | * When the control is directly exposed to userspace it is prudent to delay the |
| 28 | * decrement to avoid high frequency code modifications which can (and do) | 28 | * decrement to avoid high frequency code modifications which can (and do) |
| 29 | * cause significant performance degradation. Struct jump_label_key_deferred and | 29 | * cause significant performance degradation. Struct static_key_deferred and |
| 30 | * jump_label_dec_deferred() provide for this. | 30 | * static_key_slow_dec_deferred() provide for this. |
| 31 | * | 31 | * |
| 32 | * Lacking toolchain and or architecture support, it falls back to a simple | 32 | * Lacking toolchain and or architecture support, it falls back to a simple |
| 33 | * conditional branch. | 33 | * conditional branch. |
| 34 | */ | 34 | * |
| 35 | * struct static_key my_key = STATIC_KEY_INIT_TRUE; | ||
| 36 | * | ||
| 37 | * if (static_key_true(&my_key)) { | ||
| 38 | * } | ||
| 39 | * | ||
| 40 | * will result in the true case being in-line and starts the key with a single | ||
| 41 | * reference. Mixing static_key_true() and static_key_false() on the same key is not | ||
| 42 | * allowed. | ||
| 43 | * | ||
| 44 | * Not initializing the key (static data is initialized to 0s anyway) is the | ||
| 45 | * same as using STATIC_KEY_INIT_FALSE and static_key_false() is | ||
| 46 | * equivalent with static_branch(). | ||
| 47 | * | ||
| 48 | */ | ||
| 35 | 49 | ||
| 36 | #include <linux/types.h> | 50 | #include <linux/types.h> |
| 37 | #include <linux/compiler.h> | 51 | #include <linux/compiler.h> |
| @@ -39,16 +53,17 @@ · 1 unmodified line | |||
| 39 | 53 | ||
| 40 | #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) | 54 | #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) |
| 41 | 55 | ||
| 42 | struct jump_label_key { | 56 | struct static_key { |
| 43 | atomic_t enabled; | 57 | atomic_t enabled; |
| 58 | /* Set lsb bit to 1 if branch is default true, 0 ot */ | ||
| 44 | struct jump_entry *entries; | 59 | struct jump_entry *entries; |
| 45 | #ifdef CONFIG_MODULES | 60 | #ifdef CONFIG_MODULES |
| 46 | struct jump_label_mod *next; | 61 | struct static_key_mod *next; |
| 47 | #endif | 62 | #endif |
| 48 | }; | 63 | }; |
| 49 | 64 | ||
| 50 | struct jump_label_key_deferred { | 65 | struct static_key_deferred { |
| 51 | struct jump_label_key key; | 66 | struct static_key key; |
| 52 | unsigned long timeout; | 67 | unsigned long timeout; |
| 53 | struct delayed_work work; | 68 | struct delayed_work work; |
| 54 | }; | 69 | }; |
| @@ -66,13 +81,34 @@ struct module; · 11 unmodified lines | |||
| 66 | 81 | ||
| 67 | #ifdef HAVE_JUMP_LABEL | 82 | #ifdef HAVE_JUMP_LABEL |
| 68 | 83 | ||
| 69 | #ifdef CONFIG_MODULES | 84 | #define JUMP_LABEL_TRUE_BRANCH 1UL |
| 70 | #define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL} | 85 | |
| 71 | #else | 86 | static |
| 72 | #define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL} | 87 | inline struct jump_entry *jump_label_get_entries(struct static_key *key) |
| 73 | #endif | 88 | { |
| 89 | return (struct jump_entry *)((unsigned long)key->entries | ||
| 90 | & ~JUMP_LABEL_TRUE_BRANCH); | ||
| 91 | } | ||
| 92 | |||
| 93 | static inline bool jump_label_get_branch_default(struct static_key *key) | ||
| 94 | { | ||
| 95 | if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH) | ||
| 96 | return true; | ||
| 97 | return false; | ||
| 98 | } | ||
| 99 | |||
| 100 | static __always_inline bool static_key_false(struct static_key *key) | ||
| 101 | { | ||
| 102 | return arch_static_branch(key); | ||
| 103 | } | ||
| 74 | 104 | ||
| 75 | static __always_inline bool static_branch(struct jump_label_key *key) | 105 | static __always_inline bool static_key_true(struct static_key *key) |
| 106 | { | ||
| 107 | return !static_key_false(key); | ||
| 108 | } | ||
| 109 | |||
| 110 | /* Deprecated. Please use 'static_key_false() instead. */ | ||
| 111 | static __always_inline bool static_branch(struct static_key *key) | ||
| 76 | { | 112 | { |
| 77 | return arch_static_branch(key); | 113 | return arch_static_branch(key); |
| 78 | } | 114 | } |
| @@ -88,21 +124,24 @@ extern void arch_jump_label_transform(struct jump_entry *entry, · 9 unmodified lines | |||
| 88 | extern void arch_jump_label_transform_static(struct jump_entry *entry, | 124 | extern void arch_jump_label_transform_static(struct jump_entry *entry, |
| 89 | enum jump_label_type type); | 125 | enum jump_label_type type); |
| 90 | extern int jump_label_text_reserved(void *start, void *end); | 126 | extern int jump_label_text_reserved(void *start, void *end); |
| 91 | extern void jump_label_inc(struct jump_label_key *key); | 127 | extern void static_key_slow_inc(struct static_key *key); |
| 92 | extern void jump_label_dec(struct jump_label_key *key); | 128 | extern void static_key_slow_dec(struct static_key *key); |
| 93 | extern void jump_label_dec_deferred(struct jump_label_key_deferred *key); | 129 | extern void static_key_slow_dec_deferred(struct static_key_deferred *key); |
| 94 | extern bool jump_label_enabled(struct jump_label_key *key); | 130 | extern bool static_key_enabled(struct static_key *key); |
| 95 | extern void jump_label_apply_nops(struct module *mod); | 131 | extern void jump_label_apply_nops(struct module *mod); |
| 96 | extern void jump_label_rate_limit(struct jump_label_key_deferred *key, | 132 | extern void |
| 97 | unsigned long rl); | 133 | jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl); |
| 134 | |||
| 135 | #define STATIC_KEY_INIT_TRUE ((struct static_key) \ | ||
| 136 | { .enabled = ATOMIC_INIT(1), .entries = (void *)1 }) | ||
| 137 | #define STATIC_KEY_INIT_FALSE ((struct static_key) \ | ||
| 138 | { .enabled = ATOMIC_INIT(0), .entries = (void *)0 }) | ||
| 98 | 139 | ||
| 99 | #else /* !HAVE_JUMP_LABEL */ | 140 | #else /* !HAVE_JUMP_LABEL */ |
| 100 | 141 | ||
| 101 | #include <linux/atomic.h> | 142 | #include <linux/atomic.h> |
| 102 | 143 | ||
| 103 | #define JUMP_LABEL_INIT {ATOMIC_INIT(0)} | 144 | struct static_key { |
| 104 | |||
| 105 | struct jump_label_key { | ||
| 106 | atomic_t enabled; | 145 | atomic_t enabled; |
| 107 | }; | 146 | }; |
| 108 | 147 | ||
| @@ -110,30 +149,45 @@ static __always_inline void jump_label_init(void) · 1 unmodified line | |||
| 110 | { | 149 | { |
| 111 | } | 150 | } |
| 112 | 151 | ||
| 113 | struct jump_label_key_deferred { | 152 | struct static_key_deferred { |
| 114 | struct jump_label_key key; | 153 | struct static_key key; |
| 115 | }; | 154 | }; |
| 116 | 155 | ||
| 117 | static __always_inline bool static_branch(struct jump_label_key *key) | 156 | static __always_inline bool static_key_false(struct static_key *key) |
| 157 | { | ||
| 158 | if (unlikely(atomic_read(&key->enabled)) > 0) | ||
| 159 | return true; | ||
| 160 | return false; | ||
| 161 | } | ||
| 162 | |||
| 163 | static __always_inline bool static_key_true(struct static_key *key) | ||
| 118 | { | 164 | { |
| 119 | if (unlikely(atomic_read(&key->enabled))) | 165 | if (likely(atomic_read(&key->enabled)) > 0) |
| 120 | return true; | 166 | return true; |
| 121 | return false; | 167 | return false; |
| 122 | } | 168 | } |
| 123 | 169 | ||
| 124 | static inline void jump_label_inc(struct jump_label_key *key) | 170 | /* Deprecated. Please use 'static_key_false() instead. */ |
| 171 | static __always_inline bool static_branch(struct static_key *key) | ||
| 172 | { | ||
| 173 | if (unlikely(atomic_read(&key->enabled)) > 0) | ||
| 174 | return true; | ||
| 175 | return false; | ||
| 176 | } | ||
| 177 | |||
| 178 | static inline void static_key_slow_inc(struct static_key *key) | ||
| 125 | { | 179 | { |
| 126 | atomic_inc(&key->enabled); | 180 | atomic_inc(&key->enabled); |
| 127 | } | 181 | } |
| 128 | 182 | ||
| 129 | static inline void jump_label_dec(struct jump_label_key *key) | 183 | static inline void static_key_slow_dec(struct static_key *key) |
| 130 | { | 184 | { |
| 131 | atomic_dec(&key->enabled); | 185 | atomic_dec(&key->enabled); |
| 132 | } | 186 | } |
| 133 | 187 | ||
| 134 | static inline void jump_label_dec_deferred(struct jump_label_key_deferred *key) | 188 | static inline void static_key_slow_dec_deferred(struct static_key_deferred *key) |
| 135 | { | 189 | { |
| 136 | jump_label_dec(&key->key); | 190 | static_key_slow_dec(&key->key); |
| 137 | } | 191 | } |
| 138 | 192 | ||
| 139 | static inline int jump_label_text_reserved(void *start, void *end) | 193 | static inline int jump_label_text_reserved(void *start, void *end) |
| @@ -144,9 +198,9 @@ static inline int jump_label_text_reserved(void *start, void *end) · 4 unmodified lines | |||
| 144 | static inline void jump_label_lock(void) {} | 198 | static inline void jump_label_lock(void) {} |
| 145 | static inline void jump_label_unlock(void) {} | 199 | static inline void jump_label_unlock(void) {} |
| 146 | 200 | ||
| 147 | static inline bool jump_label_enabled(struct jump_label_key *key) | 201 | static inline bool static_key_enabled(struct static_key *key) |
| 148 | { | 202 | { |
| 149 | return !!atomic_read(&key->enabled); | 203 | return (atomic_read(&key->enabled) > 0); |
| 150 | } | 204 | } |
| 151 | 205 | ||
| 152 | static inline int jump_label_apply_nops(struct module *mod) | 206 | static inline int jump_label_apply_nops(struct module *mod) |
| @@ -154,13 +208,20 @@ static inline int jump_label_apply_nops(struct module *mod) · 1 unmodified line | |||
| 154 | return 0; | 208 | return 0; |
| 155 | } | 209 | } |
| 156 | 210 | ||
| 157 | static inline void jump_label_rate_limit(struct jump_label_key_deferred *key, | 211 | static inline void |
| 212 | jump_label_rate_limit(struct static_key_deferred *key, | ||
| 158 | unsigned long rl) | 213 | unsigned long rl) |
| 159 | { | 214 | { |
| 160 | } | 215 | } |
| 216 | |||
| 217 | #define STATIC_KEY_INIT_TRUE ((struct static_key) \ | ||
| 218 | { .enabled = ATOMIC_INIT(1) }) | ||
| 219 | #define STATIC_KEY_INIT_FALSE ((struct static_key) \ | ||
| 220 | { .enabled = ATOMIC_INIT(0) }) | ||
| 221 | |||
| 161 | #endif /* HAVE_JUMP_LABEL */ | 222 | #endif /* HAVE_JUMP_LABEL */ |
| 162 | 223 | ||
| 163 | #define jump_label_key_enabled ((struct jump_label_key){ .enabled = ATOMIC_INIT(1), }) | 224 | #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE |
| 164 | #define jump_label_key_disabled ((struct jump_label_key){ .enabled = ATOMIC_INIT(0), }) | 225 | #define jump_label_enabled static_key_enabled |
| 165 | 226 | ||
| 166 | #endif /* _LINUX_JUMP_LABEL_H */ | 227 | #endif /* _LINUX_JUMP_LABEL_H */ |
| @@ -214,8 +214,8 @@ enum { · 213 unmodified lines | |||
| 214 | #include <linux/skbuff.h> | 214 | #include <linux/skbuff.h> |
| 215 | 215 | ||
| 216 | #ifdef CONFIG_RPS | 216 | #ifdef CONFIG_RPS |
| 217 | #include <linux/jump_label.h> | 217 | #include <linux/static_key.h> |
| 218 | extern struct jump_label_key rps_needed; | 218 | extern struct static_key rps_needed; |
| 219 | #endif | 219 | #endif |
| 220 | 220 | ||
| 221 | struct neighbour; | 221 | struct neighbour; |
| @@ -163,13 +163,13 @@ extern struct ctl_path nf_net_ipv4_netfilter_sysctl_path[]; · 162 unmodified lines | |||
| 163 | extern struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; | 163 | extern struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; |
| 164 | 164 | ||
| 165 | #if defined(CONFIG_JUMP_LABEL) | 165 | #if defined(CONFIG_JUMP_LABEL) |
| 166 | #include <linux/jump_label.h> | 166 | #include <linux/static_key.h> |
| 167 | extern struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; | 167 | extern struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; |
| 168 | static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook) | 168 | static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook) |
| 169 | { | 169 | { |
| 170 | if (__builtin_constant_p(pf) && | 170 | if (__builtin_constant_p(pf) && |
| 171 | __builtin_constant_p(hook)) | 171 | __builtin_constant_p(hook)) |
| 172 | return static_branch(&nf_hooks_needed[pf][hook]); | 172 | return static_key_false(&nf_hooks_needed[pf][hook]); |
| 173 | 173 | ||
| 174 | return !list_empty(&nf_hooks[pf][hook]); | 174 | return !list_empty(&nf_hooks[pf][hook]); |
| 175 | } | 175 | } |
| @@ -514,7 +514,7 @@ struct perf_guest_info_callbacks { · 513 unmodified lines | |||
| 514 | #include <linux/ftrace.h> | 514 | #include <linux/ftrace.h> |
| 515 | #include <linux/cpu.h> | 515 | #include <linux/cpu.h> |
| 516 | #include <linux/irq_work.h> | 516 | #include <linux/irq_work.h> |
| 517 | #include <linux/jump_label.h> | 517 | #include <linux/static_key.h> |
| 518 | #include <linux/atomic.h> | 518 | #include <linux/atomic.h> |
| 519 | #include <asm/local.h> | 519 | #include <asm/local.h> |
| 520 | 520 | ||
| @@ -1041,7 +1041,7 @@ static inline int is_software_event(struct perf_event *event) · 520 unmodified lines | |||
| 1041 | return event->pmu->task_ctx_nr == perf_sw_context; | 1041 | return event->pmu->task_ctx_nr == perf_sw_context; |
| 1042 | } | 1042 | } |
| 1043 | 1043 | ||
| 1044 | extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; | 1044 | extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; |
| 1045 | 1045 | ||
| 1046 | extern void __perf_sw_event(u32, u64, struct pt_regs *, u64); | 1046 | extern void __perf_sw_event(u32, u64, struct pt_regs *, u64); |
| 1047 | 1047 | ||
| @@ -1069,7 +1069,7 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) · 21 unmodified lines | |||
| 1069 | { | 1069 | { |
| 1070 | struct pt_regs hot_regs; | 1070 | struct pt_regs hot_regs; |
| 1071 | 1071 | ||
| 1072 | if (static_branch(&perf_swevent_enabled[event_id])) { | 1072 | if (static_key_false(&perf_swevent_enabled[event_id])) { |
| 1073 | if (!regs) { | 1073 | if (!regs) { |
| 1074 | perf_fetch_caller_regs(&hot_regs); | 1074 | perf_fetch_caller_regs(&hot_regs); |
| 1075 | regs = &hot_regs; | 1075 | regs = &hot_regs; |
| @@ -1078,12 +1078,12 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) · 2 unmodified lines | |||
| 1078 | } | 1078 | } |
| 1079 | } | 1079 | } |
| 1080 | 1080 | ||
| 1081 | extern struct jump_label_key_deferred perf_sched_events; | 1081 | extern struct static_key_deferred perf_sched_events; |
| 1082 | 1082 | ||
| 1083 | static inline void perf_event_task_sched_in(struct task_struct *prev, | 1083 | static inline void perf_event_task_sched_in(struct task_struct *prev, |
| 1084 | struct task_struct *task) | 1084 | struct task_struct *task) |
| 1085 | { | 1085 | { |
| 1086 | if (static_branch(&perf_sched_events.key)) | 1086 | if (static_key_false(&perf_sched_events.key)) |
| 1087 | __perf_event_task_sched_in(prev, task); | 1087 | __perf_event_task_sched_in(prev, task); |
| 1088 | } | 1088 | } |
| 1089 | 1089 | ||
| @@ -1092,7 +1092,7 @@ static inline void perf_event_task_sched_out(struct task_struct *prev, · 2 unmodified lines | |||
| 1092 | { | 1092 | { |
| 1093 | perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0); | 1093 | perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0); |
| 1094 | 1094 | ||
| 1095 | if (static_branch(&perf_sched_events.key)) | 1095 | if (static_key_false(&perf_sched_events.key)) |
| 1096 | __perf_event_task_sched_out(prev, next); | 1096 | __perf_event_task_sched_out(prev, next); |
| 1097 | } | 1097 | } |
| 1098 | 1098 | ||
| @@ -0,0 +1 @@ | |||
| 1 | #include <linux/jump_label.h> | ||
| @@ -17,7 +17,7 @@ · 16 unmodified lines | |||
| 17 | #include <linux/errno.h> | 17 | #include <linux/errno.h> |
| 18 | #include <linux/types.h> | 18 | #include <linux/types.h> |
| 19 | #include <linux/rcupdate.h> | 19 | #include <linux/rcupdate.h> |
| 20 | #include <linux/jump_label.h> | 20 | #include <linux/static_key.h> |
| 21 | 21 | ||
| 22 | struct module; | 22 | struct module; |
| 23 | struct tracepoint; | 23 | struct tracepoint; |
| @@ -29,7 +29,7 @@ struct tracepoint_func { · 5 unmodified lines | |||
| 29 | 29 | ||
| 30 | struct tracepoint { | 30 | struct tracepoint { |
| 31 | const char *name; /* Tracepoint name */ | 31 | const char *name; /* Tracepoint name */ |
| 32 | struct jump_label_key key; | 32 | struct static_key key; |
| 33 | void (*regfunc)(void); | 33 | void (*regfunc)(void); |
| 34 | void (*unregfunc)(void); | 34 | void (*unregfunc)(void); |
| 35 | struct tracepoint_func __rcu *funcs; | 35 | struct tracepoint_func __rcu *funcs; |
| @@ -145,7 +145,7 @@ static inline void tracepoint_synchronize_unregister(void) · 109 unmodified lines | |||
| 145 | extern struct tracepoint __tracepoint_##name; \ | 145 | extern struct tracepoint __tracepoint_##name; \ |
| 146 | static inline void trace_##name(proto) \ | 146 | static inline void trace_##name(proto) \ |
| 147 | { \ | 147 | { \ |
| 148 | if (static_branch(&__tracepoint_##name.key)) \ | 148 | if (static_key_false(&__tracepoint_##name.key)) \ |
| 149 | __DO_TRACE(&__tracepoint_##name, \ | 149 | __DO_TRACE(&__tracepoint_##name, \ |
| 150 | TP_PROTO(data_proto), \ | 150 | TP_PROTO(data_proto), \ |
| 151 | TP_ARGS(data_args), \ | 151 | TP_ARGS(data_args), \ |
| @@ -188,7 +188,7 @@ static inline void tracepoint_synchronize_unregister(void) · 36 unmodified lines | |||
| 188 | __attribute__((section("__tracepoints_strings"))) = #name; \ | 188 | __attribute__((section("__tracepoints_strings"))) = #name; \ |
| 189 | struct tracepoint __tracepoint_##name \ | 189 | struct tracepoint __tracepoint_##name \ |
| 190 | __attribute__((section("__tracepoints"))) = \ | 190 | __attribute__((section("__tracepoints"))) = \ |
| 191 | { __tpstrtab_##name, JUMP_LABEL_INIT, reg, unreg, NULL };\ | 191 | { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\ |
| 192 | static struct tracepoint * const __tracepoint_ptr_##name __used \ | 192 | static struct tracepoint * const __tracepoint_ptr_##name __used \ |
| 193 | __attribute__((section("__tracepoints_ptrs"))) = \ | 193 | __attribute__((section("__tracepoints_ptrs"))) = \ |
| 194 | &__tracepoint_##name; | 194 | &__tracepoint_##name; |
| @@ -55,7 +55,7 @@ · 54 unmodified lines | |||
| 55 | #include <linux/uaccess.h> | 55 | #include <linux/uaccess.h> |
| 56 | #include <linux/memcontrol.h> | 56 | #include <linux/memcontrol.h> |
| 57 | #include <linux/res_counter.h> | 57 | #include <linux/res_counter.h> |
| 58 | #include <linux/jump_label.h> | 58 | #include <linux/static_key.h> |
| 59 | 59 | ||
| 60 | #include <linux/filter.h> | 60 | #include <linux/filter.h> |
| 61 | #include <linux/rculist_nulls.h> | 61 | #include <linux/rculist_nulls.h> |
| @@ -924,13 +924,13 @@ inline void sk_refcnt_debug_release(const struct sock *sk) · 862 unmodified lines | |||
| 924 | #endif /* SOCK_REFCNT_DEBUG */ | 924 | #endif /* SOCK_REFCNT_DEBUG */ |
| 925 | 925 | ||
| 926 | #if defined(CONFIG_CGROUP_MEM_RES_CTLR_KMEM) && defined(CONFIG_NET) | 926 | #if defined(CONFIG_CGROUP_MEM_RES_CTLR_KMEM) && defined(CONFIG_NET) |
| 927 | extern struct jump_label_key memcg_socket_limit_enabled; | 927 | extern struct static_key memcg_socket_limit_enabled; |
| 928 | static inline struct cg_proto *parent_cg_proto(struct proto *proto, | 928 | static inline struct cg_proto *parent_cg_proto(struct proto *proto, |
| 929 | struct cg_proto *cg_proto) | 929 | struct cg_proto *cg_proto) |
| 930 | { | 930 | { |
| 931 | return proto->proto_cgroup(parent_mem_cgroup(cg_proto->memcg)); | 931 | return proto->proto_cgroup(parent_mem_cgroup(cg_proto->memcg)); |
| 932 | } | 932 | } |
| 933 | #define mem_cgroup_sockets_enabled static_branch(&memcg_socket_limit_enabled) | 933 | #define mem_cgroup_sockets_enabled static_key_false(&memcg_socket_limit_enabled) |
| 934 | #else | 934 | #else |
| 935 | #define mem_cgroup_sockets_enabled 0 | 935 | #define mem_cgroup_sockets_enabled 0 |
| 936 | static inline struct cg_proto *parent_cg_proto(struct proto *proto, | 936 | static inline struct cg_proto *parent_cg_proto(struct proto *proto, |
| @@ -128,7 +128,7 @@ enum event_type_t { · 127 unmodified lines | |||
| 128 | * perf_sched_events : >0 events exist | 128 | * perf_sched_events : >0 events exist |
| 129 | * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu | 129 | * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu |
| 130 | */ | 130 | */ |
| 131 | struct jump_label_key_deferred perf_sched_events __read_mostly; | 131 | struct static_key_deferred perf_sched_events __read_mostly; |
| 132 | static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); | 132 | static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); |
| 133 | 133 | ||
| 134 | static atomic_t nr_mmap_events __read_mostly; | 134 | static atomic_t nr_mmap_events __read_mostly; |
| @@ -2769,7 +2769,7 @@ static void free_event(struct perf_event *event) · 2634 unmodified lines | |||
| 2769 | 2769 | ||
| 2770 | if (!event->parent) { | 2770 | if (!event->parent) { |
| 2771 | if (event->attach_state & PERF_ATTACH_TASK) | 2771 | if (event->attach_state & PERF_ATTACH_TASK) |
| 2772 | jump_label_dec_deferred(&perf_sched_events); | 2772 | static_key_slow_dec_deferred(&perf_sched_events); |
| 2773 | if (event->attr.mmap || event->attr.mmap_data) | 2773 | if (event->attr.mmap || event->attr.mmap_data) |
| 2774 | atomic_dec(&nr_mmap_events); | 2774 | atomic_dec(&nr_mmap_events); |
| 2775 | if (event->attr.comm) | 2775 | if (event->attr.comm) |
| @@ -2780,7 +2780,7 @@ static void free_event(struct perf_event *event) · 4 unmodified lines | |||
| 2780 | put_callchain_buffers(); | 2780 | put_callchain_buffers(); |
| 2781 | if (is_cgroup_event(event)) { | 2781 | if (is_cgroup_event(event)) { |
| 2782 | atomic_dec(&per_cpu(perf_cgroup_events, event->cpu)); | 2782 | atomic_dec(&per_cpu(perf_cgroup_events, event->cpu)); |
| 2783 | jump_label_dec_deferred(&perf_sched_events); | 2783 | static_key_slow_dec_deferred(&perf_sched_events); |
| 2784 | } | 2784 | } |
| 2785 | } | 2785 | } |
| 2786 | 2786 | ||
| @@ -4982,7 +4982,7 @@ fail: · 2195 unmodified lines | |||
| 4982 | return err; | 4982 | return err; |
| 4983 | } | 4983 | } |
| 4984 | 4984 | ||
| 4985 | struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; | 4985 | struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; |
| 4986 | 4986 | ||
| 4987 | static void sw_perf_event_destroy(struct perf_event *event) | 4987 | static void sw_perf_event_destroy(struct perf_event *event) |
| 4988 | { | 4988 | { |
| @@ -4990,7 +4990,7 @@ static void sw_perf_event_destroy(struct perf_event *event) · 1 unmodified line | |||
| 4990 | 4990 | ||
| 4991 | WARN_ON(event->parent); | 4991 | WARN_ON(event->parent); |
| 4992 | 4992 | ||
| 4993 | jump_label_dec(&perf_swevent_enabled[event_id]); | 4993 | static_key_slow_dec(&perf_swevent_enabled[event_id]); |
| 4994 | swevent_hlist_put(event); | 4994 | swevent_hlist_put(event); |
| 4995 | } | 4995 | } |
| 4996 | 4996 | ||
| @@ -5020,7 +5020,7 @@ static int perf_swevent_init(struct perf_event *event) · 23 unmodified lines | |||
| 5020 | if (err) | 5020 | if (err) |
| 5021 | return err; | 5021 | return err; |
| 5022 | 5022 | ||
| 5023 | jump_label_inc(&perf_swevent_enabled[event_id]); | 5023 | static_key_slow_inc(&perf_swevent_enabled[event_id]); |
| 5024 | event->destroy = sw_perf_event_destroy; | 5024 | event->destroy = sw_perf_event_destroy; |
| 5025 | } | 5025 | } |
| 5026 | 5026 | ||
| @@ -5843,7 +5843,7 @@ done: · 816 unmodified lines | |||
| 5843 | 5843 | ||
| 5844 | if (!event->parent) { | 5844 | if (!event->parent) { |
| 5845 | if (event->attach_state & PERF_ATTACH_TASK) | 5845 | if (event->attach_state & PERF_ATTACH_TASK) |
| 5846 | jump_label_inc(&perf_sched_events.key); | 5846 | static_key_slow_inc(&perf_sched_events.key); |
| 5847 | if (event->attr.mmap || event->attr.mmap_data) | 5847 | if (event->attr.mmap || event->attr.mmap_data) |
| 5848 | atomic_inc(&nr_mmap_events); | 5848 | atomic_inc(&nr_mmap_events); |
| 5849 | if (event->attr.comm) | 5849 | if (event->attr.comm) |
| @@ -6081,7 +6081,7 @@ SYSCALL_DEFINE5(perf_event_open, · 231 unmodified lines | |||
| 6081 | * - that may need work on context switch | 6081 | * - that may need work on context switch |
| 6082 | */ | 6082 | */ |
| 6083 | atomic_inc(&per_cpu(perf_cgroup_events, event->cpu)); | 6083 | atomic_inc(&per_cpu(perf_cgroup_events, event->cpu)); |
| 6084 | jump_label_inc(&perf_sched_events.key); | 6084 | static_key_slow_inc(&perf_sched_events.key); |
| 6085 | } | 6085 | } |
| 6086 | 6086 | ||
| 6087 | /* | 6087 | /* |
| @@ -12,7 +12,7 @@ · 11 unmodified lines | |||
| 12 | #include <linux/slab.h> | 12 | #include <linux/slab.h> |
| 13 | #include <linux/sort.h> | 13 | #include <linux/sort.h> |
| 14 | #include <linux/err.h> | 14 | #include <linux/err.h> |
| 15 | #include <linux/jump_label.h> | 15 | #include <linux/static_key.h> |
| 16 | 16 | ||
| 17 | #ifdef HAVE_JUMP_LABEL | 17 | #ifdef HAVE_JUMP_LABEL |
| 18 | 18 | ||
| @@ -29,10 +29,11 @@ void jump_label_unlock(void) · 10 unmodified lines | |||
| 29 | mutex_unlock(&jump_label_mutex); | 29 | mutex_unlock(&jump_label_mutex); |
| 30 | } | 30 | } |
| 31 | 31 | ||
| 32 | bool jump_label_enabled(struct jump_label_key *key) | 32 | bool static_key_enabled(struct static_key *key) |
| 33 | { | 33 | { |
| 34 | return !!atomic_read(&key->enabled); | 34 | return (atomic_read(&key->enabled) > 0); |
| 35 | } | 35 | } |
| 36 | EXPORT_SYMBOL_GPL(static_key_enabled); | ||
| 36 | 37 | ||
| 37 | static int jump_label_cmp(const void *a, const void *b) | 38 | static int jump_label_cmp(const void *a, const void *b) |
| 38 | { | 39 | { |
| @@ -58,56 +59,66 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) · 19 unmodified lines | |||
| 58 | sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); | 59 | sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); |
| 59 | } | 60 | } |
| 60 | 61 | ||
| 61 | static void jump_label_update(struct jump_label_key *key, int enable); | 62 | static void jump_label_update(struct static_key *key, int enable); |
| 62 | 63 | ||
| 63 | void jump_label_inc(struct jump_label_key *key) | 64 | void static_key_slow_inc(struct static_key *key) |
| 64 | { | 65 | { |
| 65 | if (atomic_inc_not_zero(&key->enabled)) | 66 | if (atomic_inc_not_zero(&key->enabled)) |
| 66 | return; | 67 | return; |
| 67 | 68 | ||
| 68 | jump_label_lock(); | 69 | jump_label_lock(); |
| 69 | if (atomic_read(&key->enabled) == 0) | 70 | if (atomic_read(&key->enabled) == 0) { |
| 70 | jump_label_update(key, JUMP_LABEL_ENABLE); | 71 | if (!jump_label_get_branch_default(key)) |
| 72 | jump_label_update(key, JUMP_LABEL_ENABLE); | ||
| 73 | else | ||
| 74 | jump_label_update(key, JUMP_LABEL_DISABLE); | ||
| 75 | } | ||
| 71 | atomic_inc(&key->enabled); | 76 | atomic_inc(&key->enabled); |
| 72 | jump_label_unlock(); | 77 | jump_label_unlock(); |
| 73 | } | 78 | } |
| 74 | EXPORT_SYMBOL_GPL(jump_label_inc); | 79 | EXPORT_SYMBOL_GPL(static_key_slow_inc); |
| 75 | 80 | ||
| 76 | static void __jump_label_dec(struct jump_label_key *key, | 81 | static void __static_key_slow_dec(struct static_key *key, |
| 77 | unsigned long rate_limit, struct delayed_work *work) | 82 | unsigned long rate_limit, struct delayed_work *work) |
| 78 | { | 83 | { |
| 79 | if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) | 84 | if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) { |
| 85 | WARN(atomic_read(&key->enabled) < 0, | ||
| 86 | "jump label: negative count!\n"); | ||
| 80 | return; | 87 | return; |
| 88 | } | ||
| 81 | 89 | ||
| 82 | if (rate_limit) { | 90 | if (rate_limit) { |
| 83 | atomic_inc(&key->enabled); | 91 | atomic_inc(&key->enabled); |
| 84 | schedule_delayed_work(work, rate_limit); | 92 | schedule_delayed_work(work, rate_limit); |
| 85 | } else | 93 | } else { |
| 86 | jump_label_update(key, JUMP_LABEL_DISABLE); | 94 | if (!jump_label_get_branch_default(key)) |
| 87 | 95 | jump_label_update(key, JUMP_LABEL_DISABLE); | |
| 96 | else | ||
| 97 | jump_label_update(key, JUMP_LABEL_ENABLE); | ||
| 98 | } | ||
| 88 | jump_label_unlock(); | 99 | jump_label_unlock(); |
| 89 | } | 100 | } |
| 90 | EXPORT_SYMBOL_GPL(jump_label_dec); | ||
| 91 | 101 | ||
| 92 | static void jump_label_update_timeout(struct work_struct *work) | 102 | static void jump_label_update_timeout(struct work_struct *work) |
| 93 | { | 103 | { |
| 94 | struct jump_label_key_deferred *key = | 104 | struct static_key_deferred *key = |
| 95 | container_of(work, struct jump_label_key_deferred, work.work); | 105 | container_of(work, struct static_key_deferred, work.work); |
| 96 | __jump_label_dec(&key->key, 0, NULL); | 106 | __static_key_slow_dec(&key->key, 0, NULL); |
| 97 | } | 107 | } |
| 98 | 108 | ||
| 99 | void jump_label_dec(struct jump_label_key *key) | 109 | void static_key_slow_dec(struct static_key *key) |
| 100 | { | 110 | { |
| 101 | __jump_label_dec(key, 0, NULL); | 111 | __static_key_slow_dec(key, 0, NULL); |
| 102 | } | 112 | } |
| 113 | EXPORT_SYMBOL_GPL(static_key_slow_dec); | ||
| 103 | 114 | ||
| 104 | void jump_label_dec_deferred(struct jump_label_key_deferred *key) | 115 | void static_key_slow_dec_deferred(struct static_key_deferred *key) |
| 105 | { | 116 | { |
| 106 | __jump_label_dec(&key->key, key->timeout, &key->work); | 117 | __static_key_slow_dec(&key->key, key->timeout, &key->work); |
| 107 | } | 118 | } |
| 119 | EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred); | ||
| 108 | 120 | ||
| 109 | 121 | void jump_label_rate_limit(struct static_key_deferred *key, | |
| 110 | void jump_label_rate_limit(struct jump_label_key_deferred *key, | ||
| 111 | unsigned long rl) | 122 | unsigned long rl) |
| 112 | { | 123 | { |
| 113 | key->timeout = rl; | 124 | key->timeout = rl; |
| @@ -150,7 +161,7 @@ void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry · 36 unmodified lines | |||
| 150 | arch_jump_label_transform(entry, type); | 161 | arch_jump_label_transform(entry, type); |
| 151 | } | 162 | } |
| 152 | 163 | ||
| 153 | static void __jump_label_update(struct jump_label_key *key, | 164 | static void __jump_label_update(struct static_key *key, |
| 154 | struct jump_entry *entry, | 165 | struct jump_entry *entry, |
| 155 | struct jump_entry *stop, int enable) | 166 | struct jump_entry *stop, int enable) |
| 156 | { | 167 | { |
| @@ -167,27 +178,40 @@ static void __jump_label_update(struct jump_label_key *key, · 10 unmodified lines | |||
| 167 | } | 178 | } |
| 168 | } | 179 | } |
| 169 | 180 | ||
| 181 | static enum jump_label_type jump_label_type(struct static_key *key) | ||
| 182 | { | ||
| 183 | bool true_branch = jump_label_get_branch_default(key); | ||
| 184 | bool state = static_key_enabled(key); | ||
| 185 | |||
| 186 | if ((!true_branch && state) || (true_branch && !state)) | ||
| 187 | return JUMP_LABEL_ENABLE; | ||
| 188 | |||
| 189 | return JUMP_LABEL_DISABLE; | ||
| 190 | } | ||
| 191 | |||
| 170 | void __init jump_label_init(void) | 192 | void __init jump_label_init(void) |
| 171 | { | 193 | { |
| 172 | struct jump_entry *iter_start = __start___jump_table; | 194 | struct jump_entry *iter_start = __start___jump_table; |
| 173 | struct jump_entry *iter_stop = __stop___jump_table; | 195 | struct jump_entry *iter_stop = __stop___jump_table; |
| 174 | struct jump_label_key *key = NULL; | 196 | struct static_key *key = NULL; |
| 175 | struct jump_entry *iter; | 197 | struct jump_entry *iter; |
| 176 | 198 | ||
| 177 | jump_label_lock(); | 199 | jump_label_lock(); |
| 178 | jump_label_sort_entries(iter_start, iter_stop); | 200 | jump_label_sort_entries(iter_start, iter_stop); |
| 179 | 201 | ||
| 180 | for (iter = iter_start; iter < iter_stop; iter++) { | 202 | for (iter = iter_start; iter < iter_stop; iter++) { |
| 181 | struct jump_label_key *iterk; | 203 | struct static_key *iterk; |
| 182 | 204 | ||
| 183 | iterk = (struct jump_label_key *)(unsigned long)iter->key; | 205 | iterk = (struct static_key *)(unsigned long)iter->key; |
| 184 | arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ? | 206 | arch_jump_label_transform_static(iter, jump_label_type(iterk)); |
| 185 | JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE); | ||
| 186 | if (iterk == key) | 207 | if (iterk == key) |
| 187 | continue; | 208 | continue; |
| 188 | 209 | ||
| 189 | key = iterk; | 210 | key = iterk; |
| 190 | key->entries = iter; | 211 | /* |
| 212 | * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH. | ||
| 213 | */ | ||
| 214 | *((unsigned long *)&key->entries) += (unsigned long)iter; | ||
| 191 | #ifdef CONFIG_MODULES | 215 | #ifdef CONFIG_MODULES |
| 192 | key->next = NULL; | 216 | key->next = NULL; |
| 193 | #endif | 217 | #endif |
| @@ -197,8 +221,8 @@ void __init jump_label_init(void) · 3 unmodified lines | |||
| 197 | 221 | ||
| 198 | #ifdef CONFIG_MODULES | 222 | #ifdef CONFIG_MODULES |
| 199 | 223 | ||
| 200 | struct jump_label_mod { | 224 | struct static_key_mod { |
| 201 | struct jump_label_mod *next; | 225 | struct static_key_mod *next; |
| 202 | struct jump_entry *entries; | 226 | struct jump_entry *entries; |
| 203 | struct module *mod; | 227 | struct module *mod; |
| 204 | }; | 228 | }; |
| @@ -218,9 +242,9 @@ static int __jump_label_mod_text_reserved(void *start, void *end) · 13 unmodified lines | |||
| 218 | start, end); | 242 | start, end); |
| 219 | } | 243 | } |
| 220 | 244 | ||
| 221 | static void __jump_label_mod_update(struct jump_label_key *key, int enable) | 245 | static void __jump_label_mod_update(struct static_key *key, int enable) |
| 222 | { | 246 | { |
| 223 | struct jump_label_mod *mod = key->next; | 247 | struct static_key_mod *mod = key->next; |
| 224 | 248 | ||
| 225 | while (mod) { | 249 | while (mod) { |
| 226 | struct module *m = mod->mod; | 250 | struct module *m = mod->mod; |
| @@ -251,11 +275,7 @@ void jump_label_apply_nops(struct module *mod) · 24 unmodified lines | |||
| 251 | return; | 275 | return; |
| 252 | 276 | ||
| 253 | for (iter = iter_start; iter < iter_stop; iter++) { | 277 | for (iter = iter_start; iter < iter_stop; iter++) { |
| 254 | struct jump_label_key *iterk; | 278 | arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE); |
| 255 | |||
| 256 | iterk = (struct jump_label_key *)(unsigned long)iter->key; | ||
| 257 | arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ? | ||
| 258 | JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE); | ||
| 259 | } | 279 | } |
| 260 | } | 280 | } |
| 261 | 281 | ||
| @@ -264,8 +284,8 @@ static int jump_label_add_module(struct module *mod) · 2 unmodified lines | |||
| 264 | struct jump_entry *iter_start = mod->jump_entries; | 284 | struct jump_entry *iter_start = mod->jump_entries; |
| 265 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; | 285 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; |
| 266 | struct jump_entry *iter; | 286 | struct jump_entry *iter; |
| 267 | struct jump_label_key *key = NULL; | 287 | struct static_key *key = NULL; |
| 268 | struct jump_label_mod *jlm; | 288 | struct static_key_mod *jlm; |
| 269 | 289 | ||
| 270 | /* if the module doesn't have jump label entries, just return */ | 290 | /* if the module doesn't have jump label entries, just return */ |
| 271 | if (iter_start == iter_stop) | 291 | if (iter_start == iter_stop) |
| @@ -274,28 +294,30 @@ static int jump_label_add_module(struct module *mod) · 2 unmodified lines | |||
| 274 | jump_label_sort_entries(iter_start, iter_stop); | 294 | jump_label_sort_entries(iter_start, iter_stop); |
| 275 | 295 | ||
| 276 | for (iter = iter_start; iter < iter_stop; iter++) { | 296 | for (iter = iter_start; iter < iter_stop; iter++) { |
| 277 | if (iter->key == (jump_label_t)(unsigned long)key) | 297 | struct static_key *iterk; |
| 278 | continue; | ||
| 279 | 298 | ||
| 280 | key = (struct jump_label_key *)(unsigned long)iter->key; | 299 | iterk = (struct static_key *)(unsigned long)iter->key; |
| 300 | if (iterk == key) | ||
| 301 | continue; | ||
| 281 | 302 | ||
| 303 | key = iterk; | ||
| 282 | if (__module_address(iter->key) == mod) { | 304 | if (__module_address(iter->key) == mod) { |
| 283 | atomic_set(&key->enabled, 0); | 305 | /* |
| 284 | key->entries = iter; | 306 | * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH. |
| 307 | */ | ||
| 308 | *((unsigned long *)&key->entries) += (unsigned long)iter; | ||
| 285 | key->next = NULL; | 309 | key->next = NULL; |
| 286 | continue; | 310 | continue; |
| 287 | } | 311 | } |
| 288 | 312 | jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL); | |
| 289 | jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL); | ||
| 290 | if (!jlm) | 313 | if (!jlm) |
| 291 | return -ENOMEM; | 314 | return -ENOMEM; |
| 292 | |||
| 293 | jlm->mod = mod; | 315 | jlm->mod = mod; |
| 294 | jlm->entries = iter; | 316 | jlm->entries = iter; |
| 295 | jlm->next = key->next; | 317 | jlm->next = key->next; |
| 296 | key->next = jlm; | 318 | key->next = jlm; |
| 297 | 319 | ||
| 298 | if (jump_label_enabled(key)) | 320 | if (jump_label_type(key) == JUMP_LABEL_ENABLE) |
| 299 | __jump_label_update(key, iter, iter_stop, JUMP_LABEL_ENABLE); | 321 | __jump_label_update(key, iter, iter_stop, JUMP_LABEL_ENABLE); |
| 300 | } | 322 | } |
| 301 | 323 | ||
| @@ -307,14 +329,14 @@ static void jump_label_del_module(struct module *mod) · 5 unmodified lines | |||
| 307 | struct jump_entry *iter_start = mod->jump_entries; | 329 | struct jump_entry *iter_start = mod->jump_entries; |
| 308 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; | 330 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; |
| 309 | struct jump_entry *iter; | 331 | struct jump_entry *iter; |
| 310 | struct jump_label_key *key = NULL; | 332 | struct static_key *key = NULL; |
| 311 | struct jump_label_mod *jlm, **prev; | 333 | struct static_key_mod *jlm, **prev; |
| 312 | 334 | ||
| 313 | for (iter = iter_start; iter < iter_stop; iter++) { | 335 | for (iter = iter_start; iter < iter_stop; iter++) { |
| 314 | if (iter->key == (jump_label_t)(unsigned long)key) | 336 | if (iter->key == (jump_label_t)(unsigned long)key) |
| 315 | continue; | 337 | continue; |
| 316 | 338 | ||
| 317 | key = (struct jump_label_key *)(unsigned long)iter->key; | 339 | key = (struct static_key *)(unsigned long)iter->key; |
| 318 | 340 | ||
| 319 | if (__module_address(iter->key) == mod) | 341 | if (__module_address(iter->key) == mod) |
| 320 | continue; | 342 | continue; |
| @@ -416,12 +438,13 @@ int jump_label_text_reserved(void *start, void *end) · 95 unmodified lines | |||
| 416 | return ret; | 438 | return ret; |
| 417 | } | 439 | } |
| 418 | 440 | ||
| 419 | static void jump_label_update(struct jump_label_key *key, int enable) | 441 | static void jump_label_update(struct static_key *key, int enable) |
| 420 | { | 442 | { |
| 421 | struct jump_entry *entry = key->entries, *stop = __stop___jump_table; | 443 | struct jump_entry *stop = __stop___jump_table; |
| 444 | struct jump_entry *entry = jump_label_get_entries(key); | ||
| 422 | 445 | ||
| 423 | #ifdef CONFIG_MODULES | 446 | #ifdef CONFIG_MODULES |
| 424 | struct module *mod = __module_address((jump_label_t)key); | 447 | struct module *mod = __module_address((unsigned long)key); |
| 425 | 448 | ||
| 426 | __jump_label_mod_update(key, enable); | 449 | __jump_label_mod_update(key, enable); |
| 427 | 450 | ||
| @@ -162,13 +162,13 @@ static int sched_feat_show(struct seq_file *m, void *v) · 161 unmodified lines | |||
| 162 | 162 | ||
| 163 | #ifdef HAVE_JUMP_LABEL | 163 | #ifdef HAVE_JUMP_LABEL |
| 164 | 164 | ||
| 165 | #define jump_label_key__true jump_label_key_enabled | 165 | #define jump_label_key__true STATIC_KEY_INIT_TRUE |
| 166 | #define jump_label_key__false jump_label_key_disabled | 166 | #define jump_label_key__false STATIC_KEY_INIT_FALSE |
| 167 | 167 | ||
| 168 | #define SCHED_FEAT(name, enabled) \ | 168 | #define SCHED_FEAT(name, enabled) \ |
| 169 | jump_label_key__##enabled , | 169 | jump_label_key__##enabled , |
| 170 | 170 | ||
| 171 | struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = { | 171 | struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { |
| 172 | #include "features.h" | 172 | #include "features.h" |
| 173 | }; | 173 | }; |
| 174 | 174 | ||
| @@ -176,14 +176,14 @@ struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR] = { · 1 unmodified line | |||
| 176 | 176 | ||
| 177 | static void sched_feat_disable(int i) | 177 | static void sched_feat_disable(int i) |
| 178 | { | 178 | { |
| 179 | if (jump_label_enabled(&sched_feat_keys[i])) | 179 | if (static_key_enabled(&sched_feat_keys[i])) |
| 180 | jump_label_dec(&sched_feat_keys[i]); | 180 | static_key_slow_dec(&sched_feat_keys[i]); |
| 181 | } | 181 | } |
| 182 | 182 | ||
| 183 | static void sched_feat_enable(int i) | 183 | static void sched_feat_enable(int i) |
| 184 | { | 184 | { |
| 185 | if (!jump_label_enabled(&sched_feat_keys[i])) | 185 | if (!static_key_enabled(&sched_feat_keys[i])) |
| 186 | jump_label_inc(&sched_feat_keys[i]); | 186 | static_key_slow_inc(&sched_feat_keys[i]); |
| 187 | } | 187 | } |
| 188 | #else | 188 | #else |
| 189 | static void sched_feat_disable(int i) { }; | 189 | static void sched_feat_disable(int i) { }; |
| @@ -894,7 +894,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) · 704 unmodified lines | |||
| 894 | delta -= irq_delta; | 894 | delta -= irq_delta; |
| 895 | #endif | 895 | #endif |
| 896 | #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING | 896 | #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING |
| 897 | if (static_branch((¶virt_steal_rq_enabled))) { | 897 | if (static_key_false((¶virt_steal_rq_enabled))) { |
| 898 | u64 st; | 898 | u64 st; |
| 899 | 899 | ||
| 900 | steal = paravirt_steal_clock(cpu_of(rq)); | 900 | steal = paravirt_steal_clock(cpu_of(rq)); |
| @@ -2756,7 +2756,7 @@ void account_idle_time(cputime_t cputime) · 1855 unmodified lines | |||
| 2756 | static __always_inline bool steal_account_process_tick(void) | 2756 | static __always_inline bool steal_account_process_tick(void) |
| 2757 | { | 2757 | { |
| 2758 | #ifdef CONFIG_PARAVIRT | 2758 | #ifdef CONFIG_PARAVIRT |
| 2759 | if (static_branch(¶virt_steal_enabled)) { | 2759 | if (static_key_false(¶virt_steal_enabled)) { |
| 2760 | u64 steal, st = 0; | 2760 | u64 steal, st = 0; |
| 2761 | 2761 | ||
| 2762 | steal = paravirt_steal_clock(smp_processor_id()); | 2762 | steal = paravirt_steal_clock(smp_processor_id()); |
| @@ -1399,20 +1399,20 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) · 1398 unmodified lines | |||
| 1399 | #ifdef CONFIG_CFS_BANDWIDTH | 1399 | #ifdef CONFIG_CFS_BANDWIDTH |
| 1400 | 1400 | ||
| 1401 | #ifdef HAVE_JUMP_LABEL | 1401 | #ifdef HAVE_JUMP_LABEL |
| 1402 | static struct jump_label_key __cfs_bandwidth_used; | 1402 | static struct static_key __cfs_bandwidth_used; |
| 1403 | 1403 | ||
| 1404 | static inline bool cfs_bandwidth_used(void) | 1404 | static inline bool cfs_bandwidth_used(void) |
| 1405 | { | 1405 | { |
| 1406 | return static_branch(&__cfs_bandwidth_used); | 1406 | return static_key_false(&__cfs_bandwidth_used); |
| 1407 | } | 1407 | } |
| 1408 | 1408 | ||
| 1409 | void account_cfs_bandwidth_used(int enabled, int was_enabled) | 1409 | void account_cfs_bandwidth_used(int enabled, int was_enabled) |
| 1410 | { | 1410 | { |
| 1411 | /* only need to count groups transitioning between enabled/!enabled */ | 1411 | /* only need to count groups transitioning between enabled/!enabled */ |
| 1412 | if (enabled && !was_enabled) | 1412 | if (enabled && !was_enabled) |
| 1413 | jump_label_inc(&__cfs_bandwidth_used); | 1413 | static_key_slow_inc(&__cfs_bandwidth_used); |
| 1414 | else if (!enabled && was_enabled) | 1414 | else if (!enabled && was_enabled) |
| 1415 | jump_label_dec(&__cfs_bandwidth_used); | 1415 | static_key_slow_dec(&__cfs_bandwidth_used); |
| 1416 | } | 1416 | } |
| 1417 | #else /* HAVE_JUMP_LABEL */ | 1417 | #else /* HAVE_JUMP_LABEL */ |
| 1418 | static bool cfs_bandwidth_used(void) | 1418 | static bool cfs_bandwidth_used(void) |
| @@ -611,7 +611,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) · 610 unmodified lines | |||
| 611 | * Tunables that become constants when CONFIG_SCHED_DEBUG is off: | 611 | * Tunables that become constants when CONFIG_SCHED_DEBUG is off: |
| 612 | */ | 612 | */ |
| 613 | #ifdef CONFIG_SCHED_DEBUG | 613 | #ifdef CONFIG_SCHED_DEBUG |
| 614 | # include <linux/jump_label.h> | 614 | # include <linux/static_key.h> |
| 615 | # define const_debug __read_mostly | 615 | # define const_debug __read_mostly |
| 616 | #else | 616 | #else |
| 617 | # define const_debug const | 617 | # define const_debug const |
| @@ -630,18 +630,18 @@ enum { · 12 unmodified lines | |||
| 630 | #undef SCHED_FEAT | 630 | #undef SCHED_FEAT |
| 631 | 631 | ||
| 632 | #if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL) | 632 | #if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL) |
| 633 | static __always_inline bool static_branch__true(struct jump_label_key *key) | 633 | static __always_inline bool static_branch__true(struct static_key *key) |
| 634 | { | 634 | { |
| 635 | return likely(static_branch(key)); /* Not out of line branch. */ | 635 | return static_key_true(key); /* Not out of line branch. */ |
| 636 | } | 636 | } |
| 637 | 637 | ||
| 638 | static __always_inline bool static_branch__false(struct jump_label_key *key) | 638 | static __always_inline bool static_branch__false(struct static_key *key) |
| 639 | { | 639 | { |
| 640 | return unlikely(static_branch(key)); /* Out of line branch. */ | 640 | return static_key_false(key); /* Out of line branch. */ |
| 641 | } | 641 | } |
| 642 | 642 | ||
| 643 | #define SCHED_FEAT(name, enabled) \ | 643 | #define SCHED_FEAT(name, enabled) \ |
| 644 | static __always_inline bool static_branch_##name(struct jump_label_key *key) \ | 644 | static __always_inline bool static_branch_##name(struct static_key *key) \ |
| 645 | { \ | 645 | { \ |
| 646 | return static_branch__##enabled(key); \ | 646 | return static_branch__##enabled(key); \ |
| 647 | } | 647 | } |
| @@ -650,7 +650,7 @@ static __always_inline bool static_branch_##name(struct jump_label_key *key) \ · 2 unmodified lines | |||
| 650 | 650 | ||
| 651 | #undef SCHED_FEAT | 651 | #undef SCHED_FEAT |
| 652 | 652 | ||
| 653 | extern struct jump_label_key sched_feat_keys[__SCHED_FEAT_NR]; | 653 | extern struct static_key sched_feat_keys[__SCHED_FEAT_NR]; |
| 654 | #define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x])) | 654 | #define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x])) |
| 655 | #else /* !(SCHED_DEBUG && HAVE_JUMP_LABEL) */ | 655 | #else /* !(SCHED_DEBUG && HAVE_JUMP_LABEL) */ |
| 656 | #define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x)) | 656 | #define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x)) |
| @@ -25,7 +25,7 @@ · 24 unmodified lines | |||
| 25 | #include <linux/err.h> | 25 | #include <linux/err.h> |
| 26 | #include <linux/slab.h> | 26 | #include <linux/slab.h> |
| 27 | #include <linux/sched.h> | 27 | #include <linux/sched.h> |
| 28 | #include <linux/jump_label.h> | 28 | #include <linux/static_key.h> |
| 29 | 29 | ||
| 30 | extern struct tracepoint * const __start___tracepoints_ptrs[]; | 30 | extern struct tracepoint * const __start___tracepoints_ptrs[]; |
| 31 | extern struct tracepoint * const __stop___tracepoints_ptrs[]; | 31 | extern struct tracepoint * const __stop___tracepoints_ptrs[]; |
| @@ -256,9 +256,9 @@ static void set_tracepoint(struct tracepoint_entry **entry, · 224 unmodified lines | |||
| 256 | { | 256 | { |
| 257 | WARN_ON(strcmp((*entry)->name, elem->name) != 0); | 257 | WARN_ON(strcmp((*entry)->name, elem->name) != 0); |
| 258 | 258 | ||
| 259 | if (elem->regfunc && !jump_label_enabled(&elem->key) && active) | 259 | if (elem->regfunc && !static_key_enabled(&elem->key) && active) |
| 260 | elem->regfunc(); | 260 | elem->regfunc(); |
| 261 | else if (elem->unregfunc && jump_label_enabled(&elem->key) && !active) | 261 | else if (elem->unregfunc && static_key_enabled(&elem->key) && !active) |
| 262 | elem->unregfunc(); | 262 | elem->unregfunc(); |
| 263 | 263 | ||
| 264 | /* | 264 | /* |
| @@ -269,10 +269,10 @@ static void set_tracepoint(struct tracepoint_entry **entry, · 4 unmodified lines | |||
| 269 | * is used. | 269 | * is used. |
| 270 | */ | 270 | */ |
| 271 | rcu_assign_pointer(elem->funcs, (*entry)->funcs); | 271 | rcu_assign_pointer(elem->funcs, (*entry)->funcs); |
| 272 | if (active && !jump_label_enabled(&elem->key)) | 272 | if (active && !static_key_enabled(&elem->key)) |
| 273 | jump_label_inc(&elem->key); | 273 | static_key_slow_inc(&elem->key); |
| 274 | else if (!active && jump_label_enabled(&elem->key)) | 274 | else if (!active && static_key_enabled(&elem->key)) |
| 275 | jump_label_dec(&elem->key); | 275 | static_key_slow_dec(&elem->key); |
| 276 | } | 276 | } |
| 277 | 277 | ||
| 278 | /* | 278 | /* |
| @@ -283,11 +283,11 @@ static void set_tracepoint(struct tracepoint_entry **entry, · 4 unmodified lines | |||
| 283 | */ | 283 | */ |
| 284 | static void disable_tracepoint(struct tracepoint *elem) | 284 | static void disable_tracepoint(struct tracepoint *elem) |
| 285 | { | 285 | { |
| 286 | if (elem->unregfunc && jump_label_enabled(&elem->key)) | 286 | if (elem->unregfunc && static_key_enabled(&elem->key)) |
| 287 | elem->unregfunc(); | 287 | elem->unregfunc(); |
| 288 | 288 | ||
| 289 | if (jump_label_enabled(&elem->key)) | 289 | if (static_key_enabled(&elem->key)) |
| 290 | jump_label_dec(&elem->key); | 290 | static_key_slow_dec(&elem->key); |
| 291 | rcu_assign_pointer(elem->funcs, NULL); | 291 | rcu_assign_pointer(elem->funcs, NULL); |
| 292 | } | 292 | } |
| 293 | 293 | ||
| @@ -134,7 +134,7 @@ · 133 unmodified lines | |||
| 134 | #include <linux/inetdevice.h> | 134 | #include <linux/inetdevice.h> |
| 135 | #include <linux/cpu_rmap.h> | 135 | #include <linux/cpu_rmap.h> |
| 136 | #include <linux/net_tstamp.h> | 136 | #include <linux/net_tstamp.h> |
| 137 | #include <linux/jump_label.h> | 137 | #include <linux/static_key.h> |
| 138 | #include <net/flow_keys.h> | 138 | #include <net/flow_keys.h> |
| 139 | 139 | ||
| 140 | #include "net-sysfs.h" | 140 | #include "net-sysfs.h" |
| @@ -1441,11 +1441,11 @@ int call_netdevice_notifiers(unsigned long val, struct net_device *dev) · 1300 unmodified lines | |||
| 1441 | } | 1441 | } |
| 1442 | EXPORT_SYMBOL(call_netdevice_notifiers); | 1442 | EXPORT_SYMBOL(call_netdevice_notifiers); |
| 1443 | 1443 | ||
| 1444 | static struct jump_label_key netstamp_needed __read_mostly; | 1444 | static struct static_key netstamp_needed __read_mostly; |
| 1445 | #ifdef HAVE_JUMP_LABEL | 1445 | #ifdef HAVE_JUMP_LABEL |
| 1446 | /* We are not allowed to call jump_label_dec() from irq context | 1446 | /* We are not allowed to call static_key_slow_dec() from irq context |
| 1447 | * If net_disable_timestamp() is called from irq context, defer the | 1447 | * If net_disable_timestamp() is called from irq context, defer the |
| 1448 | * jump_label_dec() calls. | 1448 | * static_key_slow_dec() calls. |
| 1449 | */ | 1449 | */ |
| 1450 | static atomic_t netstamp_needed_deferred; | 1450 | static atomic_t netstamp_needed_deferred; |
| 1451 | #endif | 1451 | #endif |
| @@ -1457,12 +1457,12 @@ void net_enable_timestamp(void) · 5 unmodified lines | |||
| 1457 | 1457 | ||
| 1458 | if (deferred) { | 1458 | if (deferred) { |
| 1459 | while (--deferred) | 1459 | while (--deferred) |
| 1460 | jump_label_dec(&netstamp_needed); | 1460 | static_key_slow_dec(&netstamp_needed); |
| 1461 | return; | 1461 | return; |
| 1462 | } | 1462 | } |
| 1463 | #endif | 1463 | #endif |
| 1464 | WARN_ON(in_interrupt()); | 1464 | WARN_ON(in_interrupt()); |
| 1465 | jump_label_inc(&netstamp_needed); | 1465 | static_key_slow_inc(&netstamp_needed); |
| 1466 | } | 1466 | } |
| 1467 | EXPORT_SYMBOL(net_enable_timestamp); | 1467 | EXPORT_SYMBOL(net_enable_timestamp); |
| 1468 | 1468 | ||
| @@ -1474,19 +1474,19 @@ void net_disable_timestamp(void) · 5 unmodified lines | |||
| 1474 | return; | 1474 | return; |
| 1475 | } | 1475 | } |
| 1476 | #endif | 1476 | #endif |
| 1477 | jump_label_dec(&netstamp_needed); | 1477 | static_key_slow_dec(&netstamp_needed); |
| 1478 | } | 1478 | } |
| 1479 | EXPORT_SYMBOL(net_disable_timestamp); | 1479 | EXPORT_SYMBOL(net_disable_timestamp); |
| 1480 | 1480 | ||
| 1481 | static inline void net_timestamp_set(struct sk_buff *skb) | 1481 | static inline void net_timestamp_set(struct sk_buff *skb) |
| 1482 | { | 1482 | { |
| 1483 | skb->tstamp.tv64 = 0; | 1483 | skb->tstamp.tv64 = 0; |
| 1484 | if (static_branch(&netstamp_needed)) | 1484 | if (static_key_false(&netstamp_needed)) |
| 1485 | __net_timestamp(skb); | 1485 | __net_timestamp(skb); |
| 1486 | } | 1486 | } |
| 1487 | 1487 | ||
| 1488 | #define net_timestamp_check(COND, SKB) \ | 1488 | #define net_timestamp_check(COND, SKB) \ |
| 1489 | if (static_branch(&netstamp_needed)) { \ | 1489 | if (static_key_false(&netstamp_needed)) { \ |
| 1490 | if ((COND) && !(SKB)->tstamp.tv64) \ | 1490 | if ((COND) && !(SKB)->tstamp.tv64) \ |
| 1491 | __net_timestamp(SKB); \ | 1491 | __net_timestamp(SKB); \ |
| 1492 | } \ | 1492 | } \ |
| @@ -2660,7 +2660,7 @@ EXPORT_SYMBOL(__skb_get_rxhash); · 1167 unmodified lines | |||
| 2660 | struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; | 2660 | struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; |
| 2661 | EXPORT_SYMBOL(rps_sock_flow_table); | 2661 | EXPORT_SYMBOL(rps_sock_flow_table); |
| 2662 | 2662 | ||
| 2663 | struct jump_label_key rps_needed __read_mostly; | 2663 | struct static_key rps_needed __read_mostly; |
| 2664 | 2664 | ||
| 2665 | static struct rps_dev_flow * | 2665 | static struct rps_dev_flow * |
| 2666 | set_rps_cpu(struct net_device *dev, struct sk_buff *skb, | 2666 | set_rps_cpu(struct net_device *dev, struct sk_buff *skb, |
| @@ -2945,7 +2945,7 @@ int netif_rx(struct sk_buff *skb) · 278 unmodified lines | |||
| 2945 | 2945 | ||
| 2946 | trace_netif_rx(skb); | 2946 | trace_netif_rx(skb); |
| 2947 | #ifdef CONFIG_RPS | 2947 | #ifdef CONFIG_RPS |
| 2948 | if (static_branch(&rps_needed)) { | 2948 | if (static_key_false(&rps_needed)) { |
| 2949 | struct rps_dev_flow voidflow, *rflow = &voidflow; | 2949 | struct rps_dev_flow voidflow, *rflow = &voidflow; |
| 2950 | int cpu; | 2950 | int cpu; |
| 2951 | 2951 | ||
| @@ -3309,7 +3309,7 @@ int netif_receive_skb(struct sk_buff *skb) · 357 unmodified lines | |||
| 3309 | return NET_RX_SUCCESS; | 3309 | return NET_RX_SUCCESS; |
| 3310 | 3310 | ||
| 3311 | #ifdef CONFIG_RPS | 3311 | #ifdef CONFIG_RPS |
| 3312 | if (static_branch(&rps_needed)) { | 3312 | if (static_key_false(&rps_needed)) { |
| 3313 | struct rps_dev_flow voidflow, *rflow = &voidflow; | 3313 | struct rps_dev_flow voidflow, *rflow = &voidflow; |
| 3314 | int cpu, ret; | 3314 | int cpu, ret; |
| 3315 | 3315 | ||
| @@ -608,10 +608,10 @@ static ssize_t store_rps_map(struct netdev_rx_queue *queue, · 607 unmodified lines | |||
| 608 | spin_unlock(&rps_map_lock); | 608 | spin_unlock(&rps_map_lock); |
| 609 | 609 | ||
| 610 | if (map) | 610 | if (map) |
| 611 | jump_label_inc(&rps_needed); | 611 | static_key_slow_inc(&rps_needed); |
| 612 | if (old_map) { | 612 | if (old_map) { |
| 613 | kfree_rcu(old_map, rcu); | 613 | kfree_rcu(old_map, rcu); |
| 614 | jump_label_dec(&rps_needed); | 614 | static_key_slow_dec(&rps_needed); |
| 615 | } | 615 | } |
| 616 | free_cpumask_var(mask); | 616 | free_cpumask_var(mask); |
| 617 | return len; | 617 | return len; |
| @@ -111,7 +111,7 @@ · 110 unmodified lines | |||
| 111 | #include <linux/init.h> | 111 | #include <linux/init.h> |
| 112 | #include <linux/highmem.h> | 112 | #include <linux/highmem.h> |
| 113 | #include <linux/user_namespace.h> | 113 | #include <linux/user_namespace.h> |
| 114 | #include <linux/jump_label.h> | 114 | #include <linux/static_key.h> |
| 115 | #include <linux/memcontrol.h> | 115 | #include <linux/memcontrol.h> |
| 116 | 116 | ||
| 117 | #include <asm/uaccess.h> | 117 | #include <asm/uaccess.h> |
| @@ -184,7 +184,7 @@ void mem_cgroup_sockets_destroy(struct cgroup *cgrp, struct cgroup_subsys *ss) · 66 unmodified lines | |||
| 184 | static struct lock_class_key af_family_keys[AF_MAX]; | 184 | static struct lock_class_key af_family_keys[AF_MAX]; |
| 185 | static struct lock_class_key af_family_slock_keys[AF_MAX]; | 185 | static struct lock_class_key af_family_slock_keys[AF_MAX]; |
| 186 | 186 | ||
| 187 | struct jump_label_key memcg_socket_limit_enabled; | 187 | struct static_key memcg_socket_limit_enabled; |
| 188 | EXPORT_SYMBOL(memcg_socket_limit_enabled); | 188 | EXPORT_SYMBOL(memcg_socket_limit_enabled); |
| 189 | 189 | ||
| 190 | /* | 190 | /* |
| @@ -69,9 +69,9 @@ static int rps_sock_flow_sysctl(ctl_table *table, int write, · 68 unmodified lines | |||
| 69 | if (sock_table != orig_sock_table) { | 69 | if (sock_table != orig_sock_table) { |
| 70 | rcu_assign_pointer(rps_sock_flow_table, sock_table); | 70 | rcu_assign_pointer(rps_sock_flow_table, sock_table); |
| 71 | if (sock_table) | 71 | if (sock_table) |
| 72 | jump_label_inc(&rps_needed); | 72 | static_key_slow_inc(&rps_needed); |
| 73 | if (orig_sock_table) { | 73 | if (orig_sock_table) { |
| 74 | jump_label_dec(&rps_needed); | 74 | static_key_slow_dec(&rps_needed); |
| 75 | synchronize_rcu(); | 75 | synchronize_rcu(); |
| 76 | vfree(orig_sock_table); | 76 | vfree(orig_sock_table); |
| 77 | } | 77 | } |
| @@ -111,7 +111,7 @@ void tcp_destroy_cgroup(struct cgroup *cgrp, struct cgroup_subsys *ss) · 110 unmodified lines | |||
| 111 | val = res_counter_read_u64(&tcp->tcp_memory_allocated, RES_LIMIT); | 111 | val = res_counter_read_u64(&tcp->tcp_memory_allocated, RES_LIMIT); |
| 112 | 112 | ||
| 113 | if (val != RESOURCE_MAX) | 113 | if (val != RESOURCE_MAX) |
| 114 | jump_label_dec(&memcg_socket_limit_enabled); | 114 | static_key_slow_dec(&memcg_socket_limit_enabled); |
| 115 | } | 115 | } |
| 116 | EXPORT_SYMBOL(tcp_destroy_cgroup); | 116 | EXPORT_SYMBOL(tcp_destroy_cgroup); |
| 117 | 117 | ||
| @@ -143,9 +143,9 @@ static int tcp_update_limit(struct mem_cgroup *memcg, u64 val) · 25 unmodified lines | |||
| 143 | net->ipv4.sysctl_tcp_mem[i]); | 143 | net->ipv4.sysctl_tcp_mem[i]); |
| 144 | 144 | ||
| 145 | if (val == RESOURCE_MAX && old_lim != RESOURCE_MAX) | 145 | if (val == RESOURCE_MAX && old_lim != RESOURCE_MAX) |
| 146 | jump_label_dec(&memcg_socket_limit_enabled); | 146 | static_key_slow_dec(&memcg_socket_limit_enabled); |
| 147 | else if (old_lim == RESOURCE_MAX && val != RESOURCE_MAX) | 147 | else if (old_lim == RESOURCE_MAX && val != RESOURCE_MAX) |
| 148 | jump_label_inc(&memcg_socket_limit_enabled); | 148 | static_key_slow_inc(&memcg_socket_limit_enabled); |
| 149 | 149 | ||
| 150 | return 0; | 150 | return 0; |
| 151 | } | 151 | } |
| @@ -56,7 +56,7 @@ struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS] __read_mostly; · 55 unmodified lines | |||
| 56 | EXPORT_SYMBOL(nf_hooks); | 56 | EXPORT_SYMBOL(nf_hooks); |
| 57 | 57 | ||
| 58 | #if defined(CONFIG_JUMP_LABEL) | 58 | #if defined(CONFIG_JUMP_LABEL) |
| 59 | struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; | 59 | struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; |
| 60 | EXPORT_SYMBOL(nf_hooks_needed); | 60 | EXPORT_SYMBOL(nf_hooks_needed); |
| 61 | #endif | 61 | #endif |
| 62 | 62 | ||
| @@ -77,7 +77,7 @@ int nf_register_hook(struct nf_hook_ops *reg) · 14 unmodified lines | |||
| 77 | list_add_rcu(®->list, elem->list.prev); | 77 | list_add_rcu(®->list, elem->list.prev); |
| 78 | mutex_unlock(&nf_hook_mutex); | 78 | mutex_unlock(&nf_hook_mutex); |
| 79 | #if defined(CONFIG_JUMP_LABEL) | 79 | #if defined(CONFIG_JUMP_LABEL) |
| 80 | jump_label_inc(&nf_hooks_needed[reg->pf][reg->hooknum]); | 80 | static_key_slow_inc(&nf_hooks_needed[reg->pf][reg->hooknum]); |
| 81 | #endif | 81 | #endif |
| 82 | return 0; | 82 | return 0; |
| 83 | } | 83 | } |
| @@ -89,7 +89,7 @@ void nf_unregister_hook(struct nf_hook_ops *reg) · 5 unmodified lines | |||
| 89 | list_del_rcu(®->list); | 89 | list_del_rcu(®->list); |
| 90 | mutex_unlock(&nf_hook_mutex); | 90 | mutex_unlock(&nf_hook_mutex); |
| 91 | #if defined(CONFIG_JUMP_LABEL) | 91 | #if defined(CONFIG_JUMP_LABEL) |
| 92 | jump_label_dec(&nf_hooks_needed[reg->pf][reg->hooknum]); | 92 | static_key_slow_dec(&nf_hooks_needed[reg->pf][reg->hooknum]); |
| 93 | #endif | 93 | #endif |
| 94 | synchronize_net(); | 94 | synchronize_net(); |
| 95 | } | 95 | } |