We are going to rely on the loosening of RCU callback semantics, introduced by this commit: 806f04e9fd2c: ("rcu: Allow for smp_call_function() running callbacks from idle")
Showing the changes to include/linux (15 of 68 files). Show all
Unified · Split
| @@ -120,12 +120,65 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 119 unmodified lines | |||
| 120 | /* Annotate a C jump table to allow objtool to follow the code flow */ | 120 | /* Annotate a C jump table to allow objtool to follow the code flow */ |
| 121 | #define __annotate_jump_table __section(.rodata..c_jump_table) | 121 | #define __annotate_jump_table __section(.rodata..c_jump_table) |
| 122 | 122 | ||
| 123 | #ifdef CONFIG_DEBUG_ENTRY | ||
| 124 | /* Begin/end of an instrumentation safe region */ | ||
| 125 | #define instrumentation_begin() ({ \ | ||
| 126 | asm volatile("%c0:\n\t" \ | ||
| 127 | ".pushsection .discard.instr_begin\n\t" \ | ||
| 128 | ".long %c0b - .\n\t" \ | ||
| 129 | ".popsection\n\t" : : "i" (__COUNTER__)); \ | ||
| 130 | }) | ||
| 131 | |||
| 132 | /* | ||
| 133 | * Because instrumentation_{begin,end}() can nest, objtool validation considers | ||
| 134 | * _begin() a +1 and _end() a -1 and computes a sum over the instructions. | ||
| 135 | * When the value is greater than 0, we consider instrumentation allowed. | ||
| 136 | * | ||
| 137 | * There is a problem with code like: | ||
| 138 | * | ||
| 139 | * noinstr void foo() | ||
| 140 | * { | ||
| 141 | * instrumentation_begin(); | ||
| 142 | * ... | ||
| 143 | * if (cond) { | ||
| 144 | * instrumentation_begin(); | ||
| 145 | * ... | ||
| 146 | * instrumentation_end(); | ||
| 147 | * } | ||
| 148 | * bar(); | ||
| 149 | * instrumentation_end(); | ||
| 150 | * } | ||
| 151 | * | ||
| 152 | * If instrumentation_end() would be an empty label, like all the other | ||
| 153 | * annotations, the inner _end(), which is at the end of a conditional block, | ||
| 154 | * would land on the instruction after the block. | ||
| 155 | * | ||
| 156 | * If we then consider the sum of the !cond path, we'll see that the call to | ||
| 157 | * bar() is with a 0-value, even though, we meant it to happen with a positive | ||
| 158 | * value. | ||
| 159 | * | ||
| 160 | * To avoid this, have _end() be a NOP instruction, this ensures it will be | ||
| 161 | * part of the condition block and does not escape. | ||
| 162 | */ | ||
| 163 | #define instrumentation_end() ({ \ | ||
| 164 | asm volatile("%c0: nop\n\t" \ | ||
| 165 | ".pushsection .discard.instr_end\n\t" \ | ||
| 166 | ".long %c0b - .\n\t" \ | ||
| 167 | ".popsection\n\t" : : "i" (__COUNTER__)); \ | ||
| 168 | }) | ||
| 169 | #endif /* CONFIG_DEBUG_ENTRY */ | ||
| 170 | |||
| 123 | #else | 171 | #else |
| 124 | #define annotate_reachable() | 172 | #define annotate_reachable() |
| 125 | #define annotate_unreachable() | 173 | #define annotate_unreachable() |
| 126 | #define __annotate_jump_table | 174 | #define __annotate_jump_table |
| 127 | #endif | 175 | #endif |
| 128 | 176 | ||
| 177 | #ifndef instrumentation_begin | ||
| 178 | #define instrumentation_begin() do { } while(0) | ||
| 179 | #define instrumentation_end() do { } while(0) | ||
| 180 | #endif | ||
| 181 | |||
| 129 | #ifndef ASM_UNREACHABLE | 182 | #ifndef ASM_UNREACHABLE |
| 130 | # define ASM_UNREACHABLE | 183 | # define ASM_UNREACHABLE |
| 131 | #endif | 184 | #endif |
| @@ -118,6 +118,10 @@ struct ftrace_likely_data { · 117 unmodified lines | |||
| 118 | #define notrace __attribute__((__no_instrument_function__)) | 118 | #define notrace __attribute__((__no_instrument_function__)) |
| 119 | #endif | 119 | #endif |
| 120 | 120 | ||
| 121 | /* Section for code which can't be instrumented at all */ | ||
| 122 | #define noinstr \ | ||
| 123 | noinline notrace __attribute((__section__(".noinstr.text"))) | ||
| 124 | |||
| 121 | /* | 125 | /* |
| 122 | * it doesn't make sense on ARM (currently the only user of __naked) | 126 | * it doesn't make sense on ARM (currently the only user of __naked) |
| 123 | * to trace naked functions because then mcount is called without | 127 | * to trace naked functions because then mcount is called without |
| @@ -2,15 +2,6 @@ · 1 unmodified line | |||
| 2 | #ifndef _LINUX_FTRACE_IRQ_H | 2 | #ifndef _LINUX_FTRACE_IRQ_H |
| 3 | #define _LINUX_FTRACE_IRQ_H | 3 | #define _LINUX_FTRACE_IRQ_H |
| 4 | 4 | ||
| 5 | |||
| 6 | #ifdef CONFIG_FTRACE_NMI_ENTER | ||
| 7 | extern void arch_ftrace_nmi_enter(void); | ||
| 8 | extern void arch_ftrace_nmi_exit(void); | ||
| 9 | #else | ||
| 10 | static inline void arch_ftrace_nmi_enter(void) { } | ||
| 11 | static inline void arch_ftrace_nmi_exit(void) { } | ||
| 12 | #endif | ||
| 13 | |||
| 14 | #ifdef CONFIG_HWLAT_TRACER | 5 | #ifdef CONFIG_HWLAT_TRACER |
| 15 | extern bool trace_hwlat_callback_enabled; | 6 | extern bool trace_hwlat_callback_enabled; |
| 16 | extern void trace_hwlat_callback(bool enter); | 7 | extern void trace_hwlat_callback(bool enter); |
| @@ -22,12 +13,10 @@ static inline void ftrace_nmi_enter(void) · 5 unmodified lines | |||
| 22 | if (trace_hwlat_callback_enabled) | 13 | if (trace_hwlat_callback_enabled) |
| 23 | trace_hwlat_callback(true); | 14 | trace_hwlat_callback(true); |
| 24 | #endif | 15 | #endif |
| 25 | arch_ftrace_nmi_enter(); | ||
| 26 | } | 16 | } |
| 27 | 17 | ||
| 28 | static inline void ftrace_nmi_exit(void) | 18 | static inline void ftrace_nmi_exit(void) |
| 29 | { | 19 | { |
| 30 | arch_ftrace_nmi_exit(); | ||
| 31 | #ifdef CONFIG_HWLAT_TRACER | 20 | #ifdef CONFIG_HWLAT_TRACER |
| 32 | if (trace_hwlat_callback_enabled) | 21 | if (trace_hwlat_callback_enabled) |
| 33 | trace_hwlat_callback(false); | 22 | trace_hwlat_callback(false); |
| @@ -65,14 +65,26 @@ extern void irq_exit(void); · 64 unmodified lines | |||
| 65 | #define arch_nmi_exit() do { } while (0) | 65 | #define arch_nmi_exit() do { } while (0) |
| 66 | #endif | 66 | #endif |
| 67 | 67 | ||
| 68 | /* | ||
| 69 | * NMI vs Tracing | ||
| 70 | * -------------- | ||
| 71 | * | ||
| 72 | * We must not land in a tracer until (or after) we've changed preempt_count | ||
| 73 | * such that in_nmi() becomes true. To that effect all NMI C entry points must | ||
| 74 | * be marked 'notrace' and call nmi_enter() as soon as possible. | ||
| 75 | */ | ||
| 76 | |||
| 77 | /* | ||
| 78 | * nmi_enter() can nest up to 15 times; see NMI_BITS. | ||
| 79 | */ | ||
| 68 | #define nmi_enter() \ | 80 | #define nmi_enter() \ |
| 69 | do { \ | 81 | do { \ |
| 70 | arch_nmi_enter(); \ | 82 | arch_nmi_enter(); \ |
| 71 | printk_nmi_enter(); \ | 83 | printk_nmi_enter(); \ |
| 72 | lockdep_off(); \ | 84 | lockdep_off(); \ |
| 73 | ftrace_nmi_enter(); \ | 85 | ftrace_nmi_enter(); \ |
| 74 | BUG_ON(in_nmi()); \ | 86 | BUG_ON(in_nmi() == NMI_MASK); \ |
| 75 | preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \ | 87 | __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \ |
| 76 | rcu_nmi_enter(); \ | 88 | rcu_nmi_enter(); \ |
| 77 | lockdep_hardirq_enter(); \ | 89 | lockdep_hardirq_enter(); \ |
| 78 | } while (0) | 90 | } while (0) |
| @@ -82,7 +94,7 @@ extern void irq_exit(void); · 3 unmodified lines | |||
| 82 | lockdep_hardirq_exit(); \ | 94 | lockdep_hardirq_exit(); \ |
| 83 | rcu_nmi_exit(); \ | 95 | rcu_nmi_exit(); \ |
| 84 | BUG_ON(!in_nmi()); \ | 96 | BUG_ON(!in_nmi()); \ |
| 85 | preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ | 97 | __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ |
| 86 | ftrace_nmi_exit(); \ | 98 | ftrace_nmi_exit(); \ |
| 87 | lockdep_on(); \ | 99 | lockdep_on(); \ |
| 88 | printk_nmi_exit(); \ | 100 | printk_nmi_exit(); \ |
| @@ -308,8 +308,27 @@ extern void lockdep_set_selftest_task(struct task_struct *task); · 307 unmodified lines | |||
| 308 | 308 | ||
| 309 | extern void lockdep_init_task(struct task_struct *task); | 309 | extern void lockdep_init_task(struct task_struct *task); |
| 310 | 310 | ||
| 311 | extern void lockdep_off(void); | 311 | /* |
| 312 | extern void lockdep_on(void); | 312 | * Split the recrursion counter in two to readily detect 'off' vs recursion. |
| 313 | */ | ||
| 314 | #define LOCKDEP_RECURSION_BITS 16 | ||
| 315 | #define LOCKDEP_OFF (1U << LOCKDEP_RECURSION_BITS) | ||
| 316 | #define LOCKDEP_RECURSION_MASK (LOCKDEP_OFF - 1) | ||
| 317 | |||
| 318 | /* | ||
| 319 | * lockdep_{off,on}() are macros to avoid tracing and kprobes; not inlines due | ||
| 320 | * to header dependencies. | ||
| 321 | */ | ||
| 322 | |||
| 323 | #define lockdep_off() \ | ||
| 324 | do { \ | ||
| 325 | current->lockdep_recursion += LOCKDEP_OFF; \ | ||
| 326 | } while (0) | ||
| 327 | |||
| 328 | #define lockdep_on() \ | ||
| 329 | do { \ | ||
| 330 | current->lockdep_recursion -= LOCKDEP_OFF; \ | ||
| 331 | } while (0) | ||
| 313 | 332 | ||
| 314 | extern void lockdep_register_key(struct lock_class_key *key); | 333 | extern void lockdep_register_key(struct lock_class_key *key); |
| 315 | extern void lockdep_unregister_key(struct lock_class_key *key); | 334 | extern void lockdep_unregister_key(struct lock_class_key *key); |
| @@ -26,13 +26,13 @@ · 25 unmodified lines | |||
| 26 | * PREEMPT_MASK: 0x000000ff | 26 | * PREEMPT_MASK: 0x000000ff |
| 27 | * SOFTIRQ_MASK: 0x0000ff00 | 27 | * SOFTIRQ_MASK: 0x0000ff00 |
| 28 | * HARDIRQ_MASK: 0x000f0000 | 28 | * HARDIRQ_MASK: 0x000f0000 |
| 29 | * NMI_MASK: 0x00100000 | 29 | * NMI_MASK: 0x00f00000 |
| 30 | * PREEMPT_NEED_RESCHED: 0x80000000 | 30 | * PREEMPT_NEED_RESCHED: 0x80000000 |
| 31 | */ | 31 | */ |
| 32 | #define PREEMPT_BITS 8 | 32 | #define PREEMPT_BITS 8 |
| 33 | #define SOFTIRQ_BITS 8 | 33 | #define SOFTIRQ_BITS 8 |
| 34 | #define HARDIRQ_BITS 4 | 34 | #define HARDIRQ_BITS 4 |
| 35 | #define NMI_BITS 1 | 35 | #define NMI_BITS 4 |
| 36 | 36 | ||
| 37 | #define PREEMPT_SHIFT 0 | 37 | #define PREEMPT_SHIFT 0 |
| 38 | #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) | 38 | #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) |
| @@ -371,7 +371,7 @@ static inline void list_splice_tail_init_rcu(struct list_head *list, · 370 unmodified lines | |||
| 371 | * @pos: the type * to use as a loop cursor. | 371 | * @pos: the type * to use as a loop cursor. |
| 372 | * @head: the head for your list. | 372 | * @head: the head for your list. |
| 373 | * @member: the name of the list_head within the struct. | 373 | * @member: the name of the list_head within the struct. |
| 374 | * @cond...: optional lockdep expression if called from non-RCU protection. | 374 | * @cond: optional lockdep expression if called from non-RCU protection. |
| 375 | * | 375 | * |
| 376 | * This list-traversal primitive may safely run concurrently with | 376 | * This list-traversal primitive may safely run concurrently with |
| 377 | * the _rcu list-mutation primitives such as list_add_rcu() | 377 | * the _rcu list-mutation primitives such as list_add_rcu() |
| @@ -646,7 +646,7 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n, · 268 unmodified lines | |||
| 646 | * @pos: the type * to use as a loop cursor. | 646 | * @pos: the type * to use as a loop cursor. |
| 647 | * @head: the head for your list. | 647 | * @head: the head for your list. |
| 648 | * @member: the name of the hlist_node within the struct. | 648 | * @member: the name of the hlist_node within the struct. |
| 649 | * @cond...: optional lockdep expression if called from non-RCU protection. | 649 | * @cond: optional lockdep expression if called from non-RCU protection. |
| 650 | * | 650 | * |
| 651 | * This list-traversal primitive may safely run concurrently with | 651 | * This list-traversal primitive may safely run concurrently with |
| 652 | * the _rcu list-mutation primitives such as hlist_add_head_rcu() | 652 | * the _rcu list-mutation primitives such as hlist_add_head_rcu() |
| @@ -37,6 +37,7 @@ · 36 unmodified lines | |||
| 37 | /* Exported common interfaces */ | 37 | /* Exported common interfaces */ |
| 38 | void call_rcu(struct rcu_head *head, rcu_callback_t func); | 38 | void call_rcu(struct rcu_head *head, rcu_callback_t func); |
| 39 | void rcu_barrier_tasks(void); | 39 | void rcu_barrier_tasks(void); |
| 40 | void rcu_barrier_tasks_rude(void); | ||
| 40 | void synchronize_rcu(void); | 41 | void synchronize_rcu(void); |
| 41 | 42 | ||
| 42 | #ifdef CONFIG_PREEMPT_RCU | 43 | #ifdef CONFIG_PREEMPT_RCU |
| @@ -129,25 +130,57 @@ static inline void rcu_init_nohz(void) { } · 86 unmodified lines | |||
| 129 | * Note a quasi-voluntary context switch for RCU-tasks's benefit. | 130 | * Note a quasi-voluntary context switch for RCU-tasks's benefit. |
| 130 | * This is a macro rather than an inline function to avoid #include hell. | 131 | * This is a macro rather than an inline function to avoid #include hell. |
| 131 | */ | 132 | */ |
| 132 | #ifdef CONFIG_TASKS_RCU | 133 | #ifdef CONFIG_TASKS_RCU_GENERIC |
| 133 | #define rcu_tasks_qs(t) \ | 134 | |
| 134 | do { \ | 135 | # ifdef CONFIG_TASKS_RCU |
| 135 | if (READ_ONCE((t)->rcu_tasks_holdout)) \ | 136 | # define rcu_tasks_classic_qs(t, preempt) \ |
| 136 | WRITE_ONCE((t)->rcu_tasks_holdout, false); \ | 137 | do { \ |
| 138 | if (!(preempt) && READ_ONCE((t)->rcu_tasks_holdout)) \ | ||
| 139 | WRITE_ONCE((t)->rcu_tasks_holdout, false); \ | ||
| 137 | } while (0) | 140 | } while (0) |
| 138 | #define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t) | ||
| 139 | void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func); | 141 | void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func); |
| 140 | void synchronize_rcu_tasks(void); | 142 | void synchronize_rcu_tasks(void); |
| 143 | # else | ||
| 144 | # define rcu_tasks_classic_qs(t, preempt) do { } while (0) | ||
| 145 | # define call_rcu_tasks call_rcu | ||
| 146 | # define synchronize_rcu_tasks synchronize_rcu | ||
| 147 | # endif | ||
| 148 | |||
| 149 | # ifdef CONFIG_TASKS_RCU_TRACE | ||
| 150 | # define rcu_tasks_trace_qs(t) \ | ||
| 151 | do { \ | ||
| 152 | if (!likely(READ_ONCE((t)->trc_reader_checked)) && \ | ||
| 153 | !unlikely(READ_ONCE((t)->trc_reader_nesting))) { \ | ||
| 154 | smp_store_release(&(t)->trc_reader_checked, true); \ | ||
| 155 | smp_mb(); /* Readers partitioned by store. */ \ | ||
| 156 | } \ | ||
| 157 | } while (0) | ||
| 158 | # else | ||
| 159 | # define rcu_tasks_trace_qs(t) do { } while (0) | ||
| 160 | # endif | ||
| 161 | |||
| 162 | #define rcu_tasks_qs(t, preempt) \ | ||
| 163 | do { \ | ||
| 164 | rcu_tasks_classic_qs((t), (preempt)); \ | ||
| 165 | rcu_tasks_trace_qs((t)); \ | ||
| 166 | } while (0) | ||
| 167 | |||
| 168 | # ifdef CONFIG_TASKS_RUDE_RCU | ||
| 169 | void call_rcu_tasks_rude(struct rcu_head *head, rcu_callback_t func); | ||
| 170 | void synchronize_rcu_tasks_rude(void); | ||
| 171 | # endif | ||
| 172 | |||
| 173 | #define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t, false) | ||
| 141 | void exit_tasks_rcu_start(void); | 174 | void exit_tasks_rcu_start(void); |
| 142 | void exit_tasks_rcu_finish(void); | 175 | void exit_tasks_rcu_finish(void); |
| 143 | #else /* #ifdef CONFIG_TASKS_RCU */ | 176 | #else /* #ifdef CONFIG_TASKS_RCU_GENERIC */ |
| 144 | #define rcu_tasks_qs(t) do { } while (0) | 177 | #define rcu_tasks_qs(t, preempt) do { } while (0) |
| 145 | #define rcu_note_voluntary_context_switch(t) do { } while (0) | 178 | #define rcu_note_voluntary_context_switch(t) do { } while (0) |
| 146 | #define call_rcu_tasks call_rcu | 179 | #define call_rcu_tasks call_rcu |
| 147 | #define synchronize_rcu_tasks synchronize_rcu | 180 | #define synchronize_rcu_tasks synchronize_rcu |
| 148 | static inline void exit_tasks_rcu_start(void) { } | 181 | static inline void exit_tasks_rcu_start(void) { } |
| 149 | static inline void exit_tasks_rcu_finish(void) { } | 182 | static inline void exit_tasks_rcu_finish(void) { } |
| 150 | #endif /* #else #ifdef CONFIG_TASKS_RCU */ | 183 | #endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */ |
| 151 | 184 | ||
| 152 | /** | 185 | /** |
| 153 | * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU | 186 | * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU |
| @@ -158,7 +191,7 @@ static inline void exit_tasks_rcu_finish(void) { } · 4 unmodified lines | |||
| 158 | */ | 191 | */ |
| 159 | #define cond_resched_tasks_rcu_qs() \ | 192 | #define cond_resched_tasks_rcu_qs() \ |
| 160 | do { \ | 193 | do { \ |
| 161 | rcu_tasks_qs(current); \ | 194 | rcu_tasks_qs(current, false); \ |
| 162 | cond_resched(); \ | 195 | cond_resched(); \ |
| 163 | } while (0) | 196 | } while (0) |
| 164 | 197 | ||
| @@ -0,0 +1,88 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0+ */ | ||
| 2 | /* | ||
| 3 | * Read-Copy Update mechanism for mutual exclusion, adapted for tracing. | ||
| 4 | * | ||
| 5 | * Copyright (C) 2020 Paul E. McKenney. | ||
| 6 | */ | ||
| 7 | |||
| 8 | #ifndef __LINUX_RCUPDATE_TRACE_H | ||
| 9 | #define __LINUX_RCUPDATE_TRACE_H | ||
| 10 | |||
| 11 | #include <linux/sched.h> | ||
| 12 | #include <linux/rcupdate.h> | ||
| 13 | |||
| 14 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
| 15 | |||
| 16 | extern struct lockdep_map rcu_trace_lock_map; | ||
| 17 | |||
| 18 | static inline int rcu_read_lock_trace_held(void) | ||
| 19 | { | ||
| 20 | return lock_is_held(&rcu_trace_lock_map); | ||
| 21 | } | ||
| 22 | |||
| 23 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
| 24 | |||
| 25 | static inline int rcu_read_lock_trace_held(void) | ||
| 26 | { | ||
| 27 | return 1; | ||
| 28 | } | ||
| 29 | |||
| 30 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
| 31 | |||
| 32 | #ifdef CONFIG_TASKS_TRACE_RCU | ||
| 33 | |||
| 34 | void rcu_read_unlock_trace_special(struct task_struct *t, int nesting); | ||
| 35 | |||
| 36 | /** | ||
| 37 | * rcu_read_lock_trace - mark beginning of RCU-trace read-side critical section | ||
| 38 | * | ||
| 39 | * When synchronize_rcu_trace() is invoked by one task, then that task | ||
| 40 | * is guaranteed to block until all other tasks exit their read-side | ||
| 41 | * critical sections. Similarly, if call_rcu_trace() is invoked on one | ||
| 42 | * task while other tasks are within RCU read-side critical sections, | ||
| 43 | * invocation of the corresponding RCU callback is deferred until after | ||
| 44 | * the all the other tasks exit their critical sections. | ||
| 45 | * | ||
| 46 | * For more details, please see the documentation for rcu_read_lock(). | ||
| 47 | */ | ||
| 48 | static inline void rcu_read_lock_trace(void) | ||
| 49 | { | ||
| 50 | struct task_struct *t = current; | ||
| 51 | |||
| 52 | WRITE_ONCE(t->trc_reader_nesting, READ_ONCE(t->trc_reader_nesting) + 1); | ||
| 53 | if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && | ||
| 54 | t->trc_reader_special.b.need_mb) | ||
| 55 | smp_mb(); // Pairs with update-side barriers | ||
| 56 | rcu_lock_acquire(&rcu_trace_lock_map); | ||
| 57 | } | ||
| 58 | |||
| 59 | /** | ||
| 60 | * rcu_read_unlock_trace - mark end of RCU-trace read-side critical section | ||
| 61 | * | ||
| 62 | * Pairs with a preceding call to rcu_read_lock_trace(), and nesting is | ||
| 63 | * allowed. Invoking a rcu_read_unlock_trace() when there is no matching | ||
| 64 | * rcu_read_lock_trace() is verboten, and will result in lockdep complaints. | ||
| 65 | * | ||
| 66 | * For more details, please see the documentation for rcu_read_unlock(). | ||
| 67 | */ | ||
| 68 | static inline void rcu_read_unlock_trace(void) | ||
| 69 | { | ||
| 70 | int nesting; | ||
| 71 | struct task_struct *t = current; | ||
| 72 | |||
| 73 | rcu_lock_release(&rcu_trace_lock_map); | ||
| 74 | nesting = READ_ONCE(t->trc_reader_nesting) - 1; | ||
| 75 | if (likely(!READ_ONCE(t->trc_reader_special.s)) || nesting) { | ||
| 76 | WRITE_ONCE(t->trc_reader_nesting, nesting); | ||
| 77 | return; // We assume shallow reader nesting. | ||
| 78 | } | ||
| 79 | rcu_read_unlock_trace_special(t, nesting); | ||
| 80 | } | ||
| 81 | |||
| 82 | void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func); | ||
| 83 | void synchronize_rcu_tasks_trace(void); | ||
| 84 | void rcu_barrier_tasks_trace(void); | ||
| 85 | |||
| 86 | #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ | ||
| 87 | |||
| 88 | #endif /* __LINUX_RCUPDATE_TRACE_H */ | ||
| @@ -31,4 +31,23 @@ do { \ · 30 unmodified lines | |||
| 31 | 31 | ||
| 32 | #define wait_rcu_gp(...) _wait_rcu_gp(false, __VA_ARGS__) | 32 | #define wait_rcu_gp(...) _wait_rcu_gp(false, __VA_ARGS__) |
| 33 | 33 | ||
| 34 | /** | ||
| 35 | * synchronize_rcu_mult - Wait concurrently for multiple grace periods | ||
| 36 | * @...: List of call_rcu() functions for different grace periods to wait on | ||
| 37 | * | ||
| 38 | * This macro waits concurrently for multiple types of RCU grace periods. | ||
| 39 | * For example, synchronize_rcu_mult(call_rcu, call_rcu_tasks) would wait | ||
| 40 | * on concurrent RCU and RCU-tasks grace periods. Waiting on a given SRCU | ||
| 41 | * domain requires you to write a wrapper function for that SRCU domain's | ||
| 42 | * call_srcu() function, with this wrapper supplying the pointer to the | ||
| 43 | * corresponding srcu_struct. | ||
| 44 | * | ||
| 45 | * The first argument tells Tiny RCU's _wait_rcu_gp() not to | ||
| 46 | * bother waiting for RCU. The reason for this is because anywhere | ||
| 47 | * synchronize_rcu_mult() can be called is automatically already a full | ||
| 48 | * grace period. | ||
| 49 | */ | ||
| 50 | #define synchronize_rcu_mult(...) \ | ||
| 51 | _wait_rcu_gp(IS_ENABLED(CONFIG_TINY_RCU), __VA_ARGS__) | ||
| 52 | |||
| 34 | #endif /* _LINUX_SCHED_RCUPDATE_WAIT_H */ | 53 | #endif /* _LINUX_SCHED_RCUPDATE_WAIT_H */ |
| @@ -49,7 +49,7 @@ static inline void rcu_softirq_qs(void) · 48 unmodified lines | |||
| 49 | #define rcu_note_context_switch(preempt) \ | 49 | #define rcu_note_context_switch(preempt) \ |
| 50 | do { \ | 50 | do { \ |
| 51 | rcu_qs(); \ | 51 | rcu_qs(); \ |
| 52 | rcu_tasks_qs(current); \ | 52 | rcu_tasks_qs(current, (preempt)); \ |
| 53 | } while (0) | 53 | } while (0) |
| 54 | 54 | ||
| 55 | static inline int rcu_needs_cpu(u64 basemono, u64 *nextevt) | 55 | static inline int rcu_needs_cpu(u64 basemono, u64 *nextevt) |
| @@ -71,6 +71,7 @@ static inline void rcu_irq_enter(void) { } · 15 unmodified lines | |||
| 71 | static inline void rcu_irq_exit_irqson(void) { } | 71 | static inline void rcu_irq_exit_irqson(void) { } |
| 72 | static inline void rcu_irq_enter_irqson(void) { } | 72 | static inline void rcu_irq_enter_irqson(void) { } |
| 73 | static inline void rcu_irq_exit(void) { } | 73 | static inline void rcu_irq_exit(void) { } |
| 74 | static inline void rcu_irq_exit_preempt(void) { } | ||
| 74 | static inline void exit_rcu(void) { } | 75 | static inline void exit_rcu(void) { } |
| 75 | static inline bool rcu_preempt_need_deferred_qs(struct task_struct *t) | 76 | static inline bool rcu_preempt_need_deferred_qs(struct task_struct *t) |
| 76 | { | 77 | { |
| @@ -85,8 +86,10 @@ static inline void rcu_scheduler_starting(void) { } · 8 unmodified lines | |||
| 85 | static inline void rcu_end_inkernel_boot(void) { } | 86 | static inline void rcu_end_inkernel_boot(void) { } |
| 86 | static inline bool rcu_inkernel_boot_has_ended(void) { return true; } | 87 | static inline bool rcu_inkernel_boot_has_ended(void) { return true; } |
| 87 | static inline bool rcu_is_watching(void) { return true; } | 88 | static inline bool rcu_is_watching(void) { return true; } |
| 89 | static inline bool __rcu_is_watching(void) { return true; } | ||
| 88 | static inline void rcu_momentary_dyntick_idle(void) { } | 90 | static inline void rcu_momentary_dyntick_idle(void) { } |
| 89 | static inline void kfree_rcu_scheduler_running(void) { } | 91 | static inline void kfree_rcu_scheduler_running(void) { } |
| 92 | static inline bool rcu_gp_might_be_stalled(void) { return false; } | ||
| 90 | 93 | ||
| 91 | /* Avoid RCU read-side critical sections leaking across. */ | 94 | /* Avoid RCU read-side critical sections leaking across. */ |
| 92 | static inline void rcu_all_qs(void) { barrier(); } | 95 | static inline void rcu_all_qs(void) { barrier(); } |
| @@ -39,6 +39,7 @@ void rcu_barrier(void); · 38 unmodified lines | |||
| 39 | bool rcu_eqs_special_set(int cpu); | 39 | bool rcu_eqs_special_set(int cpu); |
| 40 | void rcu_momentary_dyntick_idle(void); | 40 | void rcu_momentary_dyntick_idle(void); |
| 41 | void kfree_rcu_scheduler_running(void); | 41 | void kfree_rcu_scheduler_running(void); |
| 42 | bool rcu_gp_might_be_stalled(void); | ||
| 42 | unsigned long get_state_synchronize_rcu(void); | 43 | unsigned long get_state_synchronize_rcu(void); |
| 43 | void cond_synchronize_rcu(unsigned long oldstate); | 44 | void cond_synchronize_rcu(unsigned long oldstate); |
| 44 | 45 | ||
| @@ -46,6 +47,7 @@ void rcu_idle_enter(void); · 1 unmodified line | |||
| 46 | void rcu_idle_exit(void); | 47 | void rcu_idle_exit(void); |
| 47 | void rcu_irq_enter(void); | 48 | void rcu_irq_enter(void); |
| 48 | void rcu_irq_exit(void); | 49 | void rcu_irq_exit(void); |
| 50 | void rcu_irq_exit_preempt(void); | ||
| 49 | void rcu_irq_enter_irqson(void); | 51 | void rcu_irq_enter_irqson(void); |
| 50 | void rcu_irq_exit_irqson(void); | 52 | void rcu_irq_exit_irqson(void); |
| 51 | 53 | ||
| @@ -56,6 +58,7 @@ extern int rcu_scheduler_active __read_mostly; · 4 unmodified lines | |||
| 56 | void rcu_end_inkernel_boot(void); | 58 | void rcu_end_inkernel_boot(void); |
| 57 | bool rcu_inkernel_boot_has_ended(void); | 59 | bool rcu_inkernel_boot_has_ended(void); |
| 58 | bool rcu_is_watching(void); | 60 | bool rcu_is_watching(void); |
| 61 | bool __rcu_is_watching(void); | ||
| 59 | #ifndef CONFIG_PREEMPTION | 62 | #ifndef CONFIG_PREEMPTION |
| 60 | void rcu_all_qs(void); | 63 | void rcu_all_qs(void); |
| 61 | #endif | 64 | #endif |
| @@ -613,7 +613,7 @@ union rcu_special { · 612 unmodified lines | |||
| 613 | u8 blocked; | 613 | u8 blocked; |
| 614 | u8 need_qs; | 614 | u8 need_qs; |
| 615 | u8 exp_hint; /* Hint for performance. */ | 615 | u8 exp_hint; /* Hint for performance. */ |
| 616 | u8 deferred_qs; | 616 | u8 need_mb; /* Readers need smp_mb(). */ |
| 617 | } b; /* Bits. */ | 617 | } b; /* Bits. */ |
| 618 | u32 s; /* Set of bits. */ | 618 | u32 s; /* Set of bits. */ |
| 619 | }; | 619 | }; |
| @@ -724,6 +724,14 @@ struct task_struct { · 104 unmodified lines | |||
| 724 | struct list_head rcu_tasks_holdout_list; | 724 | struct list_head rcu_tasks_holdout_list; |
| 725 | #endif /* #ifdef CONFIG_TASKS_RCU */ | 725 | #endif /* #ifdef CONFIG_TASKS_RCU */ |
| 726 | 726 | ||
| 727 | #ifdef CONFIG_TASKS_TRACE_RCU | ||
| 728 | int trc_reader_nesting; | ||
| 729 | int trc_ipi_to_cpu; | ||
| 730 | union rcu_special trc_reader_special; | ||
| 731 | bool trc_reader_checked; | ||
| 732 | struct list_head trc_holdout_list; | ||
| 733 | #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ | ||
| 734 | |||
| 727 | struct sched_info sched_info; | 735 | struct sched_info sched_info; |
| 728 | 736 | ||
| 729 | struct list_head tasks; | 737 | struct list_head tasks; |
| @@ -1289,6 +1297,12 @@ struct task_struct { · 559 unmodified lines | |||
| 1289 | unsigned long prev_lowest_stack; | 1297 | unsigned long prev_lowest_stack; |
| 1290 | #endif | 1298 | #endif |
| 1291 | 1299 | ||
| 1300 | #ifdef CONFIG_X86_MCE | ||
| 1301 | u64 mce_addr; | ||
| 1302 | u64 mce_status; | ||
| 1303 | struct callback_head mce_kill_me; | ||
| 1304 | #endif | ||
| 1305 | |||
| 1292 | /* | 1306 | /* |
| 1293 | * New fields for task_struct should be added above here, so that | 1307 | * New fields for task_struct should be added above here, so that |
| 1294 | * they are included in the randomized portion of task_struct. | 1308 | * they are included in the randomized portion of task_struct. |
| @@ -89,7 +89,7 @@ void _torture_stop_kthread(char *m, struct task_struct **tp); · 88 unmodified lines | |||
| 89 | #ifdef CONFIG_PREEMPTION | 89 | #ifdef CONFIG_PREEMPTION |
| 90 | #define torture_preempt_schedule() preempt_schedule() | 90 | #define torture_preempt_schedule() preempt_schedule() |
| 91 | #else | 91 | #else |
| 92 | #define torture_preempt_schedule() | 92 | #define torture_preempt_schedule() do { } while (0) |
| 93 | #endif | 93 | #endif |
| 94 | 94 | ||
| 95 | #endif /* __LINUX_TORTURE_H */ | 95 | #endif /* __LINUX_TORTURE_H */ |
| @@ -1149,4 +1149,6 @@ int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, i · 1148 unmodified lines | |||
| 1149 | (wait)->flags = 0; \ | 1149 | (wait)->flags = 0; \ |
| 1150 | } while (0) | 1150 | } while (0) |
| 1151 | 1151 | ||
| 1152 | bool try_invoke_on_locked_down_task(struct task_struct *p, bool (*func)(struct task_struct *t, void *arg), void *arg); | ||
| 1153 | |||
| 1152 | #endif /* _LINUX_WAIT_H */ | 1154 | #endif /* _LINUX_WAIT_H */ |