History

Merge branch 'core/rcu' into sched/core, to pick up dependency

We are going to rely on the loosening of RCU callback semantics, introduced by this commit: 806f04e9fd2c: ("rcu: Allow for smp_call_function() running callbacks from idle")

Ingo Molnar committed 6 years ago

Commit 58ef57b2 parents 498bdcd 806f04e68 files · +2524 −936Browse files

Showing the changes to include/linux (15 of 68 files). Show all

Unified · Split

15 files
  1. include/linux/compiler.h +53 −0
  2. include/linux/compiler_types.h +4 −0
  3. include/linux/ftrace_irq.h +0 −11
  4. include/linux/hardirq.h +15 −3
  5. include/linux/lockdep.h +21 −2
  6. include/linux/preempt.h +2 −2
  7. include/linux/rculist.h +2 −2
  8. include/linux/rcupdate.h +43 −10
  9. include/linux/rcupdate_trace.h +88 −0
  10. include/linux/rcupdate_wait.h +19 −0
  11. include/linux/rcutiny.h +4 −1
  12. include/linux/rcutree.h +3 −0
  13. include/linux/sched.h +15 −1
  14. include/linux/torture.h +1 −1
  15. include/linux/wait.h +2 −0
include/linux/compiler.h+53−0
@@ -120,12 +120,65 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 119 unmodified lines
120/* Annotate a C jump table to allow objtool to follow the code flow */120/* Annotate a C jump table to allow objtool to follow the code flow */
121#define __annotate_jump_table __section(.rodata..c_jump_table)121#define __annotate_jump_table __section(.rodata..c_jump_table)
122 122
123#ifdef CONFIG_DEBUG_ENTRY
124/* Begin/end of an instrumentation safe region */
125#define instrumentation_begin() ({ \
126 asm volatile("%c0:\n\t" \
127 ".pushsection .discard.instr_begin\n\t" \
128 ".long %c0b - .\n\t" \
129 ".popsection\n\t" : : "i" (__COUNTER__)); \
130})
131
132/*
133 * Because instrumentation_{begin,end}() can nest, objtool validation considers
134 * _begin() a +1 and _end() a -1 and computes a sum over the instructions.
135 * When the value is greater than 0, we consider instrumentation allowed.
136 *
137 * There is a problem with code like:
138 *
139 * noinstr void foo()
140 * {
141 * instrumentation_begin();
142 * ...
143 * if (cond) {
144 * instrumentation_begin();
145 * ...
146 * instrumentation_end();
147 * }
148 * bar();
149 * instrumentation_end();
150 * }
151 *
152 * If instrumentation_end() would be an empty label, like all the other
153 * annotations, the inner _end(), which is at the end of a conditional block,
154 * would land on the instruction after the block.
155 *
156 * If we then consider the sum of the !cond path, we'll see that the call to
157 * bar() is with a 0-value, even though, we meant it to happen with a positive
158 * value.
159 *
160 * To avoid this, have _end() be a NOP instruction, this ensures it will be
161 * part of the condition block and does not escape.
162 */
163#define instrumentation_end() ({ \
164 asm volatile("%c0: nop\n\t" \
165 ".pushsection .discard.instr_end\n\t" \
166 ".long %c0b - .\n\t" \
167 ".popsection\n\t" : : "i" (__COUNTER__)); \
168})
169#endif /* CONFIG_DEBUG_ENTRY */
170
123#else171#else
124#define annotate_reachable()172#define annotate_reachable()
125#define annotate_unreachable()173#define annotate_unreachable()
126#define __annotate_jump_table174#define __annotate_jump_table
127#endif175#endif
128 176
177#ifndef instrumentation_begin
178#define instrumentation_begin() do { } while(0)
179#define instrumentation_end() do { } while(0)
180#endif
181
129#ifndef ASM_UNREACHABLE182#ifndef ASM_UNREACHABLE
130# define ASM_UNREACHABLE183# define ASM_UNREACHABLE
131#endif184#endif
include/linux/compiler_types.h+4−0
@@ -118,6 +118,10 @@ struct ftrace_likely_data { · 117 unmodified lines
118#define notrace __attribute__((__no_instrument_function__))118#define notrace __attribute__((__no_instrument_function__))
119#endif119#endif
120 120
121/* Section for code which can't be instrumented at all */
122#define noinstr \
123 noinline notrace __attribute((__section__(".noinstr.text")))
124
121/*125/*
122 * it doesn't make sense on ARM (currently the only user of __naked)126 * it doesn't make sense on ARM (currently the only user of __naked)
123 * to trace naked functions because then mcount is called without127 * to trace naked functions because then mcount is called without
include/linux/ftrace_irq.h+0−11
@@ -2,15 +2,6 @@ · 1 unmodified line
2#ifndef _LINUX_FTRACE_IRQ_H2#ifndef _LINUX_FTRACE_IRQ_H
3#define _LINUX_FTRACE_IRQ_H3#define _LINUX_FTRACE_IRQ_H
4 4
5
6#ifdef CONFIG_FTRACE_NMI_ENTER
7extern void arch_ftrace_nmi_enter(void);
8extern void arch_ftrace_nmi_exit(void);
9#else
10static inline void arch_ftrace_nmi_enter(void) { }
11static inline void arch_ftrace_nmi_exit(void) { }
12#endif
13
14#ifdef CONFIG_HWLAT_TRACER5#ifdef CONFIG_HWLAT_TRACER
15extern bool trace_hwlat_callback_enabled;6extern bool trace_hwlat_callback_enabled;
16extern void trace_hwlat_callback(bool enter);7extern void trace_hwlat_callback(bool enter);
@@ -22,12 +13,10 @@ static inline void ftrace_nmi_enter(void) · 5 unmodified lines
22 if (trace_hwlat_callback_enabled)13 if (trace_hwlat_callback_enabled)
23 trace_hwlat_callback(true);14 trace_hwlat_callback(true);
24#endif15#endif
25 arch_ftrace_nmi_enter();
26}16}
27 17
28static inline void ftrace_nmi_exit(void)18static inline void ftrace_nmi_exit(void)
29{19{
30 arch_ftrace_nmi_exit();
31#ifdef CONFIG_HWLAT_TRACER20#ifdef CONFIG_HWLAT_TRACER
32 if (trace_hwlat_callback_enabled)21 if (trace_hwlat_callback_enabled)
33 trace_hwlat_callback(false);22 trace_hwlat_callback(false);
include/linux/hardirq.h+15−3
@@ -65,14 +65,26 @@ extern void irq_exit(void); · 64 unmodified lines
65#define arch_nmi_exit() do { } while (0)65#define arch_nmi_exit() do { } while (0)
66#endif66#endif
67 67
68/*
69 * NMI vs Tracing
70 * --------------
71 *
72 * We must not land in a tracer until (or after) we've changed preempt_count
73 * such that in_nmi() becomes true. To that effect all NMI C entry points must
74 * be marked 'notrace' and call nmi_enter() as soon as possible.
75 */
76
77/*
78 * nmi_enter() can nest up to 15 times; see NMI_BITS.
79 */
68#define nmi_enter() \80#define nmi_enter() \
69 do { \81 do { \
70 arch_nmi_enter(); \82 arch_nmi_enter(); \
71 printk_nmi_enter(); \83 printk_nmi_enter(); \
72 lockdep_off(); \84 lockdep_off(); \
73 ftrace_nmi_enter(); \85 ftrace_nmi_enter(); \
74 BUG_ON(in_nmi()); \86 BUG_ON(in_nmi() == NMI_MASK); \
75 preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \87 __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \
76 rcu_nmi_enter(); \88 rcu_nmi_enter(); \
77 lockdep_hardirq_enter(); \89 lockdep_hardirq_enter(); \
78 } while (0)90 } while (0)
@@ -82,7 +94,7 @@ extern void irq_exit(void); · 3 unmodified lines
82 lockdep_hardirq_exit(); \94 lockdep_hardirq_exit(); \
83 rcu_nmi_exit(); \95 rcu_nmi_exit(); \
84 BUG_ON(!in_nmi()); \96 BUG_ON(!in_nmi()); \
85 preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \97 __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \
86 ftrace_nmi_exit(); \98 ftrace_nmi_exit(); \
87 lockdep_on(); \99 lockdep_on(); \
88 printk_nmi_exit(); \100 printk_nmi_exit(); \
include/linux/lockdep.h+21−2
@@ -308,8 +308,27 @@ extern void lockdep_set_selftest_task(struct task_struct *task); · 307 unmodified lines
308 308
309extern void lockdep_init_task(struct task_struct *task);309extern void lockdep_init_task(struct task_struct *task);
310 310
311extern void lockdep_off(void);311/*
312extern void lockdep_on(void);312 * Split the recrursion counter in two to readily detect 'off' vs recursion.
313 */
314#define LOCKDEP_RECURSION_BITS 16
315#define LOCKDEP_OFF (1U << LOCKDEP_RECURSION_BITS)
316#define LOCKDEP_RECURSION_MASK (LOCKDEP_OFF - 1)
317
318/*
319 * lockdep_{off,on}() are macros to avoid tracing and kprobes; not inlines due
320 * to header dependencies.
321 */
322
323#define lockdep_off() \
324do { \
325 current->lockdep_recursion += LOCKDEP_OFF; \
326} while (0)
327
328#define lockdep_on() \
329do { \
330 current->lockdep_recursion -= LOCKDEP_OFF; \
331} while (0)
313 332
314extern void lockdep_register_key(struct lock_class_key *key);333extern void lockdep_register_key(struct lock_class_key *key);
315extern void lockdep_unregister_key(struct lock_class_key *key);334extern void lockdep_unregister_key(struct lock_class_key *key);
include/linux/preempt.h+2−2
@@ -26,13 +26,13 @@ · 25 unmodified lines
26 * PREEMPT_MASK: 0x000000ff26 * PREEMPT_MASK: 0x000000ff
27 * SOFTIRQ_MASK: 0x0000ff0027 * SOFTIRQ_MASK: 0x0000ff00
28 * HARDIRQ_MASK: 0x000f000028 * HARDIRQ_MASK: 0x000f0000
29 * NMI_MASK: 0x0010000029 * NMI_MASK: 0x00f00000
30 * PREEMPT_NEED_RESCHED: 0x8000000030 * PREEMPT_NEED_RESCHED: 0x80000000
31 */31 */
32#define PREEMPT_BITS 832#define PREEMPT_BITS 8
33#define SOFTIRQ_BITS 833#define SOFTIRQ_BITS 8
34#define HARDIRQ_BITS 434#define HARDIRQ_BITS 4
35#define NMI_BITS 135#define NMI_BITS 4
36 36
37#define PREEMPT_SHIFT 037#define PREEMPT_SHIFT 0
38#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)38#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
include/linux/rculist.h+2−2
@@ -371,7 +371,7 @@ static inline void list_splice_tail_init_rcu(struct list_head *list, · 370 unmodified lines
371 * @pos: the type * to use as a loop cursor.371 * @pos: the type * to use as a loop cursor.
372 * @head: the head for your list.372 * @head: the head for your list.
373 * @member: the name of the list_head within the struct.373 * @member: the name of the list_head within the struct.
374 * @cond...: optional lockdep expression if called from non-RCU protection.374 * @cond: optional lockdep expression if called from non-RCU protection.
375 *375 *
376 * This list-traversal primitive may safely run concurrently with376 * This list-traversal primitive may safely run concurrently with
377 * the _rcu list-mutation primitives such as list_add_rcu()377 * the _rcu list-mutation primitives such as list_add_rcu()
@@ -646,7 +646,7 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n, · 268 unmodified lines
646 * @pos: the type * to use as a loop cursor.646 * @pos: the type * to use as a loop cursor.
647 * @head: the head for your list.647 * @head: the head for your list.
648 * @member: the name of the hlist_node within the struct.648 * @member: the name of the hlist_node within the struct.
649 * @cond...: optional lockdep expression if called from non-RCU protection.649 * @cond: optional lockdep expression if called from non-RCU protection.
650 *650 *
651 * This list-traversal primitive may safely run concurrently with651 * This list-traversal primitive may safely run concurrently with
652 * the _rcu list-mutation primitives such as hlist_add_head_rcu()652 * the _rcu list-mutation primitives such as hlist_add_head_rcu()
include/linux/rcupdate.h+43−10
@@ -37,6 +37,7 @@ · 36 unmodified lines
37/* Exported common interfaces */37/* Exported common interfaces */
38void call_rcu(struct rcu_head *head, rcu_callback_t func);38void call_rcu(struct rcu_head *head, rcu_callback_t func);
39void rcu_barrier_tasks(void);39void rcu_barrier_tasks(void);
40void rcu_barrier_tasks_rude(void);
40void synchronize_rcu(void);41void synchronize_rcu(void);
41 42
42#ifdef CONFIG_PREEMPT_RCU43#ifdef CONFIG_PREEMPT_RCU
@@ -129,25 +130,57 @@ static inline void rcu_init_nohz(void) { } · 86 unmodified lines
129 * Note a quasi-voluntary context switch for RCU-tasks's benefit.130 * Note a quasi-voluntary context switch for RCU-tasks's benefit.
130 * This is a macro rather than an inline function to avoid #include hell.131 * This is a macro rather than an inline function to avoid #include hell.
131 */132 */
132#ifdef CONFIG_TASKS_RCU133#ifdef CONFIG_TASKS_RCU_GENERIC
133#define rcu_tasks_qs(t) \134
134 do { \135# ifdef CONFIG_TASKS_RCU
135 if (READ_ONCE((t)->rcu_tasks_holdout)) \136# define rcu_tasks_classic_qs(t, preempt) \
136 WRITE_ONCE((t)->rcu_tasks_holdout, false); \137 do { \
138 if (!(preempt) && READ_ONCE((t)->rcu_tasks_holdout)) \
139 WRITE_ONCE((t)->rcu_tasks_holdout, false); \
137 } while (0)140 } while (0)
138#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t)
139void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func);141void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func);
140void synchronize_rcu_tasks(void);142void synchronize_rcu_tasks(void);
143# else
144# define rcu_tasks_classic_qs(t, preempt) do { } while (0)
145# define call_rcu_tasks call_rcu
146# define synchronize_rcu_tasks synchronize_rcu
147# endif
148
149# ifdef CONFIG_TASKS_RCU_TRACE
150# define rcu_tasks_trace_qs(t) \
151 do { \
152 if (!likely(READ_ONCE((t)->trc_reader_checked)) && \
153 !unlikely(READ_ONCE((t)->trc_reader_nesting))) { \
154 smp_store_release(&(t)->trc_reader_checked, true); \
155 smp_mb(); /* Readers partitioned by store. */ \
156 } \
157 } while (0)
158# else
159# define rcu_tasks_trace_qs(t) do { } while (0)
160# endif
161
162#define rcu_tasks_qs(t, preempt) \
163do { \
164 rcu_tasks_classic_qs((t), (preempt)); \
165 rcu_tasks_trace_qs((t)); \
166} while (0)
167
168# ifdef CONFIG_TASKS_RUDE_RCU
169void call_rcu_tasks_rude(struct rcu_head *head, rcu_callback_t func);
170void synchronize_rcu_tasks_rude(void);
171# endif
172
173#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t, false)
141void exit_tasks_rcu_start(void);174void exit_tasks_rcu_start(void);
142void exit_tasks_rcu_finish(void);175void exit_tasks_rcu_finish(void);
143#else /* #ifdef CONFIG_TASKS_RCU */176#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
144#define rcu_tasks_qs(t) do { } while (0)177#define rcu_tasks_qs(t, preempt) do { } while (0)
145#define rcu_note_voluntary_context_switch(t) do { } while (0)178#define rcu_note_voluntary_context_switch(t) do { } while (0)
146#define call_rcu_tasks call_rcu179#define call_rcu_tasks call_rcu
147#define synchronize_rcu_tasks synchronize_rcu180#define synchronize_rcu_tasks synchronize_rcu
148static inline void exit_tasks_rcu_start(void) { }181static inline void exit_tasks_rcu_start(void) { }
149static inline void exit_tasks_rcu_finish(void) { }182static inline void exit_tasks_rcu_finish(void) { }
150#endif /* #else #ifdef CONFIG_TASKS_RCU */183#endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
151 184
152/**185/**
153 * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU186 * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU
@@ -158,7 +191,7 @@ static inline void exit_tasks_rcu_finish(void) { } · 4 unmodified lines
158 */191 */
159#define cond_resched_tasks_rcu_qs() \192#define cond_resched_tasks_rcu_qs() \
160do { \193do { \
161 rcu_tasks_qs(current); \194 rcu_tasks_qs(current, false); \
162 cond_resched(); \195 cond_resched(); \
163} while (0)196} while (0)
164 197
include/linux/rcupdate_trace.h+88−0
@@ -0,0 +1,88 @@
1/* SPDX-License-Identifier: GPL-2.0+ */
2/*
3 * Read-Copy Update mechanism for mutual exclusion, adapted for tracing.
4 *
5 * Copyright (C) 2020 Paul E. McKenney.
6 */
7
8#ifndef __LINUX_RCUPDATE_TRACE_H
9#define __LINUX_RCUPDATE_TRACE_H
10
11#include <linux/sched.h>
12#include <linux/rcupdate.h>
13
14#ifdef CONFIG_DEBUG_LOCK_ALLOC
15
16extern struct lockdep_map rcu_trace_lock_map;
17
18static inline int rcu_read_lock_trace_held(void)
19{
20 return lock_is_held(&rcu_trace_lock_map);
21}
22
23#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
24
25static inline int rcu_read_lock_trace_held(void)
26{
27 return 1;
28}
29
30#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
31
32#ifdef CONFIG_TASKS_TRACE_RCU
33
34void rcu_read_unlock_trace_special(struct task_struct *t, int nesting);
35
36/**
37 * rcu_read_lock_trace - mark beginning of RCU-trace read-side critical section
38 *
39 * When synchronize_rcu_trace() is invoked by one task, then that task
40 * is guaranteed to block until all other tasks exit their read-side
41 * critical sections. Similarly, if call_rcu_trace() is invoked on one
42 * task while other tasks are within RCU read-side critical sections,
43 * invocation of the corresponding RCU callback is deferred until after
44 * the all the other tasks exit their critical sections.
45 *
46 * For more details, please see the documentation for rcu_read_lock().
47 */
48static inline void rcu_read_lock_trace(void)
49{
50 struct task_struct *t = current;
51
52 WRITE_ONCE(t->trc_reader_nesting, READ_ONCE(t->trc_reader_nesting) + 1);
53 if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) &&
54 t->trc_reader_special.b.need_mb)
55 smp_mb(); // Pairs with update-side barriers
56 rcu_lock_acquire(&rcu_trace_lock_map);
57}
58
59/**
60 * rcu_read_unlock_trace - mark end of RCU-trace read-side critical section
61 *
62 * Pairs with a preceding call to rcu_read_lock_trace(), and nesting is
63 * allowed. Invoking a rcu_read_unlock_trace() when there is no matching
64 * rcu_read_lock_trace() is verboten, and will result in lockdep complaints.
65 *
66 * For more details, please see the documentation for rcu_read_unlock().
67 */
68static inline void rcu_read_unlock_trace(void)
69{
70 int nesting;
71 struct task_struct *t = current;
72
73 rcu_lock_release(&rcu_trace_lock_map);
74 nesting = READ_ONCE(t->trc_reader_nesting) - 1;
75 if (likely(!READ_ONCE(t->trc_reader_special.s)) || nesting) {
76 WRITE_ONCE(t->trc_reader_nesting, nesting);
77 return; // We assume shallow reader nesting.
78 }
79 rcu_read_unlock_trace_special(t, nesting);
80}
81
82void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func);
83void synchronize_rcu_tasks_trace(void);
84void rcu_barrier_tasks_trace(void);
85
86#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
87
88#endif /* __LINUX_RCUPDATE_TRACE_H */
include/linux/rcupdate_wait.h+19−0
@@ -31,4 +31,23 @@ do { \ · 30 unmodified lines
31 31
32#define wait_rcu_gp(...) _wait_rcu_gp(false, __VA_ARGS__)32#define wait_rcu_gp(...) _wait_rcu_gp(false, __VA_ARGS__)
33 33
34/**
35 * synchronize_rcu_mult - Wait concurrently for multiple grace periods
36 * @...: List of call_rcu() functions for different grace periods to wait on
37 *
38 * This macro waits concurrently for multiple types of RCU grace periods.
39 * For example, synchronize_rcu_mult(call_rcu, call_rcu_tasks) would wait
40 * on concurrent RCU and RCU-tasks grace periods. Waiting on a given SRCU
41 * domain requires you to write a wrapper function for that SRCU domain's
42 * call_srcu() function, with this wrapper supplying the pointer to the
43 * corresponding srcu_struct.
44 *
45 * The first argument tells Tiny RCU's _wait_rcu_gp() not to
46 * bother waiting for RCU. The reason for this is because anywhere
47 * synchronize_rcu_mult() can be called is automatically already a full
48 * grace period.
49 */
50#define synchronize_rcu_mult(...) \
51 _wait_rcu_gp(IS_ENABLED(CONFIG_TINY_RCU), __VA_ARGS__)
52
34#endif /* _LINUX_SCHED_RCUPDATE_WAIT_H */53#endif /* _LINUX_SCHED_RCUPDATE_WAIT_H */
include/linux/rcutiny.h+4−1
@@ -49,7 +49,7 @@ static inline void rcu_softirq_qs(void) · 48 unmodified lines
49#define rcu_note_context_switch(preempt) \49#define rcu_note_context_switch(preempt) \
50 do { \50 do { \
51 rcu_qs(); \51 rcu_qs(); \
52 rcu_tasks_qs(current); \52 rcu_tasks_qs(current, (preempt)); \
53 } while (0)53 } while (0)
54 54
55static inline int rcu_needs_cpu(u64 basemono, u64 *nextevt)55static inline int rcu_needs_cpu(u64 basemono, u64 *nextevt)
@@ -71,6 +71,7 @@ static inline void rcu_irq_enter(void) { } · 15 unmodified lines
71static inline void rcu_irq_exit_irqson(void) { }71static inline void rcu_irq_exit_irqson(void) { }
72static inline void rcu_irq_enter_irqson(void) { }72static inline void rcu_irq_enter_irqson(void) { }
73static inline void rcu_irq_exit(void) { }73static inline void rcu_irq_exit(void) { }
74static inline void rcu_irq_exit_preempt(void) { }
74static inline void exit_rcu(void) { }75static inline void exit_rcu(void) { }
75static inline bool rcu_preempt_need_deferred_qs(struct task_struct *t)76static inline bool rcu_preempt_need_deferred_qs(struct task_struct *t)
76{77{
@@ -85,8 +86,10 @@ static inline void rcu_scheduler_starting(void) { } · 8 unmodified lines
85static inline void rcu_end_inkernel_boot(void) { }86static inline void rcu_end_inkernel_boot(void) { }
86static inline bool rcu_inkernel_boot_has_ended(void) { return true; }87static inline bool rcu_inkernel_boot_has_ended(void) { return true; }
87static inline bool rcu_is_watching(void) { return true; }88static inline bool rcu_is_watching(void) { return true; }
89static inline bool __rcu_is_watching(void) { return true; }
88static inline void rcu_momentary_dyntick_idle(void) { }90static inline void rcu_momentary_dyntick_idle(void) { }
89static inline void kfree_rcu_scheduler_running(void) { }91static inline void kfree_rcu_scheduler_running(void) { }
92static inline bool rcu_gp_might_be_stalled(void) { return false; }
90 93
91/* Avoid RCU read-side critical sections leaking across. */94/* Avoid RCU read-side critical sections leaking across. */
92static inline void rcu_all_qs(void) { barrier(); }95static inline void rcu_all_qs(void) { barrier(); }
include/linux/rcutree.h+3−0
@@ -39,6 +39,7 @@ void rcu_barrier(void); · 38 unmodified lines
39bool rcu_eqs_special_set(int cpu);39bool rcu_eqs_special_set(int cpu);
40void rcu_momentary_dyntick_idle(void);40void rcu_momentary_dyntick_idle(void);
41void kfree_rcu_scheduler_running(void);41void kfree_rcu_scheduler_running(void);
42bool rcu_gp_might_be_stalled(void);
42unsigned long get_state_synchronize_rcu(void);43unsigned long get_state_synchronize_rcu(void);
43void cond_synchronize_rcu(unsigned long oldstate);44void cond_synchronize_rcu(unsigned long oldstate);
44 45
@@ -46,6 +47,7 @@ void rcu_idle_enter(void); · 1 unmodified line
46void rcu_idle_exit(void);47void rcu_idle_exit(void);
47void rcu_irq_enter(void);48void rcu_irq_enter(void);
48void rcu_irq_exit(void);49void rcu_irq_exit(void);
50void rcu_irq_exit_preempt(void);
49void rcu_irq_enter_irqson(void);51void rcu_irq_enter_irqson(void);
50void rcu_irq_exit_irqson(void);52void rcu_irq_exit_irqson(void);
51 53
@@ -56,6 +58,7 @@ extern int rcu_scheduler_active __read_mostly; · 4 unmodified lines
56void rcu_end_inkernel_boot(void);58void rcu_end_inkernel_boot(void);
57bool rcu_inkernel_boot_has_ended(void);59bool rcu_inkernel_boot_has_ended(void);
58bool rcu_is_watching(void);60bool rcu_is_watching(void);
61bool __rcu_is_watching(void);
59#ifndef CONFIG_PREEMPTION62#ifndef CONFIG_PREEMPTION
60void rcu_all_qs(void);63void rcu_all_qs(void);
61#endif64#endif
include/linux/sched.h+15−1
@@ -613,7 +613,7 @@ union rcu_special { · 612 unmodified lines
613 u8 blocked;613 u8 blocked;
614 u8 need_qs;614 u8 need_qs;
615 u8 exp_hint; /* Hint for performance. */615 u8 exp_hint; /* Hint for performance. */
616 u8 deferred_qs;616 u8 need_mb; /* Readers need smp_mb(). */
617 } b; /* Bits. */617 } b; /* Bits. */
618 u32 s; /* Set of bits. */618 u32 s; /* Set of bits. */
619};619};
@@ -724,6 +724,14 @@ struct task_struct { · 104 unmodified lines
724 struct list_head rcu_tasks_holdout_list;724 struct list_head rcu_tasks_holdout_list;
725#endif /* #ifdef CONFIG_TASKS_RCU */725#endif /* #ifdef CONFIG_TASKS_RCU */
726 726
727#ifdef CONFIG_TASKS_TRACE_RCU
728 int trc_reader_nesting;
729 int trc_ipi_to_cpu;
730 union rcu_special trc_reader_special;
731 bool trc_reader_checked;
732 struct list_head trc_holdout_list;
733#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
734
727 struct sched_info sched_info;735 struct sched_info sched_info;
728 736
729 struct list_head tasks;737 struct list_head tasks;
@@ -1289,6 +1297,12 @@ struct task_struct { · 559 unmodified lines
1289 unsigned long prev_lowest_stack;1297 unsigned long prev_lowest_stack;
1290#endif1298#endif
1291 1299
1300#ifdef CONFIG_X86_MCE
1301 u64 mce_addr;
1302 u64 mce_status;
1303 struct callback_head mce_kill_me;
1304#endif
1305
1292 /*1306 /*
1293 * New fields for task_struct should be added above here, so that1307 * New fields for task_struct should be added above here, so that
1294 * they are included in the randomized portion of task_struct.1308 * they are included in the randomized portion of task_struct.
include/linux/torture.h+1−1
@@ -89,7 +89,7 @@ void _torture_stop_kthread(char *m, struct task_struct **tp); · 88 unmodified lines
89#ifdef CONFIG_PREEMPTION89#ifdef CONFIG_PREEMPTION
90#define torture_preempt_schedule() preempt_schedule()90#define torture_preempt_schedule() preempt_schedule()
91#else91#else
92#define torture_preempt_schedule()92#define torture_preempt_schedule() do { } while (0)
93#endif93#endif
94 94
95#endif /* __LINUX_TORTURE_H */95#endif /* __LINUX_TORTURE_H */
include/linux/wait.h+2−0
@@ -1149,4 +1149,6 @@ int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, i · 1148 unmodified lines
1149 (wait)->flags = 0; \1149 (wait)->flags = 0; \
1150 } while (0)1150 } while (0)
1151 1151
1152bool try_invoke_on_locked_down_task(struct task_struct *p, bool (*func)(struct task_struct *t, void *arg), void *arg);
1153
1152#endif /* _LINUX_WAIT_H */1154#endif /* _LINUX_WAIT_H */