History

Merge branches 'fixes.2020.04.27a', 'kfree_rcu.2020.04.27a', 'rcu-tasks.2020.04.27a', 'stall.2020.04.27a' and 'torture.2020.05.07a' into HEAD

fixes.2020.04.27a: Miscellaneous fixes. kfree_rcu.2020.04.27a: Changes related to kfree_rcu(). rcu-tasks.2020.04.27a: Addition of new RCU-tasks flavors. stall.2020.04.27a: RCU CPU stall-warning updates. torture.2020.05.07a: Torture-test updates.

Paul E. McKenney committed 6 years ago

Commit f736e0f5 parents e2f3ccf 6be7436 e5a971d 33b2b93 3c80b4040 files · +2054 −564Browse files

Showing the changes to include/linux (8 of 40 files). Show all

Unified · Split

8 files
  1. include/linux/rcupdate.h +43 −10
  2. include/linux/rcupdate_trace.h +88 −0
  3. include/linux/rcupdate_wait.h +19 −0
  4. include/linux/rcutiny.h +2 −1
  5. include/linux/rcutree.h +1 −0
  6. include/linux/sched.h +9 −1
  7. include/linux/torture.h +1 −1
  8. include/linux/wait.h +2 −0
include/linux/rcupdate.h+43−10
@@ -37,6 +37,7 @@ · 36 unmodified lines
37/* Exported common interfaces */37/* Exported common interfaces */
38void call_rcu(struct rcu_head *head, rcu_callback_t func);38void call_rcu(struct rcu_head *head, rcu_callback_t func);
39void rcu_barrier_tasks(void);39void rcu_barrier_tasks(void);
40void rcu_barrier_tasks_rude(void);
40void synchronize_rcu(void);41void synchronize_rcu(void);
41 42
42#ifdef CONFIG_PREEMPT_RCU43#ifdef CONFIG_PREEMPT_RCU
@@ -129,25 +130,57 @@ static inline void rcu_init_nohz(void) { } · 86 unmodified lines
129 * Note a quasi-voluntary context switch for RCU-tasks's benefit.130 * Note a quasi-voluntary context switch for RCU-tasks's benefit.
130 * This is a macro rather than an inline function to avoid #include hell.131 * This is a macro rather than an inline function to avoid #include hell.
131 */132 */
132#ifdef CONFIG_TASKS_RCU133#ifdef CONFIG_TASKS_RCU_GENERIC
133#define rcu_tasks_qs(t) \134
134 do { \135# ifdef CONFIG_TASKS_RCU
135 if (READ_ONCE((t)->rcu_tasks_holdout)) \136# define rcu_tasks_classic_qs(t, preempt) \
136 WRITE_ONCE((t)->rcu_tasks_holdout, false); \137 do { \
138 if (!(preempt) && READ_ONCE((t)->rcu_tasks_holdout)) \
139 WRITE_ONCE((t)->rcu_tasks_holdout, false); \
137 } while (0)140 } while (0)
138#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t)
139void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func);141void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func);
140void synchronize_rcu_tasks(void);142void synchronize_rcu_tasks(void);
143# else
144# define rcu_tasks_classic_qs(t, preempt) do { } while (0)
145# define call_rcu_tasks call_rcu
146# define synchronize_rcu_tasks synchronize_rcu
147# endif
148
149# ifdef CONFIG_TASKS_RCU_TRACE
150# define rcu_tasks_trace_qs(t) \
151 do { \
152 if (!likely(READ_ONCE((t)->trc_reader_checked)) && \
153 !unlikely(READ_ONCE((t)->trc_reader_nesting))) { \
154 smp_store_release(&(t)->trc_reader_checked, true); \
155 smp_mb(); /* Readers partitioned by store. */ \
156 } \
157 } while (0)
158# else
159# define rcu_tasks_trace_qs(t) do { } while (0)
160# endif
161
162#define rcu_tasks_qs(t, preempt) \
163do { \
164 rcu_tasks_classic_qs((t), (preempt)); \
165 rcu_tasks_trace_qs((t)); \
166} while (0)
167
168# ifdef CONFIG_TASKS_RUDE_RCU
169void call_rcu_tasks_rude(struct rcu_head *head, rcu_callback_t func);
170void synchronize_rcu_tasks_rude(void);
171# endif
172
173#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t, false)
141void exit_tasks_rcu_start(void);174void exit_tasks_rcu_start(void);
142void exit_tasks_rcu_finish(void);175void exit_tasks_rcu_finish(void);
143#else /* #ifdef CONFIG_TASKS_RCU */176#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
144#define rcu_tasks_qs(t) do { } while (0)177#define rcu_tasks_qs(t, preempt) do { } while (0)
145#define rcu_note_voluntary_context_switch(t) do { } while (0)178#define rcu_note_voluntary_context_switch(t) do { } while (0)
146#define call_rcu_tasks call_rcu179#define call_rcu_tasks call_rcu
147#define synchronize_rcu_tasks synchronize_rcu180#define synchronize_rcu_tasks synchronize_rcu
148static inline void exit_tasks_rcu_start(void) { }181static inline void exit_tasks_rcu_start(void) { }
149static inline void exit_tasks_rcu_finish(void) { }182static inline void exit_tasks_rcu_finish(void) { }
150#endif /* #else #ifdef CONFIG_TASKS_RCU */183#endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
151 184
152/**185/**
153 * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU186 * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU
@@ -158,7 +191,7 @@ static inline void exit_tasks_rcu_finish(void) { } · 4 unmodified lines
158 */191 */
159#define cond_resched_tasks_rcu_qs() \192#define cond_resched_tasks_rcu_qs() \
160do { \193do { \
161 rcu_tasks_qs(current); \194 rcu_tasks_qs(current, false); \
162 cond_resched(); \195 cond_resched(); \
163} while (0)196} while (0)
164 197
include/linux/rcupdate_trace.h+88−0
@@ -0,0 +1,88 @@
1/* SPDX-License-Identifier: GPL-2.0+ */
2/*
3 * Read-Copy Update mechanism for mutual exclusion, adapted for tracing.
4 *
5 * Copyright (C) 2020 Paul E. McKenney.
6 */
7
8#ifndef __LINUX_RCUPDATE_TRACE_H
9#define __LINUX_RCUPDATE_TRACE_H
10
11#include <linux/sched.h>
12#include <linux/rcupdate.h>
13
14#ifdef CONFIG_DEBUG_LOCK_ALLOC
15
16extern struct lockdep_map rcu_trace_lock_map;
17
18static inline int rcu_read_lock_trace_held(void)
19{
20 return lock_is_held(&rcu_trace_lock_map);
21}
22
23#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
24
25static inline int rcu_read_lock_trace_held(void)
26{
27 return 1;
28}
29
30#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
31
32#ifdef CONFIG_TASKS_TRACE_RCU
33
34void rcu_read_unlock_trace_special(struct task_struct *t, int nesting);
35
36/**
37 * rcu_read_lock_trace - mark beginning of RCU-trace read-side critical section
38 *
39 * When synchronize_rcu_trace() is invoked by one task, then that task
40 * is guaranteed to block until all other tasks exit their read-side
41 * critical sections. Similarly, if call_rcu_trace() is invoked on one
42 * task while other tasks are within RCU read-side critical sections,
43 * invocation of the corresponding RCU callback is deferred until after
44 * the all the other tasks exit their critical sections.
45 *
46 * For more details, please see the documentation for rcu_read_lock().
47 */
48static inline void rcu_read_lock_trace(void)
49{
50 struct task_struct *t = current;
51
52 WRITE_ONCE(t->trc_reader_nesting, READ_ONCE(t->trc_reader_nesting) + 1);
53 if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) &&
54 t->trc_reader_special.b.need_mb)
55 smp_mb(); // Pairs with update-side barriers
56 rcu_lock_acquire(&rcu_trace_lock_map);
57}
58
59/**
60 * rcu_read_unlock_trace - mark end of RCU-trace read-side critical section
61 *
62 * Pairs with a preceding call to rcu_read_lock_trace(), and nesting is
63 * allowed. Invoking a rcu_read_unlock_trace() when there is no matching
64 * rcu_read_lock_trace() is verboten, and will result in lockdep complaints.
65 *
66 * For more details, please see the documentation for rcu_read_unlock().
67 */
68static inline void rcu_read_unlock_trace(void)
69{
70 int nesting;
71 struct task_struct *t = current;
72
73 rcu_lock_release(&rcu_trace_lock_map);
74 nesting = READ_ONCE(t->trc_reader_nesting) - 1;
75 if (likely(!READ_ONCE(t->trc_reader_special.s)) || nesting) {
76 WRITE_ONCE(t->trc_reader_nesting, nesting);
77 return; // We assume shallow reader nesting.
78 }
79 rcu_read_unlock_trace_special(t, nesting);
80}
81
82void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func);
83void synchronize_rcu_tasks_trace(void);
84void rcu_barrier_tasks_trace(void);
85
86#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
87
88#endif /* __LINUX_RCUPDATE_TRACE_H */
include/linux/rcupdate_wait.h+19−0
@@ -31,4 +31,23 @@ do { \ · 30 unmodified lines
31 31
32#define wait_rcu_gp(...) _wait_rcu_gp(false, __VA_ARGS__)32#define wait_rcu_gp(...) _wait_rcu_gp(false, __VA_ARGS__)
33 33
34/**
35 * synchronize_rcu_mult - Wait concurrently for multiple grace periods
36 * @...: List of call_rcu() functions for different grace periods to wait on
37 *
38 * This macro waits concurrently for multiple types of RCU grace periods.
39 * For example, synchronize_rcu_mult(call_rcu, call_rcu_tasks) would wait
40 * on concurrent RCU and RCU-tasks grace periods. Waiting on a given SRCU
41 * domain requires you to write a wrapper function for that SRCU domain's
42 * call_srcu() function, with this wrapper supplying the pointer to the
43 * corresponding srcu_struct.
44 *
45 * The first argument tells Tiny RCU's _wait_rcu_gp() not to
46 * bother waiting for RCU. The reason for this is because anywhere
47 * synchronize_rcu_mult() can be called is automatically already a full
48 * grace period.
49 */
50#define synchronize_rcu_mult(...) \
51 _wait_rcu_gp(IS_ENABLED(CONFIG_TINY_RCU), __VA_ARGS__)
52
34#endif /* _LINUX_SCHED_RCUPDATE_WAIT_H */53#endif /* _LINUX_SCHED_RCUPDATE_WAIT_H */
include/linux/rcutiny.h+2−1
@@ -49,7 +49,7 @@ static inline void rcu_softirq_qs(void) · 48 unmodified lines
49#define rcu_note_context_switch(preempt) \49#define rcu_note_context_switch(preempt) \
50 do { \50 do { \
51 rcu_qs(); \51 rcu_qs(); \
52 rcu_tasks_qs(current); \52 rcu_tasks_qs(current, (preempt)); \
53 } while (0)53 } while (0)
54 54
55static inline int rcu_needs_cpu(u64 basemono, u64 *nextevt)55static inline int rcu_needs_cpu(u64 basemono, u64 *nextevt)
@@ -87,6 +87,7 @@ static inline bool rcu_inkernel_boot_has_ended(void) { return true; } · 31 unmodified lines
87static inline bool rcu_is_watching(void) { return true; }87static inline bool rcu_is_watching(void) { return true; }
88static inline void rcu_momentary_dyntick_idle(void) { }88static inline void rcu_momentary_dyntick_idle(void) { }
89static inline void kfree_rcu_scheduler_running(void) { }89static inline void kfree_rcu_scheduler_running(void) { }
90static inline bool rcu_gp_might_be_stalled(void) { return false; }
90 91
91/* Avoid RCU read-side critical sections leaking across. */92/* Avoid RCU read-side critical sections leaking across. */
92static inline void rcu_all_qs(void) { barrier(); }93static inline void rcu_all_qs(void) { barrier(); }
include/linux/rcutree.h+1−0
@@ -39,6 +39,7 @@ void rcu_barrier(void); · 38 unmodified lines
39bool rcu_eqs_special_set(int cpu);39bool rcu_eqs_special_set(int cpu);
40void rcu_momentary_dyntick_idle(void);40void rcu_momentary_dyntick_idle(void);
41void kfree_rcu_scheduler_running(void);41void kfree_rcu_scheduler_running(void);
42bool rcu_gp_might_be_stalled(void);
42unsigned long get_state_synchronize_rcu(void);43unsigned long get_state_synchronize_rcu(void);
43void cond_synchronize_rcu(unsigned long oldstate);44void cond_synchronize_rcu(unsigned long oldstate);
44 45
include/linux/sched.h+9−1
@@ -613,7 +613,7 @@ union rcu_special { · 612 unmodified lines
613 u8 blocked;613 u8 blocked;
614 u8 need_qs;614 u8 need_qs;
615 u8 exp_hint; /* Hint for performance. */615 u8 exp_hint; /* Hint for performance. */
616 u8 deferred_qs;616 u8 need_mb; /* Readers need smp_mb(). */
617 } b; /* Bits. */617 } b; /* Bits. */
618 u32 s; /* Set of bits. */618 u32 s; /* Set of bits. */
619};619};
@@ -724,6 +724,14 @@ struct task_struct { · 104 unmodified lines
724 struct list_head rcu_tasks_holdout_list;724 struct list_head rcu_tasks_holdout_list;
725#endif /* #ifdef CONFIG_TASKS_RCU */725#endif /* #ifdef CONFIG_TASKS_RCU */
726 726
727#ifdef CONFIG_TASKS_TRACE_RCU
728 int trc_reader_nesting;
729 int trc_ipi_to_cpu;
730 union rcu_special trc_reader_special;
731 bool trc_reader_checked;
732 struct list_head trc_holdout_list;
733#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
734
727 struct sched_info sched_info;735 struct sched_info sched_info;
728 736
729 struct list_head tasks;737 struct list_head tasks;
include/linux/torture.h+1−1
@@ -89,7 +89,7 @@ void _torture_stop_kthread(char *m, struct task_struct **tp); · 88 unmodified lines
89#ifdef CONFIG_PREEMPTION89#ifdef CONFIG_PREEMPTION
90#define torture_preempt_schedule() preempt_schedule()90#define torture_preempt_schedule() preempt_schedule()
91#else91#else
92#define torture_preempt_schedule()92#define torture_preempt_schedule() do { } while (0)
93#endif93#endif
94 94
95#endif /* __LINUX_TORTURE_H */95#endif /* __LINUX_TORTURE_H */
include/linux/wait.h+2−0
@@ -1149,4 +1149,6 @@ int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, i · 1148 unmodified lines
1149 (wait)->flags = 0; \1149 (wait)->flags = 0; \
1150 } while (0)1150 } while (0)
1151 1151
1152bool try_invoke_on_locked_down_task(struct task_struct *p, bool (*func)(struct task_struct *t, void *arg), void *arg);
1153
1152#endif /* _LINUX_WAIT_H */1154#endif /* _LINUX_WAIT_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t