History

Merge branches 'doc.2021.11.30c', 'exp.2021.12.07a', 'fastnohz.2021.11.30c', 'fixes.2021.11.30c', 'nocb.2021.12.09a', 'nolibc.2021.11.30c', 'tasks.2021.12.09a', 'torture.2021.12.07a' and 'torturescript.2021.11.30c' into HEAD

doc.2021.11.30c: Documentation updates. exp.2021.12.07a: Expedited-grace-period fixes. fastnohz.2021.11.30c: Remove CONFIG_RCU_FAST_NO_HZ. fixes.2021.11.30c: Miscellaneous fixes. nocb.2021.12.09a: No-CB CPU updates. nolibc.2021.11.30c: Tiny in-kernel library updates. tasks.2021.12.09a: RCU-tasks updates, including update-side scalability. torture.2021.12.07a: Torture-test in-kernel module updates. torturescript.2021.11.30c: Torture-test scripting updates.

Paul E. McKenney committed 4 years ago

Commit f80fe669 parents 5861dad 81f6d49 bc849e9 1f8da40 10d4703 b0fe9de fd796e4 53b541f 90b21bc51 files · +1089 −719Browse files

Showing the changes to kernel (17 of 51 files). Show all

Unified · Split

17 files
  1. kernel/locking/locktorture.c +2 −2
  2. kernel/rcu/Kconfig +1 −19
  3. kernel/rcu/rcu_segcblist.c +4 −6
  4. kernel/rcu/rcu_segcblist.h +7 −5
  5. kernel/rcu/rcuscale.c +7 −7
  6. kernel/rcu/rcutorture.c +174 −60
  7. kernel/rcu/refscale.c +24 −26
  8. kernel/rcu/srcutiny.c +1 −1
  9. kernel/rcu/tasks.h +394 −82
  10. kernel/rcu/tree.c +86 −45
  11. kernel/rcu/tree.h +15 −16
  12. kernel/rcu/tree_exp.h +9 −5
  13. kernel/rcu/tree_nocb.h +106 −54
  14. kernel/rcu/tree_plugin.h +33 −217
  15. kernel/rcu/tree_stall.h +2 −25
  16. kernel/scftorture.c +6 −10
  17. kernel/torture.c +2 −2
kernel/locking/locktorture.c+2−2
@@ -1047,7 +1047,7 @@ static int __init lock_torture_init(void) · 1046 unmodified lines
1047 sizeof(writer_tasks[0]),1047 sizeof(writer_tasks[0]),
1048 GFP_KERNEL);1048 GFP_KERNEL);
1049 if (writer_tasks == NULL) {1049 if (writer_tasks == NULL) {
1050 VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory");1050 TOROUT_ERRSTRING("writer_tasks: Out of memory");
1051 firsterr = -ENOMEM;1051 firsterr = -ENOMEM;
1052 goto unwind;1052 goto unwind;
1053 }1053 }
@@ -1058,7 +1058,7 @@ static int __init lock_torture_init(void) · 4 unmodified lines
1058 sizeof(reader_tasks[0]),1058 sizeof(reader_tasks[0]),
1059 GFP_KERNEL);1059 GFP_KERNEL);
1060 if (reader_tasks == NULL) {1060 if (reader_tasks == NULL) {
1061 VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory");1061 TOROUT_ERRSTRING("reader_tasks: Out of memory");
1062 kfree(writer_tasks);1062 kfree(writer_tasks);
1063 writer_tasks = NULL;1063 writer_tasks = NULL;
1064 firsterr = -ENOMEM;1064 firsterr = -ENOMEM;
kernel/rcu/Kconfig+1−19
@@ -112,7 +112,7 @@ config RCU_STALL_COMMON · 111 unmodified lines
112 making these warnings mandatory for the tree variants.112 making these warnings mandatory for the tree variants.
113 113
114config RCU_NEED_SEGCBLIST114config RCU_NEED_SEGCBLIST
115 def_bool ( TREE_RCU || TREE_SRCU )115 def_bool ( TREE_RCU || TREE_SRCU || TASKS_RCU_GENERIC )
116 116
117config RCU_FANOUT117config RCU_FANOUT
118 int "Tree-based hierarchical RCU fanout value"118 int "Tree-based hierarchical RCU fanout value"
@@ -169,24 +169,6 @@ config RCU_FANOUT_LEAF · 50 unmodified lines
169 169
170 Take the default if unsure.170 Take the default if unsure.
171 171
172config RCU_FAST_NO_HZ
173 bool "Accelerate last non-dyntick-idle CPU's grace periods"
174 depends on NO_HZ_COMMON && SMP && RCU_EXPERT
175 default n
176 help
177 This option permits CPUs to enter dynticks-idle state even if
178 they have RCU callbacks queued, and prevents RCU from waking
179 these CPUs up more than roughly once every four jiffies (by
180 default, you can adjust this using the rcutree.rcu_idle_gp_delay
181 parameter), thus improving energy efficiency. On the other
182 hand, this option increases the duration of RCU grace periods,
183 for example, slowing down synchronize_rcu().
184
185 Say Y if energy efficiency is critically important, and you
186 don't care about increased grace-period durations.
187
188 Say N if you are unsure.
189
190config RCU_BOOST172config RCU_BOOST
191 bool "Enable RCU priority boosting"173 bool "Enable RCU priority boosting"
192 depends on (RT_MUTEXES && PREEMPT_RCU && RCU_EXPERT) || PREEMPT_RT174 depends on (RT_MUTEXES && PREEMPT_RCU && RCU_EXPERT) || PREEMPT_RT
kernel/rcu/rcu_segcblist.c+4−6
@@ -261,16 +261,14 @@ void rcu_segcblist_disable(struct rcu_segcblist *rsclp) · 260 unmodified lines
261}261}
262 262
263/*263/*
264 * Mark the specified rcu_segcblist structure as offloaded.264 * Mark the specified rcu_segcblist structure as offloaded (or not)
265 */265 */
266void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload)266void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload)
267{267{
268 if (offload) {268 if (offload)
269 rcu_segcblist_clear_flags(rsclp, SEGCBLIST_SOFTIRQ_ONLY);269 rcu_segcblist_set_flags(rsclp, SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED);
270 rcu_segcblist_set_flags(rsclp, SEGCBLIST_OFFLOADED);270 else
271 } else {
272 rcu_segcblist_clear_flags(rsclp, SEGCBLIST_OFFLOADED);271 rcu_segcblist_clear_flags(rsclp, SEGCBLIST_OFFLOADED);
273 }
274}272}
275 273
276/*274/*
kernel/rcu/rcu_segcblist.h+7−5
@@ -80,11 +80,14 @@ static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) · 79 unmodified lines
80 return rcu_segcblist_test_flags(rsclp, SEGCBLIST_ENABLED);80 return rcu_segcblist_test_flags(rsclp, SEGCBLIST_ENABLED);
81}81}
82 82
83/* Is the specified rcu_segcblist offloaded, or is SEGCBLIST_SOFTIRQ_ONLY set? */83/*
84 * Is the specified rcu_segcblist NOCB offloaded (or in the middle of the
85 * [de]offloading process)?
86 */
84static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)87static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)
85{88{
86 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&89 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
87 !rcu_segcblist_test_flags(rsclp, SEGCBLIST_SOFTIRQ_ONLY))90 rcu_segcblist_test_flags(rsclp, SEGCBLIST_LOCKING))
88 return true;91 return true;
89 92
90 return false;93 return false;
@@ -92,9 +95,8 @@ static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) · 1 unmodified line
92 95
93static inline bool rcu_segcblist_completely_offloaded(struct rcu_segcblist *rsclp)96static inline bool rcu_segcblist_completely_offloaded(struct rcu_segcblist *rsclp)
94{97{
95 int flags = SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP | SEGCBLIST_OFFLOADED;98 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
96 99 !rcu_segcblist_test_flags(rsclp, SEGCBLIST_RCU_CORE))
97 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) && (rsclp->flags & flags) == flags)
98 return true;100 return true;
99 101
100 return false;102 return false;
kernel/rcu/rcuscale.c+7−7
@@ -50,8 +50,8 @@ MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>"); · 49 unmodified lines
50 pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s)50 pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s)
51#define VERBOSE_SCALEOUT_STRING(s) \51#define VERBOSE_SCALEOUT_STRING(s) \
52 do { if (verbose) pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s); } while (0)52 do { if (verbose) pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s); } while (0)
53#define VERBOSE_SCALEOUT_ERRSTRING(s) \53#define SCALEOUT_ERRSTRING(s) \
54 do { if (verbose) pr_alert("%s" SCALE_FLAG "!!! %s\n", scale_type, s); } while (0)54 pr_alert("%s" SCALE_FLAG "!!! %s\n", scale_type, s)
55 55
56/*56/*
57 * The intended use cases for the nreaders and nwriters module parameters57 * The intended use cases for the nreaders and nwriters module parameters
@@ -514,11 +514,11 @@ rcu_scale_cleanup(void) · 456 unmodified lines
514 * during the mid-boot phase, so have to wait till the end.514 * during the mid-boot phase, so have to wait till the end.
515 */515 */
516 if (rcu_gp_is_expedited() && !rcu_gp_is_normal() && !gp_exp)516 if (rcu_gp_is_expedited() && !rcu_gp_is_normal() && !gp_exp)
517 VERBOSE_SCALEOUT_ERRSTRING("All grace periods expedited, no normal ones to measure!");517 SCALEOUT_ERRSTRING("All grace periods expedited, no normal ones to measure!");
518 if (rcu_gp_is_normal() && gp_exp)518 if (rcu_gp_is_normal() && gp_exp)
519 VERBOSE_SCALEOUT_ERRSTRING("All grace periods normal, no expedited ones to measure!");519 SCALEOUT_ERRSTRING("All grace periods normal, no expedited ones to measure!");
520 if (gp_exp && gp_async)520 if (gp_exp && gp_async)
521 VERBOSE_SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!");521 SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!");
522 522
523 if (torture_cleanup_begin())523 if (torture_cleanup_begin())
524 return;524 return;
@@ -845,7 +845,7 @@ rcu_scale_init(void) · 320 unmodified lines
845 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]),845 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]),
846 GFP_KERNEL);846 GFP_KERNEL);
847 if (reader_tasks == NULL) {847 if (reader_tasks == NULL) {
848 VERBOSE_SCALEOUT_ERRSTRING("out of memory");848 SCALEOUT_ERRSTRING("out of memory");
849 firsterr = -ENOMEM;849 firsterr = -ENOMEM;
850 goto unwind;850 goto unwind;
851 }851 }
@@ -865,7 +865,7 @@ rcu_scale_init(void) · 13 unmodified lines
865 kcalloc(nrealwriters, sizeof(*writer_n_durations),865 kcalloc(nrealwriters, sizeof(*writer_n_durations),
866 GFP_KERNEL);866 GFP_KERNEL);
867 if (!writer_tasks || !writer_durations || !writer_n_durations) {867 if (!writer_tasks || !writer_durations || !writer_n_durations) {
868 VERBOSE_SCALEOUT_ERRSTRING("out of memory");868 SCALEOUT_ERRSTRING("out of memory");
869 firsterr = -ENOMEM;869 firsterr = -ENOMEM;
870 goto unwind;870 goto unwind;
871 }871 }
kernel/rcu/rcutorture.c+174−60
@@ -46,6 +46,7 @@ · 45 unmodified lines
46#include <linux/oom.h>46#include <linux/oom.h>
47#include <linux/tick.h>47#include <linux/tick.h>
48#include <linux/rcupdate_trace.h>48#include <linux/rcupdate_trace.h>
49#include <linux/nmi.h>
49 50
50#include "rcu.h"51#include "rcu.h"
51 52
@@ -53,15 +54,18 @@ MODULE_LICENSE("GPL"); · 1 unmodified line
53MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>");54MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
54 55
55/* Bits for ->extendables field, extendables param, and related definitions. */56/* Bits for ->extendables field, extendables param, and related definitions. */
56#define RCUTORTURE_RDR_SHIFT 8 /* Put SRCU index in upper bits. */57#define RCUTORTURE_RDR_SHIFT_1 8 /* Put SRCU index in upper bits. */
57#define RCUTORTURE_RDR_MASK ((1 << RCUTORTURE_RDR_SHIFT) - 1)58#define RCUTORTURE_RDR_MASK_1 (1 << RCUTORTURE_RDR_SHIFT_1)
59#define RCUTORTURE_RDR_SHIFT_2 9 /* Put SRCU index in upper bits. */
60#define RCUTORTURE_RDR_MASK_2 (1 << RCUTORTURE_RDR_SHIFT_2)
58#define RCUTORTURE_RDR_BH 0x01 /* Extend readers by disabling bh. */61#define RCUTORTURE_RDR_BH 0x01 /* Extend readers by disabling bh. */
59#define RCUTORTURE_RDR_IRQ 0x02 /* ... disabling interrupts. */62#define RCUTORTURE_RDR_IRQ 0x02 /* ... disabling interrupts. */
60#define RCUTORTURE_RDR_PREEMPT 0x04 /* ... disabling preemption. */63#define RCUTORTURE_RDR_PREEMPT 0x04 /* ... disabling preemption. */
61#define RCUTORTURE_RDR_RBH 0x08 /* ... rcu_read_lock_bh(). */64#define RCUTORTURE_RDR_RBH 0x08 /* ... rcu_read_lock_bh(). */
62#define RCUTORTURE_RDR_SCHED 0x10 /* ... rcu_read_lock_sched(). */65#define RCUTORTURE_RDR_SCHED 0x10 /* ... rcu_read_lock_sched(). */
63#define RCUTORTURE_RDR_RCU 0x20 /* ... entering another RCU reader. */66#define RCUTORTURE_RDR_RCU_1 0x20 /* ... entering another RCU reader. */
64#define RCUTORTURE_RDR_NBITS 6 /* Number of bits defined above. */67#define RCUTORTURE_RDR_RCU_2 0x40 /* ... entering another RCU reader. */
68#define RCUTORTURE_RDR_NBITS 7 /* Number of bits defined above. */
65#define RCUTORTURE_MAX_EXTEND \69#define RCUTORTURE_MAX_EXTEND \
66 (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \70 (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \
67 RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED)71 RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED)
@@ -75,7 +79,7 @@ torture_param(int, fqs_duration, 0, · 7 unmodified lines
75 "Duration of fqs bursts (us), 0 to disable");79 "Duration of fqs bursts (us), 0 to disable");
76torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)");80torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)");
77torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)");81torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)");
78torture_param(bool, fwd_progress, 1, "Test grace-period forward progress");82torture_param(int, fwd_progress, 1, "Test grace-period forward progress");
79torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait");83torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait");
80torture_param(int, fwd_progress_holdoff, 60,84torture_param(int, fwd_progress_holdoff, 60,
81 "Time between forward-progress tests (s)");85 "Time between forward-progress tests (s)");
@@ -109,6 +113,8 @@ torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); · 27 unmodified lines
109torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable.");113torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable.");
110torture_param(int, stall_cpu_holdoff, 10,114torture_param(int, stall_cpu_holdoff, 10,
111 "Time to wait before starting stall (s).");115 "Time to wait before starting stall (s).");
116torture_param(bool, stall_no_softlockup, false,
117 "Avoid softlockup warning during cpu stall.");
112torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling.");118torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling.");
113torture_param(int, stall_cpu_block, 0, "Sleep while stalling.");119torture_param(int, stall_cpu_block, 0, "Sleep while stalling.");
114torture_param(int, stall_gp_kthread, 0,120torture_param(int, stall_gp_kthread, 0,
@@ -140,7 +146,7 @@ static struct task_struct *stats_task; · 25 unmodified lines
140static struct task_struct *fqs_task;146static struct task_struct *fqs_task;
141static struct task_struct *boost_tasks[NR_CPUS];147static struct task_struct *boost_tasks[NR_CPUS];
142static struct task_struct *stall_task;148static struct task_struct *stall_task;
143static struct task_struct *fwd_prog_task;149static struct task_struct **fwd_prog_tasks;
144static struct task_struct **barrier_cbs_tasks;150static struct task_struct **barrier_cbs_tasks;
145static struct task_struct *barrier_task;151static struct task_struct *barrier_task;
146static struct task_struct *read_exit_task;152static struct task_struct *read_exit_task;
@@ -342,10 +348,12 @@ struct rcu_torture_ops { · 195 unmodified lines
342 void (*gp_kthread_dbg)(void);348 void (*gp_kthread_dbg)(void);
343 bool (*check_boost_failed)(unsigned long gp_state, int *cpup);349 bool (*check_boost_failed)(unsigned long gp_state, int *cpup);
344 int (*stall_dur)(void);350 int (*stall_dur)(void);
351 long cbflood_max;
345 int irq_capable;352 int irq_capable;
346 int can_boost;353 int can_boost;
347 int extendables;354 int extendables;
348 int slow_gps;355 int slow_gps;
356 int no_pi_lock;
349 const char *name;357 const char *name;
350};358};
351 359
@@ -667,6 +675,7 @@ static struct rcu_torture_ops srcu_ops = { · 315 unmodified lines
667 .cb_barrier = srcu_torture_barrier,675 .cb_barrier = srcu_torture_barrier,
668 .stats = srcu_torture_stats,676 .stats = srcu_torture_stats,
669 .irq_capable = 1,677 .irq_capable = 1,
678 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU),
670 .name = "srcu"679 .name = "srcu"
671};680};
672 681
@@ -700,6 +709,7 @@ static struct rcu_torture_ops srcud_ops = { · 27 unmodified lines
700 .cb_barrier = srcu_torture_barrier,709 .cb_barrier = srcu_torture_barrier,
701 .stats = srcu_torture_stats,710 .stats = srcu_torture_stats,
702 .irq_capable = 1,711 .irq_capable = 1,
712 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU),
703 .name = "srcud"713 .name = "srcud"
704};714};
705 715
@@ -720,6 +730,7 @@ static struct rcu_torture_ops busted_srcud_ops = { · 14 unmodified lines
720 .cb_barrier = srcu_torture_barrier,730 .cb_barrier = srcu_torture_barrier,
721 .stats = srcu_torture_stats,731 .stats = srcu_torture_stats,
722 .irq_capable = 1,732 .irq_capable = 1,
733 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU),
723 .extendables = RCUTORTURE_MAX_EXTEND,734 .extendables = RCUTORTURE_MAX_EXTEND,
724 .name = "busted_srcud"735 .name = "busted_srcud"
725};736};
@@ -831,6 +842,7 @@ static struct rcu_torture_ops tasks_rude_ops = { · 105 unmodified lines
831 .call = call_rcu_tasks_rude,842 .call = call_rcu_tasks_rude,
832 .cb_barrier = rcu_barrier_tasks_rude,843 .cb_barrier = rcu_barrier_tasks_rude,
833 .gp_kthread_dbg = show_rcu_tasks_rude_gp_kthread,844 .gp_kthread_dbg = show_rcu_tasks_rude_gp_kthread,
845 .cbflood_max = 50000,
834 .fqs = NULL,846 .fqs = NULL,
835 .stats = NULL,847 .stats = NULL,
836 .irq_capable = 1,848 .irq_capable = 1,
@@ -871,6 +883,7 @@ static struct rcu_torture_ops tasks_tracing_ops = { · 34 unmodified lines
871 .call = call_rcu_tasks_trace,883 .call = call_rcu_tasks_trace,
872 .cb_barrier = rcu_barrier_tasks_trace,884 .cb_barrier = rcu_barrier_tasks_trace,
873 .gp_kthread_dbg = show_rcu_tasks_trace_gp_kthread,885 .gp_kthread_dbg = show_rcu_tasks_trace_gp_kthread,
886 .cbflood_max = 50000,
874 .fqs = NULL,887 .fqs = NULL,
875 .stats = NULL,888 .stats = NULL,
876 .irq_capable = 1,889 .irq_capable = 1,
@@ -1420,13 +1433,15 @@ static void rcutorture_one_extend(int *readstate, int newstate, · 543 unmodified lines
1420 struct rt_read_seg *rtrsp)1433 struct rt_read_seg *rtrsp)
1421{1434{
1422 unsigned long flags;1435 unsigned long flags;
1423 int idxnew = -1;1436 int idxnew1 = -1;
1424 int idxold = *readstate;1437 int idxnew2 = -1;
1438 int idxold1 = *readstate;
1439 int idxold2 = idxold1;
1425 int statesnew = ~*readstate & newstate;1440 int statesnew = ~*readstate & newstate;
1426 int statesold = *readstate & ~newstate;1441 int statesold = *readstate & ~newstate;
1427 1442
1428 WARN_ON_ONCE(idxold < 0);1443 WARN_ON_ONCE(idxold2 < 0);
1429 WARN_ON_ONCE((idxold >> RCUTORTURE_RDR_SHIFT) > 1);1444 WARN_ON_ONCE((idxold2 >> RCUTORTURE_RDR_SHIFT_2) > 1);
1430 rtrsp->rt_readstate = newstate;1445 rtrsp->rt_readstate = newstate;
1431 1446
1432 /* First, put new protection in place to avoid critical-section gap. */1447 /* First, put new protection in place to avoid critical-section gap. */
@@ -1440,8 +1455,10 @@ static void rcutorture_one_extend(int *readstate, int newstate, · 7 unmodified lines
1440 preempt_disable();1455 preempt_disable();
1441 if (statesnew & RCUTORTURE_RDR_SCHED)1456 if (statesnew & RCUTORTURE_RDR_SCHED)
1442 rcu_read_lock_sched();1457 rcu_read_lock_sched();
1443 if (statesnew & RCUTORTURE_RDR_RCU)1458 if (statesnew & RCUTORTURE_RDR_RCU_1)
1444 idxnew = cur_ops->readlock() << RCUTORTURE_RDR_SHIFT;1459 idxnew1 = (cur_ops->readlock() & 0x1) << RCUTORTURE_RDR_SHIFT_1;
1460 if (statesnew & RCUTORTURE_RDR_RCU_2)
1461 idxnew2 = (cur_ops->readlock() & 0x1) << RCUTORTURE_RDR_SHIFT_2;
1445 1462
1446 /*1463 /*
1447 * Next, remove old protection, in decreasing order of strength1464 * Next, remove old protection, in decreasing order of strength
@@ -1460,12 +1477,20 @@ static void rcutorture_one_extend(int *readstate, int newstate, · 12 unmodified lines
1460 local_bh_enable();1477 local_bh_enable();
1461 if (statesold & RCUTORTURE_RDR_RBH)1478 if (statesold & RCUTORTURE_RDR_RBH)
1462 rcu_read_unlock_bh();1479 rcu_read_unlock_bh();
1463 if (statesold & RCUTORTURE_RDR_RCU) {1480 if (statesold & RCUTORTURE_RDR_RCU_2) {
1464 bool lockit = !statesnew && !(torture_random(trsp) & 0xffff);1481 cur_ops->readunlock((idxold2 >> RCUTORTURE_RDR_SHIFT_2) & 0x1);
1482 WARN_ON_ONCE(idxnew2 != -1);
1483 idxold2 = 0;
1484 }
1485 if (statesold & RCUTORTURE_RDR_RCU_1) {
1486 bool lockit;
1465 1487
1488 lockit = !cur_ops->no_pi_lock && !statesnew && !(torture_random(trsp) & 0xffff);
1466 if (lockit)1489 if (lockit)
1467 raw_spin_lock_irqsave(&current->pi_lock, flags);1490 raw_spin_lock_irqsave(&current->pi_lock, flags);
1468 cur_ops->readunlock(idxold >> RCUTORTURE_RDR_SHIFT);1491 cur_ops->readunlock((idxold1 >> RCUTORTURE_RDR_SHIFT_1) & 0x1);
1492 WARN_ON_ONCE(idxnew1 != -1);
1493 idxold1 = 0;
1469 if (lockit)1494 if (lockit)
1470 raw_spin_unlock_irqrestore(&current->pi_lock, flags);1495 raw_spin_unlock_irqrestore(&current->pi_lock, flags);
1471 }1496 }
@@ -1475,13 +1500,19 @@ static void rcutorture_one_extend(int *readstate, int newstate, · 3 unmodified lines
1475 cur_ops->read_delay(trsp, rtrsp);1500 cur_ops->read_delay(trsp, rtrsp);
1476 1501
1477 /* Update the reader state. */1502 /* Update the reader state. */
1478 if (idxnew == -1)1503 if (idxnew1 == -1)
1479 idxnew = idxold & ~RCUTORTURE_RDR_MASK;1504 idxnew1 = idxold1 & RCUTORTURE_RDR_MASK_1;
1480 WARN_ON_ONCE(idxnew < 0);1505 WARN_ON_ONCE(idxnew1 < 0);
1481 WARN_ON_ONCE((idxnew >> RCUTORTURE_RDR_SHIFT) > 1);1506 if (WARN_ON_ONCE((idxnew1 >> RCUTORTURE_RDR_SHIFT_1) > 1))
1482 *readstate = idxnew | newstate;1507 pr_info("Unexpected idxnew1 value of %#x\n", idxnew1);
1483 WARN_ON_ONCE((*readstate >> RCUTORTURE_RDR_SHIFT) < 0);1508 if (idxnew2 == -1)
1484 WARN_ON_ONCE((*readstate >> RCUTORTURE_RDR_SHIFT) > 1);1509 idxnew2 = idxold2 & RCUTORTURE_RDR_MASK_2;
1510 WARN_ON_ONCE(idxnew2 < 0);
1511 WARN_ON_ONCE((idxnew2 >> RCUTORTURE_RDR_SHIFT_2) > 1);
1512 *readstate = idxnew1 | idxnew2 | newstate;
1513 WARN_ON_ONCE(*readstate < 0);
1514 if (WARN_ON_ONCE((*readstate >> RCUTORTURE_RDR_SHIFT_2) > 1))
1515 pr_info("Unexpected idxnew2 value of %#x\n", idxnew2);
1485}1516}
1486 1517
1487/* Return the biggest extendables mask given current RCU and boot parameters. */1518/* Return the biggest extendables mask given current RCU and boot parameters. */
@@ -1491,7 +1522,7 @@ static int rcutorture_extend_mask_max(void) · 3 unmodified lines
1491 1522
1492 WARN_ON_ONCE(extendables & ~RCUTORTURE_MAX_EXTEND);1523 WARN_ON_ONCE(extendables & ~RCUTORTURE_MAX_EXTEND);
1493 mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables;1524 mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables;
1494 mask = mask | RCUTORTURE_RDR_RCU;1525 mask = mask | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2;
1495 return mask;1526 return mask;
1496}1527}
1497 1528
@@ -1506,13 +1537,21 @@ rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) · 8 unmodified lines
1506 unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ;1537 unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ;
1507 unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH;1538 unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH;
1508 1539
1509 WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT);1540 WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT_1);
1510 /* Mostly only one bit (need preemption!), sometimes lots of bits. */1541 /* Mostly only one bit (need preemption!), sometimes lots of bits. */
1511 if (!(randmask1 & 0x7))1542 if (!(randmask1 & 0x7))
1512 mask = mask & randmask2;1543 mask = mask & randmask2;
1513 else1544 else
1514 mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS));1545 mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS));
1515 1546
1547 // Can't have nested RCU reader without outer RCU reader.
1548 if (!(mask & RCUTORTURE_RDR_RCU_1) && (mask & RCUTORTURE_RDR_RCU_2)) {
1549 if (oldmask & RCUTORTURE_RDR_RCU_1)
1550 mask &= ~RCUTORTURE_RDR_RCU_2;
1551 else
1552 mask |= RCUTORTURE_RDR_RCU_1;
1553 }
1554
1516 /*1555 /*
1517 * Can't enable bh w/irq disabled.1556 * Can't enable bh w/irq disabled.
1518 */1557 */
@@ -1532,7 +1571,7 @@ rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) · 13 unmodified lines
1532 mask |= oldmask & bhs;1571 mask |= oldmask & bhs;
1533 }1572 }
1534 1573
1535 return mask ?: RCUTORTURE_RDR_RCU;1574 return mask ?: RCUTORTURE_RDR_RCU_1;
1536}1575}
1537 1576
1538/*1577/*
@@ -1626,7 +1665,7 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) · 87 unmodified lines
1626 rcu_torture_writer_state,1665 rcu_torture_writer_state,
1627 cookie, cur_ops->get_gp_state());1666 cookie, cur_ops->get_gp_state());
1628 rcutorture_one_extend(&readstate, 0, trsp, rtrsp);1667 rcutorture_one_extend(&readstate, 0, trsp, rtrsp);
1629 WARN_ON_ONCE(readstate & RCUTORTURE_RDR_MASK);1668 WARN_ON_ONCE(readstate);
1630 // This next splat is expected behavior if leakpointer, especially1669 // This next splat is expected behavior if leakpointer, especially
1631 // for CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels.1670 // for CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels.
1632 WARN_ON_ONCE(leakpointer && READ_ONCE(p->rtort_pipe_count) > 1);1671 WARN_ON_ONCE(leakpointer && READ_ONCE(p->rtort_pipe_count) > 1);
@@ -2052,6 +2091,8 @@ static int rcu_torture_stall(void *args) · 419 unmodified lines
2052#else2091#else
2053 schedule_timeout_uninterruptible(HZ);2092 schedule_timeout_uninterruptible(HZ);
2054#endif2093#endif
2094 } else if (stall_no_softlockup) {
2095 touch_softlockup_watchdog();
2055 }2096 }
2056 if (stall_cpu_irqsoff)2097 if (stall_cpu_irqsoff)
2057 local_irq_enable();2098 local_irq_enable();
@@ -2123,10 +2164,13 @@ struct rcu_fwd { · 65 unmodified lines
2123 unsigned long rcu_fwd_startat;2164 unsigned long rcu_fwd_startat;
2124 struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST];2165 struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST];
2125 unsigned long rcu_launder_gp_seq_start;2166 unsigned long rcu_launder_gp_seq_start;
2167 int rcu_fwd_id;
2126};2168};
2127 2169
2128static DEFINE_MUTEX(rcu_fwd_mutex);2170static DEFINE_MUTEX(rcu_fwd_mutex);
2129static struct rcu_fwd *rcu_fwds;2171static struct rcu_fwd *rcu_fwds;
2172static unsigned long rcu_fwd_seq;
2173static atomic_long_t rcu_fwd_max_cbs;
2130static bool rcu_fwd_emergency_stop;2174static bool rcu_fwd_emergency_stop;
2131 2175
2132static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp)2176static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp)
@@ -2139,8 +2183,9 @@ static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp) · 6 unmodified lines
2139 for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--)2183 for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--)
2140 if (rfp->n_launders_hist[i].n_launders > 0)2184 if (rfp->n_launders_hist[i].n_launders > 0)
2141 break;2185 break;
2142 pr_alert("%s: Callback-invocation histogram (duration %lu jiffies):",2186 mutex_lock(&rcu_fwd_mutex); // Serialize histograms.
2143 __func__, jiffies - rfp->rcu_fwd_startat);2187 pr_alert("%s: Callback-invocation histogram %d (duration %lu jiffies):",
2188 __func__, rfp->rcu_fwd_id, jiffies - rfp->rcu_fwd_startat);
2144 gps_old = rfp->rcu_launder_gp_seq_start;2189 gps_old = rfp->rcu_launder_gp_seq_start;
2145 for (j = 0; j <= i; j++) {2190 for (j = 0; j <= i; j++) {
2146 gps = rfp->n_launders_hist[j].launder_gp_seq;2191 gps = rfp->n_launders_hist[j].launder_gp_seq;
@@ -2151,6 +2196,7 @@ static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp) · 4 unmodified lines
2151 gps_old = gps;2196 gps_old = gps;
2152 }2197 }
2153 pr_cont("\n");2198 pr_cont("\n");
2199 mutex_unlock(&rcu_fwd_mutex);
2154}2200}
2155 2201
2156/* Callback function for continuous-flood RCU callbacks. */2202/* Callback function for continuous-flood RCU callbacks. */
@@ -2276,7 +2322,8 @@ static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp, · 119 unmodified lines
2276 cver = READ_ONCE(rcu_torture_current_version) - cver;2322 cver = READ_ONCE(rcu_torture_current_version) - cver;
2277 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);2323 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
2278 WARN_ON(!cver && gps < 2);2324 WARN_ON(!cver && gps < 2);
2279 pr_alert("%s: Duration %ld cver %ld gps %ld\n", __func__, dur, cver, gps);2325 pr_alert("%s: %d Duration %ld cver %ld gps %ld\n", __func__,
2326 rfp->rcu_fwd_id, dur, cver, gps);
2280 }2327 }
2281 if (selfpropcb) {2328 if (selfpropcb) {
2282 WRITE_ONCE(fcs.stop, 1);2329 WRITE_ONCE(fcs.stop, 1);
@@ -2344,7 +2391,7 @@ static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) · 61 unmodified lines
2344 rfp->rcu_fwd_cb_head = rfcpn;2391 rfp->rcu_fwd_cb_head = rfcpn;
2345 n_launders++;2392 n_launders++;
2346 n_launders_sa++;2393 n_launders_sa++;
2347 } else {2394 } else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) {
2348 rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL);2395 rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL);
2349 if (WARN_ON_ONCE(!rfcp)) {2396 if (WARN_ON_ONCE(!rfcp)) {
2350 schedule_timeout_interruptible(1);2397 schedule_timeout_interruptible(1);
@@ -2354,8 +2401,11 @@ static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) · 3 unmodified lines
2354 n_launders_sa = 0;2401 n_launders_sa = 0;
2355 rfcp->rfc_gps = 0;2402 rfcp->rfc_gps = 0;
2356 rfcp->rfc_rfp = rfp;2403 rfcp->rfc_rfp = rfp;
2404 } else {
2405 rfcp = NULL;
2357 }2406 }
2358 cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);2407 if (rfcp)
2408 cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);
2359 rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs);2409 rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs);
2360 if (tick_nohz_full_enabled()) {2410 if (tick_nohz_full_enabled()) {
2361 local_irq_save(flags);2411 local_irq_save(flags);
@@ -2379,6 +2429,7 @@ static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) · 17 unmodified lines
2379 n_launders + n_max_cbs - n_launders_cb_snap,2429 n_launders + n_max_cbs - n_launders_cb_snap,
2380 n_launders, n_launders_sa,2430 n_launders, n_launders_sa,
2381 n_max_gps, n_max_cbs, cver, gps);2431 n_max_gps, n_max_cbs, cver, gps);
2432 atomic_long_add(n_max_cbs, &rcu_fwd_max_cbs);
2382 rcu_torture_fwd_cb_hist(rfp);2433 rcu_torture_fwd_cb_hist(rfp);
2383 }2434 }
2384 schedule_timeout_uninterruptible(HZ); /* Let CBs drain. */2435 schedule_timeout_uninterruptible(HZ); /* Let CBs drain. */
@@ -2394,6 +2445,8 @@ static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) · 9 unmodified lines
2394static int rcutorture_oom_notify(struct notifier_block *self,2445static int rcutorture_oom_notify(struct notifier_block *self,
2395 unsigned long notused, void *nfreed)2446 unsigned long notused, void *nfreed)
2396{2447{
2448 int i;
2449 long ncbs;
2397 struct rcu_fwd *rfp;2450 struct rcu_fwd *rfp;
2398 2451
2399 mutex_lock(&rcu_fwd_mutex);2452 mutex_lock(&rcu_fwd_mutex);
@@ -2404,18 +2457,26 @@ static int rcutorture_oom_notify(struct notifier_block *self, · 4 unmodified lines
2404 }2457 }
2405 WARN(1, "%s invoked upon OOM during forward-progress testing.\n",2458 WARN(1, "%s invoked upon OOM during forward-progress testing.\n",
2406 __func__);2459 __func__);
2407 rcu_torture_fwd_cb_hist(rfp);2460 for (i = 0; i < fwd_progress; i++) {
2408 rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp->rcu_fwd_startat)) / 2);2461 rcu_torture_fwd_cb_hist(&rfp[i]);
2462 rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp[i].rcu_fwd_startat)) / 2);
2463 }
2409 WRITE_ONCE(rcu_fwd_emergency_stop, true);2464 WRITE_ONCE(rcu_fwd_emergency_stop, true);
2410 smp_mb(); /* Emergency stop before free and wait to avoid hangs. */2465 smp_mb(); /* Emergency stop before free and wait to avoid hangs. */
2411 pr_info("%s: Freed %lu RCU callbacks.\n",2466 ncbs = 0;
2412 __func__, rcu_torture_fwd_prog_cbfree(rfp));2467 for (i = 0; i < fwd_progress; i++)
2468 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);
2469 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs);
2413 rcu_barrier();2470 rcu_barrier();
2414 pr_info("%s: Freed %lu RCU callbacks.\n",2471 ncbs = 0;
2415 __func__, rcu_torture_fwd_prog_cbfree(rfp));2472 for (i = 0; i < fwd_progress; i++)
2473 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);
2474 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs);
2416 rcu_barrier();2475 rcu_barrier();
2417 pr_info("%s: Freed %lu RCU callbacks.\n",2476 ncbs = 0;
2418 __func__, rcu_torture_fwd_prog_cbfree(rfp));2477 for (i = 0; i < fwd_progress; i++)
2478 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);
2479 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs);
2419 smp_mb(); /* Frees before return to avoid redoing OOM. */2480 smp_mb(); /* Frees before return to avoid redoing OOM. */
2420 (*(unsigned long *)nfreed)++; /* Forward progress CBs freed! */2481 (*(unsigned long *)nfreed)++; /* Forward progress CBs freed! */
2421 pr_info("%s returning after OOM processing.\n", __func__);2482 pr_info("%s returning after OOM processing.\n", __func__);
@@ -2430,7 +2491,10 @@ static struct notifier_block rcutorture_oom_nb = { · 8 unmodified lines
2430/* Carry out grace-period forward-progress testing. */2491/* Carry out grace-period forward-progress testing. */
2431static int rcu_torture_fwd_prog(void *args)2492static int rcu_torture_fwd_prog(void *args)
2432{2493{
2494 bool firsttime = true;
2495 long max_cbs;
2433 int oldnice = task_nice(current);2496 int oldnice = task_nice(current);
2497 unsigned long oldseq = READ_ONCE(rcu_fwd_seq);
2434 struct rcu_fwd *rfp = args;2498 struct rcu_fwd *rfp = args;
2435 int tested = 0;2499 int tested = 0;
2436 int tested_tries = 0;2500 int tested_tries = 0;
@@ -2440,21 +2504,38 @@ static int rcu_torture_fwd_prog(void *args) · 3 unmodified lines
2440 if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST))2504 if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST))
2441 set_user_nice(current, MAX_NICE);2505 set_user_nice(current, MAX_NICE);
2442 do {2506 do {
2443 schedule_timeout_interruptible(fwd_progress_holdoff * HZ);2507 if (!rfp->rcu_fwd_id) {
2444 WRITE_ONCE(rcu_fwd_emergency_stop, false);2508 schedule_timeout_interruptible(fwd_progress_holdoff * HZ);
2445 if (!IS_ENABLED(CONFIG_TINY_RCU) ||2509 WRITE_ONCE(rcu_fwd_emergency_stop, false);
2446 rcu_inkernel_boot_has_ended())2510 if (!firsttime) {
2447 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);2511 max_cbs = atomic_long_xchg(&rcu_fwd_max_cbs, 0);
2448 if (rcu_inkernel_boot_has_ended())2512 pr_alert("%s n_max_cbs: %ld\n", __func__, max_cbs);
2513 }
2514 firsttime = false;
2515 WRITE_ONCE(rcu_fwd_seq, rcu_fwd_seq + 1);
2516 } else {
2517 while (READ_ONCE(rcu_fwd_seq) == oldseq)
2518 schedule_timeout_interruptible(1);
2519 oldseq = READ_ONCE(rcu_fwd_seq);
2520 }
2521 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);
2522 if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id)
2449 rcu_torture_fwd_prog_cr(rfp);2523 rcu_torture_fwd_prog_cr(rfp);
2524 if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) &&
2525 (!IS_ENABLED(CONFIG_TINY_RCU) ||
2526 (rcu_inkernel_boot_has_ended() &&
2527 torture_num_online_cpus() > rfp->rcu_fwd_id)))
2528 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);
2450 2529
2451 /* Avoid slow periods, better to test when busy. */2530 /* Avoid slow periods, better to test when busy. */
2452 if (stutter_wait("rcu_torture_fwd_prog"))2531 if (stutter_wait("rcu_torture_fwd_prog"))
2453 sched_set_normal(current, oldnice);2532 sched_set_normal(current, oldnice);
2454 } while (!torture_must_stop());2533 } while (!torture_must_stop());
2455 /* Short runs might not contain a valid forward-progress attempt. */2534 /* Short runs might not contain a valid forward-progress attempt. */
2456 WARN_ON(!tested && tested_tries >= 5);2535 if (!rfp->rcu_fwd_id) {
2457 pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries);2536 WARN_ON(!tested && tested_tries >= 5);
2537 pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries);
2538 }
2458 torture_kthread_stopping("rcu_torture_fwd_prog");2539 torture_kthread_stopping("rcu_torture_fwd_prog");
2459 return 0;2540 return 0;
2460}2541}
@@ -2462,17 +2543,28 @@ static int rcu_torture_fwd_prog(void *args) · 1 unmodified line
2462/* If forward-progress checking is requested and feasible, spawn the thread. */2543/* If forward-progress checking is requested and feasible, spawn the thread. */
2463static int __init rcu_torture_fwd_prog_init(void)2544static int __init rcu_torture_fwd_prog_init(void)
2464{2545{
2546 int i;
2547 int ret = 0;
2465 struct rcu_fwd *rfp;2548 struct rcu_fwd *rfp;
2466 2549
2467 if (!fwd_progress)2550 if (!fwd_progress)
2468 return 0; /* Not requested, so don't do it. */2551 return 0; /* Not requested, so don't do it. */
2552 if (fwd_progress >= nr_cpu_ids) {
2553 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Limiting fwd_progress to # CPUs.\n");
2554 fwd_progress = nr_cpu_ids;
2555 } else if (fwd_progress < 0) {
2556 fwd_progress = nr_cpu_ids;
2557 }
2469 if ((!cur_ops->sync && !cur_ops->call) ||2558 if ((!cur_ops->sync && !cur_ops->call) ||
2470 !cur_ops->stall_dur || cur_ops->stall_dur() <= 0 || cur_ops == &rcu_busted_ops) {2559 (!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) ||
2560 cur_ops == &rcu_busted_ops) {
2471 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test");2561 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test");
2562 fwd_progress = 0;
2472 return 0;2563 return 0;
2473 }2564 }
2474 if (stall_cpu > 0) {2565 if (stall_cpu > 0) {
2475 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall testing");2566 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall testing");
2567 fwd_progress = 0;
2476 if (IS_MODULE(CONFIG_RCU_TORTURE_TEST))2568 if (IS_MODULE(CONFIG_RCU_TORTURE_TEST))
2477 return -EINVAL; /* In module, can fail back to user. */2569 return -EINVAL; /* In module, can fail back to user. */
2478 WARN_ON(1); /* Make sure rcutorture notices conflict. */2570 WARN_ON(1); /* Make sure rcutorture notices conflict. */
@@ -2482,29 +2574,51 @@ static int __init rcu_torture_fwd_prog_init(void) · 3 unmodified lines
2482 fwd_progress_holdoff = 1;2574 fwd_progress_holdoff = 1;
2483 if (fwd_progress_div <= 0)2575 if (fwd_progress_div <= 0)
2484 fwd_progress_div = 4;2576 fwd_progress_div = 4;
2485 rfp = kzalloc(sizeof(*rfp), GFP_KERNEL);2577 rfp = kcalloc(fwd_progress, sizeof(*rfp), GFP_KERNEL);
2486 if (!rfp)2578 fwd_prog_tasks = kcalloc(fwd_progress, sizeof(*fwd_prog_tasks), GFP_KERNEL);
2579 if (!rfp || !fwd_prog_tasks) {
2580 kfree(rfp);
2581 kfree(fwd_prog_tasks);
2582 fwd_prog_tasks = NULL;
2583 fwd_progress = 0;
2487 return -ENOMEM;2584 return -ENOMEM;
2488 spin_lock_init(&rfp->rcu_fwd_lock);2585 }
2489 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head;2586 for (i = 0; i < fwd_progress; i++) {
2587 spin_lock_init(&rfp[i].rcu_fwd_lock);
2588 rfp[i].rcu_fwd_cb_tail = &rfp[i].rcu_fwd_cb_head;
2589 rfp[i].rcu_fwd_id = i;
2590 }
2490 mutex_lock(&rcu_fwd_mutex);2591 mutex_lock(&rcu_fwd_mutex);
2491 rcu_fwds = rfp;2592 rcu_fwds = rfp;
2492 mutex_unlock(&rcu_fwd_mutex);2593 mutex_unlock(&rcu_fwd_mutex);
2493 register_oom_notifier(&rcutorture_oom_nb);2594 register_oom_notifier(&rcutorture_oom_nb);
2494 return torture_create_kthread(rcu_torture_fwd_prog, rfp, fwd_prog_task);2595 for (i = 0; i < fwd_progress; i++) {
2596 ret = torture_create_kthread(rcu_torture_fwd_prog, &rcu_fwds[i], fwd_prog_tasks[i]);
2597 if (ret) {
2598 fwd_progress = i;
2599 return ret;
2600 }
2601 }
2602 return 0;
2495}2603}
2496 2604
2497static void rcu_torture_fwd_prog_cleanup(void)2605static void rcu_torture_fwd_prog_cleanup(void)
2498{2606{
2607 int i;
2499 struct rcu_fwd *rfp;2608 struct rcu_fwd *rfp;
2500 2609
2501 torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_task);2610 if (!rcu_fwds || !fwd_prog_tasks)
2502 rfp = rcu_fwds;2611 return;
2612 for (i = 0; i < fwd_progress; i++)
2613 torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_tasks[i]);
2614 unregister_oom_notifier(&rcutorture_oom_nb);
2503 mutex_lock(&rcu_fwd_mutex);2615 mutex_lock(&rcu_fwd_mutex);
2616 rfp = rcu_fwds;
2504 rcu_fwds = NULL;2617 rcu_fwds = NULL;
2505 mutex_unlock(&rcu_fwd_mutex);2618 mutex_unlock(&rcu_fwd_mutex);
2506 unregister_oom_notifier(&rcutorture_oom_nb);
2507 kfree(rfp);2619 kfree(rfp);
2620 kfree(fwd_prog_tasks);
2621 fwd_prog_tasks = NULL;
2508}2622}
2509 2623
2510/* Callback function for RCU barrier testing. */2624/* Callback function for RCU barrier testing. */
@@ -2741,7 +2855,7 @@ static int rcu_torture_read_exit(void *unused) · 230 unmodified lines
2741 &trs, "%s",2855 &trs, "%s",
2742 "rcu_torture_read_exit_child");2856 "rcu_torture_read_exit_child");
2743 if (IS_ERR(tsp)) {2857 if (IS_ERR(tsp)) {
2744 VERBOSE_TOROUT_ERRSTRING("out of memory");2858 TOROUT_ERRSTRING("out of memory");
2745 errexit = true;2859 errexit = true;
2746 tsp = NULL;2860 tsp = NULL;
2747 break;2861 break;
@@ -3068,7 +3182,7 @@ rcu_torture_init(void) · 320 unmodified lines
3068 sizeof(fakewriter_tasks[0]),3182 sizeof(fakewriter_tasks[0]),
3069 GFP_KERNEL);3183 GFP_KERNEL);
3070 if (fakewriter_tasks == NULL) {3184 if (fakewriter_tasks == NULL) {
3071 VERBOSE_TOROUT_ERRSTRING("out of memory");3185 TOROUT_ERRSTRING("out of memory");
3072 firsterr = -ENOMEM;3186 firsterr = -ENOMEM;
3073 goto unwind;3187 goto unwind;
3074 }3188 }
@@ -3084,7 +3198,7 @@ rcu_torture_init(void) · 9 unmodified lines
3084 rcu_torture_reader_mbchk = kcalloc(nrealreaders, sizeof(*rcu_torture_reader_mbchk),3198 rcu_torture_reader_mbchk = kcalloc(nrealreaders, sizeof(*rcu_torture_reader_mbchk),
3085 GFP_KERNEL);3199 GFP_KERNEL);
3086 if (!reader_tasks || !rcu_torture_reader_mbchk) {3200 if (!reader_tasks || !rcu_torture_reader_mbchk) {
3087 VERBOSE_TOROUT_ERRSTRING("out of memory");3201 TOROUT_ERRSTRING("out of memory");
3088 firsterr = -ENOMEM;3202 firsterr = -ENOMEM;
3089 goto unwind;3203 goto unwind;
3090 }3204 }
@@ -3103,7 +3217,7 @@ rcu_torture_init(void) · 12 unmodified lines
3103 if (nrealnocbers > 0) {3217 if (nrealnocbers > 0) {
3104 nocb_tasks = kcalloc(nrealnocbers, sizeof(nocb_tasks[0]), GFP_KERNEL);3218 nocb_tasks = kcalloc(nrealnocbers, sizeof(nocb_tasks[0]), GFP_KERNEL);
3105 if (nocb_tasks == NULL) {3219 if (nocb_tasks == NULL) {
3106 VERBOSE_TOROUT_ERRSTRING("out of memory");3220 TOROUT_ERRSTRING("out of memory");
3107 firsterr = -ENOMEM;3221 firsterr = -ENOMEM;
3108 goto unwind;3222 goto unwind;
3109 }3223 }
kernel/rcu/refscale.c+24−26
@@ -44,7 +44,10 @@ · 43 unmodified lines
44 pr_alert("%s" SCALE_FLAG s, scale_type, ## x)44 pr_alert("%s" SCALE_FLAG s, scale_type, ## x)
45 45
46#define VERBOSE_SCALEOUT(s, x...) \46#define VERBOSE_SCALEOUT(s, x...) \
47 do { if (verbose) pr_alert("%s" SCALE_FLAG s, scale_type, ## x); } while (0)47 do { \
48 if (verbose) \
49 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
50 } while (0)
48 51
49static atomic_t verbose_batch_ctr;52static atomic_t verbose_batch_ctr;
50 53
@@ -54,12 +57,11 @@ do { \ · 3 unmodified lines
54 (verbose_batched <= 0 || \57 (verbose_batched <= 0 || \
55 !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \58 !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \
56 schedule_timeout_uninterruptible(1); \59 schedule_timeout_uninterruptible(1); \
57 pr_alert("%s" SCALE_FLAG s, scale_type, ## x); \60 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
58 } \61 } \
59} while (0)62} while (0)
60 63
61#define VERBOSE_SCALEOUT_ERRSTRING(s, x...) \64#define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x)
62 do { if (verbose) pr_alert("%s" SCALE_FLAG "!!! " s, scale_type, ## x); } while (0)
63 65
64MODULE_LICENSE("GPL");66MODULE_LICENSE("GPL");
65MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");67MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");
@@ -604,7 +606,7 @@ static u64 process_durations(int n) · 538 unmodified lines
604 char *buf;606 char *buf;
605 u64 sum = 0;607 u64 sum = 0;
606 608
607 buf = kmalloc(128 + nreaders * 32, GFP_KERNEL);609 buf = kmalloc(800 + 64, GFP_KERNEL);
608 if (!buf)610 if (!buf)
609 return 0;611 return 0;
610 buf[0] = 0;612 buf[0] = 0;
@@ -617,13 +619,15 @@ static u64 process_durations(int n) · 6 unmodified lines
617 619
618 if (i % 5 == 0)620 if (i % 5 == 0)
619 strcat(buf, "\n");621 strcat(buf, "\n");
622 if (strlen(buf) >= 800) {
623 pr_alert("%s", buf);
624 buf[0] = 0;
625 }
620 strcat(buf, buf1);626 strcat(buf, buf1);
621 627
622 sum += rt->last_duration_ns;628 sum += rt->last_duration_ns;
623 }629 }
624 strcat(buf, "\n");630 pr_alert("%s\n", buf);
625
626 SCALEOUT("%s\n", buf);
627 631
628 kfree(buf);632 kfree(buf);
629 return sum;633 return sum;
@@ -637,7 +641,6 @@ static u64 process_durations(int n) · 7 unmodified lines
637// point all the timestamps are printed.641// point all the timestamps are printed.
638static int main_func(void *arg)642static int main_func(void *arg)
639{643{
640 bool errexit = false;
641 int exp, r;644 int exp, r;
642 char buf1[64];645 char buf1[64];
643 char *buf;646 char *buf;
@@ -648,10 +651,10 @@ static int main_func(void *arg) · 4 unmodified lines
648 651
649 VERBOSE_SCALEOUT("main_func task started");652 VERBOSE_SCALEOUT("main_func task started");
650 result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL);653 result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL);
651 buf = kzalloc(64 + nruns * 32, GFP_KERNEL);654 buf = kzalloc(800 + 64, GFP_KERNEL);
652 if (!result_avg || !buf) {655 if (!result_avg || !buf) {
653 VERBOSE_SCALEOUT_ERRSTRING("out of memory");656 SCALEOUT_ERRSTRING("out of memory");
654 errexit = true;657 goto oom_exit;
655 }658 }
656 if (holdoff)659 if (holdoff)
657 schedule_timeout_interruptible(holdoff * HZ);660 schedule_timeout_interruptible(holdoff * HZ);
@@ -663,8 +666,6 @@ static int main_func(void *arg) · 5 unmodified lines
663 666
664 // Start exp readers up per experiment667 // Start exp readers up per experiment
665 for (exp = 0; exp < nruns && !torture_must_stop(); exp++) {668 for (exp = 0; exp < nruns && !torture_must_stop(); exp++) {
666 if (errexit)
667 break;
668 if (torture_must_stop())669 if (torture_must_stop())
669 goto end;670 goto end;
670 671
@@ -698,26 +699,23 @@ static int main_func(void *arg) · 27 unmodified lines
698 // Print the average of all experiments699 // Print the average of all experiments
699 SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");700 SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");
700 701
701 if (!errexit) {702 pr_alert("Runs\tTime(ns)\n");
702 buf[0] = 0;
703 strcat(buf, "\n");
704 strcat(buf, "Runs\tTime(ns)\n");
705 }
706
707 for (exp = 0; exp < nruns; exp++) {703 for (exp = 0; exp < nruns; exp++) {
708 u64 avg;704 u64 avg;
709 u32 rem;705 u32 rem;
710 706
711 if (errexit)
712 break;
713 avg = div_u64_rem(result_avg[exp], 1000, &rem);707 avg = div_u64_rem(result_avg[exp], 1000, &rem);
714 sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem);708 sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem);
715 strcat(buf, buf1);709 strcat(buf, buf1);
710 if (strlen(buf) >= 800) {
711 pr_alert("%s", buf);
712 buf[0] = 0;
713 }
716 }714 }
717 715
718 if (!errexit)716 pr_alert("%s", buf);
719 SCALEOUT("%s", buf);
720 717
718oom_exit:
721 // This will shutdown everything including us.719 // This will shutdown everything including us.
722 if (shutdown) {720 if (shutdown) {
723 shutdown_start = 1;721 shutdown_start = 1;
@@ -841,12 +839,12 @@ ref_scale_init(void) · 117 unmodified lines
841 reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]),839 reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]),
842 GFP_KERNEL);840 GFP_KERNEL);
843 if (!reader_tasks) {841 if (!reader_tasks) {
844 VERBOSE_SCALEOUT_ERRSTRING("out of memory");842 SCALEOUT_ERRSTRING("out of memory");
845 firsterr = -ENOMEM;843 firsterr = -ENOMEM;
846 goto unwind;844 goto unwind;
847 }845 }
848 846
849 VERBOSE_SCALEOUT("Starting %d reader threads\n", nreaders);847 VERBOSE_SCALEOUT("Starting %d reader threads", nreaders);
850 848
851 for (i = 0; i < nreaders; i++) {849 for (i = 0; i < nreaders; i++) {
852 firsterr = torture_create_kthread(ref_scale_reader, (void *)i,850 firsterr = torture_create_kthread(ref_scale_reader, (void *)i,
kernel/rcu/srcutiny.c+1−1
@@ -99,7 +99,7 @@ void __srcu_read_unlock(struct srcu_struct *ssp, int idx) · 98 unmodified lines
99 int newval = READ_ONCE(ssp->srcu_lock_nesting[idx]) - 1;99 int newval = READ_ONCE(ssp->srcu_lock_nesting[idx]) - 1;
100 100
101 WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval);101 WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval);
102 if (!newval && READ_ONCE(ssp->srcu_gp_waiting))102 if (!newval && READ_ONCE(ssp->srcu_gp_waiting) && in_task())
103 swake_up_one(&ssp->srcu_wq);103 swake_up_one(&ssp->srcu_wq);
104}104}
105EXPORT_SYMBOL_GPL(__srcu_read_unlock);105EXPORT_SYMBOL_GPL(__srcu_read_unlock);
kernel/rcu/tasks.h+394−82
@@ -6,6 +6,7 @@ · 5 unmodified lines
6 */6 */
7 7
8#ifdef CONFIG_TASKS_RCU_GENERIC8#ifdef CONFIG_TASKS_RCU_GENERIC
9#include "rcu_segcblist.h"
9 10
10////////////////////////////////////////////////////////////////////////11////////////////////////////////////////////////////////////////////////
11//12//
@@ -19,12 +20,34 @@ typedef void (*postscan_func_t)(struct list_head *hop); · 7 unmodified lines
19typedef void (*holdouts_func_t)(struct list_head *hop, bool ndrpt, bool *frptp);20typedef void (*holdouts_func_t)(struct list_head *hop, bool ndrpt, bool *frptp);
20typedef void (*postgp_func_t)(struct rcu_tasks *rtp);21typedef void (*postgp_func_t)(struct rcu_tasks *rtp);
21 22
23/**
24 * struct rcu_tasks_percpu - Per-CPU component of definition for a Tasks-RCU-like mechanism.
25 * @cblist: Callback list.
26 * @lock: Lock protecting per-CPU callback list.
27 * @rtp_jiffies: Jiffies counter value for statistics.
28 * @rtp_n_lock_retries: Rough lock-contention statistic.
29 * @rtp_work: Work queue for invoking callbacks.
30 * @rtp_irq_work: IRQ work queue for deferred wakeups.
31 * @barrier_q_head: RCU callback for barrier operation.
32 * @cpu: CPU number corresponding to this entry.
33 * @rtpp: Pointer to the rcu_tasks structure.
34 */
35struct rcu_tasks_percpu {
36 struct rcu_segcblist cblist;
37 raw_spinlock_t __private lock;
38 unsigned long rtp_jiffies;
39 unsigned long rtp_n_lock_retries;
40 struct work_struct rtp_work;
41 struct irq_work rtp_irq_work;
42 struct rcu_head barrier_q_head;
43 int cpu;
44 struct rcu_tasks *rtpp;
45};
46
22/**47/**
23 * struct rcu_tasks - Definition for a Tasks-RCU-like mechanism.48 * struct rcu_tasks - Definition for a Tasks-RCU-like mechanism.
24 * @cbs_head: Head of callback list.
25 * @cbs_tail: Tail pointer for callback list.
26 * @cbs_wq: Wait queue allowing new callback to get kthread's attention.49 * @cbs_wq: Wait queue allowing new callback to get kthread's attention.
27 * @cbs_lock: Lock protecting callback list.50 * @cbs_gbl_lock: Lock protecting callback list.
28 * @kthread_ptr: This flavor's grace-period/callback-invocation kthread.51 * @kthread_ptr: This flavor's grace-period/callback-invocation kthread.
29 * @gp_func: This flavor's grace-period-wait function.52 * @gp_func: This flavor's grace-period-wait function.
30 * @gp_state: Grace period's most recent state transition (debugging).53 * @gp_state: Grace period's most recent state transition (debugging).
@@ -32,7 +55,7 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp); · 1 unmodified line
32 * @init_fract: Initial backoff sleep interval.55 * @init_fract: Initial backoff sleep interval.
33 * @gp_jiffies: Time of last @gp_state transition.56 * @gp_jiffies: Time of last @gp_state transition.
34 * @gp_start: Most recent grace-period start in jiffies.57 * @gp_start: Most recent grace-period start in jiffies.
35 * @n_gps: Number of grace periods completed since boot.58 * @tasks_gp_seq: Number of grace periods completed since boot.
36 * @n_ipis: Number of IPIs sent to encourage grace periods to end.59 * @n_ipis: Number of IPIs sent to encourage grace periods to end.
37 * @n_ipis_fails: Number of IPI-send failures.60 * @n_ipis_fails: Number of IPI-send failures.
38 * @pregp_func: This flavor's pre-grace-period function (optional).61 * @pregp_func: This flavor's pre-grace-period function (optional).
@@ -41,20 +64,27 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp); · 2 unmodified lines
41 * @holdouts_func: This flavor's holdout-list scan function (optional).64 * @holdouts_func: This flavor's holdout-list scan function (optional).
42 * @postgp_func: This flavor's post-grace-period function (optional).65 * @postgp_func: This flavor's post-grace-period function (optional).
43 * @call_func: This flavor's call_rcu()-equivalent function.66 * @call_func: This flavor's call_rcu()-equivalent function.
67 * @rtpcpu: This flavor's rcu_tasks_percpu structure.
68 * @percpu_enqueue_shift: Shift down CPU ID this much when enqueuing callbacks.
69 * @percpu_enqueue_lim: Number of per-CPU callback queues in use for enqueuing.
70 * @percpu_dequeue_lim: Number of per-CPU callback queues in use for dequeuing.
71 * @percpu_dequeue_gpseq: RCU grace-period number to propagate enqueue limit to dequeuers.
72 * @barrier_q_mutex: Serialize barrier operations.
73 * @barrier_q_count: Number of queues being waited on.
74 * @barrier_q_completion: Barrier wait/wakeup mechanism.
75 * @barrier_q_seq: Sequence number for barrier operations.
44 * @name: This flavor's textual name.76 * @name: This flavor's textual name.
45 * @kname: This flavor's kthread name.77 * @kname: This flavor's kthread name.
46 */78 */
47struct rcu_tasks {79struct rcu_tasks {
48 struct rcu_head *cbs_head;
49 struct rcu_head **cbs_tail;
50 struct wait_queue_head cbs_wq;80 struct wait_queue_head cbs_wq;
51 raw_spinlock_t cbs_lock;81 raw_spinlock_t cbs_gbl_lock;
52 int gp_state;82 int gp_state;
53 int gp_sleep;83 int gp_sleep;
54 int init_fract;84 int init_fract;
55 unsigned long gp_jiffies;85 unsigned long gp_jiffies;
56 unsigned long gp_start;86 unsigned long gp_start;
57 unsigned long n_gps;87 unsigned long tasks_gp_seq;
58 unsigned long n_ipis;88 unsigned long n_ipis;
59 unsigned long n_ipis_fails;89 unsigned long n_ipis_fails;
60 struct task_struct *kthread_ptr;90 struct task_struct *kthread_ptr;
@@ -65,20 +95,40 @@ struct rcu_tasks { · 4 unmodified lines
65 holdouts_func_t holdouts_func;95 holdouts_func_t holdouts_func;
66 postgp_func_t postgp_func;96 postgp_func_t postgp_func;
67 call_rcu_func_t call_func;97 call_rcu_func_t call_func;
98 struct rcu_tasks_percpu __percpu *rtpcpu;
99 int percpu_enqueue_shift;
100 int percpu_enqueue_lim;
101 int percpu_dequeue_lim;
102 unsigned long percpu_dequeue_gpseq;
103 struct mutex barrier_q_mutex;
104 atomic_t barrier_q_count;
105 struct completion barrier_q_completion;
106 unsigned long barrier_q_seq;
68 char *name;107 char *name;
69 char *kname;108 char *kname;
70};109};
71 110
72#define DEFINE_RCU_TASKS(rt_name, gp, call, n) \111static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp);
73static struct rcu_tasks rt_name = \112
74{ \113#define DEFINE_RCU_TASKS(rt_name, gp, call, n) \
75 .cbs_tail = &rt_name.cbs_head, \114static DEFINE_PER_CPU(struct rcu_tasks_percpu, rt_name ## __percpu) = { \
76 .cbs_wq = __WAIT_QUEUE_HEAD_INITIALIZER(rt_name.cbs_wq), \115 .lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name ## __percpu.cbs_pcpu_lock), \
77 .cbs_lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name.cbs_lock), \116 .rtp_irq_work = IRQ_WORK_INIT(call_rcu_tasks_iw_wakeup), \
78 .gp_func = gp, \117}; \
79 .call_func = call, \118static struct rcu_tasks rt_name = \
80 .name = n, \119{ \
81 .kname = #rt_name, \120 .cbs_wq = __WAIT_QUEUE_HEAD_INITIALIZER(rt_name.cbs_wq), \
121 .cbs_gbl_lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name.cbs_gbl_lock), \
122 .gp_func = gp, \
123 .call_func = call, \
124 .rtpcpu = &rt_name ## __percpu, \
125 .name = n, \
126 .percpu_enqueue_shift = ilog2(CONFIG_NR_CPUS), \
127 .percpu_enqueue_lim = 1, \
128 .percpu_dequeue_lim = 1, \
129 .barrier_q_mutex = __MUTEX_INITIALIZER(rt_name.barrier_q_mutex), \
130 .barrier_q_seq = (0UL - 50UL) << RCU_SEQ_CTR_SHIFT, \
131 .kname = #rt_name, \
82}132}
83 133
84/* Track exiting tasks in order to allow them to be waited for. */134/* Track exiting tasks in order to allow them to be waited for. */
@@ -94,6 +144,15 @@ module_param(rcu_task_ipi_delay, int, 0644); · 9 unmodified lines
94static int rcu_task_stall_timeout __read_mostly = RCU_TASK_STALL_TIMEOUT;144static int rcu_task_stall_timeout __read_mostly = RCU_TASK_STALL_TIMEOUT;
95module_param(rcu_task_stall_timeout, int, 0644);145module_param(rcu_task_stall_timeout, int, 0644);
96 146
147static int rcu_task_enqueue_lim __read_mostly = -1;
148module_param(rcu_task_enqueue_lim, int, 0444);
149
150static bool rcu_task_cb_adjust;
151static int rcu_task_contend_lim __read_mostly = 100;
152module_param(rcu_task_contend_lim, int, 0444);
153static int rcu_task_collapse_lim __read_mostly = 10;
154module_param(rcu_task_collapse_lim, int, 0444);
155
97/* RCU tasks grace-period state for debugging. */156/* RCU tasks grace-period state for debugging. */
98#define RTGS_INIT 0157#define RTGS_INIT 0
99#define RTGS_WAIT_WAIT_CBS 1158#define RTGS_WAIT_WAIT_CBS 1
@@ -128,6 +187,8 @@ static const char * const rcu_tasks_gp_state_names[] = { · 28 unmodified lines
128//187//
129// Generic code.188// Generic code.
130 189
190static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp);
191
131/* Record grace-period phase and time. */192/* Record grace-period phase and time. */
132static void set_tasks_gp_state(struct rcu_tasks *rtp, int newstate)193static void set_tasks_gp_state(struct rcu_tasks *rtp, int newstate)
133{194{
@@ -148,23 +209,106 @@ static const char *tasks_gp_state_getname(struct rcu_tasks *rtp) · 14 unmodified lines
148}209}
149#endif /* #ifndef CONFIG_TINY_RCU */210#endif /* #ifndef CONFIG_TINY_RCU */
150 211
212// Initialize per-CPU callback lists for the specified flavor of
213// Tasks RCU.
214static void cblist_init_generic(struct rcu_tasks *rtp)
215{
216 int cpu;
217 unsigned long flags;
218 int lim;
219
220 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
221 if (rcu_task_enqueue_lim < 0) {
222 rcu_task_enqueue_lim = 1;
223 rcu_task_cb_adjust = true;
224 pr_info("%s: Setting adjustable number of callback queues.\n", __func__);
225 } else if (rcu_task_enqueue_lim == 0) {
226 rcu_task_enqueue_lim = 1;
227 }
228 lim = rcu_task_enqueue_lim;
229
230 if (lim > nr_cpu_ids)
231 lim = nr_cpu_ids;
232 WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids / lim));
233 WRITE_ONCE(rtp->percpu_dequeue_lim, lim);
234 smp_store_release(&rtp->percpu_enqueue_lim, lim);
235 for_each_possible_cpu(cpu) {
236 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
237
238 WARN_ON_ONCE(!rtpcp);
239 if (cpu)
240 raw_spin_lock_init(&ACCESS_PRIVATE(rtpcp, lock));
241 raw_spin_lock_rcu_node(rtpcp); // irqs already disabled.
242 if (rcu_segcblist_empty(&rtpcp->cblist))
243 rcu_segcblist_init(&rtpcp->cblist);
244 INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq);
245 rtpcp->cpu = cpu;
246 rtpcp->rtpp = rtp;
247 raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled.
248 }
249 raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags);
250 pr_info("%s: Setting shift to %d and lim to %d.\n", __func__, data_race(rtp->percpu_enqueue_shift), data_race(rtp->percpu_enqueue_lim));
251}
252
253// IRQ-work handler that does deferred wakeup for call_rcu_tasks_generic().
254static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp)
255{
256 struct rcu_tasks *rtp;
257 struct rcu_tasks_percpu *rtpcp = container_of(iwp, struct rcu_tasks_percpu, rtp_irq_work);
258
259 rtp = rtpcp->rtpp;
260 wake_up(&rtp->cbs_wq);
261}
262
151// Enqueue a callback for the specified flavor of Tasks RCU.263// Enqueue a callback for the specified flavor of Tasks RCU.
152static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func,264static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func,
153 struct rcu_tasks *rtp)265 struct rcu_tasks *rtp)
154{266{
155 unsigned long flags;267 unsigned long flags;
268 unsigned long j;
269 bool needadjust = false;
156 bool needwake;270 bool needwake;
271 struct rcu_tasks_percpu *rtpcp;
157 272
158 rhp->next = NULL;273 rhp->next = NULL;
159 rhp->func = func;274 rhp->func = func;
160 raw_spin_lock_irqsave(&rtp->cbs_lock, flags);275 local_irq_save(flags);
161 needwake = !rtp->cbs_head;276 rcu_read_lock();
162 WRITE_ONCE(*rtp->cbs_tail, rhp);277 rtpcp = per_cpu_ptr(rtp->rtpcpu,
163 rtp->cbs_tail = &rhp->next;278 smp_processor_id() >> READ_ONCE(rtp->percpu_enqueue_shift));
164 raw_spin_unlock_irqrestore(&rtp->cbs_lock, flags);279 if (!raw_spin_trylock_rcu_node(rtpcp)) { // irqs already disabled.
280 raw_spin_lock_rcu_node(rtpcp); // irqs already disabled.
281 j = jiffies;
282 if (rtpcp->rtp_jiffies != j) {
283 rtpcp->rtp_jiffies = j;
284 rtpcp->rtp_n_lock_retries = 0;
285 }
286 if (rcu_task_cb_adjust && ++rtpcp->rtp_n_lock_retries > rcu_task_contend_lim &&
287 READ_ONCE(rtp->percpu_enqueue_lim) != nr_cpu_ids)
288 needadjust = true; // Defer adjustment to avoid deadlock.
289 }
290 if (!rcu_segcblist_is_enabled(&rtpcp->cblist)) {
291 raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled.
292 cblist_init_generic(rtp);
293 raw_spin_lock_rcu_node(rtpcp); // irqs already disabled.
294 }
295 needwake = rcu_segcblist_empty(&rtpcp->cblist);
296 rcu_segcblist_enqueue(&rtpcp->cblist, rhp);
297 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
298 if (unlikely(needadjust)) {
299 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
300 if (rtp->percpu_enqueue_lim != nr_cpu_ids) {
301 WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids));
302 WRITE_ONCE(rtp->percpu_dequeue_lim, nr_cpu_ids);
303 smp_store_release(&rtp->percpu_enqueue_lim, nr_cpu_ids);
304 pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name);
305 }
306 raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags);
307 }
308 rcu_read_unlock();
165 /* We can't create the thread unless interrupts are enabled. */309 /* We can't create the thread unless interrupts are enabled. */
166 if (needwake && READ_ONCE(rtp->kthread_ptr))310 if (needwake && READ_ONCE(rtp->kthread_ptr))
167 wake_up(&rtp->cbs_wq);311 irq_work_queue(&rtpcp->rtp_irq_work);
168}312}
169 313
170// Wait for a grace period for the specified flavor of Tasks RCU.314// Wait for a grace period for the specified flavor of Tasks RCU.
@@ -178,12 +322,173 @@ static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp) · 7 unmodified lines
178 wait_rcu_gp(rtp->call_func);322 wait_rcu_gp(rtp->call_func);
179}323}
180 324
325// RCU callback function for rcu_barrier_tasks_generic().
326static void rcu_barrier_tasks_generic_cb(struct rcu_head *rhp)
327{
328 struct rcu_tasks *rtp;
329 struct rcu_tasks_percpu *rtpcp;
330
331 rtpcp = container_of(rhp, struct rcu_tasks_percpu, barrier_q_head);
332 rtp = rtpcp->rtpp;
333 if (atomic_dec_and_test(&rtp->barrier_q_count))
334 complete(&rtp->barrier_q_completion);
335}
336
337// Wait for all in-flight callbacks for the specified RCU Tasks flavor.
338// Operates in a manner similar to rcu_barrier().
339static void rcu_barrier_tasks_generic(struct rcu_tasks *rtp)
340{
341 int cpu;
342 unsigned long flags;
343 struct rcu_tasks_percpu *rtpcp;
344 unsigned long s = rcu_seq_snap(&rtp->barrier_q_seq);
345
346 mutex_lock(&rtp->barrier_q_mutex);
347 if (rcu_seq_done(&rtp->barrier_q_seq, s)) {
348 smp_mb();
349 mutex_unlock(&rtp->barrier_q_mutex);
350 return;
351 }
352 rcu_seq_start(&rtp->barrier_q_seq);
353 init_completion(&rtp->barrier_q_completion);
354 atomic_set(&rtp->barrier_q_count, 2);
355 for_each_possible_cpu(cpu) {
356 if (cpu >= smp_load_acquire(&rtp->percpu_dequeue_lim))
357 break;
358 rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
359 rtpcp->barrier_q_head.func = rcu_barrier_tasks_generic_cb;
360 raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
361 if (rcu_segcblist_entrain(&rtpcp->cblist, &rtpcp->barrier_q_head))
362 atomic_inc(&rtp->barrier_q_count);
363 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
364 }
365 if (atomic_sub_and_test(2, &rtp->barrier_q_count))
366 complete(&rtp->barrier_q_completion);
367 wait_for_completion(&rtp->barrier_q_completion);
368 rcu_seq_end(&rtp->barrier_q_seq);
369 mutex_unlock(&rtp->barrier_q_mutex);
370}
371
372// Advance callbacks and indicate whether either a grace period or
373// callback invocation is needed.
374static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp)
375{
376 int cpu;
377 unsigned long flags;
378 long n;
379 long ncbs = 0;
380 long ncbsnz = 0;
381 int needgpcb = 0;
382
383 for (cpu = 0; cpu < smp_load_acquire(&rtp->percpu_dequeue_lim); cpu++) {
384 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
385
386 /* Advance and accelerate any new callbacks. */
387 if (!rcu_segcblist_n_cbs(&rtpcp->cblist))
388 continue;
389 raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
390 // Should we shrink down to a single callback queue?
391 n = rcu_segcblist_n_cbs(&rtpcp->cblist);
392 if (n) {
393 ncbs += n;
394 if (cpu > 0)
395 ncbsnz += n;
396 }
397 rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq));
398 (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq));
399 if (rcu_segcblist_pend_cbs(&rtpcp->cblist))
400 needgpcb |= 0x3;
401 if (!rcu_segcblist_empty(&rtpcp->cblist))
402 needgpcb |= 0x1;
403 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
404 }
405
406 // Shrink down to a single callback queue if appropriate.
407 // This is done in two stages: (1) If there are no more than
408 // rcu_task_collapse_lim callbacks on CPU 0 and none on any other
409 // CPU, limit enqueueing to CPU 0. (2) After an RCU grace period,
410 // if there has not been an increase in callbacks, limit dequeuing
411 // to CPU 0. Note the matching RCU read-side critical section in
412 // call_rcu_tasks_generic().
413 if (rcu_task_cb_adjust && ncbs <= rcu_task_collapse_lim) {
414 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
415 if (rtp->percpu_enqueue_lim > 1) {
416 WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids));
417 smp_store_release(&rtp->percpu_enqueue_lim, 1);
418 rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu();
419 pr_info("Starting switch %s to CPU-0 callback queuing.\n", rtp->name);
420 }
421 raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags);
422 }
423 if (rcu_task_cb_adjust && !ncbsnz &&
424 poll_state_synchronize_rcu(rtp->percpu_dequeue_gpseq)) {
425 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
426 if (rtp->percpu_enqueue_lim < rtp->percpu_dequeue_lim) {
427 WRITE_ONCE(rtp->percpu_dequeue_lim, 1);
428 pr_info("Completing switch %s to CPU-0 callback queuing.\n", rtp->name);
429 }
430 raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags);
431 }
432
433 return needgpcb;
434}
435
436// Advance callbacks and invoke any that are ready.
437static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu *rtpcp)
438{
439 int cpu;
440 int cpunext;
441 unsigned long flags;
442 int len;
443 struct rcu_head *rhp;
444 struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
445 struct rcu_tasks_percpu *rtpcp_next;
446
447 cpu = rtpcp->cpu;
448 cpunext = cpu * 2 + 1;
449 if (cpunext < smp_load_acquire(&rtp->percpu_dequeue_lim)) {
450 rtpcp_next = per_cpu_ptr(rtp->rtpcpu, cpunext);
451 queue_work_on(cpunext, system_wq, &rtpcp_next->rtp_work);
452 cpunext++;
453 if (cpunext < smp_load_acquire(&rtp->percpu_dequeue_lim)) {
454 rtpcp_next = per_cpu_ptr(rtp->rtpcpu, cpunext);
455 queue_work_on(cpunext, system_wq, &rtpcp_next->rtp_work);
456 }
457 }
458
459 if (rcu_segcblist_empty(&rtpcp->cblist))
460 return;
461 raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
462 rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq));
463 rcu_segcblist_extract_done_cbs(&rtpcp->cblist, &rcl);
464 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
465 len = rcl.len;
466 for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = rcu_cblist_dequeue(&rcl)) {
467 local_bh_disable();
468 rhp->func(rhp);
469 local_bh_enable();
470 cond_resched();
471 }
472 raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
473 rcu_segcblist_add_len(&rtpcp->cblist, -len);
474 (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq));
475 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
476}
477
478// Workqueue flood to advance callbacks and invoke any that are ready.
479static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp)
480{
481 struct rcu_tasks *rtp;
482 struct rcu_tasks_percpu *rtpcp = container_of(wp, struct rcu_tasks_percpu, rtp_work);
483
484 rtp = rtpcp->rtpp;
485 rcu_tasks_invoke_cbs(rtp, rtpcp);
486}
487
181/* RCU-tasks kthread that detects grace periods and invokes callbacks. */488/* RCU-tasks kthread that detects grace periods and invokes callbacks. */
182static int __noreturn rcu_tasks_kthread(void *arg)489static int __noreturn rcu_tasks_kthread(void *arg)
183{490{
184 unsigned long flags;491 int needgpcb;
185 struct rcu_head *list;
186 struct rcu_head *next;
187 struct rcu_tasks *rtp = arg;492 struct rcu_tasks *rtp = arg;
188 493
189 /* Run on housekeeping CPUs by default. Sysadm can move if desired. */494 /* Run on housekeeping CPUs by default. Sysadm can move if desired. */
@@ -199,42 +504,22 @@ static int __noreturn rcu_tasks_kthread(void *arg) · 9 unmodified lines
199 for (;;) {504 for (;;) {
200 set_tasks_gp_state(rtp, RTGS_WAIT_CBS);505 set_tasks_gp_state(rtp, RTGS_WAIT_CBS);
201 506
202 /* Pick up any new callbacks. */
203 raw_spin_lock_irqsave(&rtp->cbs_lock, flags);
204 smp_mb__after_spinlock(); // Order updates vs. GP.
205 list = rtp->cbs_head;
206 rtp->cbs_head = NULL;
207 rtp->cbs_tail = &rtp->cbs_head;
208 raw_spin_unlock_irqrestore(&rtp->cbs_lock, flags);
209
210 /* If there were none, wait a bit and start over. */507 /* If there were none, wait a bit and start over. */
211 if (!list) {508 wait_event_idle(rtp->cbs_wq, (needgpcb = rcu_tasks_need_gpcb(rtp)));
212 wait_event_interruptible(rtp->cbs_wq,509
213 READ_ONCE(rtp->cbs_head));510 if (needgpcb & 0x2) {
214 if (!rtp->cbs_head) {511 // Wait for one grace period.
215 WARN_ON(signal_pending(current));512 set_tasks_gp_state(rtp, RTGS_WAIT_GP);
216 set_tasks_gp_state(rtp, RTGS_WAIT_WAIT_CBS);513 rtp->gp_start = jiffies;
217 schedule_timeout_idle(HZ/10);514 rcu_seq_start(&rtp->tasks_gp_seq);
218 }515 rtp->gp_func(rtp);
219 continue;516 rcu_seq_end(&rtp->tasks_gp_seq);
220 }517 }
221 518
222 // Wait for one grace period.519 /* Invoke callbacks. */
223 set_tasks_gp_state(rtp, RTGS_WAIT_GP);
224 rtp->gp_start = jiffies;
225 rtp->gp_func(rtp);
226 rtp->n_gps++;
227
228 /* Invoke the callbacks. */
229 set_tasks_gp_state(rtp, RTGS_INVOKE_CBS);520 set_tasks_gp_state(rtp, RTGS_INVOKE_CBS);
230 while (list) {521 rcu_tasks_invoke_cbs(rtp, per_cpu_ptr(rtp->rtpcpu, 0));
231 next = list->next;522
232 local_bh_disable();
233 list->func(list);
234 local_bh_enable();
235 list = next;
236 cond_resched();
237 }
238 /* Paranoid sleep to keep this from entering a tight loop */523 /* Paranoid sleep to keep this from entering a tight loop */
239 schedule_timeout_idle(rtp->gp_sleep);524 schedule_timeout_idle(rtp->gp_sleep);
240 }525 }
@@ -279,14 +564,15 @@ static void __init rcu_tasks_bootup_oddness(void) · 38 unmodified lines
279/* Dump out rcutorture-relevant state common to all RCU-tasks flavors. */564/* Dump out rcutorture-relevant state common to all RCU-tasks flavors. */
280static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s)565static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s)
281{566{
567 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, 0); // for_each...
282 pr_info("%s: %s(%d) since %lu g:%lu i:%lu/%lu %c%c %s\n",568 pr_info("%s: %s(%d) since %lu g:%lu i:%lu/%lu %c%c %s\n",
283 rtp->kname,569 rtp->kname,
284 tasks_gp_state_getname(rtp), data_race(rtp->gp_state),570 tasks_gp_state_getname(rtp), data_race(rtp->gp_state),
285 jiffies - data_race(rtp->gp_jiffies),571 jiffies - data_race(rtp->gp_jiffies),
286 data_race(rtp->n_gps),572 data_race(rcu_seq_current(&rtp->tasks_gp_seq)),
287 data_race(rtp->n_ipis_fails), data_race(rtp->n_ipis),573 data_race(rtp->n_ipis_fails), data_race(rtp->n_ipis),
288 ".k"[!!data_race(rtp->kthread_ptr)],574 ".k"[!!data_race(rtp->kthread_ptr)],
289 ".C"[!!data_race(rtp->cbs_head)],575 ".C"[!data_race(rcu_segcblist_empty(&rtpcp->cblist))],
290 s);576 s);
291}577}
292#endif // #ifndef CONFIG_TINY_RCU578#endif // #ifndef CONFIG_TINY_RCU
@@ -411,10 +697,10 @@ static void rcu_tasks_wait_gp(struct rcu_tasks *rtp) · 118 unmodified lines
411// exit_tasks_rcu_finish() functions begin and end, respectively, the SRCU697// exit_tasks_rcu_finish() functions begin and end, respectively, the SRCU
412// read-side critical sections waited for by rcu_tasks_postscan().698// read-side critical sections waited for by rcu_tasks_postscan().
413//699//
414// Pre-grace-period update-side code is ordered before the grace via the700// Pre-grace-period update-side code is ordered before the grace
415// ->cbs_lock and the smp_mb__after_spinlock(). Pre-grace-period read-side701// via the raw_spin_lock.*rcu_node(). Pre-grace-period read-side code
416// code is ordered before the grace period via synchronize_rcu() call702// is ordered before the grace period via synchronize_rcu() call in
417// in rcu_tasks_pregp_step() and by the scheduler's locks and interrupt703// rcu_tasks_pregp_step() and by the scheduler's locks and interrupt
418// disabling.704// disabling.
419 705
420/* Pre-grace-period preparation. */706/* Pre-grace-period preparation. */
@@ -586,13 +872,13 @@ EXPORT_SYMBOL_GPL(synchronize_rcu_tasks); · 165 unmodified lines
586 */872 */
587void rcu_barrier_tasks(void)873void rcu_barrier_tasks(void)
588{874{
589 /* There is only one callback queue, so this is easy. ;-) */875 rcu_barrier_tasks_generic(&rcu_tasks);
590 synchronize_rcu_tasks();
591}876}
592EXPORT_SYMBOL_GPL(rcu_barrier_tasks);877EXPORT_SYMBOL_GPL(rcu_barrier_tasks);
593 878
594static int __init rcu_spawn_tasks_kthread(void)879static int __init rcu_spawn_tasks_kthread(void)
595{880{
881 cblist_init_generic(&rcu_tasks);
596 rcu_tasks.gp_sleep = HZ / 10;882 rcu_tasks.gp_sleep = HZ / 10;
597 rcu_tasks.init_fract = HZ / 10;883 rcu_tasks.init_fract = HZ / 10;
598 rcu_tasks.pregp_func = rcu_tasks_pregp_step;884 rcu_tasks.pregp_func = rcu_tasks_pregp_step;
@@ -724,13 +1010,13 @@ EXPORT_SYMBOL_GPL(synchronize_rcu_tasks_rude); · 125 unmodified lines
724 */1010 */
725void rcu_barrier_tasks_rude(void)1011void rcu_barrier_tasks_rude(void)
726{1012{
727 /* There is only one callback queue, so this is easy. ;-) */1013 rcu_barrier_tasks_generic(&rcu_tasks_rude);
728 synchronize_rcu_tasks_rude();
729}1014}
730EXPORT_SYMBOL_GPL(rcu_barrier_tasks_rude);1015EXPORT_SYMBOL_GPL(rcu_barrier_tasks_rude);
731 1016
732static int __init rcu_spawn_tasks_rude_kthread(void)1017static int __init rcu_spawn_tasks_rude_kthread(void)
733{1018{
1019 cblist_init_generic(&rcu_tasks_rude);
734 rcu_tasks_rude.gp_sleep = HZ / 10;1020 rcu_tasks_rude.gp_sleep = HZ / 10;
735 rcu_spawn_tasks_kthread_generic(&rcu_tasks_rude);1021 rcu_spawn_tasks_kthread_generic(&rcu_tasks_rude);
736 return 0;1022 return 0;
@@ -1073,25 +1359,50 @@ static void rcu_tasks_trace_postscan(struct list_head *hop) · 336 unmodified lines
1073 // Any tasks that exit after this point will set ->trc_reader_checked.1359 // Any tasks that exit after this point will set ->trc_reader_checked.
1074}1360}
1075 1361
1362/* Communicate task state back to the RCU tasks trace stall warning request. */
1363struct trc_stall_chk_rdr {
1364 int nesting;
1365 int ipi_to_cpu;
1366 u8 needqs;
1367};
1368
1369static int trc_check_slow_task(struct task_struct *t, void *arg)
1370{
1371 struct trc_stall_chk_rdr *trc_rdrp = arg;
1372
1373 if (task_curr(t))
1374 return false; // It is running, so decline to inspect it.
1375 trc_rdrp->nesting = READ_ONCE(t->trc_reader_nesting);
1376 trc_rdrp->ipi_to_cpu = READ_ONCE(t->trc_ipi_to_cpu);
1377 trc_rdrp->needqs = READ_ONCE(t->trc_reader_special.b.need_qs);
1378 return true;
1379}
1380
1076/* Show the state of a task stalling the current RCU tasks trace GP. */1381/* Show the state of a task stalling the current RCU tasks trace GP. */
1077static void show_stalled_task_trace(struct task_struct *t, bool *firstreport)1382static void show_stalled_task_trace(struct task_struct *t, bool *firstreport)
1078{1383{
1079 int cpu;1384 int cpu;
1385 struct trc_stall_chk_rdr trc_rdr;
1386 bool is_idle_tsk = is_idle_task(t);
1080 1387
1081 if (*firstreport) {1388 if (*firstreport) {
1082 pr_err("INFO: rcu_tasks_trace detected stalls on tasks:\n");1389 pr_err("INFO: rcu_tasks_trace detected stalls on tasks:\n");
1083 *firstreport = false;1390 *firstreport = false;
1084 }1391 }
1085 // FIXME: This should attempt to use try_invoke_on_nonrunning_task().
1086 cpu = task_cpu(t);1392 cpu = task_cpu(t);
1087 pr_alert("P%d: %c%c%c nesting: %d%c cpu: %d\n",1393 if (!task_call_func(t, trc_check_slow_task, &trc_rdr))
1088 t->pid,1394 pr_alert("P%d: %c\n",
1089 ".I"[READ_ONCE(t->trc_ipi_to_cpu) >= 0],1395 t->pid,
1090 ".i"[is_idle_task(t)],1396 ".i"[is_idle_tsk]);
1091 ".N"[cpu >= 0 && tick_nohz_full_cpu(cpu)],1397 else
1092 READ_ONCE(t->trc_reader_nesting),1398 pr_alert("P%d: %c%c%c nesting: %d%c cpu: %d\n",
1093 " N"[!!READ_ONCE(t->trc_reader_special.b.need_qs)],1399 t->pid,
1094 cpu);1400 ".I"[trc_rdr.ipi_to_cpu >= 0],
1401 ".i"[is_idle_tsk],
1402 ".N"[cpu >= 0 && tick_nohz_full_cpu(cpu)],
1403 trc_rdr.nesting,
1404 " N"[!!trc_rdr.needqs],
1405 cpu);
1095 sched_show_task(t);1406 sched_show_task(t);
1096}1407}
1097 1408
@@ -1121,7 +1432,8 @@ static void check_all_holdout_tasks_trace(struct list_head *hop, · 23 unmodified lines
1121 trc_wait_for_one_reader(t, hop);1432 trc_wait_for_one_reader(t, hop);
1122 1433
1123 // If check succeeded, remove this task from the list.1434 // If check succeeded, remove this task from the list.
1124 if (READ_ONCE(t->trc_reader_checked))1435 if (smp_load_acquire(&t->trc_ipi_to_cpu) == -1 &&
1436 READ_ONCE(t->trc_reader_checked))
1125 trc_del_holdout(t);1437 trc_del_holdout(t);
1126 else if (needreport)1438 else if (needreport)
1127 show_stalled_task_trace(t, firstreport);1439 show_stalled_task_trace(t, firstreport);
@@ -1156,7 +1468,7 @@ static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) · 28 unmodified lines
1156 // Yes, this assumes that CPUs process IPIs in order. If that ever1468 // Yes, this assumes that CPUs process IPIs in order. If that ever
1157 // changes, there will need to be a recheck and/or timed wait.1469 // changes, there will need to be a recheck and/or timed wait.
1158 for_each_online_cpu(cpu)1470 for_each_online_cpu(cpu)
1159 if (smp_load_acquire(per_cpu_ptr(&trc_ipi_to_cpu, cpu)))1471 if (WARN_ON_ONCE(smp_load_acquire(per_cpu_ptr(&trc_ipi_to_cpu, cpu))))
1160 smp_call_function_single(cpu, rcu_tasks_trace_empty_fn, NULL, 1);1472 smp_call_function_single(cpu, rcu_tasks_trace_empty_fn, NULL, 1);
1161 1473
1162 // Remove the safety count.1474 // Remove the safety count.
@@ -1256,13 +1568,13 @@ EXPORT_SYMBOL_GPL(synchronize_rcu_tasks_trace); · 93 unmodified lines
1256 */1568 */
1257void rcu_barrier_tasks_trace(void)1569void rcu_barrier_tasks_trace(void)
1258{1570{
1259 /* There is only one callback queue, so this is easy. ;-) */1571 rcu_barrier_tasks_generic(&rcu_tasks_trace);
1260 synchronize_rcu_tasks_trace();
1261}1572}
1262EXPORT_SYMBOL_GPL(rcu_barrier_tasks_trace);1573EXPORT_SYMBOL_GPL(rcu_barrier_tasks_trace);
1263 1574
1264static int __init rcu_spawn_tasks_trace_kthread(void)1575static int __init rcu_spawn_tasks_trace_kthread(void)
1265{1576{
1577 cblist_init_generic(&rcu_tasks_trace);
1266 if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) {1578 if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) {
1267 rcu_tasks_trace.gp_sleep = HZ / 10;1579 rcu_tasks_trace.gp_sleep = HZ / 10;
1268 rcu_tasks_trace.init_fract = HZ / 10;1580 rcu_tasks_trace.init_fract = HZ / 10;
kernel/rcu/tree.c+86−45
@@ -79,7 +79,7 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { · 78 unmodified lines
79 .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,79 .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
80 .dynticks = ATOMIC_INIT(1),80 .dynticks = ATOMIC_INIT(1),
81#ifdef CONFIG_RCU_NOCB_CPU81#ifdef CONFIG_RCU_NOCB_CPU
82 .cblist.flags = SEGCBLIST_SOFTIRQ_ONLY,82 .cblist.flags = SEGCBLIST_RCU_CORE,
83#endif83#endif
84};84};
85static struct rcu_state rcu_state = {85static struct rcu_state rcu_state = {
@@ -624,7 +624,6 @@ static noinstr void rcu_eqs_enter(bool user) · 538 unmodified lines
624 instrumentation_begin();624 instrumentation_begin();
625 trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks));625 trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks));
626 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));626 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
627 rcu_prepare_for_idle();
628 rcu_preempt_deferred_qs(current);627 rcu_preempt_deferred_qs(current);
629 628
630 // instrumentation for the noinstr rcu_dynticks_eqs_enter()629 // instrumentation for the noinstr rcu_dynticks_eqs_enter()
@@ -768,9 +767,6 @@ noinstr void rcu_nmi_exit(void) · 137 unmodified lines
768 trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks));767 trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks));
769 WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */768 WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
770 769
771 if (!in_nmi())
772 rcu_prepare_for_idle();
773
774 // instrumentation for the noinstr rcu_dynticks_eqs_enter()770 // instrumentation for the noinstr rcu_dynticks_eqs_enter()
775 instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));771 instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
776 instrumentation_end();772 instrumentation_end();
@@ -872,7 +868,6 @@ static void noinstr rcu_eqs_exit(bool user) · 95 unmodified lines
872 // instrumentation for the noinstr rcu_dynticks_eqs_exit()868 // instrumentation for the noinstr rcu_dynticks_eqs_exit()
873 instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));869 instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
874 870
875 rcu_cleanup_after_idle();
876 trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks));871 trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks));
877 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));872 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
878 WRITE_ONCE(rdp->dynticks_nesting, 1);873 WRITE_ONCE(rdp->dynticks_nesting, 1);
@@ -1014,12 +1009,6 @@ noinstr void rcu_nmi_enter(void) · 135 unmodified lines
1014 rcu_dynticks_eqs_exit();1009 rcu_dynticks_eqs_exit();
1015 // ... but is watching here.1010 // ... but is watching here.
1016 1011
1017 if (!in_nmi()) {
1018 instrumentation_begin();
1019 rcu_cleanup_after_idle();
1020 instrumentation_end();
1021 }
1022
1023 instrumentation_begin();1012 instrumentation_begin();
1024 // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs()1013 // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs()
1025 instrument_atomic_read(&rdp->dynticks, sizeof(rdp->dynticks));1014 instrument_atomic_read(&rdp->dynticks, sizeof(rdp->dynticks));
@@ -1086,6 +1075,24 @@ void rcu_irq_enter_irqson(void) · 60 unmodified lines
1086 local_irq_restore(flags);1075 local_irq_restore(flags);
1087}1076}
1088 1077
1078/*
1079 * Check to see if any future non-offloaded RCU-related work will need
1080 * to be done by the current CPU, even if none need be done immediately,
1081 * returning 1 if so. This function is part of the RCU implementation;
1082 * it is -not- an exported member of the RCU API. This is used by
1083 * the idle-entry code to figure out whether it is safe to disable the
1084 * scheduler-clock interrupt.
1085 *
1086 * Just check whether or not this CPU has non-offloaded RCU callbacks
1087 * queued.
1088 */
1089int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1090{
1091 *nextevt = KTIME_MAX;
1092 return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) &&
1093 !rcu_rdp_is_offloaded(this_cpu_ptr(&rcu_data));
1094}
1095
1089/*1096/*
1090 * If any sort of urgency was applied to the current CPU (for example,1097 * If any sort of urgency was applied to the current CPU (for example,
1091 * the scheduler-clock interrupt was enabled on a nohz_full CPU) in order1098 * the scheduler-clock interrupt was enabled on a nohz_full CPU) in order
@@ -1467,7 +1474,7 @@ static void rcu_gp_kthread_wake(void) · 375 unmodified lines
1467{1474{
1468 struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);1475 struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
1469 1476
1470 if ((current == t && !in_irq() && !in_serving_softirq()) ||1477 if ((current == t && !in_hardirq() && !in_serving_softirq()) ||
1471 !READ_ONCE(rcu_state.gp_flags) || !t)1478 !READ_ONCE(rcu_state.gp_flags) || !t)
1472 return;1479 return;
1473 WRITE_ONCE(rcu_state.gp_wake_time, jiffies);1480 WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
@@ -1590,10 +1597,11 @@ static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp, · 116 unmodified lines
1590 struct rcu_data *rdp)1597 struct rcu_data *rdp)
1591{1598{
1592 rcu_lockdep_assert_cblist_protected(rdp);1599 rcu_lockdep_assert_cblist_protected(rdp);
1593 if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) ||1600 if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) || !raw_spin_trylock_rcu_node(rnp))
1594 !raw_spin_trylock_rcu_node(rnp))
1595 return;1601 return;
1596 WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp));1602 // The grace period cannot end while we hold the rcu_node lock.
1603 if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))
1604 WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp));
1597 raw_spin_unlock_rcu_node(rnp);1605 raw_spin_unlock_rcu_node(rnp);
1598}1606}
1599 1607
@@ -2277,7 +2285,7 @@ rcu_report_qs_rdp(struct rcu_data *rdp) · 677 unmodified lines
2277 unsigned long flags;2285 unsigned long flags;
2278 unsigned long mask;2286 unsigned long mask;
2279 bool needwake = false;2287 bool needwake = false;
2280 const bool offloaded = rcu_rdp_is_offloaded(rdp);2288 bool needacc = false;
2281 struct rcu_node *rnp;2289 struct rcu_node *rnp;
2282 2290
2283 WARN_ON_ONCE(rdp->cpu != smp_processor_id());2291 WARN_ON_ONCE(rdp->cpu != smp_processor_id());
@@ -2304,15 +2312,30 @@ rcu_report_qs_rdp(struct rcu_data *rdp) · 20 unmodified lines
2304 /*2312 /*
2305 * This GP can't end until cpu checks in, so all of our2313 * This GP can't end until cpu checks in, so all of our
2306 * callbacks can be processed during the next GP.2314 * callbacks can be processed during the next GP.
2315 *
2316 * NOCB kthreads have their own way to deal with that...
2307 */2317 */
2308 if (!offloaded)2318 if (!rcu_rdp_is_offloaded(rdp)) {
2309 needwake = rcu_accelerate_cbs(rnp, rdp);2319 needwake = rcu_accelerate_cbs(rnp, rdp);
2320 } else if (!rcu_segcblist_completely_offloaded(&rdp->cblist)) {
2321 /*
2322 * ...but NOCB kthreads may miss or delay callbacks acceleration
2323 * if in the middle of a (de-)offloading process.
2324 */
2325 needacc = true;
2326 }
2310 2327
2311 rcu_disable_urgency_upon_qs(rdp);2328 rcu_disable_urgency_upon_qs(rdp);
2312 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);2329 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2313 /* ^^^ Released rnp->lock */2330 /* ^^^ Released rnp->lock */
2314 if (needwake)2331 if (needwake)
2315 rcu_gp_kthread_wake();2332 rcu_gp_kthread_wake();
2333
2334 if (needacc) {
2335 rcu_nocb_lock_irqsave(rdp, flags);
2336 rcu_accelerate_cbs_unlocked(rnp, rdp);
2337 rcu_nocb_unlock_irqrestore(rdp, flags);
2338 }
2316 }2339 }
2317}2340}
2318 2341
@@ -2444,7 +2467,6 @@ static void rcu_do_batch(struct rcu_data *rdp) · 125 unmodified lines
2444 int div;2467 int div;
2445 bool __maybe_unused empty;2468 bool __maybe_unused empty;
2446 unsigned long flags;2469 unsigned long flags;
2447 const bool offloaded = rcu_rdp_is_offloaded(rdp);
2448 struct rcu_head *rhp;2470 struct rcu_head *rhp;
2449 struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);2471 struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
2450 long bl, count = 0;2472 long bl, count = 0;
@@ -2462,18 +2484,17 @@ static void rcu_do_batch(struct rcu_data *rdp) · 11 unmodified lines
2462 }2484 }
2463 2485
2464 /*2486 /*
2465 * Extract the list of ready callbacks, disabling to prevent2487 * Extract the list of ready callbacks, disabling IRQs to prevent
2466 * races with call_rcu() from interrupt handlers. Leave the2488 * races with call_rcu() from interrupt handlers. Leave the
2467 * callback counts, as rcu_barrier() needs to be conservative.2489 * callback counts, as rcu_barrier() needs to be conservative.
2468 */2490 */
2469 local_irq_save(flags);2491 rcu_nocb_lock_irqsave(rdp, flags);
2470 rcu_nocb_lock(rdp);
2471 WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));2492 WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
2472 pending = rcu_segcblist_n_cbs(&rdp->cblist);2493 pending = rcu_segcblist_n_cbs(&rdp->cblist);
2473 div = READ_ONCE(rcu_divisor);2494 div = READ_ONCE(rcu_divisor);
2474 div = div < 0 ? 7 : div > sizeof(long) * 8 - 2 ? sizeof(long) * 8 - 2 : div;2495 div = div < 0 ? 7 : div > sizeof(long) * 8 - 2 ? sizeof(long) * 8 - 2 : div;
2475 bl = max(rdp->blimit, pending >> div);2496 bl = max(rdp->blimit, pending >> div);
2476 if (unlikely(bl > 100)) {2497 if (in_serving_softirq() && unlikely(bl > 100)) {
2477 long rrn = READ_ONCE(rcu_resched_ns);2498 long rrn = READ_ONCE(rcu_resched_ns);
2478 2499
2479 rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;2500 rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;
@@ -2482,7 +2503,7 @@ static void rcu_do_batch(struct rcu_data *rdp) · 2 unmodified lines
2482 trace_rcu_batch_start(rcu_state.name,2503 trace_rcu_batch_start(rcu_state.name,
2483 rcu_segcblist_n_cbs(&rdp->cblist), bl);2504 rcu_segcblist_n_cbs(&rdp->cblist), bl);
2484 rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl);2505 rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl);
2485 if (offloaded)2506 if (rcu_rdp_is_offloaded(rdp))
2486 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);2507 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
2487 2508
2488 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbDequeued"));2509 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbDequeued"));
@@ -2510,18 +2531,21 @@ static void rcu_do_batch(struct rcu_data *rdp) · 21 unmodified lines
2510 /*2531 /*
2511 * Stop only if limit reached and CPU has something to do.2532 * Stop only if limit reached and CPU has something to do.
2512 */2533 */
2513 if (count >= bl && !offloaded &&2534 if (in_serving_softirq()) {
2514 (need_resched() ||2535 if (count >= bl && (need_resched() || !is_idle_task(current)))
2515 (!is_idle_task(current) && !rcu_is_callbacks_kthread())))2536 break;
2516 break;2537 /*
2517 if (unlikely(tlimit)) {2538 * Make sure we don't spend too much time here and deprive other
2518 /* only call local_clock() every 32 callbacks */2539 * softirq vectors of CPU cycles.
2519 if (likely((count & 31) || local_clock() < tlimit))2540 */
2520 continue;2541 if (unlikely(tlimit)) {
2521 /* Exceeded the time limit, so leave. */2542 /* only call local_clock() every 32 callbacks */
2522 break;2543 if (likely((count & 31) || local_clock() < tlimit))
2523 }2544 continue;
2524 if (!in_serving_softirq()) {2545 /* Exceeded the time limit, so leave. */
2546 break;
2547 }
2548 } else {
2525 local_bh_enable();2549 local_bh_enable();
2526 lockdep_assert_irqs_enabled();2550 lockdep_assert_irqs_enabled();
2527 cond_resched_tasks_rcu_qs();2551 cond_resched_tasks_rcu_qs();
@@ -2530,8 +2554,7 @@ static void rcu_do_batch(struct rcu_data *rdp) · 2 unmodified lines
2530 }2554 }
2531 }2555 }
2532 2556
2533 local_irq_save(flags);2557 rcu_nocb_lock_irqsave(rdp, flags);
2534 rcu_nocb_lock(rdp);
2535 rdp->n_cbs_invoked += count;2558 rdp->n_cbs_invoked += count;
2536 trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),2559 trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),
2537 is_idle_task(current), rcu_is_callbacks_kthread());2560 is_idle_task(current), rcu_is_callbacks_kthread());
@@ -2565,9 +2588,6 @@ static void rcu_do_batch(struct rcu_data *rdp) · 27 unmodified lines
2565 2588
2566 rcu_nocb_unlock_irqrestore(rdp, flags);2589 rcu_nocb_unlock_irqrestore(rdp, flags);
2567 2590
2568 /* Re-invoke RCU core processing if there are callbacks remaining. */
2569 if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist))
2570 invoke_rcu_core();
2571 tick_dep_clear_task(current, TICK_DEP_BIT_RCU);2591 tick_dep_clear_task(current, TICK_DEP_BIT_RCU);
2572}2592}
2573 2593
@@ -2706,6 +2726,23 @@ static __latent_entropy void rcu_core(void) · 132 unmodified lines
2706 unsigned long flags;2726 unsigned long flags;
2707 struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);2727 struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
2708 struct rcu_node *rnp = rdp->mynode;2728 struct rcu_node *rnp = rdp->mynode;
2729 /*
2730 * On RT rcu_core() can be preempted when IRQs aren't disabled.
2731 * Therefore this function can race with concurrent NOCB (de-)offloading
2732 * on this CPU and the below condition must be considered volatile.
2733 * However if we race with:
2734 *
2735 * _ Offloading: In the worst case we accelerate or process callbacks
2736 * concurrently with NOCB kthreads. We are guaranteed to
2737 * call rcu_nocb_lock() if that happens.
2738 *
2739 * _ Deoffloading: In the worst case we miss callbacks acceleration or
2740 * processing. This is fine because the early stage
2741 * of deoffloading invokes rcu_core() after setting
2742 * SEGCBLIST_RCU_CORE. So we guarantee that we'll process
2743 * what could have been dismissed without the need to wait
2744 * for the next rcu_pending() check in the next jiffy.
2745 */
2709 const bool do_batch = !rcu_segcblist_completely_offloaded(&rdp->cblist);2746 const bool do_batch = !rcu_segcblist_completely_offloaded(&rdp->cblist);
2710 2747
2711 if (cpu_is_offline(smp_processor_id()))2748 if (cpu_is_offline(smp_processor_id()))
@@ -2714,7 +2751,7 @@ static __latent_entropy void rcu_core(void) · 2 unmodified lines
2714 WARN_ON_ONCE(!rdp->beenonline);2751 WARN_ON_ONCE(!rdp->beenonline);
2715 2752
2716 /* Report any deferred quiescent states if preemption enabled. */2753 /* Report any deferred quiescent states if preemption enabled. */
2717 if (!(preempt_count() & PREEMPT_MASK)) {2754 if (IS_ENABLED(CONFIG_PREEMPT_COUNT) && (!(preempt_count() & PREEMPT_MASK))) {
2718 rcu_preempt_deferred_qs(current);2755 rcu_preempt_deferred_qs(current);
2719 } else if (rcu_preempt_need_deferred_qs(current)) {2756 } else if (rcu_preempt_need_deferred_qs(current)) {
2720 set_tsk_need_resched(current);2757 set_tsk_need_resched(current);
@@ -2737,8 +2774,12 @@ static __latent_entropy void rcu_core(void) · 16 unmodified lines
2737 2774
2738 /* If there are callbacks ready, invoke them. */2775 /* If there are callbacks ready, invoke them. */
2739 if (do_batch && rcu_segcblist_ready_cbs(&rdp->cblist) &&2776 if (do_batch && rcu_segcblist_ready_cbs(&rdp->cblist) &&
2740 likely(READ_ONCE(rcu_scheduler_fully_active)))2777 likely(READ_ONCE(rcu_scheduler_fully_active))) {
2741 rcu_do_batch(rdp);2778 rcu_do_batch(rdp);
2779 /* Re-invoke RCU core processing if there are callbacks remaining. */
2780 if (rcu_segcblist_ready_cbs(&rdp->cblist))
2781 invoke_rcu_core();
2782 }
2742 2783
2743 /* Do any needed deferred wakeups of rcuo kthreads. */2784 /* Do any needed deferred wakeups of rcuo kthreads. */
2744 do_nocb_deferred_wakeup(rdp);2785 do_nocb_deferred_wakeup(rdp);
@@ -2982,7 +3023,7 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func) · 237 unmodified lines
2982 head->func = func;3023 head->func = func;
2983 head->next = NULL;3024 head->next = NULL;
2984 local_irq_save(flags);3025 local_irq_save(flags);
2985 kasan_record_aux_stack(head);3026 kasan_record_aux_stack_noalloc(head);
2986 rdp = this_cpu_ptr(&rcu_data);3027 rdp = this_cpu_ptr(&rcu_data);
2987 3028
2988 /* Add the callback to our list. */3029 /* Add the callback to our list. */
@@ -3547,7 +3588,7 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) · 558 unmodified lines
3547 return;3588 return;
3548 }3589 }
3549 3590
3550 kasan_record_aux_stack(ptr);3591 kasan_record_aux_stack_noalloc(ptr);
3551 success = add_ptr_to_bulk_krc_lock(&krcp, &flags, ptr, !head);3592 success = add_ptr_to_bulk_krc_lock(&krcp, &flags, ptr, !head);
3552 if (!success) {3593 if (!success) {
3553 run_page_cache_worker(krcp);3594 run_page_cache_worker(krcp);
kernel/rcu/tree.h+15−16
@@ -157,7 +157,6 @@ struct rcu_data { · 156 unmodified lines
157 bool core_needs_qs; /* Core waits for quiescent state. */157 bool core_needs_qs; /* Core waits for quiescent state. */
158 bool beenonline; /* CPU online at least once. */158 bool beenonline; /* CPU online at least once. */
159 bool gpwrap; /* Possible ->gp_seq wrap. */159 bool gpwrap; /* Possible ->gp_seq wrap. */
160 bool exp_deferred_qs; /* This CPU awaiting a deferred QS? */
161 bool cpu_started; /* RCU watching this onlining CPU. */160 bool cpu_started; /* RCU watching this onlining CPU. */
162 struct rcu_node *mynode; /* This CPU's leaf of hierarchy */161 struct rcu_node *mynode; /* This CPU's leaf of hierarchy */
163 unsigned long grpmask; /* Mask to apply to leaf qsmask. */162 unsigned long grpmask; /* Mask to apply to leaf qsmask. */
@@ -189,11 +188,6 @@ struct rcu_data { · 25 unmodified lines
189 bool rcu_urgent_qs; /* GP old need light quiescent state. */188 bool rcu_urgent_qs; /* GP old need light quiescent state. */
190 bool rcu_forced_tick; /* Forced tick to provide QS. */189 bool rcu_forced_tick; /* Forced tick to provide QS. */
191 bool rcu_forced_tick_exp; /* ... provide QS to expedited GP. */190 bool rcu_forced_tick_exp; /* ... provide QS to expedited GP. */
192#ifdef CONFIG_RCU_FAST_NO_HZ
193 unsigned long last_accelerate; /* Last jiffy CBs were accelerated. */
194 unsigned long last_advance_all; /* Last jiffy CBs were all advanced. */
195 int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */
196#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
197 191
198 /* 4) rcu_barrier(), OOM callbacks, and expediting. */192 /* 4) rcu_barrier(), OOM callbacks, and expediting. */
199 struct rcu_head barrier_head;193 struct rcu_head barrier_head;
@@ -227,8 +221,11 @@ struct rcu_data { · 27 unmodified lines
227 struct swait_queue_head nocb_gp_wq; /* For nocb kthreads to sleep on. */221 struct swait_queue_head nocb_gp_wq; /* For nocb kthreads to sleep on. */
228 bool nocb_cb_sleep; /* Is the nocb CB thread asleep? */222 bool nocb_cb_sleep; /* Is the nocb CB thread asleep? */
229 struct task_struct *nocb_cb_kthread;223 struct task_struct *nocb_cb_kthread;
230 struct rcu_data *nocb_next_cb_rdp;224 struct list_head nocb_head_rdp; /*
231 /* Next rcu_data in wakeup chain. */225 * Head of rcu_data list in wakeup chain,
226 * if rdp_gp.
227 */
228 struct list_head nocb_entry_rdp; /* rcu_data node in wakeup chain. */
232 229
233 /* The following fields are used by CB kthread, hence new cacheline. */230 /* The following fields are used by CB kthread, hence new cacheline. */
234 struct rcu_data *nocb_gp_rdp ____cacheline_internodealigned_in_smp;231 struct rcu_data *nocb_gp_rdp ____cacheline_internodealigned_in_smp;
@@ -419,8 +416,6 @@ static bool rcu_is_callbacks_kthread(void); · 184 unmodified lines
419static void rcu_cpu_kthread_setup(unsigned int cpu);416static void rcu_cpu_kthread_setup(unsigned int cpu);
420static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp);417static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp);
421static void __init rcu_spawn_boost_kthreads(void);418static void __init rcu_spawn_boost_kthreads(void);
422static void rcu_cleanup_after_idle(void);
423static void rcu_prepare_for_idle(void);
424static bool rcu_preempt_has_tasks(struct rcu_node *rnp);419static bool rcu_preempt_has_tasks(struct rcu_node *rnp);
425static bool rcu_preempt_need_deferred_qs(struct task_struct *t);420static bool rcu_preempt_need_deferred_qs(struct task_struct *t);
426static void rcu_preempt_deferred_qs(struct task_struct *t);421static void rcu_preempt_deferred_qs(struct task_struct *t);
@@ -447,12 +442,16 @@ static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, · 20 unmodified lines
447static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp);442static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp);
448#ifdef CONFIG_RCU_NOCB_CPU443#ifdef CONFIG_RCU_NOCB_CPU
449static void __init rcu_organize_nocb_kthreads(void);444static void __init rcu_organize_nocb_kthreads(void);
450#define rcu_nocb_lock_irqsave(rdp, flags) \445
451do { \446/*
452 if (!rcu_segcblist_is_offloaded(&(rdp)->cblist)) \447 * Disable IRQs before checking offloaded state so that local
453 local_irq_save(flags); \448 * locking is safe against concurrent de-offloading.
454 else \449 */
455 raw_spin_lock_irqsave(&(rdp)->nocb_lock, (flags)); \450#define rcu_nocb_lock_irqsave(rdp, flags) \
451do { \
452 local_irq_save(flags); \
453 if (rcu_segcblist_is_offloaded(&(rdp)->cblist)) \
454 raw_spin_lock(&(rdp)->nocb_lock); \
456} while (0)455} while (0)
457#else /* #ifdef CONFIG_RCU_NOCB_CPU */456#else /* #ifdef CONFIG_RCU_NOCB_CPU */
458#define rcu_nocb_lock_irqsave(rdp, flags) local_irq_save(flags)457#define rcu_nocb_lock_irqsave(rdp, flags) local_irq_save(flags)
kernel/rcu/tree_exp.h+9−5
@@ -255,7 +255,7 @@ static void rcu_report_exp_cpu_mult(struct rcu_node *rnp, · 254 unmodified lines
255 */255 */
256static void rcu_report_exp_rdp(struct rcu_data *rdp)256static void rcu_report_exp_rdp(struct rcu_data *rdp)
257{257{
258 WRITE_ONCE(rdp->exp_deferred_qs, false);258 WRITE_ONCE(rdp->cpu_no_qs.b.exp, false);
259 rcu_report_exp_cpu_mult(rdp->mynode, rdp->grpmask, true);259 rcu_report_exp_cpu_mult(rdp->mynode, rdp->grpmask, true);
260}260}
261 261
@@ -387,6 +387,7 @@ retry_ipi: · 125 unmodified lines
387 continue;387 continue;
388 }388 }
389 if (get_cpu() == cpu) {389 if (get_cpu() == cpu) {
390 mask_ofl_test |= mask;
390 put_cpu();391 put_cpu();
391 continue;392 continue;
392 }393 }
@@ -506,7 +507,10 @@ static void synchronize_rcu_expedited_wait(void) · 113 unmodified lines
506 if (rdp->rcu_forced_tick_exp)507 if (rdp->rcu_forced_tick_exp)
507 continue;508 continue;
508 rdp->rcu_forced_tick_exp = true;509 rdp->rcu_forced_tick_exp = true;
509 tick_dep_set_cpu(cpu, TICK_DEP_BIT_RCU_EXP);510 preempt_disable();
511 if (cpu_online(cpu))
512 tick_dep_set_cpu(cpu, TICK_DEP_BIT_RCU_EXP);
513 preempt_enable();
510 }514 }
511 }515 }
512 j = READ_ONCE(jiffies_till_first_fqs);516 j = READ_ONCE(jiffies_till_first_fqs);
@@ -655,7 +659,7 @@ static void rcu_exp_handler(void *unused) · 142 unmodified lines
655 rcu_dynticks_curr_cpu_in_eqs()) {659 rcu_dynticks_curr_cpu_in_eqs()) {
656 rcu_report_exp_rdp(rdp);660 rcu_report_exp_rdp(rdp);
657 } else {661 } else {
658 rdp->exp_deferred_qs = true;662 WRITE_ONCE(rdp->cpu_no_qs.b.exp, true);
659 set_tsk_need_resched(t);663 set_tsk_need_resched(t);
660 set_preempt_need_resched();664 set_preempt_need_resched();
661 }665 }
@@ -677,7 +681,7 @@ static void rcu_exp_handler(void *unused) · 15 unmodified lines
677 if (depth > 0) {681 if (depth > 0) {
678 raw_spin_lock_irqsave_rcu_node(rnp, flags);682 raw_spin_lock_irqsave_rcu_node(rnp, flags);
679 if (rnp->expmask & rdp->grpmask) {683 if (rnp->expmask & rdp->grpmask) {
680 rdp->exp_deferred_qs = true;684 WRITE_ONCE(rdp->cpu_no_qs.b.exp, true);
681 t->rcu_read_unlock_special.b.exp_hint = true;685 t->rcu_read_unlock_special.b.exp_hint = true;
682 }686 }
683 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);687 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
@@ -759,7 +763,7 @@ static void sync_sched_exp_online_cleanup(int cpu) · 75 unmodified lines
759 my_cpu = get_cpu();763 my_cpu = get_cpu();
760 /* Quiescent state either not needed or already requested, leave. */764 /* Quiescent state either not needed or already requested, leave. */
761 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) ||765 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) ||
762 rdp->cpu_no_qs.b.exp) {766 READ_ONCE(rdp->cpu_no_qs.b.exp)) {
763 put_cpu();767 put_cpu();
764 return;768 return;
765 }769 }
kernel/rcu/tree_nocb.h+106−54
@@ -60,16 +60,22 @@ static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp) · 59 unmodified lines
60 * Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters.60 * Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters.
61 * If the list is invalid, a warning is emitted and all CPUs are offloaded.61 * If the list is invalid, a warning is emitted and all CPUs are offloaded.
62 */62 */
63
64static bool rcu_nocb_is_setup;
65
63static int __init rcu_nocb_setup(char *str)66static int __init rcu_nocb_setup(char *str)
64{67{
65 alloc_bootmem_cpumask_var(&rcu_nocb_mask);68 alloc_bootmem_cpumask_var(&rcu_nocb_mask);
66 if (cpulist_parse(str, rcu_nocb_mask)) {69 if (*str == '=') {
67 pr_warn("rcu_nocbs= bad CPU range, all CPUs set\n");70 if (cpulist_parse(++str, rcu_nocb_mask)) {
68 cpumask_setall(rcu_nocb_mask);71 pr_warn("rcu_nocbs= bad CPU range, all CPUs set\n");
72 cpumask_setall(rcu_nocb_mask);
73 }
69 }74 }
75 rcu_nocb_is_setup = true;
70 return 1;76 return 1;
71}77}
72__setup("rcu_nocbs=", rcu_nocb_setup);78__setup("rcu_nocbs", rcu_nocb_setup);
73 79
74static int __init parse_rcu_nocb_poll(char *arg)80static int __init parse_rcu_nocb_poll(char *arg)
75{81{
@@ -625,7 +631,21 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) · 549 unmodified lines
625 * and the global grace-period kthread are awakened if needed.631 * and the global grace-period kthread are awakened if needed.
626 */632 */
627 WARN_ON_ONCE(my_rdp->nocb_gp_rdp != my_rdp);633 WARN_ON_ONCE(my_rdp->nocb_gp_rdp != my_rdp);
628 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_cb_rdp) {634 /*
635 * An rcu_data structure is removed from the list after its
636 * CPU is de-offloaded and added to the list before that CPU is
637 * (re-)offloaded. If the following loop happens to be referencing
638 * that rcu_data structure during the time that the corresponding
639 * CPU is de-offloaded and then immediately re-offloaded, this
640 * loop's rdp pointer will be carried to the end of the list by
641 * the resulting pair of list operations. This can cause the loop
642 * to skip over some of the rcu_data structures that were supposed
643 * to have been scanned. Fortunately a new iteration through the
644 * entire loop is forced after a given CPU's rcu_data structure
645 * is added to the list, so the skipped-over rcu_data structures
646 * won't be ignored for long.
647 */
648 list_for_each_entry_rcu(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp, 1) {
629 bool needwake_state = false;649 bool needwake_state = false;
630 650
631 if (!nocb_gp_enabled_cb(rdp))651 if (!nocb_gp_enabled_cb(rdp))
@@ -789,6 +809,18 @@ static void nocb_cb_wait(struct rcu_data *rdp) · 157 unmodified lines
789 bool can_sleep = true;809 bool can_sleep = true;
790 struct rcu_node *rnp = rdp->mynode;810 struct rcu_node *rnp = rdp->mynode;
791 811
812 do {
813 swait_event_interruptible_exclusive(rdp->nocb_cb_wq,
814 nocb_cb_wait_cond(rdp));
815
816 // VVV Ensure CB invocation follows _sleep test.
817 if (smp_load_acquire(&rdp->nocb_cb_sleep)) { // ^^^
818 WARN_ON(signal_pending(current));
819 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty"));
820 }
821 } while (!nocb_cb_can_run(rdp));
822
823
792 local_irq_save(flags);824 local_irq_save(flags);
793 rcu_momentary_dyntick_idle();825 rcu_momentary_dyntick_idle();
794 local_irq_restore(flags);826 local_irq_restore(flags);
@@ -841,17 +873,6 @@ static void nocb_cb_wait(struct rcu_data *rdp) · 46 unmodified lines
841 873
842 if (needwake_state)874 if (needwake_state)
843 swake_up_one(&rdp->nocb_state_wq);875 swake_up_one(&rdp->nocb_state_wq);
844
845 do {
846 swait_event_interruptible_exclusive(rdp->nocb_cb_wq,
847 nocb_cb_wait_cond(rdp));
848
849 // VVV Ensure CB invocation follows _sleep test.
850 if (smp_load_acquire(&rdp->nocb_cb_sleep)) { // ^^^
851 WARN_ON(signal_pending(current));
852 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty"));
853 }
854 } while (!nocb_cb_can_run(rdp));
855}876}
856 877
857/*878/*
@@ -990,22 +1011,33 @@ static long rcu_nocb_rdp_deoffload(void *arg) · 132 unmodified lines
990 * will refuse to put anything into the bypass.1011 * will refuse to put anything into the bypass.
991 */1012 */
992 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));1013 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));
1014 /*
1015 * Start with invoking rcu_core() early. This way if the current thread
1016 * happens to preempt an ongoing call to rcu_core() in the middle,
1017 * leaving some work dismissed because rcu_core() still thinks the rdp is
1018 * completely offloaded, we are guaranteed a nearby future instance of
1019 * rcu_core() to catch up.
1020 */
1021 rcu_segcblist_set_flags(cblist, SEGCBLIST_RCU_CORE);
1022 invoke_rcu_core();
993 ret = rdp_offload_toggle(rdp, false, flags);1023 ret = rdp_offload_toggle(rdp, false, flags);
994 swait_event_exclusive(rdp->nocb_state_wq,1024 swait_event_exclusive(rdp->nocb_state_wq,
995 !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB |1025 !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB |
996 SEGCBLIST_KTHREAD_GP));1026 SEGCBLIST_KTHREAD_GP));
1027 /* Stop nocb_gp_wait() from iterating over this structure. */
1028 list_del_rcu(&rdp->nocb_entry_rdp);
997 /*1029 /*
998 * Lock one last time to acquire latest callback updates from kthreads1030 * Lock one last time to acquire latest callback updates from kthreads
999 * so we can later handle callbacks locally without locking.1031 * so we can later handle callbacks locally without locking.
1000 */1032 */
1001 rcu_nocb_lock_irqsave(rdp, flags);1033 rcu_nocb_lock_irqsave(rdp, flags);
1002 /*1034 /*
1003 * Theoretically we could set SEGCBLIST_SOFTIRQ_ONLY after the nocb1035 * Theoretically we could clear SEGCBLIST_LOCKING after the nocb
1004 * lock is released but how about being paranoid for once?1036 * lock is released but how about being paranoid for once?
1005 */1037 */
1006 rcu_segcblist_set_flags(cblist, SEGCBLIST_SOFTIRQ_ONLY);1038 rcu_segcblist_clear_flags(cblist, SEGCBLIST_LOCKING);
1007 /*1039 /*
1008 * With SEGCBLIST_SOFTIRQ_ONLY, we can't use1040 * Without SEGCBLIST_LOCKING, we can't use
1009 * rcu_nocb_unlock_irqrestore() anymore.1041 * rcu_nocb_unlock_irqrestore() anymore.
1010 */1042 */
1011 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);1043 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
@@ -1057,15 +1089,26 @@ static long rcu_nocb_rdp_offload(void *arg) · 45 unmodified lines
1057 return -EINVAL;1089 return -EINVAL;
1058 1090
1059 pr_info("Offloading %d\n", rdp->cpu);1091 pr_info("Offloading %d\n", rdp->cpu);
1092
1093 /*
1094 * Cause future nocb_gp_wait() invocations to iterate over
1095 * structure, resetting ->nocb_gp_sleep and waking up the related
1096 * "rcuog". Since nocb_gp_wait() in turn locks ->nocb_gp_lock
1097 * before setting ->nocb_gp_sleep again, we are guaranteed to
1098 * iterate this newly added structure before "rcuog" goes to
1099 * sleep again.
1100 */
1101 list_add_tail_rcu(&rdp->nocb_entry_rdp, &rdp->nocb_gp_rdp->nocb_head_rdp);
1102
1060 /*1103 /*
1061 * Can't use rcu_nocb_lock_irqsave() while we are in1104 * Can't use rcu_nocb_lock_irqsave() before SEGCBLIST_LOCKING
1062 * SEGCBLIST_SOFTIRQ_ONLY mode.1105 * is set.
1063 */1106 */
1064 raw_spin_lock_irqsave(&rdp->nocb_lock, flags);1107 raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
1065 1108
1066 /*1109 /*
1067 * We didn't take the nocb lock while working on the1110 * We didn't take the nocb lock while working on the
1068 * rdp->cblist in SEGCBLIST_SOFTIRQ_ONLY mode.1111 * rdp->cblist with SEGCBLIST_LOCKING cleared (pure softirq/rcuc mode).
1069 * Every modifications that have been done previously on1112 * Every modifications that have been done previously on
1070 * rdp->cblist must be visible remotely by the nocb kthreads1113 * rdp->cblist must be visible remotely by the nocb kthreads
1071 * upon wake up after reading the cblist flags.1114 * upon wake up after reading the cblist flags.
@@ -1084,6 +1127,14 @@ static long rcu_nocb_rdp_offload(void *arg) · 12 unmodified lines
1084 rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB) &&1127 rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB) &&
1085 rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP));1128 rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP));
1086 1129
1130 /*
1131 * All kthreads are ready to work, we can finally relieve rcu_core() and
1132 * enable nocb bypass.
1133 */
1134 rcu_nocb_lock_irqsave(rdp, flags);
1135 rcu_segcblist_clear_flags(cblist, SEGCBLIST_RCU_CORE);
1136 rcu_nocb_unlock_irqrestore(rdp, flags);
1137
1087 return ret;1138 return ret;
1088}1139}
1089 1140
@@ -1122,13 +1173,17 @@ void __init rcu_init_nohz(void) · 32 unmodified lines
1122 need_rcu_nocb_mask = true;1173 need_rcu_nocb_mask = true;
1123#endif /* #if defined(CONFIG_NO_HZ_FULL) */1174#endif /* #if defined(CONFIG_NO_HZ_FULL) */
1124 1175
1125 if (!cpumask_available(rcu_nocb_mask) && need_rcu_nocb_mask) {1176 if (need_rcu_nocb_mask) {
1126 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {1177 if (!cpumask_available(rcu_nocb_mask)) {
1127 pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");1178 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
1128 return;1179 pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
1180 return;
1181 }
1129 }1182 }
1183 rcu_nocb_is_setup = true;
1130 }1184 }
1131 if (!cpumask_available(rcu_nocb_mask))1185
1186 if (!rcu_nocb_is_setup)
1132 return;1187 return;
1133 1188
1134#if defined(CONFIG_NO_HZ_FULL)1189#if defined(CONFIG_NO_HZ_FULL)
@@ -1154,8 +1209,8 @@ void __init rcu_init_nohz(void) · 19 unmodified lines
1154 if (rcu_segcblist_empty(&rdp->cblist))1209 if (rcu_segcblist_empty(&rdp->cblist))
1155 rcu_segcblist_init(&rdp->cblist);1210 rcu_segcblist_init(&rdp->cblist);
1156 rcu_segcblist_offload(&rdp->cblist, true);1211 rcu_segcblist_offload(&rdp->cblist, true);
1157 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB);1212 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP);
1158 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_GP);1213 rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_RCU_CORE);
1159 }1214 }
1160 rcu_organize_nocb_kthreads();1215 rcu_organize_nocb_kthreads();
1161}1216}
@@ -1178,17 +1233,17 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) · 16 unmodified lines
1178 * rcuo CB kthread, spawn it. Additionally, if the rcuo GP kthread1233 * rcuo CB kthread, spawn it. Additionally, if the rcuo GP kthread
1179 * for this CPU's group has not yet been created, spawn it as well.1234 * for this CPU's group has not yet been created, spawn it as well.
1180 */1235 */
1181static void rcu_spawn_one_nocb_kthread(int cpu)1236static void rcu_spawn_cpu_nocb_kthread(int cpu)
1182{1237{
1183 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);1238 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
1184 struct rcu_data *rdp_gp;1239 struct rcu_data *rdp_gp;
1185 struct task_struct *t;1240 struct task_struct *t;
1186 1241
1187 /*1242 if (!rcu_scheduler_fully_active || !rcu_nocb_is_setup)
1188 * If this isn't a no-CBs CPU or if it already has an rcuo kthread,1243 return;
1189 * then nothing to do.1244
1190 */1245 /* If there already is an rcuo kthread, then nothing to do. */
1191 if (!rcu_is_nocb_cpu(cpu) || rdp->nocb_cb_kthread)1246 if (rdp->nocb_cb_kthread)
1192 return;1247 return;
1193 1248
1194 /* If we didn't spawn the GP kthread first, reorganize! */1249 /* If we didn't spawn the GP kthread first, reorganize! */
@@ -1210,16 +1265,6 @@ static void rcu_spawn_one_nocb_kthread(int cpu) · 15 unmodified lines
1210 WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread);1265 WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread);
1211}1266}
1212 1267
1213/*
1214 * If the specified CPU is a no-CBs CPU that does not already have its
1215 * rcuo kthread, spawn it.
1216 */
1217static void rcu_spawn_cpu_nocb_kthread(int cpu)
1218{
1219 if (rcu_scheduler_fully_active)
1220 rcu_spawn_one_nocb_kthread(cpu);
1221}
1222
1223/*1268/*
1224 * Once the scheduler is running, spawn rcuo kthreads for all online1269 * Once the scheduler is running, spawn rcuo kthreads for all online
1225 * no-CBs CPUs. This assumes that the early_initcall()s happen before1270 * no-CBs CPUs. This assumes that the early_initcall()s happen before
@@ -1230,8 +1275,10 @@ static void __init rcu_spawn_nocb_kthreads(void) · 4 unmodified lines
1230{1275{
1231 int cpu;1276 int cpu;
1232 1277
1233 for_each_online_cpu(cpu)1278 if (rcu_nocb_is_setup) {
1234 rcu_spawn_cpu_nocb_kthread(cpu);1279 for_each_online_cpu(cpu)
1280 rcu_spawn_cpu_nocb_kthread(cpu);
1281 }
1235}1282}
1236 1283
1237/* How many CB CPU IDs per GP kthread? Default of -1 for sqrt(nr_cpu_ids). */1284/* How many CB CPU IDs per GP kthread? Default of -1 for sqrt(nr_cpu_ids). */
@@ -1251,7 +1298,6 @@ static void __init rcu_organize_nocb_kthreads(void) · 13 unmodified lines
1251 int nl = 0; /* Next GP kthread. */1298 int nl = 0; /* Next GP kthread. */
1252 struct rcu_data *rdp;1299 struct rcu_data *rdp;
1253 struct rcu_data *rdp_gp = NULL; /* Suppress misguided gcc warn. */1300 struct rcu_data *rdp_gp = NULL; /* Suppress misguided gcc warn. */
1254 struct rcu_data *rdp_prev = NULL;
1255 1301
1256 if (!cpumask_available(rcu_nocb_mask))1302 if (!cpumask_available(rcu_nocb_mask))
1257 return;1303 return;
@@ -1265,14 +1311,14 @@ static void __init rcu_organize_nocb_kthreads(void) · 7 unmodified lines
1265 * Should the corresponding CPU come online in the future, then1311 * Should the corresponding CPU come online in the future, then
1266 * we will spawn the needed set of rcu_nocb_kthread() kthreads.1312 * we will spawn the needed set of rcu_nocb_kthread() kthreads.
1267 */1313 */
1268 for_each_cpu(cpu, rcu_nocb_mask) {1314 for_each_possible_cpu(cpu) {
1269 rdp = per_cpu_ptr(&rcu_data, cpu);1315 rdp = per_cpu_ptr(&rcu_data, cpu);
1270 if (rdp->cpu >= nl) {1316 if (rdp->cpu >= nl) {
1271 /* New GP kthread, set up for CBs & next GP. */1317 /* New GP kthread, set up for CBs & next GP. */
1272 gotnocbs = true;1318 gotnocbs = true;
1273 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;1319 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;
1274 rdp->nocb_gp_rdp = rdp;
1275 rdp_gp = rdp;1320 rdp_gp = rdp;
1321 INIT_LIST_HEAD(&rdp->nocb_head_rdp);
1276 if (dump_tree) {1322 if (dump_tree) {
1277 if (!firsttime)1323 if (!firsttime)
1278 pr_cont("%s\n", gotnocbscbs1324 pr_cont("%s\n", gotnocbscbs
@@ -1285,12 +1331,12 @@ static void __init rcu_organize_nocb_kthreads(void) · 6 unmodified lines
1285 } else {1331 } else {
1286 /* Another CB kthread, link to previous GP kthread. */1332 /* Another CB kthread, link to previous GP kthread. */
1287 gotnocbscbs = true;1333 gotnocbscbs = true;
1288 rdp->nocb_gp_rdp = rdp_gp;
1289 rdp_prev->nocb_next_cb_rdp = rdp;
1290 if (dump_tree)1334 if (dump_tree)
1291 pr_cont(" %d", cpu);1335 pr_cont(" %d", cpu);
1292 }1336 }
1293 rdp_prev = rdp;1337 rdp->nocb_gp_rdp = rdp_gp;
1338 if (cpumask_test_cpu(cpu, rcu_nocb_mask))
1339 list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp);
1294 }1340 }
1295 if (gotnocbs && dump_tree)1341 if (gotnocbs && dump_tree)
1296 pr_cont("%s\n", gotnocbscbs ? "" : " (self only)");1342 pr_cont("%s\n", gotnocbscbs ? "" : " (self only)");
@@ -1352,6 +1398,7 @@ static void show_rcu_nocb_state(struct rcu_data *rdp) · 55 unmodified lines
1352{1398{
1353 char bufw[20];1399 char bufw[20];
1354 char bufr[20];1400 char bufr[20];
1401 struct rcu_data *nocb_next_rdp;
1355 struct rcu_segcblist *rsclp = &rdp->cblist;1402 struct rcu_segcblist *rsclp = &rdp->cblist;
1356 bool waslocked;1403 bool waslocked;
1357 bool wassleep;1404 bool wassleep;
@@ -1359,11 +1406,16 @@ static void show_rcu_nocb_state(struct rcu_data *rdp) · 1 unmodified line
1359 if (rdp->nocb_gp_rdp == rdp)1406 if (rdp->nocb_gp_rdp == rdp)
1360 show_rcu_nocb_gp_state(rdp);1407 show_rcu_nocb_gp_state(rdp);
1361 1408
1409 nocb_next_rdp = list_next_or_null_rcu(&rdp->nocb_gp_rdp->nocb_head_rdp,
1410 &rdp->nocb_entry_rdp,
1411 typeof(*rdp),
1412 nocb_entry_rdp);
1413
1362 sprintf(bufw, "%ld", rsclp->gp_seq[RCU_WAIT_TAIL]);1414 sprintf(bufw, "%ld", rsclp->gp_seq[RCU_WAIT_TAIL]);
1363 sprintf(bufr, "%ld", rsclp->gp_seq[RCU_NEXT_READY_TAIL]);1415 sprintf(bufr, "%ld", rsclp->gp_seq[RCU_NEXT_READY_TAIL]);
1364 pr_info(" CB %d^%d->%d %c%c%c%c%c%c F%ld L%ld C%d %c%c%s%c%s%c%c q%ld %c CPU %d%s\n",1416 pr_info(" CB %d^%d->%d %c%c%c%c%c%c F%ld L%ld C%d %c%c%s%c%s%c%c q%ld %c CPU %d%s\n",
1365 rdp->cpu, rdp->nocb_gp_rdp->cpu,1417 rdp->cpu, rdp->nocb_gp_rdp->cpu,
1366 rdp->nocb_next_cb_rdp ? rdp->nocb_next_cb_rdp->cpu : -1,1418 nocb_next_rdp ? nocb_next_rdp->cpu : -1,
1367 "kK"[!!rdp->nocb_cb_kthread],1419 "kK"[!!rdp->nocb_cb_kthread],
1368 "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)],1420 "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)],
1369 "cC"[!!atomic_read(&rdp->nocb_lock_contended)],1421 "cC"[!!atomic_read(&rdp->nocb_lock_contended)],
kernel/rcu/tree_plugin.h+33−217
@@ -16,7 +16,7 @@ · 15 unmodified lines
16static bool rcu_rdp_is_offloaded(struct rcu_data *rdp)16static bool rcu_rdp_is_offloaded(struct rcu_data *rdp)
17{17{
18 /*18 /*
19 * In order to read the offloaded state of an rdp is a safe19 * In order to read the offloaded state of an rdp in a safe
20 * and stable way and prevent from its value to be changed20 * and stable way and prevent from its value to be changed
21 * under us, we must either hold the barrier mutex, the cpu21 * under us, we must either hold the barrier mutex, the cpu
22 * hotplug lock (read or write) or the nocb lock. Local22 * hotplug lock (read or write) or the nocb lock. Local
@@ -51,12 +51,10 @@ static void __init rcu_bootup_announce_oddness(void) · 28 unmodified lines
51 RCU_FANOUT);51 RCU_FANOUT);
52 if (rcu_fanout_exact)52 if (rcu_fanout_exact)
53 pr_info("\tHierarchical RCU autobalancing is disabled.\n");53 pr_info("\tHierarchical RCU autobalancing is disabled.\n");
54 if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ))
55 pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
56 if (IS_ENABLED(CONFIG_PROVE_RCU))54 if (IS_ENABLED(CONFIG_PROVE_RCU))
57 pr_info("\tRCU lockdep checking is enabled.\n");55 pr_info("\tRCU lockdep checking is enabled.\n");
58 if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))56 if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
59 pr_info("\tRCU strict (and thus non-scalable) grace periods enabled.\n");57 pr_info("\tRCU strict (and thus non-scalable) grace periods are enabled.\n");
60 if (RCU_NUM_LVLS >= 4)58 if (RCU_NUM_LVLS >= 4)
61 pr_info("\tFour(or more)-level hierarchy is enabled.\n");59 pr_info("\tFour(or more)-level hierarchy is enabled.\n");
62 if (RCU_FANOUT_LEAF != 16)60 if (RCU_FANOUT_LEAF != 16)
@@ -88,13 +86,13 @@ static void __init rcu_bootup_announce_oddness(void) · 25 unmodified lines
88 if (rcu_kick_kthreads)86 if (rcu_kick_kthreads)
89 pr_info("\tKick kthreads if too-long grace period.\n");87 pr_info("\tKick kthreads if too-long grace period.\n");
90 if (IS_ENABLED(CONFIG_DEBUG_OBJECTS_RCU_HEAD))88 if (IS_ENABLED(CONFIG_DEBUG_OBJECTS_RCU_HEAD))
91 pr_info("\tRCU callback double-/use-after-free debug enabled.\n");89 pr_info("\tRCU callback double-/use-after-free debug is enabled.\n");
92 if (gp_preinit_delay)90 if (gp_preinit_delay)
93 pr_info("\tRCU debug GP pre-init slowdown %d jiffies.\n", gp_preinit_delay);91 pr_info("\tRCU debug GP pre-init slowdown %d jiffies.\n", gp_preinit_delay);
94 if (gp_init_delay)92 if (gp_init_delay)
95 pr_info("\tRCU debug GP init slowdown %d jiffies.\n", gp_init_delay);93 pr_info("\tRCU debug GP init slowdown %d jiffies.\n", gp_init_delay);
96 if (gp_cleanup_delay)94 if (gp_cleanup_delay)
97 pr_info("\tRCU debug GP init slowdown %d jiffies.\n", gp_cleanup_delay);95 pr_info("\tRCU debug GP cleanup slowdown %d jiffies.\n", gp_cleanup_delay);
98 if (!use_softirq)96 if (!use_softirq)
99 pr_info("\tRCU_SOFTIRQ processing moved to rcuc kthreads.\n");97 pr_info("\tRCU_SOFTIRQ processing moved to rcuc kthreads.\n");
100 if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG))98 if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG))
@@ -260,10 +258,10 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) · 159 unmodified lines
260 * no need to check for a subsequent expedited GP. (Though we are258 * no need to check for a subsequent expedited GP. (Though we are
261 * still in a quiescent state in any case.)259 * still in a quiescent state in any case.)
262 */260 */
263 if (blkd_state & RCU_EXP_BLKD && rdp->exp_deferred_qs)261 if (blkd_state & RCU_EXP_BLKD && rdp->cpu_no_qs.b.exp)
264 rcu_report_exp_rdp(rdp);262 rcu_report_exp_rdp(rdp);
265 else263 else
266 WARN_ON_ONCE(rdp->exp_deferred_qs);264 WARN_ON_ONCE(rdp->cpu_no_qs.b.exp);
267}265}
268 266
269/*267/*
@@ -277,12 +275,16 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) · 7 unmodified lines
277 * current task, there might be any number of other tasks blocked while275 * current task, there might be any number of other tasks blocked while
278 * in an RCU read-side critical section.276 * in an RCU read-side critical section.
279 *277 *
278 * Unlike non-preemptible-RCU, quiescent state reports for expedited
279 * grace periods are handled separately via deferred quiescent states
280 * and context switch events.
281 *
280 * Callers to this function must disable preemption.282 * Callers to this function must disable preemption.
281 */283 */
282static void rcu_qs(void)284static void rcu_qs(void)
283{285{
284 RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!\n");286 RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!\n");
285 if (__this_cpu_read(rcu_data.cpu_no_qs.s)) {287 if (__this_cpu_read(rcu_data.cpu_no_qs.b.norm)) {
286 trace_rcu_grace_period(TPS("rcu_preempt"),288 trace_rcu_grace_period(TPS("rcu_preempt"),
287 __this_cpu_read(rcu_data.gp_seq),289 __this_cpu_read(rcu_data.gp_seq),
288 TPS("cpuqs"));290 TPS("cpuqs"));
@@ -350,7 +352,7 @@ void rcu_note_context_switch(bool preempt) · 61 unmodified lines
350 * means that we continue to block the current grace period.352 * means that we continue to block the current grace period.
351 */353 */
352 rcu_qs();354 rcu_qs();
353 if (rdp->exp_deferred_qs)355 if (rdp->cpu_no_qs.b.exp)
354 rcu_report_exp_rdp(rdp);356 rcu_report_exp_rdp(rdp);
355 rcu_tasks_qs(current, preempt);357 rcu_tasks_qs(current, preempt);
356 trace_rcu_utilization(TPS("End context switch"));358 trace_rcu_utilization(TPS("End context switch"));
@@ -477,7 +479,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) · 120 unmodified lines
477 */479 */
478 special = t->rcu_read_unlock_special;480 special = t->rcu_read_unlock_special;
479 rdp = this_cpu_ptr(&rcu_data);481 rdp = this_cpu_ptr(&rcu_data);
480 if (!special.s && !rdp->exp_deferred_qs) {482 if (!special.s && !rdp->cpu_no_qs.b.exp) {
481 local_irq_restore(flags);483 local_irq_restore(flags);
482 return;484 return;
483 }485 }
@@ -497,7 +499,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) · 13 unmodified lines
497 * tasks are handled when removing the task from the499 * tasks are handled when removing the task from the
498 * blocked-tasks list below.500 * blocked-tasks list below.
499 */501 */
500 if (rdp->exp_deferred_qs)502 if (rdp->cpu_no_qs.b.exp)
501 rcu_report_exp_rdp(rdp);503 rcu_report_exp_rdp(rdp);
502 504
503 /* Clean up if blocked during RCU read-side critical section. */505 /* Clean up if blocked during RCU read-side critical section. */
@@ -580,7 +582,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) · 76 unmodified lines
580 */582 */
581static bool rcu_preempt_need_deferred_qs(struct task_struct *t)583static bool rcu_preempt_need_deferred_qs(struct task_struct *t)
582{584{
583 return (__this_cpu_read(rcu_data.exp_deferred_qs) ||585 return (__this_cpu_read(rcu_data.cpu_no_qs.b.exp) ||
584 READ_ONCE(t->rcu_read_unlock_special.s)) &&586 READ_ONCE(t->rcu_read_unlock_special.s)) &&
585 rcu_preempt_depth() == 0;587 rcu_preempt_depth() == 0;
586}588}
@@ -642,7 +644,7 @@ static void rcu_read_unlock_special(struct task_struct *t) · 55 unmodified lines
642 (IS_ENABLED(CONFIG_RCU_BOOST) && irqs_were_disabled &&644 (IS_ENABLED(CONFIG_RCU_BOOST) && irqs_were_disabled &&
643 t->rcu_blocked_node);645 t->rcu_blocked_node);
644 // Need to defer quiescent state until everything is enabled.646 // Need to defer quiescent state until everything is enabled.
645 if (use_softirq && (in_irq() || (expboost && !irqs_were_disabled))) {647 if (use_softirq && (in_hardirq() || (expboost && !irqs_were_disabled))) {
646 // Using softirq, safe to awaken, and either the648 // Using softirq, safe to awaken, and either the
647 // wakeup is free or there is either an expedited649 // wakeup is free or there is either an expedited
648 // GP in flight or a potential need to deboost.650 // GP in flight or a potential need to deboost.
@@ -845,10 +847,8 @@ static void rcu_qs(void) · 196 unmodified lines
845 trace_rcu_grace_period(TPS("rcu_sched"),847 trace_rcu_grace_period(TPS("rcu_sched"),
846 __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs"));848 __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs"));
847 __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false);849 __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false);
848 if (!__this_cpu_read(rcu_data.cpu_no_qs.b.exp))850 if (__this_cpu_read(rcu_data.cpu_no_qs.b.exp))
849 return;851 rcu_report_exp_rdp(this_cpu_ptr(&rcu_data));
850 __this_cpu_write(rcu_data.cpu_no_qs.b.exp, false);
851 rcu_report_exp_rdp(this_cpu_ptr(&rcu_data));
852}852}
853 853
854/*854/*
@@ -925,7 +925,18 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t) · 70 unmodified lines
925{925{
926 return false;926 return false;
927}927}
928static void rcu_preempt_deferred_qs(struct task_struct *t) { }928
929// Except that we do need to respond to a request by an expedited grace
930// period for a quiescent state from this CPU. Note that requests from
931// tasks are handled when removing the task from the blocked-tasks list
932// below.
933static void rcu_preempt_deferred_qs(struct task_struct *t)
934{
935 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
936
937 if (rdp->cpu_no_qs.b.exp)
938 rcu_report_exp_rdp(rdp);
939}
929 940
930/*941/*
931 * Because there is no preemptible RCU, there can be no readers blocked,942 * Because there is no preemptible RCU, there can be no readers blocked,
@@ -1153,7 +1164,6 @@ static void rcu_preempt_boost_start_gp(struct rcu_node *rnp) · 221 unmodified lines
1153/*1164/*
1154 * Create an RCU-boost kthread for the specified node if one does not1165 * Create an RCU-boost kthread for the specified node if one does not
1155 * already exist. We only create this kthread for preemptible RCU.1166 * already exist. We only create this kthread for preemptible RCU.
1156 * Returns zero if all is well, a negated errno otherwise.
1157 */1167 */
1158static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp)1168static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp)
1159{1169{
@@ -1204,8 +1214,9 @@ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu) · 44 unmodified lines
1204 if ((mask & leaf_node_cpu_bit(rnp, cpu)) &&1214 if ((mask & leaf_node_cpu_bit(rnp, cpu)) &&
1205 cpu != outgoingcpu)1215 cpu != outgoingcpu)
1206 cpumask_set_cpu(cpu, cm);1216 cpumask_set_cpu(cpu, cm);
1217 cpumask_and(cm, cm, housekeeping_cpumask(HK_FLAG_RCU));
1207 if (cpumask_weight(cm) == 0)1218 if (cpumask_weight(cm) == 0)
1208 cpumask_setall(cm);1219 cpumask_copy(cm, housekeeping_cpumask(HK_FLAG_RCU));
1209 set_cpus_allowed_ptr(t, cm);1220 set_cpus_allowed_ptr(t, cm);
1210 free_cpumask_var(cm);1221 free_cpumask_var(cm);
1211}1222}
@@ -1253,201 +1264,6 @@ static void __init rcu_spawn_boost_kthreads(void) · 41 unmodified lines
1253 1264
1254#endif /* #else #ifdef CONFIG_RCU_BOOST */1265#endif /* #else #ifdef CONFIG_RCU_BOOST */
1255 1266
1256#if !defined(CONFIG_RCU_FAST_NO_HZ)
1257
1258/*
1259 * Check to see if any future non-offloaded RCU-related work will need
1260 * to be done by the current CPU, even if none need be done immediately,
1261 * returning 1 if so. This function is part of the RCU implementation;
1262 * it is -not- an exported member of the RCU API.
1263 *
1264 * Because we not have RCU_FAST_NO_HZ, just check whether or not this
1265 * CPU has RCU callbacks queued.
1266 */
1267int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1268{
1269 *nextevt = KTIME_MAX;
1270 return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) &&
1271 !rcu_rdp_is_offloaded(this_cpu_ptr(&rcu_data));
1272}
1273
1274/*
1275 * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
1276 * after it.
1277 */
1278static void rcu_cleanup_after_idle(void)
1279{
1280}
1281
1282/*
1283 * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n,
1284 * is nothing.
1285 */
1286static void rcu_prepare_for_idle(void)
1287{
1288}
1289
1290#else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1291
1292/*
1293 * This code is invoked when a CPU goes idle, at which point we want
1294 * to have the CPU do everything required for RCU so that it can enter
1295 * the energy-efficient dyntick-idle mode.
1296 *
1297 * The following preprocessor symbol controls this:
1298 *
1299 * RCU_IDLE_GP_DELAY gives the number of jiffies that a CPU is permitted
1300 * to sleep in dyntick-idle mode with RCU callbacks pending. This
1301 * is sized to be roughly one RCU grace period. Those energy-efficiency
1302 * benchmarkers who might otherwise be tempted to set this to a large
1303 * number, be warned: Setting RCU_IDLE_GP_DELAY too high can hang your
1304 * system. And if you are -that- concerned about energy efficiency,
1305 * just power the system down and be done with it!
1306 *
1307 * The value below works well in practice. If future workloads require
1308 * adjustment, they can be converted into kernel config parameters, though
1309 * making the state machine smarter might be a better option.
1310 */
1311#define RCU_IDLE_GP_DELAY 4 /* Roughly one grace period. */
1312
1313static int rcu_idle_gp_delay = RCU_IDLE_GP_DELAY;
1314module_param(rcu_idle_gp_delay, int, 0644);
1315
1316/*
1317 * Try to advance callbacks on the current CPU, but only if it has been
1318 * awhile since the last time we did so. Afterwards, if there are any
1319 * callbacks ready for immediate invocation, return true.
1320 */
1321static bool __maybe_unused rcu_try_advance_all_cbs(void)
1322{
1323 bool cbs_ready = false;
1324 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1325 struct rcu_node *rnp;
1326
1327 /* Exit early if we advanced recently. */
1328 if (jiffies == rdp->last_advance_all)
1329 return false;
1330 rdp->last_advance_all = jiffies;
1331
1332 rnp = rdp->mynode;
1333
1334 /*
1335 * Don't bother checking unless a grace period has
1336 * completed since we last checked and there are
1337 * callbacks not yet ready to invoke.
1338 */
1339 if ((rcu_seq_completed_gp(rdp->gp_seq,
1340 rcu_seq_current(&rnp->gp_seq)) ||
1341 unlikely(READ_ONCE(rdp->gpwrap))) &&
1342 rcu_segcblist_pend_cbs(&rdp->cblist))
1343 note_gp_changes(rdp);
1344
1345 if (rcu_segcblist_ready_cbs(&rdp->cblist))
1346 cbs_ready = true;
1347 return cbs_ready;
1348}
1349
1350/*
1351 * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready
1352 * to invoke. If the CPU has callbacks, try to advance them. Tell the
1353 * caller about what to set the timeout.
1354 *
1355 * The caller must have disabled interrupts.
1356 */
1357int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1358{
1359 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1360 unsigned long dj;
1361
1362 lockdep_assert_irqs_disabled();
1363
1364 /* If no non-offloaded callbacks, RCU doesn't need the CPU. */
1365 if (rcu_segcblist_empty(&rdp->cblist) ||
1366 rcu_rdp_is_offloaded(rdp)) {
1367 *nextevt = KTIME_MAX;
1368 return 0;
1369 }
1370
1371 /* Attempt to advance callbacks. */
1372 if (rcu_try_advance_all_cbs()) {
1373 /* Some ready to invoke, so initiate later invocation. */
1374 invoke_rcu_core();
1375 return 1;
1376 }
1377 rdp->last_accelerate = jiffies;
1378
1379 /* Request timer and round. */
1380 dj = round_up(rcu_idle_gp_delay + jiffies, rcu_idle_gp_delay) - jiffies;
1381
1382 *nextevt = basemono + dj * TICK_NSEC;
1383 return 0;
1384}
1385
1386/*
1387 * Prepare a CPU for idle from an RCU perspective. The first major task is to
1388 * sense whether nohz mode has been enabled or disabled via sysfs. The second
1389 * major task is to accelerate (that is, assign grace-period numbers to) any
1390 * recently arrived callbacks.
1391 *
1392 * The caller must have disabled interrupts.
1393 */
1394static void rcu_prepare_for_idle(void)
1395{
1396 bool needwake;
1397 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1398 struct rcu_node *rnp;
1399 int tne;
1400
1401 lockdep_assert_irqs_disabled();
1402 if (rcu_rdp_is_offloaded(rdp))
1403 return;
1404
1405 /* Handle nohz enablement switches conservatively. */
1406 tne = READ_ONCE(tick_nohz_active);
1407 if (tne != rdp->tick_nohz_enabled_snap) {
1408 if (!rcu_segcblist_empty(&rdp->cblist))
1409 invoke_rcu_core(); /* force nohz to see update. */
1410 rdp->tick_nohz_enabled_snap = tne;
1411 return;
1412 }
1413 if (!tne)
1414 return;
1415
1416 /*
1417 * If we have not yet accelerated this jiffy, accelerate all
1418 * callbacks on this CPU.
1419 */
1420 if (rdp->last_accelerate == jiffies)
1421 return;
1422 rdp->last_accelerate = jiffies;
1423 if (rcu_segcblist_pend_cbs(&rdp->cblist)) {
1424 rnp = rdp->mynode;
1425 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
1426 needwake = rcu_accelerate_cbs(rnp, rdp);
1427 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
1428 if (needwake)
1429 rcu_gp_kthread_wake();
1430 }
1431}
1432
1433/*
1434 * Clean up for exit from idle. Attempt to advance callbacks based on
1435 * any grace periods that elapsed while the CPU was idle, and if any
1436 * callbacks are now ready to invoke, initiate invocation.
1437 */
1438static void rcu_cleanup_after_idle(void)
1439{
1440 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1441
1442 lockdep_assert_irqs_disabled();
1443 if (rcu_rdp_is_offloaded(rdp))
1444 return;
1445 if (rcu_try_advance_all_cbs())
1446 invoke_rcu_core();
1447}
1448
1449#endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1450
1451/*1267/*
1452 * Is this CPU a NO_HZ_FULL CPU that should ignore RCU so that the1268 * Is this CPU a NO_HZ_FULL CPU that should ignore RCU so that the
1453 * grace-period kthread will do force_quiescent_state() processing?1269 * grace-period kthread will do force_quiescent_state() processing?
@@ -1455,7 +1271,7 @@ static void rcu_cleanup_after_idle(void) · 1 unmodified line
1455 * CPU unless the grace period has extended for too long.1271 * CPU unless the grace period has extended for too long.
1456 *1272 *
1457 * This code relies on the fact that all NO_HZ_FULL CPUs are also1273 * This code relies on the fact that all NO_HZ_FULL CPUs are also
1458 * CONFIG_RCU_NOCB_CPU CPUs.1274 * RCU_NOCB_CPU CPUs.
1459 */1275 */
1460static bool rcu_nohz_full_cpu(void)1276static bool rcu_nohz_full_cpu(void)
1461{1277{
kernel/rcu/tree_stall.h+2−25
@@ -347,26 +347,6 @@ static void rcu_dump_cpu_stacks(void) · 346 unmodified lines
347 }347 }
348}348}
349 349
350#ifdef CONFIG_RCU_FAST_NO_HZ
351
352static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
353{
354 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
355
356 sprintf(cp, "last_accelerate: %04lx/%04lx dyntick_enabled: %d",
357 rdp->last_accelerate & 0xffff, jiffies & 0xffff,
358 !!rdp->tick_nohz_enabled_snap);
359}
360
361#else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
362
363static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
364{
365 *cp = '\0';
366}
367
368#endif /* #else #ifdef CONFIG_RCU_FAST_NO_HZ */
369
370static const char * const gp_state_names[] = {350static const char * const gp_state_names[] = {
371 [RCU_GP_IDLE] = "RCU_GP_IDLE",351 [RCU_GP_IDLE] = "RCU_GP_IDLE",
372 [RCU_GP_WAIT_GPS] = "RCU_GP_WAIT_GPS",352 [RCU_GP_WAIT_GPS] = "RCU_GP_WAIT_GPS",
@@ -408,13 +388,12 @@ static bool rcu_is_gp_kthread_starving(unsigned long *jp) · 35 unmodified lines
408 * of RCU grace periods that this CPU is ignorant of, for example, "1"388 * of RCU grace periods that this CPU is ignorant of, for example, "1"
409 * if the CPU was aware of the previous grace period.389 * if the CPU was aware of the previous grace period.
410 *390 *
411 * Also print out idle and (if CONFIG_RCU_FAST_NO_HZ) idle-entry info.391 * Also print out idle info.
412 */392 */
413static void print_cpu_stall_info(int cpu)393static void print_cpu_stall_info(int cpu)
414{394{
415 unsigned long delta;395 unsigned long delta;
416 bool falsepositive;396 bool falsepositive;
417 char fast_no_hz[72];
418 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);397 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
419 char *ticks_title;398 char *ticks_title;
420 unsigned long ticks_value;399 unsigned long ticks_value;
@@ -432,11 +411,10 @@ static void print_cpu_stall_info(int cpu) · 11 unmodified lines
432 ticks_title = "ticks this GP";411 ticks_title = "ticks this GP";
433 ticks_value = rdp->ticks_this_gp;412 ticks_value = rdp->ticks_this_gp;
434 }413 }
435 print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
436 delta = rcu_seq_ctr(rdp->mynode->gp_seq - rdp->rcu_iw_gp_seq);414 delta = rcu_seq_ctr(rdp->mynode->gp_seq - rdp->rcu_iw_gp_seq);
437 falsepositive = rcu_is_gp_kthread_starving(NULL) &&415 falsepositive = rcu_is_gp_kthread_starving(NULL) &&
438 rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp));416 rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp));
439 pr_err("\t%d-%c%c%c%c: (%lu %s) idle=%03x/%ld/%#lx softirq=%u/%u fqs=%ld %s%s\n",417 pr_err("\t%d-%c%c%c%c: (%lu %s) idle=%03x/%ld/%#lx softirq=%u/%u fqs=%ld %s\n",
440 cpu,418 cpu,
441 "O."[!!cpu_online(cpu)],419 "O."[!!cpu_online(cpu)],
442 "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],420 "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],
@@ -449,7 +427,6 @@ static void print_cpu_stall_info(int cpu) · 6 unmodified lines
449 rdp->dynticks_nesting, rdp->dynticks_nmi_nesting,427 rdp->dynticks_nesting, rdp->dynticks_nmi_nesting,
450 rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),428 rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
451 data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart,429 data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart,
452 fast_no_hz,
453 falsepositive ? " (false positive?)" : "");430 falsepositive ? " (false positive?)" : "");
454}431}
455 432
kernel/scftorture.c+6−10
@@ -38,14 +38,10 @@ · 37 unmodified lines
38#define SCFTORT_STRING "scftorture"38#define SCFTORT_STRING "scftorture"
39#define SCFTORT_FLAG SCFTORT_STRING ": "39#define SCFTORT_FLAG SCFTORT_STRING ": "
40 40
41#define SCFTORTOUT(s, x...) \
42 pr_alert(SCFTORT_FLAG s, ## x)
43
44#define VERBOSE_SCFTORTOUT(s, x...) \41#define VERBOSE_SCFTORTOUT(s, x...) \
45 do { if (verbose) pr_alert(SCFTORT_FLAG s, ## x); } while (0)42 do { if (verbose) pr_alert(SCFTORT_FLAG s "\n", ## x); } while (0)
46 43
47#define VERBOSE_SCFTORTOUT_ERRSTRING(s, x...) \44#define SCFTORTOUT_ERRSTRING(s, x...) pr_alert(SCFTORT_FLAG "!!! " s "\n", ## x)
48 do { if (verbose) pr_alert(SCFTORT_FLAG "!!! " s, ## x); } while (0)
49 45
50MODULE_LICENSE("GPL");46MODULE_LICENSE("GPL");
51MODULE_AUTHOR("Paul E. McKenney <paulmck@kernel.org>");47MODULE_AUTHOR("Paul E. McKenney <paulmck@kernel.org>");
@@ -587,14 +583,14 @@ static int __init scf_torture_init(void) · 535 unmodified lines
587 if (weight_resched1 == 0 && weight_single1 == 0 && weight_single_rpc1 == 0 &&583 if (weight_resched1 == 0 && weight_single1 == 0 && weight_single_rpc1 == 0 &&
588 weight_single_wait1 == 0 && weight_many1 == 0 && weight_many_wait1 == 0 &&584 weight_single_wait1 == 0 && weight_many1 == 0 && weight_many_wait1 == 0 &&
589 weight_all1 == 0 && weight_all_wait1 == 0) {585 weight_all1 == 0 && weight_all_wait1 == 0) {
590 VERBOSE_SCFTORTOUT_ERRSTRING("all zero weights makes no sense");586 SCFTORTOUT_ERRSTRING("all zero weights makes no sense");
591 firsterr = -EINVAL;587 firsterr = -EINVAL;
592 goto unwind;588 goto unwind;
593 }589 }
594 if (IS_BUILTIN(CONFIG_SCF_TORTURE_TEST))590 if (IS_BUILTIN(CONFIG_SCF_TORTURE_TEST))
595 scf_sel_add(weight_resched1, SCF_PRIM_RESCHED, false);591 scf_sel_add(weight_resched1, SCF_PRIM_RESCHED, false);
596 else if (weight_resched1)592 else if (weight_resched1)
597 VERBOSE_SCFTORTOUT_ERRSTRING("built as module, weight_resched ignored");593 SCFTORTOUT_ERRSTRING("built as module, weight_resched ignored");
598 scf_sel_add(weight_single1, SCF_PRIM_SINGLE, false);594 scf_sel_add(weight_single1, SCF_PRIM_SINGLE, false);
599 scf_sel_add(weight_single_rpc1, SCF_PRIM_SINGLE_RPC, true);595 scf_sel_add(weight_single_rpc1, SCF_PRIM_SINGLE_RPC, true);
600 scf_sel_add(weight_single_wait1, SCF_PRIM_SINGLE, true);596 scf_sel_add(weight_single_wait1, SCF_PRIM_SINGLE, true);
@@ -625,12 +621,12 @@ static int __init scf_torture_init(void) · 24 unmodified lines
625 nthreads = num_online_cpus();621 nthreads = num_online_cpus();
626 scf_stats_p = kcalloc(nthreads, sizeof(scf_stats_p[0]), GFP_KERNEL);622 scf_stats_p = kcalloc(nthreads, sizeof(scf_stats_p[0]), GFP_KERNEL);
627 if (!scf_stats_p) {623 if (!scf_stats_p) {
628 VERBOSE_SCFTORTOUT_ERRSTRING("out of memory");624 SCFTORTOUT_ERRSTRING("out of memory");
629 firsterr = -ENOMEM;625 firsterr = -ENOMEM;
630 goto unwind;626 goto unwind;
631 }627 }
632 628
633 VERBOSE_SCFTORTOUT("Starting %d smp_call_function() threads\n", nthreads);629 VERBOSE_SCFTORTOUT("Starting %d smp_call_function() threads", nthreads);
634 630
635 atomic_set(&n_started, nthreads);631 atomic_set(&n_started, nthreads);
636 for (i = 0; i < nthreads; i++) {632 for (i = 0; i < nthreads; i++) {
kernel/torture.c+2−2
@@ -570,7 +570,7 @@ int torture_shuffle_init(long shuffint) · 569 unmodified lines
570 shuffle_idle_cpu = -1;570 shuffle_idle_cpu = -1;
571 571
572 if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {572 if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
573 VERBOSE_TOROUT_ERRSTRING("Failed to alloc mask");573 TOROUT_ERRSTRING("Failed to alloc mask");
574 return -ENOMEM;574 return -ENOMEM;
575 }575 }
576 576
@@ -934,7 +934,7 @@ int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m, · 357 unmodified lines
934 *tp = kthread_run(fn, arg, "%s", s);934 *tp = kthread_run(fn, arg, "%s", s);
935 if (IS_ERR(*tp)) {935 if (IS_ERR(*tp)) {
936 ret = PTR_ERR(*tp);936 ret = PTR_ERR(*tp);
937 VERBOSE_TOROUT_ERRSTRING(f);937 TOROUT_ERRSTRING(f);
938 *tp = NULL;938 *tp = NULL;
939 }939 }
940 torture_shuffle_task_register(*tp);940 torture_shuffle_task_register(*tp);