History

Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull RCU updates from Ingo Molnar: "The main RCU related changes in this cycle were: - Additional cleanups after RCU flavor consolidation - Grace-period forward-progress cleanups and improvements - Documentation updates - Miscellaneous fixes - spin_is_locked() conversions to lockdep - SPDX changes to RCU source and header files - SRCU updates - Torture-test updates, including nolibc updates and moving nolibc to tools/include" * 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (71 commits) locking/locktorture: Convert to SPDX license identifier linux/torture: Convert to SPDX license identifier torture: Convert to SPDX license identifier linux/srcu: Convert to SPDX license identifier linux/rcutree: Convert to SPDX license identifier linux/rcutiny: Convert to SPDX license identifier linux/rcu_sync: Convert to SPDX license identifier linux/rcu_segcblist: Convert to SPDX license identifier linux/rcupdate: Convert to SPDX license identifier linux/rcu_node_tree: Convert to SPDX license identifier rcu/update: Convert to SPDX license identifier rcu/tree: Convert to SPDX license identifier rcu/tiny: Convert to SPDX license identifier rcu/sync: Convert to SPDX license identifier rcu/srcu: Convert to SPDX license identifier rcu/rcutorture: Convert to SPDX license identifier rcu/rcu_segcblist: Convert to SPDX license identifier rcu/rcuperf: Convert to SPDX license identifier rcu/rcu.h: Convert to SPDX license identifier RCU/torture.txt: Remove section MODULE PARAMETERS ...

Linus Torvalds committed 7 years ago

Commit 3717f612 parents b1b988a cae45e163 files · +1395 −1268Browse files

Unified · Split

63 files
  1. Documentation/RCU/Design/Expedited-Grace-Periods/ExpSchedFlow.svg +11 −7
  2. Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.html +13 −13
  3. Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.html +3 −3
  4. Documentation/RCU/Design/Memory-Ordering/TreeRCU-callback-invocation.svg +1 −1
  5. Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg +4 −4
  6. Documentation/RCU/Design/Memory-Ordering/TreeRCU-qs.svg +3 −3
  7. Documentation/RCU/Design/Requirements/Requirements.html +13 −7
  8. Documentation/RCU/stallwarn.txt +8 −7
  9. Documentation/RCU/torture.txt +2 −167
  10. Documentation/RCU/whatisRCU.txt +2 −2
  11. Documentation/admin-guide/kernel-parameters.txt +19 −13
  12. MAINTAINERS +6 −0
  13. include/asm-generic/bug.h +0 −3
  14. include/linux/rcu_node_tree.h +2 −15
  15. include/linux/rcu_segcblist.h +2 −15
  16. include/linux/rcu_sync.h +1 −14
  17. include/linux/rcupdate.h +13 −78
  18. include/linux/rcutiny.h +2 −15
  19. include/linux/rcutree.h +3 −16
  20. include/linux/srcu.h +3 −15
  21. include/linux/srcutiny.h +2 −15
  22. include/linux/srcutree.h +4 −16
  23. include/linux/torture.h +4 −16
  24. kernel/locking/locktorture.c +4 −17
  25. kernel/rcu/rcu.h +3 −18
  26. kernel/rcu/rcu_segcblist.c +2 −15
  27. kernel/rcu/rcu_segcblist.h +2 −15
  28. kernel/rcu/rcuperf.c +10 −17
  29. kernel/rcu/rcutorture.c +35 −24
  30. kernel/rcu/srcutiny.c +2 −15
  31. kernel/rcu/srcutree.c +26 −46
  32. kernel/rcu/sync.c +1 −14
  33. kernel/rcu/tiny.c +3 −16
  34. kernel/rcu/tree.c +162 −105
  35. kernel/rcu/tree.h +14 −39
  36. kernel/rcu/tree_exp.h +71 −130
  37. kernel/rcu/tree_plugin.h +57 −181
  38. kernel/rcu/update.c +2 −15
  39. kernel/sched/cpufreq.c +2 −2
  40. kernel/sched/cpufreq_schedutil.c +1 −1
  41. kernel/sched/sched.h +1 −1
  42. kernel/sched/topology.c +2 −2
  43. kernel/time/timer.c +1 −1
  44. kernel/torture.c +8 −17
  45. tools/include/nolibc/nolibc.h +92 −26
  46. tools/memory-model/.gitignore +1 −0
  47. tools/memory-model/README +2 −0
  48. tools/memory-model/linux-kernel.bell +2 −1
  49. tools/memory-model/linux-kernel.cat +3 −1
  50. tools/memory-model/linux-kernel.def +1 −0
  51. tools/memory-model/scripts/README +70 −0
  52. tools/memory-model/scripts/checkalllitmus.sh +23 −30
  53. tools/memory-model/scripts/checkghlitmus.sh +65 −0
  54. tools/memory-model/scripts/checklitmus.sh +11 −63
  55. tools/memory-model/scripts/checklitmushist.sh +60 −0
  56. tools/memory-model/scripts/cmplitmushist.sh +87 −0
  57. tools/memory-model/scripts/initlitmushist.sh +68 −0
  58. tools/memory-model/scripts/judgelitmus.sh +78 −0
  59. tools/memory-model/scripts/newlitmushist.sh +61 −0
  60. tools/memory-model/scripts/parseargs.sh +136 −0
  61. tools/memory-model/scripts/runlitmushist.sh +87 −0
  62. tools/testing/selftests/rcutorture/bin/mkinitrd.sh +17 −10
  63. virt/kvm/kvm_main.c +1 −1
Documentation/RCU/Design/Expedited-Grace-Periods/ExpSchedFlow.svg+11−7
@@ -328,13 +328,13 @@ · 327 unmodified lines
328328 inkscape:window-height="1148"
329329 id="namedview90"
330330 showgrid="true"
331− inkscape:zoom="0.80021373"
332− inkscape:cx="462.49289"
333− inkscape:cy="473.6718"
331+ inkscape:zoom="0.69092787"
332+ inkscape:cx="476.34085"
333+ inkscape:cy="712.80957"
334334 inkscape:window-x="770"
335335 inkscape:window-y="24"
336336 inkscape:window-maximized="0"
337− inkscape:current-layer="g4114-9-3-9"
337+ inkscape:current-layer="g4"
338338 inkscape:snap-grids="false"
339339 fit-margin-top="5"
340340 fit-margin-right="5"
@@ -813,14 +813,18 @@ · 472 unmodified lines
813813 <text
814814 sodipodi:linespacing="125%"
815815 id="text4110-5-7-6-2-4-0"
816− y="841.88086"
816+ y="670.74316"
817817 x="1460.1007"
818818 style="font-size:267.24359131px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
819819 xml:space="preserve"><tspan
820− y="841.88086"
820+ y="670.74316"
821+ x="1460.1007"
822+ sodipodi:role="line"
823+ id="tspan4925-1-2-4-5">Request</tspan><tspan
824+ y="1004.7976"
821825 x="1460.1007"
822826 sodipodi:role="line"
823− id="tspan4925-1-2-4-5">reched_cpu()</tspan></text>
827+ id="tspan3100">context switch</tspan></text>
824828 </g>
825829 </g>
826830 </svg>
Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.html+13−13
@@ -72,10 +72,10 @@ will ignore it because idle and offline CPUs are already residing · 71 unmodified lines
7272 in quiescent states.
7373 Otherwise, the expedited grace period will use
7474 <tt>smp_call_function_single()</tt> to send the CPU an IPI, which
75−is handled by <tt>sync_rcu_exp_handler()</tt>.
75+is handled by <tt>rcu_exp_handler()</tt>.
7676
7777 <p>
78−However, because this is preemptible RCU, <tt>sync_rcu_exp_handler()</tt>
78+However, because this is preemptible RCU, <tt>rcu_exp_handler()</tt>
7979 can check to see if the CPU is currently running in an RCU read-side
8080 critical section.
8181 If not, the handler can immediately report a quiescent state.
@@ -145,19 +145,18 @@ expedited grace period is shown in the following diagram: · 63 unmodified lines
145145 <p><img src="ExpSchedFlow.svg" alt="ExpSchedFlow.svg" width="55%">
146146
147147 <p>
148−As with RCU-preempt's <tt>synchronize_rcu_expedited()</tt>,
148+As with RCU-preempt, RCU-sched's
149149 <tt>synchronize_sched_expedited()</tt> ignores offline and
150150 idle CPUs, again because they are in remotely detectable
151151 quiescent states.
152−However, the <tt>synchronize_rcu_expedited()</tt> handler
153−is <tt>sync_sched_exp_handler()</tt>, and because the
152+However, because the
154153 <tt>rcu_read_lock_sched()</tt> and <tt>rcu_read_unlock_sched()</tt>
155154 leave no trace of their invocation, in general it is not possible to tell
156155 whether or not the current CPU is in an RCU read-side critical section.
157−The best that <tt>sync_sched_exp_handler()</tt> can do is to check
156+The best that RCU-sched's <tt>rcu_exp_handler()</tt> can do is to check
158157 for idle, on the off-chance that the CPU went idle while the IPI
159158 was in flight.
160−If the CPU is idle, then <tt>sync_sched_exp_handler()</tt> reports
159+If the CPU is idle, then <tt>rcu_exp_handler()</tt> reports
161160 the quiescent state.
162161
163162 <p> Otherwise, the handler forces a future context switch by setting the
@@ -298,19 +297,18 @@ Instead, the task pushing the grace period forward will include the · 134 unmodified lines
298297 idle CPUs in the mask passed to <tt>rcu_report_exp_cpu_mult()</tt>.
299298
300299 <p>
301−For RCU-sched, there is an additional check for idle in the IPI
302−handler, <tt>sync_sched_exp_handler()</tt>.
300+For RCU-sched, there is an additional check:
303301 If the IPI has interrupted the idle loop, then
304−<tt>sync_sched_exp_handler()</tt> invokes <tt>rcu_report_exp_rdp()</tt>
302+<tt>rcu_exp_handler()</tt> invokes <tt>rcu_report_exp_rdp()</tt>
305303 to report the corresponding quiescent state.
306304
307305 <p>
308306 For RCU-preempt, there is no specific check for idle in the
309−IPI handler (<tt>sync_rcu_exp_handler()</tt>), but because
307+IPI handler (<tt>rcu_exp_handler()</tt>), but because
310308 RCU read-side critical sections are not permitted within the
311−idle loop, if <tt>sync_rcu_exp_handler()</tt> sees that the CPU is within
309+idle loop, if <tt>rcu_exp_handler()</tt> sees that the CPU is within
312310 RCU read-side critical section, the CPU cannot possibly be idle.
313−Otherwise, <tt>sync_rcu_exp_handler()</tt> invokes
311+Otherwise, <tt>rcu_exp_handler()</tt> invokes
314312 <tt>rcu_report_exp_rdp()</tt> to report the corresponding quiescent
315313 state, regardless of whether or not that quiescent state was due to
316314 the CPU being idle.
@@ -625,6 +623,8 @@ checks, but only during the mid-boot dead zone. · 308 unmodified lines
625623 <p>
626624 With this refinement, synchronous grace periods can now be used from
627625 task context pretty much any time during the life of the kernel.
626+That is, aside from some points in the suspend, hibernate, or shutdown
627+code path.
628628
629629 <h3><a name="Summary">
630630 Summary</a></h3>
Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.html+3−3
@@ -485,13 +485,13 @@ section that the grace period must wait on. · 484 unmodified lines
485485 noted by <tt>rcu_node_context_switch()</tt> on the left.
486486 On the other hand, if the CPU takes a scheduler-clock interrupt
487487 while executing in usermode, a quiescent state will be noted by
488−<tt>rcu_check_callbacks()</tt> on the right.
488+<tt>rcu_sched_clock_irq()</tt> on the right.
489489 Either way, the passage through a quiescent state will be noted
490490 in a per-CPU variable.
491491
492492 <p>The next time an <tt>RCU_SOFTIRQ</tt> handler executes on
493493 this CPU (for example, after the next scheduler-clock
494−interrupt), <tt>__rcu_process_callbacks()</tt> will invoke
494+interrupt), <tt>rcu_core()</tt> will invoke
495495 <tt>rcu_check_quiescent_state()</tt>, which will notice the
496496 recorded quiescent state, and invoke
497497 <tt>rcu_report_qs_rdp()</tt>.
@@ -651,7 +651,7 @@ to end. · 153 unmodified lines
651651 These callbacks are identified by <tt>rcu_advance_cbs()</tt>,
652652 which is usually invoked by <tt>__note_gp_changes()</tt>.
653653 As shown in the diagram below, this invocation can be triggered by
654−the scheduling-clock interrupt (<tt>rcu_check_callbacks()</tt> on
654+the scheduling-clock interrupt (<tt>rcu_sched_clock_irq()</tt> on
655655 the left) or by idle entry (<tt>rcu_cleanup_after_idle()</tt> on
656656 the right, but only for kernels build with
657657 <tt>CONFIG_RCU_FAST_NO_HZ=y</tt>).
Documentation/RCU/Design/Memory-Ordering/TreeRCU-callback-invocation.svg+1−1
@@ -349,7 +349,7 @@ · 348 unmodified lines
349349 font-weight="bold"
350350 font-size="192"
351351 id="text202-7-5"
352− style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">rcu_check_callbacks()</text>
352+ style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">rcu_sched_clock_irq()</text>
353353 <rect
354354 x="7069.6187"
355355 y="5087.4678"
Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg+4−4
@@ -3902,7 +3902,7 @@ · 3901 unmodified lines
39023902 font-style="normal"
39033903 y="-4418.6582"
39043904 x="3745.7725"
3905− xml:space="preserve">rcu_check_callbacks()</text>
3905+ xml:space="preserve">rcu_sched_clock_irq()</text>
39063906 </g>
39073907 <g
39083908 transform="translate(-850.30204,55463.106)"
@@ -3924,7 +3924,7 @@ · 15 unmodified lines
39243924 font-style="normal"
39253925 y="-4418.6582"
39263926 x="3745.7725"
3927− xml:space="preserve">rcu_process_callbacks()</text>
3927+ xml:space="preserve">rcu_core()</text>
39283928 <text
39293929 style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"
39303930 id="text202-7-5-3-27-0"
@@ -3933,7 +3933,7 @@ · 2 unmodified lines
39333933 font-style="normal"
39343934 y="-4165.7954"
39353935 x="3745.7725"
3936− xml:space="preserve">rcu_check_quiescent_state())</text>
3936+ xml:space="preserve">rcu_check_quiescent_state()</text>
39373937 <text
39383938 style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"
39393939 id="text202-7-5-3-27-0-9"
@@ -4968,7 +4968,7 @@ · 1028 unmodified lines
49684968 font-weight="bold"
49694969 font-size="192"
49704970 id="text202-7-5-19"
4971− style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">rcu_check_callbacks()</text>
4971+ style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">rcu_sched_clock_irq()</text>
49724972 <rect
49734973 x="5314.2671"
49744974 y="82817.688"
Documentation/RCU/Design/Memory-Ordering/TreeRCU-qs.svg+3−3
@@ -775,7 +775,7 @@ · 774 unmodified lines
775775 font-style="normal"
776776 y="-4418.6582"
777777 x="3745.7725"
778− xml:space="preserve">rcu_check_callbacks()</text>
778+ xml:space="preserve">rcu_sched_clock_irq()</text>
779779 </g>
780780 <g
781781 transform="translate(399.7744,828.86448)"
@@ -797,7 +797,7 @@ · 15 unmodified lines
797797 font-style="normal"
798798 y="-4418.6582"
799799 x="3745.7725"
800− xml:space="preserve">rcu_process_callbacks()</text>
800+ xml:space="preserve">rcu_core()</text>
801801 <text
802802 style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"
803803 id="text202-7-5-3-27-0"
@@ -806,7 +806,7 @@ · 2 unmodified lines
806806 font-style="normal"
807807 y="-4165.7954"
808808 x="3745.7725"
809− xml:space="preserve">rcu_check_quiescent_state())</text>
809+ xml:space="preserve">rcu_check_quiescent_state()</text>
810810 <text
811811 style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"
812812 id="text202-7-5-3-27-0-9"
Documentation/RCU/Design/Requirements/Requirements.html+13−7
@@ -3099,7 +3099,7 @@ If you block forever in one of a given domain's SRCU read-side critical · 3098 unmodified lines
30993099 sections, then that domain's grace periods will also be blocked forever.
31003100 Of course, one good way to block forever is to deadlock, which can
31013101 happen if any operation in a given domain's SRCU read-side critical
3102−section can block waiting, either directly or indirectly, for that domain's
3102+section can wait, either directly or indirectly, for that domain's
31033103 grace period to elapse.
31043104 For example, this results in a self-deadlock:
31053105
@@ -3139,12 +3139,18 @@ API, which, in combination with <tt>srcu_read_unlock()</tt>, · 33 unmodified lines
31393139 guarantees a full memory barrier.
31403140
31413141 <p>
3142−Also unlike other RCU flavors, SRCU's callbacks-wait function
3143−<tt>srcu_barrier()</tt> may be invoked from CPU-hotplug notifiers,
3144−though this is not necessarily a good idea.
3145−The reason that this is possible is that SRCU is insensitive
3146−to whether or not a CPU is online, which means that <tt>srcu_barrier()</tt>
3147−need not exclude CPU-hotplug operations.
3142+Also unlike other RCU flavors, <tt>synchronize_srcu()</tt> may <b>not</b>
3143+be invoked from CPU-hotplug notifiers, due to the fact that SRCU grace
3144+periods make use of timers and the possibility of timers being temporarily
3145+&ldquo;stranded&rdquo; on the outgoing CPU.
3146+This stranding of timers means that timers posted to the outgoing CPU
3147+will not fire until late in the CPU-hotplug process.
3148+The problem is that if a notifier is waiting on an SRCU grace period,
3149+that grace period is waiting on a timer, and that timer is stranded on the
3150+outgoing CPU, then the notifier will never be awakened, in other words,
3151+deadlock has occurred.
3152+This same situation of course also prohibits <tt>srcu_barrier()</tt>
3153+from being invoked from CPU-hotplug notifiers.
31483154
31493155 <p>
31503156 SRCU also differs from other RCU flavors in that SRCU's expedited and
Documentation/RCU/stallwarn.txt+8−7
@@ -219,17 +219,18 @@ an estimate of the total number of RCU callbacks queued across all CPUs · 218 unmodified lines
219219 In kernels with CONFIG_RCU_FAST_NO_HZ, more information is printed
220220 for each CPU:
221221
222− 0: (64628 ticks this GP) idle=dd5/3fffffffffffffff/0 softirq=82/543 last_accelerate: a345/d342 nonlazy_posted: 25 .D
222+ 0: (64628 ticks this GP) idle=dd5/3fffffffffffffff/0 softirq=82/543 last_accelerate: a345/d342 Nonlazy posted: ..D
223223
224224 The "last_accelerate:" prints the low-order 16 bits (in hex) of the
225225 jiffies counter when this CPU last invoked rcu_try_advance_all_cbs()
226226 from rcu_needs_cpu() or last invoked rcu_accelerate_cbs() from
227−rcu_prepare_for_idle(). The "nonlazy_posted:" prints the number
228−of non-lazy callbacks posted since the last call to rcu_needs_cpu().
229−Finally, an "L" indicates that there are currently no non-lazy callbacks
230−("." is printed otherwise, as shown above) and "D" indicates that
231−dyntick-idle processing is enabled ("." is printed otherwise, for example,
232−if disabled via the "nohz=" kernel boot parameter).
227+rcu_prepare_for_idle(). The "Nonlazy posted:" indicates lazy-callback
228+status, so that an "l" indicates that all callbacks were lazy at the start
229+of the last idle period and an "L" indicates that there are currently
230+no non-lazy callbacks (in both cases, "." is printed otherwise, as
231+shown above) and "D" indicates that dyntick-idle processing is enabled
232+("." is printed otherwise, for example, if disabled via the "nohz="
233+kernel boot parameter).
233234
234235 If the grace period ends just as the stall warning starts printing,
235236 there will be a spurious stall-warning message, which will include
Documentation/RCU/torture.txt+2−167
@@ -10,173 +10,8 @@ status messages via printk(), which can be examined via the dmesg · 9 unmodified lines
1010 command (perhaps grepping for "torture"). The test is started
1111 when the module is loaded, and stops when the module is unloaded.
1212
13−
14−MODULE PARAMETERS
15−
16−This module has the following parameters:
17−
18−fqs_duration Duration (in microseconds) of artificially induced bursts
19− of force_quiescent_state() invocations. In RCU
20− implementations having force_quiescent_state(), these
21− bursts help force races between forcing a given grace
22− period and that grace period ending on its own.
23−
24−fqs_holdoff Holdoff time (in microseconds) between consecutive calls
25− to force_quiescent_state() within a burst.
26−
27−fqs_stutter Wait time (in seconds) between consecutive bursts
28− of calls to force_quiescent_state().
29−
30−gp_normal Make the fake writers use normal synchronous grace-period
31− primitives.
32−
33−gp_exp Make the fake writers use expedited synchronous grace-period
34− primitives. If both gp_normal and gp_exp are set, or
35− if neither gp_normal nor gp_exp are set, then randomly
36− choose the primitive so that about 50% are normal and
37− 50% expedited. By default, neither are set, which
38− gives best overall test coverage.
39−
40−irqreader Says to invoke RCU readers from irq level. This is currently
41− done via timers. Defaults to "1" for variants of RCU that
42− permit this. (Or, more accurately, variants of RCU that do
43− -not- permit this know to ignore this variable.)
44−
45−n_barrier_cbs If this is nonzero, RCU barrier testing will be conducted,
46− in which case n_barrier_cbs specifies the number of
47− RCU callbacks (and corresponding kthreads) to use for
48− this testing. The value cannot be negative. If you
49− specify this to be non-zero when torture_type indicates a
50− synchronous RCU implementation (one for which a member of
51− the synchronize_rcu() rather than the call_rcu() family is
52− used -- see the documentation for torture_type below), an
53− error will be reported and no testing will be carried out.
54−
55−nfakewriters This is the number of RCU fake writer threads to run. Fake
56− writer threads repeatedly use the synchronous "wait for
57− current readers" function of the interface selected by
58− torture_type, with a delay between calls to allow for various
59− different numbers of writers running in parallel.
60− nfakewriters defaults to 4, which provides enough parallelism
61− to trigger special cases caused by multiple writers, such as
62− the synchronize_srcu() early return optimization.
63−
64−nreaders This is the number of RCU reading threads supported.
65− The default is twice the number of CPUs. Why twice?
66− To properly exercise RCU implementations with preemptible
67− read-side critical sections.
68−
69−onoff_interval
70− The number of seconds between each attempt to execute a
71− randomly selected CPU-hotplug operation. Defaults to
72− zero, which disables CPU hotplugging. In HOTPLUG_CPU=n
73− kernels, rcutorture will silently refuse to do any
74− CPU-hotplug operations regardless of what value is
75− specified for onoff_interval.
76−
77−onoff_holdoff The number of seconds to wait until starting CPU-hotplug
78− operations. This would normally only be used when
79− rcutorture was built into the kernel and started
80− automatically at boot time, in which case it is useful
81− in order to avoid confusing boot-time code with CPUs
82− coming and going.
83−
84−shuffle_interval
85− The number of seconds to keep the test threads affinitied
86− to a particular subset of the CPUs, defaults to 3 seconds.
87− Used in conjunction with test_no_idle_hz.
88−
89−shutdown_secs The number of seconds to run the test before terminating
90− the test and powering off the system. The default is
91− zero, which disables test termination and system shutdown.
92− This capability is useful for automated testing.
93−
94−stall_cpu The number of seconds that a CPU should be stalled while
95− within both an rcu_read_lock() and a preempt_disable().
96− This stall happens only once per rcutorture run.
97− If you need multiple stalls, use modprobe and rmmod to
98− repeatedly run rcutorture. The default for stall_cpu
99− is zero, which prevents rcutorture from stalling a CPU.
100−
101− Note that attempts to rmmod rcutorture while the stall
102− is ongoing will hang, so be careful what value you
103− choose for this module parameter! In addition, too-large
104− values for stall_cpu might well induce failures and
105− warnings in other parts of the kernel. You have been
106− warned!
107−
108−stall_cpu_holdoff
109− The number of seconds to wait after rcutorture starts
110− before stalling a CPU. Defaults to 10 seconds.
111−
112−stat_interval The number of seconds between output of torture
113− statistics (via printk()). Regardless of the interval,
114− statistics are printed when the module is unloaded.
115− Setting the interval to zero causes the statistics to
116− be printed -only- when the module is unloaded, and this
117− is the default.
118−
119−stutter The length of time to run the test before pausing for this
120− same period of time. Defaults to "stutter=5", so as
121− to run and pause for (roughly) five-second intervals.
122− Specifying "stutter=0" causes the test to run continuously
123− without pausing, which is the old default behavior.
124−
125−test_boost Whether or not to test the ability of RCU to do priority
126− boosting. Defaults to "test_boost=1", which performs
127− RCU priority-inversion testing only if the selected
128− RCU implementation supports priority boosting. Specifying
129− "test_boost=0" never performs RCU priority-inversion
130− testing. Specifying "test_boost=2" performs RCU
131− priority-inversion testing even if the selected RCU
132− implementation does not support RCU priority boosting,
133− which can be used to test rcutorture's ability to
134− carry out RCU priority-inversion testing.
135−
136−test_boost_interval
137− The number of seconds in an RCU priority-inversion test
138− cycle. Defaults to "test_boost_interval=7". It is
139− usually wise for this value to be relatively prime to
140− the value selected for "stutter".
141−
142−test_boost_duration
143− The number of seconds to do RCU priority-inversion testing
144− within any given "test_boost_interval". Defaults to
145− "test_boost_duration=4".
146−
147−test_no_idle_hz Whether or not to test the ability of RCU to operate in
148− a kernel that disables the scheduling-clock interrupt to
149− idle CPUs. Boolean parameter, "1" to test, "0" otherwise.
150− Defaults to omitting this test.
151−
152−torture_type The type of RCU to test, with string values as follows:
153−
154− "rcu": rcu_read_lock(), rcu_read_unlock() and call_rcu(),
155− along with expedited, synchronous, and polling
156− variants.
157−
158− "rcu_bh": rcu_read_lock_bh(), rcu_read_unlock_bh(), and
159− call_rcu_bh(), along with expedited and synchronous
160− variants.
161−
162− "rcu_busted": This tests an intentionally incorrect version
163− of RCU in order to help test rcutorture itself.
164−
165− "srcu": srcu_read_lock(), srcu_read_unlock() and
166− call_srcu(), along with expedited and
167− synchronous variants.
168−
169− "sched": preempt_disable(), preempt_enable(), and
170− call_rcu_sched(), along with expedited,
171− synchronous, and polling variants.
172−
173− "tasks": voluntary context switch and call_rcu_tasks(),
174− along with expedited and synchronous variants.
175−
176− Defaults to "rcu".
177−
178−verbose Enable debug printk()s. Default is disabled.
179−
13+Module parameters are prefixed by "rcutorture." in
14+Documentation/admin-guide/kernel-parameters.txt.
18015
18116 OUTPUT
18217
Documentation/RCU/whatisRCU.txt+2−2
@@ -302,7 +302,7 @@ rcu_dereference() · 301 unmodified lines
302302 must prohibit. The rcu_dereference_protected() variant takes
303303 a lockdep expression to indicate which locks must be acquired
304304 by the caller. If the indicated protection is not provided,
305− a lockdep splat is emitted. See RCU/Design/Requirements.html
305+ a lockdep splat is emitted. See RCU/Design/Requirements/Requirements.html
306306 and the API's code comments for more details and example usage.
307307
308308 The following diagram shows how each API communicates among the
@@ -560,7 +560,7 @@ presents two such "toy" implementations of RCU, one that is implemented · 251 unmodified lines
560560 in terms of familiar locking primitives, and another that more closely
561561 resembles "classic" RCU. Both are way too simple for real-world use,
562562 lacking both functionality and performance. However, they are useful
563−in getting a feel for how RCU works. See kernel/rcupdate.c for a
563+in getting a feel for how RCU works. See kernel/rcu/update.c for a
564564 production-quality implementation, and see:
565565
566566 http://www.rdrop.com/users/paulmck/RCU
Documentation/admin-guide/kernel-parameters.txt+19−13
@@ -3658,19 +3658,6 @@ · 3657 unmodified lines
36583658 latencies, which will choose a value aligned
36593659 with the appropriate hardware boundaries.
36603660
3661− rcutree.jiffies_till_sched_qs= [KNL]
3662− Set required age in jiffies for a
3663− given grace period before RCU starts
3664− soliciting quiescent-state help from
3665− rcu_note_context_switch(). If not specified, the
3666− kernel will calculate a value based on the most
3667− recent settings of rcutree.jiffies_till_first_fqs
3668− and rcutree.jiffies_till_next_fqs.
3669− This calculated value may be viewed in
3670− rcutree.jiffies_to_sched_qs. Any attempt to
3671− set rcutree.jiffies_to_sched_qs will be
3672− cheerfully overwritten.
3673−
36743661 rcutree.jiffies_till_first_fqs= [KNL]
36753662 Set delay from grace-period initialization to
36763663 first attempt to force quiescent states.
@@ -3682,6 +3669,20 @@ · 5 unmodified lines
36823669 quiescent states. Units are jiffies, minimum
36833670 value is one, and maximum value is HZ.
36843671
3672+ rcutree.jiffies_till_sched_qs= [KNL]
3673+ Set required age in jiffies for a
3674+ given grace period before RCU starts
3675+ soliciting quiescent-state help from
3676+ rcu_note_context_switch() and cond_resched().
3677+ If not specified, the kernel will calculate
3678+ a value based on the most recent settings
3679+ of rcutree.jiffies_till_first_fqs
3680+ and rcutree.jiffies_till_next_fqs.
3681+ This calculated value may be viewed in
3682+ rcutree.jiffies_to_sched_qs. Any attempt to set
3683+ rcutree.jiffies_to_sched_qs will be cheerfully
3684+ overwritten.
3685+
36853686 rcutree.kthread_prio= [KNL,BOOT]
36863687 Set the SCHED_FIFO priority of the RCU per-CPU
36873688 kthreads (rcuc/N). This value is also used for
@@ -3725,6 +3726,11 @@ · 37 unmodified lines
37253726 This wake_up() will be accompanied by a
37263727 WARN_ONCE() splat and an ftrace_dump().
37273728
3729+ rcutree.sysrq_rcu= [KNL]
3730+ Commandeer a sysrq key to dump out Tree RCU's
3731+ rcu_node tree with an eye towards determining
3732+ why a new grace period has not yet started.
3733+
37283734 rcuperf.gp_async= [KNL]
37293735 Measure performance of asynchronous
37303736 grace-period primitives such as call_rcu().
MAINTAINERS+6−0
@@ -10827,6 +10827,12 @@ F: drivers/power/supply/bq27xxx_battery_i2c.c · 10826 unmodified lines
1082710827 F: drivers/power/supply/isp1704_charger.c
1082810828 F: drivers/power/supply/rx51_battery.c
1082910829
10830+NOLIBC HEADER FILE
10831+M: Willy Tarreau <w@1wt.eu>
10832+S: Maintained
10833+T: git git://git.kernel.org/pub/scm/linux/kernel/git/wtarreau/nolibc.git
10834+F: tools/include/nolibc/
10835+
1083010836 NTB AMD DRIVER
1083110837 M: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
1083210838 L: linux-ntb@googlegroups.com
include/asm-generic/bug.h+0−3
@@ -211,9 +211,6 @@ void __warn(const char *file, int line, void *caller, unsigned taint, · 210 unmodified lines
211211 /*
212212 * WARN_ON_SMP() is for cases that the warning is either
213213 * meaningless for !SMP or may even cause failures.
214− * This is usually used for cases that we have
215− * WARN_ON(!spin_is_locked(&lock)) checks, as spin_is_locked()
216− * returns 0 for uniprocessor settings.
217214 * It can also be used with values that are only defined
218215 * on SMP:
219216 *
include/linux/rcu_node_tree.h+2−15
@@ -1,3 +1,4 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * RCU node combining tree definitions. These are used to compute
34 * global attributes while avoiding common-case global contention. A key
@@ -11,23 +12,9 @@ · 7 unmodified lines
1112 * because the size of the TREE SRCU srcu_struct structure depends
1213 * on these definitions.
1314 *
14− * This program is free software; you can redistribute it and/or modify
15− * it under the terms of the GNU General Public License as published by
16− * the Free Software Foundation; either version 2 of the License, or
17− * (at your option) any later version.
18− *
19− * This program is distributed in the hope that it will be useful,
20− * but WITHOUT ANY WARRANTY; without even the implied warranty of
21− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22− * GNU General Public License for more details.
23− *
24− * You should have received a copy of the GNU General Public License
25− * along with this program; if not, you can access it online at
26− * http://www.gnu.org/licenses/gpl-2.0.html.
27− *
2815 * Copyright IBM Corporation, 2017
2916 *
30− * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
17+ * Author: Paul E. McKenney <paulmck@linux.ibm.com>
3118 */
3219
3320 #ifndef __LINUX_RCU_NODE_TREE_H
include/linux/rcu_segcblist.h+2−15
@@ -1,3 +1,4 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * RCU segmented callback lists
34 *
@@ -5,23 +6,9 @@ · 1 unmodified line
56 * because the size of the TREE SRCU srcu_struct structure depends
67 * on these definitions.
78 *
8− * This program is free software; you can redistribute it and/or modify
9− * it under the terms of the GNU General Public License as published by
10− * the Free Software Foundation; either version 2 of the License, or
11− * (at your option) any later version.
12− *
13− * This program is distributed in the hope that it will be useful,
14− * but WITHOUT ANY WARRANTY; without even the implied warranty of
15− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16− * GNU General Public License for more details.
17− *
18− * You should have received a copy of the GNU General Public License
19− * along with this program; if not, you can access it online at
20− * http://www.gnu.org/licenses/gpl-2.0.html.
21− *
229 * Copyright IBM Corporation, 2017
2310 *
24− * Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
11+ * Authors: Paul E. McKenney <paulmck@linux.net.ibm.com>
2512 */
2613
2714 #ifndef __INCLUDE_LINUX_RCU_SEGCBLIST_H
include/linux/rcu_sync.h+1−14
@@ -1,20 +1,7 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * RCU-based infrastructure for lightweight reader-writer locking
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright (c) 2015, Red Hat, Inc.
196 *
207 * Author: Oleg Nesterov <oleg@redhat.com>
include/linux/rcupdate.h+13−78
@@ -1,25 +1,12 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * Read-Copy Update mechanism for mutual exclusion
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright IBM Corporation, 2001
196 *
207 * Author: Dipankar Sarma <dipankar@in.ibm.com>
218 *
22− * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
9+ * Based on the original work by Paul McKenney <paulmck@vnet.ibm.com>
2310 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
2411 * Papers:
2512 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
@@ -89,7 +76,7 @@ static inline int rcu_preempt_depth(void) · 63 unmodified lines
8976 /* Internal to kernel */
9077 void rcu_init(void);
9178 extern int rcu_scheduler_active __read_mostly;
92−void rcu_check_callbacks(int user);
79+void rcu_sched_clock_irq(int user);
9380 void rcu_report_dead(unsigned int cpu);
9481 void rcutree_migrate_callbacks(int cpu);
9582
@@ -309,16 +296,16 @@ static inline void rcu_preempt_sleep_check(void) { } · 213 unmodified lines
309296 */
310297
311298 #ifdef __CHECKER__
312−#define rcu_dereference_sparse(p, space) \
299+#define rcu_check_sparse(p, space) \
313300 ((void)(((typeof(*p) space *)p) == p))
314301 #else /* #ifdef __CHECKER__ */
315−#define rcu_dereference_sparse(p, space)
302+#define rcu_check_sparse(p, space)
316303 #endif /* #else #ifdef __CHECKER__ */
317304
318305 #define __rcu_access_pointer(p, space) \
319306 ({ \
320307 typeof(*p) *_________p1 = (typeof(*p) *__force)READ_ONCE(p); \
321− rcu_dereference_sparse(p, space); \
308+ rcu_check_sparse(p, space); \
322309 ((typeof(*p) __force __kernel *)(_________p1)); \
323310 })
324311 #define __rcu_dereference_check(p, c, space) \
@@ -326,13 +313,13 @@ static inline void rcu_preempt_sleep_check(void) { } · 1 unmodified line
326313 /* Dependency order vs. p above. */ \
327314 typeof(*p) *________p1 = (typeof(*p) *__force)READ_ONCE(p); \
328315 RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \
329− rcu_dereference_sparse(p, space); \
316+ rcu_check_sparse(p, space); \
330317 ((typeof(*p) __force __kernel *)(________p1)); \
331318 })
332319 #define __rcu_dereference_protected(p, c, space) \
333320 ({ \
334321 RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \
335− rcu_dereference_sparse(p, space); \
322+ rcu_check_sparse(p, space); \
336323 ((typeof(*p) __force __kernel *)(p)); \
337324 })
338325 #define rcu_dereference_raw(p) \
@@ -382,6 +369,7 @@ static inline void rcu_preempt_sleep_check(void) { } · 43 unmodified lines
382369 #define rcu_assign_pointer(p, v) \
383370 ({ \
384371 uintptr_t _r_a_p__v = (uintptr_t)(v); \
372+ rcu_check_sparse(p, __rcu); \
385373 \
386374 if (__builtin_constant_p(v) && (_r_a_p__v) == (uintptr_t)NULL) \
387375 WRITE_ONCE((p), (typeof(p))(_r_a_p__v)); \
@@ -785,7 +773,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) · 397 unmodified lines
785773 */
786774 #define RCU_INIT_POINTER(p, v) \
787775 do { \
788− rcu_dereference_sparse(p, __rcu); \
776+ rcu_check_sparse(p, __rcu); \
789777 WRITE_ONCE(p, RCU_INITIALIZER(v)); \
790778 } while (0)
791779
@@ -859,7 +847,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) · 67 unmodified lines
859847
860848 /* Has the specified rcu_head structure been handed to call_rcu()? */
861849
862−/*
850+/**
863851 * rcu_head_init - Initialize rcu_head for rcu_head_after_call_rcu()
864852 * @rhp: The rcu_head structure to initialize.
865853 *
@@ -874,10 +862,10 @@ static inline void rcu_head_init(struct rcu_head *rhp) · 8 unmodified lines
874862 rhp->func = (rcu_callback_t)~0L;
875863 }
876864
877−/*
865+/**
878866 * rcu_head_after_call_rcu - Has this rcu_head been passed to call_rcu()?
879867 * @rhp: The rcu_head structure to test.
880− * @func: The function passed to call_rcu() along with @rhp.
868+ * @f: The function passed to call_rcu() along with @rhp.
881869 *
882870 * Returns @true if the @rhp has been passed to call_rcu() with @func,
883871 * and @false otherwise. Emits a warning in any other case, including
@@ -896,57 +884,4 @@ rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f) · 12 unmodified lines
896884 return false;
897885 }
898886
899−
900−/* Transitional pre-consolidation compatibility definitions. */
901−
902−static inline void synchronize_rcu_bh(void)
903−{
904− synchronize_rcu();
905−}
906−
907−static inline void synchronize_rcu_bh_expedited(void)
908−{
909− synchronize_rcu_expedited();
910−}
911−
912−static inline void call_rcu_bh(struct rcu_head *head, rcu_callback_t func)
913−{
914− call_rcu(head, func);
915−}
916−
917−static inline void rcu_barrier_bh(void)
918−{
919− rcu_barrier();
920−}
921−
922−static inline void synchronize_sched(void)
923−{
924− synchronize_rcu();
925−}
926−
927−static inline void synchronize_sched_expedited(void)
928−{
929− synchronize_rcu_expedited();
930−}
931−
932−static inline void call_rcu_sched(struct rcu_head *head, rcu_callback_t func)
933−{
934− call_rcu(head, func);
935−}
936−
937−static inline void rcu_barrier_sched(void)
938−{
939− rcu_barrier();
940−}
941−
942−static inline unsigned long get_state_synchronize_sched(void)
943−{
944− return get_state_synchronize_rcu();
945−}
946−
947−static inline void cond_synchronize_sched(unsigned long oldstate)
948−{
949− cond_synchronize_rcu(oldstate);
950−}
951−
952887 #endif /* __LINUX_RCUPDATE_H */
include/linux/rcutiny.h+2−15
@@ -1,23 +1,10 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition.
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright IBM Corporation, 2008
196 *
20− * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
7+ * Author: Paul E. McKenney <paulmck@linux.ibm.com>
218 *
229 * For detailed explanation of Read-Copy Update mechanism see -
2310 * Documentation/RCU
include/linux/rcutree.h+3−16
@@ -1,26 +1,13 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright IBM Corporation, 2008
196 *
207 * Author: Dipankar Sarma <dipankar@in.ibm.com>
21− * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical algorithm
8+ * Paul E. McKenney <paulmck@linux.ibm.com> Hierarchical algorithm
229 *
23− * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
10+ * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
2411 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
2512 *
2613 * For detailed explanation of Read-Copy Update mechanism see -
include/linux/srcu.h+3−15
@@ -1,24 +1,11 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * Sleepable Read-Copy Update mechanism for mutual exclusion
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright (C) IBM Corporation, 2006
196 * Copyright (C) Fujitsu, 2012
207 *
21− * Author: Paul McKenney <paulmck@us.ibm.com>
8+ * Author: Paul McKenney <paulmck@linux.ibm.com>
229 * Lai Jiangshan <laijs@cn.fujitsu.com>
2310 *
2411 * For detailed explanation of Read-Copy Update mechanism see -
@@ -223,6 +210,7 @@ srcu_read_lock_notrace(struct srcu_struct *ssp) __acquires(ssp) · 198 unmodified lines
223210 static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)
224211 __releases(ssp)
225212 {
213+ WARN_ON_ONCE(idx & ~0x1);
226214 rcu_lock_release(&(ssp)->dep_map);
227215 __srcu_read_unlock(ssp, idx);
228216 }
include/linux/srcutiny.h+2−15
@@ -1,24 +1,11 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * Sleepable Read-Copy Update mechanism for mutual exclusion,
34 * tiny variant.
45 *
5− * This program is free software; you can redistribute it and/or modify
6− * it under the terms of the GNU General Public License as published by
7− * the Free Software Foundation; either version 2 of the License, or
8− * (at your option) any later version.
9− *
10− * This program is distributed in the hope that it will be useful,
11− * but WITHOUT ANY WARRANTY; without even the implied warranty of
12− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13− * GNU General Public License for more details.
14− *
15− * You should have received a copy of the GNU General Public License
16− * along with this program; if not, you can access it online at
17− * http://www.gnu.org/licenses/gpl-2.0.html.
18− *
196 * Copyright (C) IBM Corporation, 2017
207 *
21− * Author: Paul McKenney <paulmck@us.ibm.com>
8+ * Author: Paul McKenney <paulmck@linux.ibm.com>
229 */
2310
2411 #ifndef _LINUX_SRCU_TINY_H
include/linux/srcutree.h+4−16
@@ -1,24 +1,11 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * Sleepable Read-Copy Update mechanism for mutual exclusion,
34 * tree variant.
45 *
5− * This program is free software; you can redistribute it and/or modify
6− * it under the terms of the GNU General Public License as published by
7− * the Free Software Foundation; either version 2 of the License, or
8− * (at your option) any later version.
9− *
10− * This program is distributed in the hope that it will be useful,
11− * but WITHOUT ANY WARRANTY; without even the implied warranty of
12− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13− * GNU General Public License for more details.
14− *
15− * You should have received a copy of the GNU General Public License
16− * along with this program; if not, you can access it online at
17− * http://www.gnu.org/licenses/gpl-2.0.html.
18− *
196 * Copyright (C) IBM Corporation, 2017
207 *
21− * Author: Paul McKenney <paulmck@us.ibm.com>
8+ * Author: Paul McKenney <paulmck@linux.ibm.com>
229 */
2310
2411 #ifndef _LINUX_SRCU_TREE_H
@@ -45,7 +32,8 @@ struct srcu_data { · 20 unmodified lines
4532 unsigned long srcu_gp_seq_needed; /* Furthest future GP needed. */
4633 unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */
4734 bool srcu_cblist_invoking; /* Invoking these CBs? */
48− struct delayed_work work; /* Context for CB invoking. */
35+ struct timer_list delay_work; /* Delay for CB invoking */
36+ struct work_struct work; /* Context for CB invoking. */
4937 struct rcu_head srcu_barrier_head; /* For srcu_barrier() use. */
5038 struct srcu_node *mynode; /* Leaf srcu_node. */
5139 unsigned long grpmask; /* Mask for leaf srcu_node */
include/linux/torture.h+4−16
@@ -1,23 +1,10 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * Common functions for in-kernel torture tests.
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright IBM Corporation, 2014
196 *
20− * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
7+ * Author: Paul E. McKenney <paulmck@linux.ibm.com>
218 */
229
2310 #ifndef __LINUX_TORTURE_H
@@ -50,11 +37,12 @@ · 26 unmodified lines
5037 do { if (verbose) pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s); } while (0)
5138
5239 /* Definitions for online/offline exerciser. */
40+typedef void torture_ofl_func(void);
5341 bool torture_offline(int cpu, long *n_onl_attempts, long *n_onl_successes,
5442 unsigned long *sum_offl, int *min_onl, int *max_onl);
5543 bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes,
5644 unsigned long *sum_onl, int *min_onl, int *max_onl);
57−int torture_onoff_init(long ooholdoff, long oointerval);
45+int torture_onoff_init(long ooholdoff, long oointerval, torture_ofl_func *f);
5846 void torture_onoff_stats(void);
5947 bool torture_onoff_failures(void);
6048
kernel/locking/locktorture.c+4−17
@@ -1,23 +1,10 @@
1+// SPDX-License-Identifier: GPL-2.0+
12 /*
23 * Module-based torture test facility for locking
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright (C) IBM Corporation, 2014
196 *
20− * Authors: Paul E. McKenney <paulmck@us.ibm.com>
7+ * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
218 * Davidlohr Bueso <dave@stgolabs.net>
229 * Based on kernel/rcu/torture.c.
2310 */
@@ -45,7 +32,7 @@ · 21 unmodified lines
4532 #include <linux/torture.h>
4633
4734 MODULE_LICENSE("GPL");
48−MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
35+MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
4936
5037 torture_param(int, nwriters_stress, -1,
5138 "Number of write-locking stress-test threads");
@@ -970,7 +957,7 @@ static int __init lock_torture_init(void) · 918 unmodified lines
970957 /* Prepare torture context. */
971958 if (onoff_interval > 0) {
972959 firsterr = torture_onoff_init(onoff_holdoff * HZ,
973− onoff_interval * HZ);
960+ onoff_interval * HZ, NULL);
974961 if (firsterr)
975962 goto unwind;
976963 }
kernel/rcu/rcu.h+3−18
@@ -1,23 +1,10 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * Read-Copy Update definitions shared among RCU implementations.
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright IBM Corporation, 2011
196 *
20− * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
7+ * Author: Paul E. McKenney <paulmck@linux.ibm.com>
218 */
229
2310 #ifndef __LINUX_RCU_H
@@ -30,7 +17,7 @@ · 6 unmodified lines
3017 #define RCU_TRACE(stmt)
3118 #endif /* #else #ifdef CONFIG_RCU_TRACE */
3219
33−/* Offset to allow for unmatched rcu_irq_{enter,exit}(). */
20+/* Offset to allow distinguishing irq vs. task-based idle entry/exit. */
3421 #define DYNTICK_IRQ_NONIDLE ((LONG_MAX / 2) + 1)
3522
3623
@@ -462,8 +449,6 @@ void rcu_request_urgent_qs_task(struct task_struct *t); · 425 unmodified lines
462449
463450 enum rcutorture_type {
464451 RCU_FLAVOR,
465− RCU_BH_FLAVOR,
466− RCU_SCHED_FLAVOR,
467452 RCU_TASKS_FLAVOR,
468453 SRCU_FLAVOR,
469454 INVALID_RCU_FLAVOR
kernel/rcu/rcu_segcblist.c+2−15
@@ -1,23 +1,10 @@
1+// SPDX-License-Identifier: GPL-2.0+
12 /*
23 * RCU segmented callback lists, function definitions
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright IBM Corporation, 2017
196 *
20− * Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
7+ * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
218 */
229
2310 #include <linux/types.h>
kernel/rcu/rcu_segcblist.h+2−15
@@ -1,23 +1,10 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * RCU segmented callback lists, internal-to-rcu header file
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright IBM Corporation, 2017
196 *
20− * Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
7+ * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
218 */
229
2310 #include <linux/rcu_segcblist.h>
kernel/rcu/rcuperf.c+10−17
@@ -1,23 +1,10 @@
1+// SPDX-License-Identifier: GPL-2.0+
12 /*
23 * Read-Copy Update module-based performance-test facility
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright (C) IBM Corporation, 2015
196 *
20− * Authors: Paul E. McKenney <paulmck@us.ibm.com>
7+ * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
218 */
229
2310 #define pr_fmt(fmt) fmt
@@ -54,7 +41,7 @@ · 30 unmodified lines
5441 #include "rcu.h"
5542
5643 MODULE_LICENSE("GPL");
57−MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.vnet.ibm.com>");
44+MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
5845
5946 #define PERF_FLAG "-perf:"
6047 #define PERFOUT_STRING(s) \
@@ -83,13 +70,19 @@ MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.vnet.ibm.com>"); · 22 unmodified lines
8370 * Various other use cases may of course be specified.
8471 */
8572
73+#ifdef MODULE
74+# define RCUPERF_SHUTDOWN 0
75+#else
76+# define RCUPERF_SHUTDOWN 1
77+#endif
78+
8679 torture_param(bool, gp_async, false, "Use asynchronous GP wait primitives");
8780 torture_param(int, gp_async_max, 1000, "Max # outstanding waits per reader");
8881 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
8982 torture_param(int, holdoff, 10, "Holdoff time before test start (s)");
9083 torture_param(int, nreaders, -1, "Number of RCU reader threads");
9184 torture_param(int, nwriters, -1, "Number of RCU updater threads");
92−torture_param(bool, shutdown, !IS_ENABLED(MODULE),
85+torture_param(bool, shutdown, RCUPERF_SHUTDOWN,
9386 "Shutdown at end of performance tests.");
9487 torture_param(int, verbose, 1, "Enable verbose debugging printk()s");
9588 torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable");
kernel/rcu/rcutorture.c+35−24
@@ -1,23 +1,10 @@
1+// SPDX-License-Identifier: GPL-2.0+
12 /*
23 * Read-Copy Update module-based torture test facility
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright (C) IBM Corporation, 2005, 2006
196 *
20− * Authors: Paul E. McKenney <paulmck@us.ibm.com>
7+ * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
218 * Josh Triplett <josh@joshtriplett.org>
229 *
2310 * See also: Documentation/RCU/torture.txt
@@ -61,7 +48,7 @@ · 37 unmodified lines
6148 #include "rcu.h"
6249
6350 MODULE_LICENSE("GPL");
64−MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
51+MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
6552
6653
6754 /* Bits for ->extendables field, extendables param, and related definitions. */
@@ -1630,21 +1617,34 @@ static bool rcu_fwd_emergency_stop; · 1562 unmodified lines
16301617 #define MIN_FWD_CB_LAUNDERS 3 /* This many CB invocations to count. */
16311618 #define MIN_FWD_CBS_LAUNDERED 100 /* Number of counted CBs. */
16321619 #define FWD_CBS_HIST_DIV 10 /* Histogram buckets/second. */
1633−static long n_launders_hist[2 * MAX_FWD_CB_JIFFIES / (HZ / FWD_CBS_HIST_DIV)];
1620+struct rcu_launder_hist {
1621+ long n_launders;
1622+ unsigned long launder_gp_seq;
1623+};
1624+#define N_LAUNDERS_HIST (2 * MAX_FWD_CB_JIFFIES / (HZ / FWD_CBS_HIST_DIV))
1625+static struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST];
1626+static unsigned long rcu_launder_gp_seq_start;
16341627
16351628 static void rcu_torture_fwd_cb_hist(void)
16361629 {
1630+ unsigned long gps;
1631+ unsigned long gps_old;
16371632 int i;
16381633 int j;
16391634
16401635 for (i = ARRAY_SIZE(n_launders_hist) - 1; i > 0; i--)
1641− if (n_launders_hist[i] > 0)
1636+ if (n_launders_hist[i].n_launders > 0)
16421637 break;
16431638 pr_alert("%s: Callback-invocation histogram (duration %lu jiffies):",
16441639 __func__, jiffies - rcu_fwd_startat);
1645− for (j = 0; j <= i; j++)
1646− pr_cont(" %ds/%d: %ld",
1647− j + 1, FWD_CBS_HIST_DIV, n_launders_hist[j]);
1640+ gps_old = rcu_launder_gp_seq_start;
1641+ for (j = 0; j <= i; j++) {
1642+ gps = n_launders_hist[j].launder_gp_seq;
1643+ pr_cont(" %ds/%d: %ld:%ld",
1644+ j + 1, FWD_CBS_HIST_DIV, n_launders_hist[j].n_launders,
1645+ rcutorture_seq_diff(gps, gps_old));
1646+ gps_old = gps;
1647+ }
16481648 pr_cont("\n");
16491649 }
16501650
@@ -1666,7 +1666,8 @@ static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp) · 15 unmodified lines
16661666 i = ((jiffies - rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV));
16671667 if (i >= ARRAY_SIZE(n_launders_hist))
16681668 i = ARRAY_SIZE(n_launders_hist) - 1;
1669− n_launders_hist[i]++;
1669+ n_launders_hist[i].n_launders++;
1670+ n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq();
16701671 spin_unlock_irqrestore(&rcu_fwd_lock, flags);
16711672 }
16721673
@@ -1786,9 +1787,10 @@ static void rcu_torture_fwd_prog_cr(void) · 113 unmodified lines
17861787 n_max_cbs = 0;
17871788 n_max_gps = 0;
17881789 for (i = 0; i < ARRAY_SIZE(n_launders_hist); i++)
1789− n_launders_hist[i] = 0;
1790+ n_launders_hist[i].n_launders = 0;
17901791 cver = READ_ONCE(rcu_torture_current_version);
17911792 gps = cur_ops->get_gp_seq();
1793+ rcu_launder_gp_seq_start = gps;
17921794 while (time_before(jiffies, stopat) &&
17931795 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
17941796 rfcp = READ_ONCE(rcu_fwd_cb_head);
@@ -2228,6 +2230,14 @@ static void rcu_test_debug_objects(void) · 433 unmodified lines
22282230 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
22292231 }
22302232
2233+static void rcutorture_sync(void)
2234+{
2235+ static unsigned long n;
2236+
2237+ if (cur_ops->sync && !(++n & 0xfff))
2238+ cur_ops->sync();
2239+}
2240+
22312241 static int __init
22322242 rcu_torture_init(void)
22332243 {
@@ -2389,7 +2399,8 @@ rcu_torture_init(void) · 155 unmodified lines
23892399 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup);
23902400 if (firsterr)
23912401 goto unwind;
2392− firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval);
2402+ firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval,
2403+ rcutorture_sync);
23932404 if (firsterr)
23942405 goto unwind;
23952406 firsterr = rcu_torture_stall_init();
kernel/rcu/srcutiny.c+2−15
@@ -1,24 +1,11 @@
1+// SPDX-License-Identifier: GPL-2.0+
12 /*
23 * Sleepable Read-Copy Update mechanism for mutual exclusion,
34 * tiny version for non-preemptible single-CPU use.
45 *
5− * This program is free software; you can redistribute it and/or modify
6− * it under the terms of the GNU General Public License as published by
7− * the Free Software Foundation; either version 2 of the License, or
8− * (at your option) any later version.
9− *
10− * This program is distributed in the hope that it will be useful,
11− * but WITHOUT ANY WARRANTY; without even the implied warranty of
12− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13− * GNU General Public License for more details.
14− *
15− * You should have received a copy of the GNU General Public License
16− * along with this program; if not, you can access it online at
17− * http://www.gnu.org/licenses/gpl-2.0.html.
18− *
196 * Copyright (C) IBM Corporation, 2017
207 *
21− * Author: Paul McKenney <paulmck@us.ibm.com>
8+ * Author: Paul McKenney <paulmck@linux.ibm.com>
229 */
2310
2411 #include <linux/export.h>
kernel/rcu/srcutree.c+26−46
@@ -1,24 +1,11 @@
1+// SPDX-License-Identifier: GPL-2.0+
12 /*
23 * Sleepable Read-Copy Update mechanism for mutual exclusion.
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright (C) IBM Corporation, 2006
196 * Copyright (C) Fujitsu, 2012
207 *
21− * Author: Paul McKenney <paulmck@us.ibm.com>
8+ * Author: Paul McKenney <paulmck@linux.ibm.com>
229 * Lai Jiangshan <laijs@cn.fujitsu.com>
2310 *
2411 * For detailed explanation of Read-Copy Update mechanism see -
@@ -58,6 +45,7 @@ static bool __read_mostly srcu_init_done; · 33 unmodified lines
5845 static void srcu_invoke_callbacks(struct work_struct *work);
5946 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay);
6047 static void process_srcu(struct work_struct *work);
48+static void srcu_delay_timer(struct timer_list *t);
6149
6250 /* Wrappers for lock acquisition and release, see raw_spin_lock_rcu_node(). */
6351 #define spin_lock_rcu_node(p) \
@@ -156,7 +144,8 @@ static void init_srcu_struct_nodes(struct srcu_struct *ssp, bool is_static) · 92 unmodified lines
156144 snp->grphi = cpu;
157145 }
158146 sdp->cpu = cpu;
159− INIT_DELAYED_WORK(&sdp->work, srcu_invoke_callbacks);
147+ INIT_WORK(&sdp->work, srcu_invoke_callbacks);
148+ timer_setup(&sdp->delay_work, srcu_delay_timer, 0);
160149 sdp->ssp = ssp;
161150 sdp->grpmask = 1 << (cpu - sdp->mynode->grplo);
162151 if (is_static)
@@ -386,13 +375,19 @@ void _cleanup_srcu_struct(struct srcu_struct *ssp, bool quiesced) · 223 unmodified lines
386375 } else {
387376 flush_delayed_work(&ssp->work);
388377 }
389− for_each_possible_cpu(cpu)
378+ for_each_possible_cpu(cpu) {
379+ struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);
380+
390381 if (quiesced) {
391− if (WARN_ON(delayed_work_pending(&per_cpu_ptr(ssp->sda, cpu)->work)))
382+ if (WARN_ON(timer_pending(&sdp->delay_work)))
383+ return; /* Just leak it! */
384+ if (WARN_ON(work_pending(&sdp->work)))
392385 return; /* Just leak it! */
393386 } else {
394− flush_delayed_work(&per_cpu_ptr(ssp->sda, cpu)->work);
387+ del_timer_sync(&sdp->delay_work);
388+ flush_work(&sdp->work);
395389 }
390+ }
396391 if (WARN_ON(rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) != SRCU_STATE_IDLE) ||
397392 WARN_ON(srcu_readers_active(ssp))) {
398393 pr_info("%s: Active srcu_struct %p state: %d\n",
@@ -463,39 +458,23 @@ static void srcu_gp_start(struct srcu_struct *ssp) · 64 unmodified lines
463458 WARN_ON_ONCE(state != SRCU_STATE_SCAN1);
464459 }
465460
466−/*
467− * Track online CPUs to guide callback workqueue placement.
468− */
469−DEFINE_PER_CPU(bool, srcu_online);
470461
471−void srcu_online_cpu(unsigned int cpu)
462+static void srcu_delay_timer(struct timer_list *t)
472463 {
473− WRITE_ONCE(per_cpu(srcu_online, cpu), true);
474−}
464+ struct srcu_data *sdp = container_of(t, struct srcu_data, delay_work);
475465
476−void srcu_offline_cpu(unsigned int cpu)
477−{
478− WRITE_ONCE(per_cpu(srcu_online, cpu), false);
466+ queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work);
479467 }
480468
481−/*
482− * Place the workqueue handler on the specified CPU if online, otherwise
483− * just run it whereever. This is useful for placing workqueue handlers
484− * that are to invoke the specified CPU's callbacks.
485− */
486−static bool srcu_queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
487− struct delayed_work *dwork,
469+static void srcu_queue_delayed_work_on(struct srcu_data *sdp,
488470 unsigned long delay)
489471 {
490− bool ret;
472+ if (!delay) {
473+ queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work);
474+ return;
475+ }
491476
492− preempt_disable();
493− if (READ_ONCE(per_cpu(srcu_online, cpu)))
494− ret = queue_delayed_work_on(cpu, wq, dwork, delay);
495− else
496− ret = queue_delayed_work(wq, dwork, delay);
497− preempt_enable();
498− return ret;
477+ timer_reduce(&sdp->delay_work, jiffies + delay);
499478 }
500479
501480 /*
@@ -504,7 +483,7 @@ static bool srcu_queue_delayed_work_on(int cpu, struct workqueue_struct *wq, · 2 unmodified lines
504483 */
505484 static void srcu_schedule_cbs_sdp(struct srcu_data *sdp, unsigned long delay)
506485 {
507− srcu_queue_delayed_work_on(sdp->cpu, rcu_gp_wq, &sdp->work, delay);
486+ srcu_queue_delayed_work_on(sdp, delay);
508487 }
509488
510489 /*
@@ -1186,7 +1165,8 @@ static void srcu_invoke_callbacks(struct work_struct *work) · 675 unmodified lines
11861165 struct srcu_data *sdp;
11871166 struct srcu_struct *ssp;
11881167
1189− sdp = container_of(work, struct srcu_data, work.work);
1168+ sdp = container_of(work, struct srcu_data, work);
1169+
11901170 ssp = sdp->ssp;
11911171 rcu_cblist_init(&ready_cbs);
11921172 spin_lock_irq_rcu_node(sdp);
kernel/rcu/sync.c+1−14
@@ -1,20 +1,7 @@
1+// SPDX-License-Identifier: GPL-2.0+
12 /*
23 * RCU-based infrastructure for lightweight reader-writer locking
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright (c) 2015, Red Hat, Inc.
196 *
207 * Author: Oleg Nesterov <oleg@redhat.com>
kernel/rcu/tiny.c+3−16
@@ -1,23 +1,10 @@
1+// SPDX-License-Identifier: GPL-2.0+
12 /*
23 * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition.
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright IBM Corporation, 2008
196 *
20− * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
7+ * Author: Paul E. McKenney <paulmck@linux.ibm.com>
218 *
229 * For detailed explanation of Read-Copy Update mechanism see -
2310 * Documentation/RCU
@@ -76,7 +63,7 @@ void rcu_qs(void) · 52 unmodified lines
7663 * be called from hardirq context. It is normally called from the
7764 * scheduling-clock interrupt.
7865 */
79−void rcu_check_callbacks(int user)
66+void rcu_sched_clock_irq(int user)
8067 {
8168 if (user) {
8269 rcu_qs();
kernel/rcu/tree.c+162−105
@@ -1,27 +1,14 @@
1+// SPDX-License-Identifier: GPL-2.0+
12 /*
23 * Read-Copy Update mechanism for mutual exclusion
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright IBM Corporation, 2008
196 *
207 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
218 * Manfred Spraul <manfred@colorfullife.com>
22− * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version
9+ * Paul E. McKenney <paulmck@linux.ibm.com> Hierarchical version
2310 *
24− * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
11+ * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
2512 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
2613 *
2714 * For detailed explanation of Read-Copy Update mechanism see -
@@ -62,6 +49,7 @@ · 34 unmodified lines
6249 #include <linux/suspend.h>
6350 #include <linux/ftrace.h>
6451 #include <linux/tick.h>
52+#include <linux/sysrq.h>
6553
6654 #include "tree.h"
6755 #include "rcu.h"
@@ -115,6 +103,9 @@ int num_rcu_lvl[] = NUM_RCU_LVL_INIT; · 47 unmodified lines
115103 int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */
116104 /* panic() on RCU Stall sysctl. */
117105 int sysctl_panic_on_rcu_stall __read_mostly;
106+/* Commandeer a sysrq key to dump RCU's tree. */
107+static bool sysrq_rcu;
108+module_param(sysrq_rcu, bool, 0444);
118109
119110 /*
120111 * The rcu_scheduler_active variable is initialized to the value
@@ -479,7 +470,6 @@ module_param_cb(jiffies_till_next_fqs, &next_fqs_jiffies_ops, &jiffies_till_next · 358 unmodified lines
479470 module_param(rcu_kick_kthreads, bool, 0644);
480471
481472 static void force_qs_rnp(int (*f)(struct rcu_data *rdp));
482−static void force_quiescent_state(void);
483473 static int rcu_pending(void);
484474
485475 /*
@@ -504,13 +494,12 @@ unsigned long rcu_exp_batches_completed(void) · 18 unmodified lines
504494 EXPORT_SYMBOL_GPL(rcu_exp_batches_completed);
505495
506496 /*
507− * Force a quiescent state.
497+ * Return the root node of the rcu_state structure.
508498 */
509−void rcu_force_quiescent_state(void)
499+static struct rcu_node *rcu_get_root(void)
510500 {
511− force_quiescent_state();
501+ return &rcu_state.node[0];
512502 }
513−EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
514503
515504 /*
516505 * Convert a ->gp_state value to a character string.
@@ -529,19 +518,30 @@ void show_rcu_gp_kthreads(void) · 12 unmodified lines
529518 {
530519 int cpu;
531520 unsigned long j;
521+ unsigned long ja;
522+ unsigned long jr;
523+ unsigned long jw;
532524 struct rcu_data *rdp;
533525 struct rcu_node *rnp;
534526
535− j = jiffies - READ_ONCE(rcu_state.gp_activity);
536− pr_info("%s: wait state: %s(%d) ->state: %#lx delta ->gp_activity %ld\n",
527+ j = jiffies;
528+ ja = j - READ_ONCE(rcu_state.gp_activity);
529+ jr = j - READ_ONCE(rcu_state.gp_req_activity);
530+ jw = j - READ_ONCE(rcu_state.gp_wake_time);
531+ pr_info("%s: wait state: %s(%d) ->state: %#lx delta ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_flags %#x\n",
537532 rcu_state.name, gp_state_getname(rcu_state.gp_state),
538− rcu_state.gp_state, rcu_state.gp_kthread->state, j);
533+ rcu_state.gp_state,
534+ rcu_state.gp_kthread ? rcu_state.gp_kthread->state : 0x1ffffL,
535+ ja, jr, jw, (long)READ_ONCE(rcu_state.gp_wake_seq),
536+ (long)READ_ONCE(rcu_state.gp_seq),
537+ (long)READ_ONCE(rcu_get_root()->gp_seq_needed),
538+ READ_ONCE(rcu_state.gp_flags));
539539 rcu_for_each_node_breadth_first(rnp) {
540540 if (ULONG_CMP_GE(rcu_state.gp_seq, rnp->gp_seq_needed))
541541 continue;
542− pr_info("\trcu_node %d:%d ->gp_seq %lu ->gp_seq_needed %lu\n",
543− rnp->grplo, rnp->grphi, rnp->gp_seq,
544− rnp->gp_seq_needed);
542+ pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld\n",
543+ rnp->grplo, rnp->grphi, (long)rnp->gp_seq,
544+ (long)rnp->gp_seq_needed);
545545 if (!rcu_is_leaf_node(rnp))
546546 continue;
547547 for_each_leaf_node_possible_cpu(rnp, cpu) {
@@ -550,14 +550,35 @@ void show_rcu_gp_kthreads(void) · 2 unmodified lines
550550 ULONG_CMP_GE(rcu_state.gp_seq,
551551 rdp->gp_seq_needed))
552552 continue;
553− pr_info("\tcpu %d ->gp_seq_needed %lu\n",
554− cpu, rdp->gp_seq_needed);
553+ pr_info("\tcpu %d ->gp_seq_needed %ld\n",
554+ cpu, (long)rdp->gp_seq_needed);
555555 }
556556 }
557557 /* sched_show_task(rcu_state.gp_kthread); */
558558 }
559559 EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads);
560560
561+/* Dump grace-period-request information due to commandeered sysrq. */
562+static void sysrq_show_rcu(int key)
563+{
564+ show_rcu_gp_kthreads();
565+}
566+
567+static struct sysrq_key_op sysrq_rcudump_op = {
568+ .handler = sysrq_show_rcu,
569+ .help_msg = "show-rcu(y)",
570+ .action_msg = "Show RCU tree",
571+ .enable_mask = SYSRQ_ENABLE_DUMP,
572+};
573+
574+static int __init rcu_sysrq_init(void)
575+{
576+ if (sysrq_rcu)
577+ return register_sysrq_key('y', &sysrq_rcudump_op);
578+ return 0;
579+}
580+early_initcall(rcu_sysrq_init);
581+
561582 /*
562583 * Send along grace-period-related data for rcutorture diagnostics.
563584 */
@@ -566,8 +587,6 @@ void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags, · 2 unmodified lines
566587 {
567588 switch (test_type) {
568589 case RCU_FLAVOR:
569− case RCU_BH_FLAVOR:
570− case RCU_SCHED_FLAVOR:
571590 *flags = READ_ONCE(rcu_state.gp_flags);
572591 *gp_seq = rcu_seq_current(&rcu_state.gp_seq);
573592 break;
@@ -577,14 +596,6 @@ void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags, · 3 unmodified lines
577596 }
578597 EXPORT_SYMBOL_GPL(rcutorture_get_gp_data);
579598
580−/*
581− * Return the root node of the rcu_state structure.
582− */
583−static struct rcu_node *rcu_get_root(void)
584−{
585− return &rcu_state.node[0];
586−}
587−
588599 /*
589600 * Enter an RCU extended quiescent state, which can be either the
590601 * idle loop or adaptive-tickless usermode execution.
@@ -701,7 +712,6 @@ static __always_inline void rcu_nmi_exit_common(bool irq) · 110 unmodified lines
701712
702713 /**
703714 * rcu_nmi_exit - inform RCU of exit from NMI context
704− * @irq: Is this call from rcu_irq_exit?
705715 *
706716 * If you add or remove a call to rcu_nmi_exit(), be sure to test
707717 * with CONFIG_RCU_EQS_DEBUG=y.
@@ -1115,7 +1125,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) · 407 unmodified lines
11151125 }
11161126
11171127 /*
1118− * NO_HZ_FULL CPUs can run in-kernel without rcu_check_callbacks!
1128+ * NO_HZ_FULL CPUs can run in-kernel without rcu_sched_clock_irq!
11191129 * The above code handles this, but only for straight cond_resched().
11201130 * And some in-kernel loops check need_resched() before calling
11211131 * cond_resched(), which defeats the above code for CPUs that are
@@ -1181,7 +1191,7 @@ static void rcu_check_gp_kthread_starvation(void) · 59 unmodified lines
11811191 pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#lx ->cpu=%d\n",
11821192 rcu_state.name, j,
11831193 (long)rcu_seq_current(&rcu_state.gp_seq),
1184− rcu_state.gp_flags,
1194+ READ_ONCE(rcu_state.gp_flags),
11851195 gp_state_getname(rcu_state.gp_state), rcu_state.gp_state,
11861196 gpk ? gpk->state : ~0, gpk ? task_cpu(gpk) : -1);
11871197 if (gpk) {
@@ -1310,7 +1320,7 @@ static void print_other_cpu_stall(unsigned long gp_seq) · 122 unmodified lines
13101320
13111321 panic_on_rcu_stall();
13121322
1313− force_quiescent_state(); /* Kick them all. */
1323+ rcu_force_quiescent_state(); /* Kick them all. */
13141324 }
13151325
13161326 static void print_cpu_stall(void)
@@ -1557,17 +1567,28 @@ static bool rcu_future_gp_cleanup(struct rcu_node *rnp) · 240 unmodified lines
15571567 }
15581568
15591569 /*
1560− * Awaken the grace-period kthread. Don't do a self-awaken, and don't
1561− * bother awakening when there is nothing for the grace-period kthread
1562− * to do (as in several CPUs raced to awaken, and we lost), and finally
1563− * don't try to awaken a kthread that has not yet been created.
1570+ * Awaken the grace-period kthread. Don't do a self-awaken (unless in
1571+ * an interrupt or softirq handler), and don't bother awakening when there
1572+ * is nothing for the grace-period kthread to do (as in several CPUs raced
1573+ * to awaken, and we lost), and finally don't try to awaken a kthread that
1574+ * has not yet been created. If all those checks are passed, track some
1575+ * debug information and awaken.
1576+ *
1577+ * So why do the self-wakeup when in an interrupt or softirq handler
1578+ * in the grace-period kthread's context? Because the kthread might have
1579+ * been interrupted just as it was going to sleep, and just after the final
1580+ * pre-sleep check of the awaken condition. In this case, a wakeup really
1581+ * is required, and is therefore supplied.
15641582 */
15651583 static void rcu_gp_kthread_wake(void)
15661584 {
1567− if (current == rcu_state.gp_kthread ||
1585+ if ((current == rcu_state.gp_kthread &&
1586+ !in_interrupt() && !in_serving_softirq()) ||
15681587 !READ_ONCE(rcu_state.gp_flags) ||
15691588 !rcu_state.gp_kthread)
15701589 return;
1590+ WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
1591+ WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
15711592 swake_up_one(&rcu_state.gp_wq);
15721593 }
15731594
@@ -1711,7 +1732,7 @@ static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp) · 137 unmodified lines
17111732 zero_cpu_stall_ticks(rdp);
17121733 }
17131734 rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */
1714− if (ULONG_CMP_GE(rnp->gp_seq_needed, rdp->gp_seq_needed) || rdp->gpwrap)
1735+ if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) || rdp->gpwrap)
17151736 rdp->gp_seq_needed = rnp->gp_seq_needed;
17161737 WRITE_ONCE(rdp->gpwrap, false);
17171738 rcu_gpnum_ovf(rnp, rdp);
@@ -1939,7 +1960,7 @@ static void rcu_gp_fqs_loop(void) · 221 unmodified lines
19391960 if (!ret) {
19401961 rcu_state.jiffies_force_qs = jiffies + j;
19411962 WRITE_ONCE(rcu_state.jiffies_kick_kthreads,
1942− jiffies + 3 * j);
1963+ jiffies + (j ? 3 * j : 2));
19431964 }
19441965 trace_rcu_grace_period(rcu_state.name,
19451966 READ_ONCE(rcu_state.gp_seq),
@@ -2497,14 +2518,14 @@ static void rcu_do_batch(struct rcu_data *rdp) · 551 unmodified lines
24972518 }
24982519
24992520 /*
2500− * Check to see if this CPU is in a non-context-switch quiescent state
2501− * (user mode or idle loop for rcu, non-softirq execution for rcu_bh).
2502− * Also schedule RCU core processing.
2503− *
2504− * This function must be called from hardirq context. It is normally
2505− * invoked from the scheduling-clock interrupt.
2521+ * This function is invoked from each scheduling-clock interrupt,
2522+ * and checks to see if this CPU is in a non-context-switch quiescent
2523+ * state, for example, user mode or idle loop. It also schedules RCU
2524+ * core processing. If the current grace period has gone on too long,
2525+ * it will ask the scheduler to manufacture a context switch for the sole
2526+ * purpose of providing a providing the needed quiescent state.
25062527 */
2507−void rcu_check_callbacks(int user)
2528+void rcu_sched_clock_irq(int user)
25082529 {
25092530 trace_rcu_utilization(TPS("Start scheduler-tick"));
25102531 raw_cpu_inc(rcu_data.ticks_this_gp);
@@ -2517,7 +2538,7 @@ void rcu_check_callbacks(int user) · 6 unmodified lines
25172538 }
25182539 __this_cpu_write(rcu_data.rcu_urgent_qs, false);
25192540 }
2520− rcu_flavor_check_callbacks(user);
2541+ rcu_flavor_sched_clock_irq(user);
25212542 if (rcu_pending())
25222543 invoke_rcu_core();
25232544
@@ -2578,7 +2599,7 @@ static void force_qs_rnp(int (*f)(struct rcu_data *rdp)) · 54 unmodified lines
25782599 * Force quiescent states on reluctant CPUs, and also detect which
25792600 * CPUs are in dyntick-idle mode.
25802601 */
2581−static void force_quiescent_state(void)
2602+void rcu_force_quiescent_state(void)
25822603 {
25832604 unsigned long flags;
25842605 bool ret;
@@ -2610,6 +2631,7 @@ static void force_quiescent_state(void) · 25 unmodified lines
26102631 raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
26112632 rcu_gp_kthread_wake();
26122633 }
2634+EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
26132635
26142636 /*
26152637 * This function checks for grace-period requests that fail to motivate
@@ -2657,16 +2679,11 @@ rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, · 41 unmodified lines
26572679 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
26582680 return;
26592681 }
2660− pr_alert("%s: g%ld->%ld gar:%lu ga:%lu f%#x gs:%d %s->state:%#lx\n",
2661− __func__, (long)READ_ONCE(rcu_state.gp_seq),
2662− (long)READ_ONCE(rnp_root->gp_seq_needed),
2663− j - rcu_state.gp_req_activity, j - rcu_state.gp_activity,
2664− rcu_state.gp_flags, rcu_state.gp_state, rcu_state.name,
2665− rcu_state.gp_kthread ? rcu_state.gp_kthread->state : 0x1ffffL);
26662682 WARN_ON(1);
26672683 if (rnp_root != rnp)
26682684 raw_spin_unlock_rcu_node(rnp_root);
26692685 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2686+ show_rcu_gp_kthreads();
26702687 }
26712688
26722689 /*
@@ -2711,12 +2728,8 @@ void rcu_fwd_progress_check(unsigned long j) · 38 unmodified lines
27112728 }
27122729 EXPORT_SYMBOL_GPL(rcu_fwd_progress_check);
27132730
2714−/*
2715− * This does the RCU core processing work for the specified rcu_data
2716− * structures. This may be called only from the CPU to whom the rdp
2717− * belongs.
2718− */
2719−static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused)
2731+/* Perform RCU core processing work for the current CPU. */
2732+static __latent_entropy void rcu_core(struct softirq_action *unused)
27202733 {
27212734 unsigned long flags;
27222735 struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
@@ -2801,9 +2814,9 @@ static void __call_rcu_core(struct rcu_data *rdp, struct rcu_head *head, · 78 unmodified lines
28012814
28022815 /*
28032816 * Force the grace period if too many callbacks or too long waiting.
2804− * Enforce hysteresis, and don't invoke force_quiescent_state()
2817+ * Enforce hysteresis, and don't invoke rcu_force_quiescent_state()
28052818 * if some other CPU has recently done so. Also, don't bother
2806− * invoking force_quiescent_state() if the newly enqueued callback
2819+ * invoking rcu_force_quiescent_state() if the newly enqueued callback
28072820 * is the only one waiting for a grace period to complete.
28082821 */
28092822 if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) >
@@ -2820,7 +2833,7 @@ static void __call_rcu_core(struct rcu_data *rdp, struct rcu_head *head, · 10 unmodified lines
28202833 rdp->blimit = LONG_MAX;
28212834 if (rcu_state.n_force_qs == rdp->n_force_qs_snap &&
28222835 rcu_segcblist_first_pend_cb(&rdp->cblist) != head)
2823− force_quiescent_state();
2836+ rcu_force_quiescent_state();
28242837 rdp->n_force_qs_snap = rcu_state.n_force_qs;
28252838 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
28262839 }
@@ -2889,9 +2902,6 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func, int cpu, bool lazy) · 62 unmodified lines
28892902 rcu_segcblist_init(&rdp->cblist);
28902903 }
28912904 rcu_segcblist_enqueue(&rdp->cblist, head, lazy);
2892− if (!lazy)
2893− rcu_idle_count_callbacks_posted();
2894−
28952905 if (__is_kfree_rcu_offset((unsigned long)func))
28962906 trace_rcu_kfree_callback(rcu_state.name, head,
28972907 (unsigned long)func,
@@ -2961,6 +2971,79 @@ void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func) · 63 unmodified lines
29612971 }
29622972 EXPORT_SYMBOL_GPL(kfree_call_rcu);
29632973
2974+/*
2975+ * During early boot, any blocking grace-period wait automatically
2976+ * implies a grace period. Later on, this is never the case for PREEMPT.
2977+ *
2978+ * Howevr, because a context switch is a grace period for !PREEMPT, any
2979+ * blocking grace-period wait automatically implies a grace period if
2980+ * there is only one CPU online at any point time during execution of
2981+ * either synchronize_rcu() or synchronize_rcu_expedited(). It is OK to
2982+ * occasionally incorrectly indicate that there are multiple CPUs online
2983+ * when there was in fact only one the whole time, as this just adds some
2984+ * overhead: RCU still operates correctly.
2985+ */
2986+static int rcu_blocking_is_gp(void)
2987+{
2988+ int ret;
2989+
2990+ if (IS_ENABLED(CONFIG_PREEMPT))
2991+ return rcu_scheduler_active == RCU_SCHEDULER_INACTIVE;
2992+ might_sleep(); /* Check for RCU read-side critical section. */
2993+ preempt_disable();
2994+ ret = num_online_cpus() <= 1;
2995+ preempt_enable();
2996+ return ret;
2997+}
2998+
2999+/**
3000+ * synchronize_rcu - wait until a grace period has elapsed.
3001+ *
3002+ * Control will return to the caller some time after a full grace
3003+ * period has elapsed, in other words after all currently executing RCU
3004+ * read-side critical sections have completed. Note, however, that
3005+ * upon return from synchronize_rcu(), the caller might well be executing
3006+ * concurrently with new RCU read-side critical sections that began while
3007+ * synchronize_rcu() was waiting. RCU read-side critical sections are
3008+ * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
3009+ * In addition, regions of code across which interrupts, preemption, or
3010+ * softirqs have been disabled also serve as RCU read-side critical
3011+ * sections. This includes hardware interrupt handlers, softirq handlers,
3012+ * and NMI handlers.
3013+ *
3014+ * Note that this guarantee implies further memory-ordering guarantees.
3015+ * On systems with more than one CPU, when synchronize_rcu() returns,
3016+ * each CPU is guaranteed to have executed a full memory barrier since
3017+ * the end of its last RCU read-side critical section whose beginning
3018+ * preceded the call to synchronize_rcu(). In addition, each CPU having
3019+ * an RCU read-side critical section that extends beyond the return from
3020+ * synchronize_rcu() is guaranteed to have executed a full memory barrier
3021+ * after the beginning of synchronize_rcu() and before the beginning of
3022+ * that RCU read-side critical section. Note that these guarantees include
3023+ * CPUs that are offline, idle, or executing in user mode, as well as CPUs
3024+ * that are executing in the kernel.
3025+ *
3026+ * Furthermore, if CPU A invoked synchronize_rcu(), which returned
3027+ * to its caller on CPU B, then both CPU A and CPU B are guaranteed
3028+ * to have executed a full memory barrier during the execution of
3029+ * synchronize_rcu() -- even if CPU A and CPU B are the same CPU (but
3030+ * again only if the system has more than one CPU).
3031+ */
3032+void synchronize_rcu(void)
3033+{
3034+ RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
3035+ lock_is_held(&rcu_lock_map) ||
3036+ lock_is_held(&rcu_sched_lock_map),
3037+ "Illegal synchronize_rcu() in RCU read-side critical section");
3038+ if (rcu_blocking_is_gp())
3039+ return;
3040+ if (rcu_gp_is_expedited())
3041+ synchronize_rcu_expedited();
3042+ else
3043+ wait_rcu_gp(call_rcu);
3044+}
3045+EXPORT_SYMBOL_GPL(synchronize_rcu);
3046+
29643047 /**
29653048 * get_state_synchronize_rcu - Snapshot current RCU state
29663049 *
@@ -3048,28 +3131,6 @@ static int rcu_pending(void) · 81 unmodified lines
30483131 return 0;
30493132 }
30503133
3051−/*
3052− * Return true if the specified CPU has any callback. If all_lazy is
3053− * non-NULL, store an indication of whether all callbacks are lazy.
3054− * (If there are no callbacks, all of them are deemed to be lazy.)
3055− */
3056−static bool rcu_cpu_has_callbacks(bool *all_lazy)
3057−{
3058− bool al = true;
3059− bool hc = false;
3060− struct rcu_data *rdp;
3061−
3062− rdp = this_cpu_ptr(&rcu_data);
3063− if (!rcu_segcblist_empty(&rdp->cblist)) {
3064− hc = true;
3065− if (rcu_segcblist_n_nonlazy_cbs(&rdp->cblist))
3066− al = false;
3067− }
3068− if (all_lazy)
3069− *all_lazy = al;
3070− return hc;
3071−}
3072−
30733134 /*
30743135 * Helper function for rcu_barrier() tracing. If tracing is disabled,
30753136 * the compiler is expected to optimize this away.
@@ -3299,7 +3360,7 @@ int rcutree_prepare_cpu(unsigned int cpu) · 223 unmodified lines
32993360 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
33003361 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
33013362 rcu_prepare_kthreads(cpu);
3302− rcu_spawn_all_nocb_kthreads(cpu);
3363+ rcu_spawn_cpu_nocb_kthread(cpu);
33033364
33043365 return 0;
33053366 }
@@ -3329,8 +3390,6 @@ int rcutree_online_cpu(unsigned int cpu) · 23 unmodified lines
33293390 raw_spin_lock_irqsave_rcu_node(rnp, flags);
33303391 rnp->ffmask |= rdp->grpmask;
33313392 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3332− if (IS_ENABLED(CONFIG_TREE_SRCU))
3333− srcu_online_cpu(cpu);
33343393 if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
33353394 return 0; /* Too early in boot for scheduler work. */
33363395 sync_sched_exp_online_cleanup(cpu);
@@ -3355,8 +3414,6 @@ int rcutree_offline_cpu(unsigned int cpu) · 18 unmodified lines
33553414 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
33563415
33573416 rcutree_affinity_setting(cpu, cpu);
3358− if (IS_ENABLED(CONFIG_TREE_SRCU))
3359− srcu_offline_cpu(cpu);
33603417 return 0;
33613418 }
33623419
@@ -3777,7 +3834,7 @@ void __init rcu_init(void) · 414 unmodified lines
37773834 rcu_init_one();
37783835 if (dump_tree)
37793836 rcu_dump_rcu_node_tree();
3780− open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
3837+ open_softirq(RCU_SOFTIRQ, rcu_core);
37813838
37823839 /*
37833840 * We don't need protection against CPU-hotplug here because
kernel/rcu/tree.h+14−39
@@ -1,25 +1,12 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
34 * Internal non-public definitions.
45 *
5− * This program is free software; you can redistribute it and/or modify
6− * it under the terms of the GNU General Public License as published by
7− * the Free Software Foundation; either version 2 of the License, or
8− * (at your option) any later version.
9− *
10− * This program is distributed in the hope that it will be useful,
11− * but WITHOUT ANY WARRANTY; without even the implied warranty of
12− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13− * GNU General Public License for more details.
14− *
15− * You should have received a copy of the GNU General Public License
16− * along with this program; if not, you can access it online at
17− * http://www.gnu.org/licenses/gpl-2.0.html.
18− *
196 * Copyright IBM Corporation, 2008
207 *
218 * Author: Ingo Molnar <mingo@elte.hu>
22− * Paul E. McKenney <paulmck@linux.vnet.ibm.com>
9+ * Paul E. McKenney <paulmck@linux.ibm.com>
2310 */
2411
2512 #include <linux/cache.h>
@@ -36,7 +23,6 @@ · 10 unmodified lines
3623
3724 /* Communicate arguments to a workqueue handler. */
3825 struct rcu_exp_work {
39− smp_call_func_t rew_func;
4026 unsigned long rew_s;
4127 struct work_struct rew_work;
4228 };
@@ -194,10 +180,7 @@ struct rcu_data { · 151 unmodified lines
194180 bool rcu_need_heavy_qs; /* GP old, so heavy quiescent state! */
195181 bool rcu_urgent_qs; /* GP old need light quiescent state. */
196182 #ifdef CONFIG_RCU_FAST_NO_HZ
197− bool all_lazy; /* Are all CPU's CBs lazy? */
198− unsigned long nonlazy_posted; /* # times non-lazy CB posted to CPU. */
199− unsigned long nonlazy_posted_snap;
200− /* Nonlazy_posted snapshot. */
183+ bool all_lazy; /* All CPU's CBs lazy at idle start? */
201184 unsigned long last_accelerate; /* Last jiffy CBs were accelerated. */
202185 unsigned long last_advance_all; /* Last jiffy CBs were all advanced. */
203186 int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */
@@ -234,7 +217,13 @@ struct rcu_data { · 30 unmodified lines
234217 /* Leader CPU takes GP-end wakeups. */
235218 #endif /* #ifdef CONFIG_RCU_NOCB_CPU */
236219
237− /* 6) Diagnostic data, including RCU CPU stall warnings. */
220+ /* 6) RCU priority boosting. */
221+ struct task_struct *rcu_cpu_kthread_task;
222+ /* rcuc per-CPU kthread or NULL. */
223+ unsigned int rcu_cpu_kthread_status;
224+ char rcu_cpu_has_work;
225+
226+ /* 7) Diagnostic data, including RCU CPU stall warnings. */
238227 unsigned int softirq_snap; /* Snapshot of softirq activity. */
239228 /* ->rcu_iw* fields protected by leaf rcu_node ->lock. */
240229 struct irq_work rcu_iw; /* Check for non-irq activity. */
@@ -303,6 +292,8 @@ struct rcu_state { · 62 unmodified lines
303292 struct swait_queue_head gp_wq; /* Where GP task waits. */
304293 short gp_flags; /* Commands for GP task. */
305294 short gp_state; /* GP kthread sleep state. */
295+ unsigned long gp_wake_time; /* Last GP kthread wake. */
296+ unsigned long gp_wake_seq; /* ->gp_seq at ^^^. */
306297
307298 /* End of fields guarded by root rcu_node's lock. */
308299
@@ -402,13 +393,6 @@ static const char *tp_rcu_varname __used __tracepoint_string = rcu_name; · 93 unmodified lines
402393
403394 int rcu_dynticks_snap(struct rcu_data *rdp);
404395
405−#ifdef CONFIG_RCU_BOOST
406−DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
407−DECLARE_PER_CPU(int, rcu_cpu_kthread_cpu);
408−DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
409−DECLARE_PER_CPU(char, rcu_cpu_has_work);
410−#endif /* #ifdef CONFIG_RCU_BOOST */
411−
412396 /* Forward declarations for rcutree_plugin.h */
413397 static void rcu_bootup_announce(void);
414398 static void rcu_qs(void);
@@ -420,7 +404,7 @@ static void rcu_print_detail_task_stall(void); · 5 unmodified lines
420404 static int rcu_print_task_stall(struct rcu_node *rnp);
421405 static int rcu_print_task_exp_stall(struct rcu_node *rnp);
422406 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp);
423−static void rcu_flavor_check_callbacks(int user);
407+static void rcu_flavor_sched_clock_irq(int user);
424408 void call_rcu(struct rcu_head *head, rcu_callback_t func);
425409 static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck);
426410 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);
@@ -431,7 +415,6 @@ static void __init rcu_spawn_boost_kthreads(void); · 4 unmodified lines
431415 static void rcu_prepare_kthreads(int cpu);
432416 static void rcu_cleanup_after_idle(void);
433417 static void rcu_prepare_for_idle(void);
434−static void rcu_idle_count_callbacks_posted(void);
435418 static bool rcu_preempt_has_tasks(struct rcu_node *rnp);
436419 static bool rcu_preempt_need_deferred_qs(struct task_struct *t);
437420 static void rcu_preempt_deferred_qs(struct task_struct *t);
@@ -451,7 +434,7 @@ static bool rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp, · 13 unmodified lines
451434 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp);
452435 static void do_nocb_deferred_wakeup(struct rcu_data *rdp);
453436 static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp);
454−static void rcu_spawn_all_nocb_kthreads(int cpu);
437+static void rcu_spawn_cpu_nocb_kthread(int cpu);
455438 static void __init rcu_spawn_nocb_kthreads(void);
456439 #ifdef CONFIG_RCU_NOCB_CPU
457440 static void __init rcu_organize_nocb_kthreads(void);
@@ -462,11 +445,3 @@ static void rcu_bind_gp_kthread(void); · 4 unmodified lines
462445 static bool rcu_nohz_full_cpu(void);
463446 static void rcu_dynticks_task_enter(void);
464447 static void rcu_dynticks_task_exit(void);
465−
466−#ifdef CONFIG_SRCU
467−void srcu_online_cpu(unsigned int cpu);
468−void srcu_offline_cpu(unsigned int cpu);
469−#else /* #ifdef CONFIG_SRCU */
470−void srcu_online_cpu(unsigned int cpu) { }
471−void srcu_offline_cpu(unsigned int cpu) { }
472−#endif /* #else #ifdef CONFIG_SRCU */
kernel/rcu/tree_exp.h+71−130
@@ -1,27 +1,16 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * RCU expedited grace periods
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright IBM Corporation, 2016
196 *
20− * Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
7+ * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
218 */
229
2310 #include <linux/lockdep.h>
2411
12+static void rcu_exp_handler(void *unused);
13+
2514 /*
2615 * Record the start of an expedited grace period.
2716 */
@@ -344,7 +333,6 @@ static void sync_rcu_exp_select_node_cpus(struct work_struct *wp) · 316 unmodified lines
344333 {
345334 int cpu;
346335 unsigned long flags;
347− smp_call_func_t func;
348336 unsigned long mask_ofl_test;
349337 unsigned long mask_ofl_ipi;
350338 int ret;
@@ -352,7 +340,6 @@ static void sync_rcu_exp_select_node_cpus(struct work_struct *wp) · 1 unmodified line
352340 container_of(wp, struct rcu_exp_work, rew_work);
353341 struct rcu_node *rnp = container_of(rewp, struct rcu_node, rew);
354342
355− func = rewp->rew_func;
356343 raw_spin_lock_irqsave_rcu_node(rnp, flags);
357344
358345 /* Each pass checks a CPU for identity, offline, and idle. */
@@ -396,7 +383,7 @@ retry_ipi: · 37 unmodified lines
396383 mask_ofl_test |= mask;
397384 continue;
398385 }
399− ret = smp_call_function_single(cpu, func, NULL, 0);
386+ ret = smp_call_function_single(cpu, rcu_exp_handler, NULL, 0);
400387 if (!ret) {
401388 mask_ofl_ipi &= ~mask;
402389 continue;
@@ -426,7 +413,7 @@ retry_ipi: · 23 unmodified lines
426413 * Select the nodes that the upcoming expedited grace period needs
427414 * to wait for.
428415 */
429−static void sync_rcu_exp_select_cpus(smp_call_func_t func)
416+static void sync_rcu_exp_select_cpus(void)
430417 {
431418 int cpu;
432419 struct rcu_node *rnp;
@@ -440,7 +427,6 @@ static void sync_rcu_exp_select_cpus(smp_call_func_t func) · 7 unmodified lines
440427 rnp->exp_need_flush = false;
441428 if (!READ_ONCE(rnp->expmask))
442429 continue; /* Avoid early boot non-existent wq. */
443− rnp->rew.rew_func = func;
444430 if (!READ_ONCE(rcu_par_gp_wq) ||
445431 rcu_scheduler_active != RCU_SCHEDULER_RUNNING ||
446432 rcu_is_last_leaf_node(rnp)) {
@@ -449,7 +435,6 @@ static void sync_rcu_exp_select_cpus(smp_call_func_t func) · 2 unmodified lines
449435 continue;
450436 }
451437 INIT_WORK(&rnp->rew.rew_work, sync_rcu_exp_select_node_cpus);
452− preempt_disable();
453438 cpu = find_next_bit(&rnp->ffmask, BITS_PER_LONG, -1);
454439 /* If all offline, queue the work on an unbound CPU. */
455440 if (unlikely(cpu > rnp->grphi - rnp->grplo))
@@ -457,7 +442,6 @@ static void sync_rcu_exp_select_cpus(smp_call_func_t func) · 1 unmodified line
457442 else
458443 cpu += rnp->grplo;
459444 queue_work_on(cpu, rcu_par_gp_wq, &rnp->rew.rew_work);
460− preempt_enable();
461445 rnp->exp_need_flush = true;
462446 }
463447
@@ -580,10 +564,10 @@ static void rcu_exp_wait_wake(unsigned long s) · 116 unmodified lines
580564 * Common code to drive an expedited grace period forward, used by
581565 * workqueues and mid-boot-time tasks.
582566 */
583−static void rcu_exp_sel_wait_wake(smp_call_func_t func, unsigned long s)
567+static void rcu_exp_sel_wait_wake(unsigned long s)
584568 {
585569 /* Initialize the rcu_node tree in preparation for the wait. */
586− sync_rcu_exp_select_cpus(func);
570+ sync_rcu_exp_select_cpus();
587571
588572 /* Wait and clean up, including waking everyone. */
589573 rcu_exp_wait_wake(s);
@@ -597,52 +581,7 @@ static void wait_rcu_exp_gp(struct work_struct *wp) · 7 unmodified lines
597581 struct rcu_exp_work *rewp;
598582
599583 rewp = container_of(wp, struct rcu_exp_work, rew_work);
600− rcu_exp_sel_wait_wake(rewp->rew_func, rewp->rew_s);
601−}
602−
603−/*
604− * Given a smp_call_function() handler, kick off the specified
605− * implementation of expedited grace period.
606− */
607−static void _synchronize_rcu_expedited(smp_call_func_t func)
608−{
609− struct rcu_data *rdp;
610− struct rcu_exp_work rew;
611− struct rcu_node *rnp;
612− unsigned long s;
613−
614− /* If expedited grace periods are prohibited, fall back to normal. */
615− if (rcu_gp_is_normal()) {
616− wait_rcu_gp(call_rcu);
617− return;
618− }
619−
620− /* Take a snapshot of the sequence number. */
621− s = rcu_exp_gp_seq_snap();
622− if (exp_funnel_lock(s))
623− return; /* Someone else did our work for us. */
624−
625− /* Ensure that load happens before action based on it. */
626− if (unlikely(rcu_scheduler_active == RCU_SCHEDULER_INIT)) {
627− /* Direct call during scheduler init and early_initcalls(). */
628− rcu_exp_sel_wait_wake(func, s);
629− } else {
630− /* Marshall arguments & schedule the expedited grace period. */
631− rew.rew_func = func;
632− rew.rew_s = s;
633− INIT_WORK_ONSTACK(&rew.rew_work, wait_rcu_exp_gp);
634− queue_work(rcu_gp_wq, &rew.rew_work);
635− }
636−
637− /* Wait for expedited grace period to complete. */
638− rdp = per_cpu_ptr(&rcu_data, raw_smp_processor_id());
639− rnp = rcu_get_root();
640− wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3],
641− sync_exp_work_done(s));
642− smp_mb(); /* Workqueue actions happen before return. */
643−
644− /* Let the next expedited grace period start. */
645− mutex_unlock(&rcu_state.exp_mutex);
584+ rcu_exp_sel_wait_wake(rewp->rew_s);
646585 }
647586
648587 #ifdef CONFIG_PREEMPT_RCU
@@ -654,7 +593,7 @@ static void _synchronize_rcu_expedited(smp_call_func_t func) · 5 unmodified lines
654593 * ->expmask fields in the rcu_node tree. Otherwise, immediately
655594 * report the quiescent state.
656595 */
657−static void sync_rcu_exp_handler(void *unused)
596+static void rcu_exp_handler(void *unused)
658597 {
659598 unsigned long flags;
660599 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
@@ -697,6 +636,7 @@ static void sync_rcu_exp_handler(void *unused) · 36 unmodified lines
697636 WRITE_ONCE(t->rcu_read_unlock_special.b.exp_hint, true);
698637 }
699638 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
639+ return;
700640 }
701641
702642 /*
@@ -730,43 +670,10 @@ static void sync_sched_exp_online_cleanup(int cpu) · 27 unmodified lines
730670 {
731671 }
732672
733−/**
734− * synchronize_rcu_expedited - Brute-force RCU grace period
735− *
736− * Wait for an RCU-preempt grace period, but expedite it. The basic
737− * idea is to IPI all non-idle non-nohz online CPUs. The IPI handler
738− * checks whether the CPU is in an RCU-preempt critical section, and
739− * if so, it sets a flag that causes the outermost rcu_read_unlock()
740− * to report the quiescent state. On the other hand, if the CPU is
741− * not in an RCU read-side critical section, the IPI handler reports
742− * the quiescent state immediately.
743− *
744− * Although this is a greate improvement over previous expedited
745− * implementations, it is still unfriendly to real-time workloads, so is
746− * thus not recommended for any sort of common-case code. In fact, if
747− * you are using synchronize_rcu_expedited() in a loop, please restructure
748− * your code to batch your updates, and then Use a single synchronize_rcu()
749− * instead.
750− *
751− * This has the same semantics as (but is more brutal than) synchronize_rcu().
752− */
753−void synchronize_rcu_expedited(void)
754−{
755− RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
756− lock_is_held(&rcu_lock_map) ||
757− lock_is_held(&rcu_sched_lock_map),
758− "Illegal synchronize_rcu_expedited() in RCU read-side critical section");
759−
760− if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
761− return;
762− _synchronize_rcu_expedited(sync_rcu_exp_handler);
763−}
764−EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
765−
766673 #else /* #ifdef CONFIG_PREEMPT_RCU */
767674
768675 /* Invoked on each online non-idle CPU for expedited quiescent state. */
769−static void sync_sched_exp_handler(void *unused)
676+static void rcu_exp_handler(void *unused)
770677 {
771678 struct rcu_data *rdp;
772679 struct rcu_node *rnp;
@@ -798,44 +705,78 @@ static void sync_sched_exp_online_cleanup(int cpu) · 25 unmodified lines
798705 rnp = rdp->mynode;
799706 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask))
800707 return;
801− ret = smp_call_function_single(cpu, sync_sched_exp_handler, NULL, 0);
708+ ret = smp_call_function_single(cpu, rcu_exp_handler, NULL, 0);
802709 WARN_ON_ONCE(ret);
803710 }
804711
805−/*
806− * Because a context switch is a grace period for !PREEMPT, any
807− * blocking grace-period wait automatically implies a grace period if
808− * there is only one CPU online at any point time during execution of
809− * either synchronize_rcu() or synchronize_rcu_expedited(). It is OK to
810− * occasionally incorrectly indicate that there are multiple CPUs online
811− * when there was in fact only one the whole time, as this just adds some
812− * overhead: RCU still operates correctly.
813− */
814−static int rcu_blocking_is_gp(void)
815−{
816− int ret;
817−
818− might_sleep(); /* Check for RCU read-side critical section. */
819− preempt_disable();
820− ret = num_online_cpus() <= 1;
821− preempt_enable();
822− return ret;
823−}
712+#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
824713
825−/* PREEMPT=n implementation of synchronize_rcu_expedited(). */
714+/**
715+ * synchronize_rcu_expedited - Brute-force RCU grace period
716+ *
717+ * Wait for an RCU grace period, but expedite it. The basic idea is to
718+ * IPI all non-idle non-nohz online CPUs. The IPI handler checks whether
719+ * the CPU is in an RCU critical section, and if so, it sets a flag that
720+ * causes the outermost rcu_read_unlock() to report the quiescent state
721+ * for RCU-preempt or asks the scheduler for help for RCU-sched. On the
722+ * other hand, if the CPU is not in an RCU read-side critical section,
723+ * the IPI handler reports the quiescent state immediately.
724+ *
725+ * Although this is a greate improvement over previous expedited
726+ * implementations, it is still unfriendly to real-time workloads, so is
727+ * thus not recommended for any sort of common-case code. In fact, if
728+ * you are using synchronize_rcu_expedited() in a loop, please restructure
729+ * your code to batch your updates, and then Use a single synchronize_rcu()
730+ * instead.
731+ *
732+ * This has the same semantics as (but is more brutal than) synchronize_rcu().
733+ */
826734 void synchronize_rcu_expedited(void)
827735 {
736+ struct rcu_data *rdp;
737+ struct rcu_exp_work rew;
738+ struct rcu_node *rnp;
739+ unsigned long s;
740+
828741 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
829742 lock_is_held(&rcu_lock_map) ||
830743 lock_is_held(&rcu_sched_lock_map),
831744 "Illegal synchronize_rcu_expedited() in RCU read-side critical section");
832745
833− /* If only one CPU, this is automatically a grace period. */
746+ /* Is the state is such that the call is a grace period? */
834747 if (rcu_blocking_is_gp())
835748 return;
836749
837− _synchronize_rcu_expedited(sync_sched_exp_handler);
750+ /* If expedited grace periods are prohibited, fall back to normal. */
751+ if (rcu_gp_is_normal()) {
752+ wait_rcu_gp(call_rcu);
753+ return;
754+ }
755+
756+ /* Take a snapshot of the sequence number. */
757+ s = rcu_exp_gp_seq_snap();
758+ if (exp_funnel_lock(s))
759+ return; /* Someone else did our work for us. */
760+
761+ /* Ensure that load happens before action based on it. */
762+ if (unlikely(rcu_scheduler_active == RCU_SCHEDULER_INIT)) {
763+ /* Direct call during scheduler init and early_initcalls(). */
764+ rcu_exp_sel_wait_wake(s);
765+ } else {
766+ /* Marshall arguments & schedule the expedited grace period. */
767+ rew.rew_s = s;
768+ INIT_WORK_ONSTACK(&rew.rew_work, wait_rcu_exp_gp);
769+ queue_work(rcu_gp_wq, &rew.rew_work);
770+ }
771+
772+ /* Wait for expedited grace period to complete. */
773+ rdp = per_cpu_ptr(&rcu_data, raw_smp_processor_id());
774+ rnp = rcu_get_root();
775+ wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3],
776+ sync_exp_work_done(s));
777+ smp_mb(); /* Workqueue actions happen before return. */
778+
779+ /* Let the next expedited grace period start. */
780+ mutex_unlock(&rcu_state.exp_mutex);
838781 }
839782 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
840−
841−#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
kernel/rcu/tree_plugin.h+57−181
@@ -1,27 +1,14 @@
1+/* SPDX-License-Identifier: GPL-2.0+ */
12 /*
23 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
34 * Internal non-public definitions that provide either classic
45 * or preemptible semantics.
56 *
6− * This program is free software; you can redistribute it and/or modify
7− * it under the terms of the GNU General Public License as published by
8− * the Free Software Foundation; either version 2 of the License, or
9− * (at your option) any later version.
10− *
11− * This program is distributed in the hope that it will be useful,
12− * but WITHOUT ANY WARRANTY; without even the implied warranty of
13− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14− * GNU General Public License for more details.
15− *
16− * You should have received a copy of the GNU General Public License
17− * along with this program; if not, you can access it online at
18− * http://www.gnu.org/licenses/gpl-2.0.html.
19− *
207 * Copyright Red Hat, 2009
218 * Copyright IBM Corporation, 2009
229 *
2310 * Author: Ingo Molnar <mingo@elte.hu>
24− * Paul E. McKenney <paulmck@linux.vnet.ibm.com>
11+ * Paul E. McKenney <paulmck@linux.ibm.com>
2512 */
2613
2714 #include <linux/delay.h>
@@ -34,17 +21,7 @@ · 6 unmodified lines
3421 #include "../time/tick-internal.h"
3522
3623 #ifdef CONFIG_RCU_BOOST
37−
3824 #include "../locking/rtmutex_common.h"
39−
40−/*
41− * Control variables for per-CPU and per-rcu_node kthreads.
42− */
43−static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
44−DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
45−DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
46−DEFINE_PER_CPU(char, rcu_cpu_has_work);
47−
4825 #else /* #ifdef CONFIG_RCU_BOOST */
4926
5027 /*
@@ -307,7 +284,7 @@ static void rcu_qs(void) · 256 unmodified lines
307284 __this_cpu_read(rcu_data.gp_seq),
308285 TPS("cpuqs"));
309286 __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false);
310− barrier(); /* Coordinate with rcu_flavor_check_callbacks(). */
287+ barrier(); /* Coordinate with rcu_flavor_sched_clock_irq(). */
311288 current->rcu_read_unlock_special.b.need_qs = false;
312289 }
313290 }
@@ -788,13 +765,13 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp) · 474 unmodified lines
788765 }
789766
790767 /*
791− * Check for a quiescent state from the current CPU. When a task blocks,
792− * the task is recorded in the corresponding CPU's rcu_node structure,
793− * which is checked elsewhere.
794− *
795− * Caller must disable hard irqs.
768+ * Check for a quiescent state from the current CPU, including voluntary
769+ * context switches for Tasks RCU. When a task blocks, the task is
770+ * recorded in the corresponding CPU's rcu_node structure, which is checked
771+ * elsewhere, hence this function need only check for quiescent states
772+ * related to the current CPU, not to those related to tasks.
796773 */
797−static void rcu_flavor_check_callbacks(int user)
774+static void rcu_flavor_sched_clock_irq(int user)
798775 {
799776 struct task_struct *t = current;
800777
@@ -825,54 +802,6 @@ static void rcu_flavor_check_callbacks(int user) · 24 unmodified lines
825802 t->rcu_read_unlock_special.b.need_qs = true;
826803 }
827804
828−/**
829− * synchronize_rcu - wait until a grace period has elapsed.
830− *
831− * Control will return to the caller some time after a full grace
832− * period has elapsed, in other words after all currently executing RCU
833− * read-side critical sections have completed. Note, however, that
834− * upon return from synchronize_rcu(), the caller might well be executing
835− * concurrently with new RCU read-side critical sections that began while
836− * synchronize_rcu() was waiting. RCU read-side critical sections are
837− * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
838− * In addition, regions of code across which interrupts, preemption, or
839− * softirqs have been disabled also serve as RCU read-side critical
840− * sections. This includes hardware interrupt handlers, softirq handlers,
841− * and NMI handlers.
842− *
843− * Note that this guarantee implies further memory-ordering guarantees.
844− * On systems with more than one CPU, when synchronize_rcu() returns,
845− * each CPU is guaranteed to have executed a full memory barrier since
846− * the end of its last RCU read-side critical section whose beginning
847− * preceded the call to synchronize_rcu(). In addition, each CPU having
848− * an RCU read-side critical section that extends beyond the return from
849− * synchronize_rcu() is guaranteed to have executed a full memory barrier
850− * after the beginning of synchronize_rcu() and before the beginning of
851− * that RCU read-side critical section. Note that these guarantees include
852− * CPUs that are offline, idle, or executing in user mode, as well as CPUs
853− * that are executing in the kernel.
854− *
855− * Furthermore, if CPU A invoked synchronize_rcu(), which returned
856− * to its caller on CPU B, then both CPU A and CPU B are guaranteed
857− * to have executed a full memory barrier during the execution of
858− * synchronize_rcu() -- even if CPU A and CPU B are the same CPU (but
859− * again only if the system has more than one CPU).
860− */
861−void synchronize_rcu(void)
862−{
863− RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
864− lock_is_held(&rcu_lock_map) ||
865− lock_is_held(&rcu_sched_lock_map),
866− "Illegal synchronize_rcu() in RCU read-side critical section");
867− if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
868− return;
869− if (rcu_gp_is_expedited())
870− synchronize_rcu_expedited();
871− else
872− wait_rcu_gp(call_rcu);
873−}
874−EXPORT_SYMBOL_GPL(synchronize_rcu);
875−
876805 /*
877806 * Check for a task exiting while in a preemptible-RCU read-side
878807 * critical section, clean up if so. No need to issue warnings,
@@ -1088,14 +1017,10 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp) · 209 unmodified lines
10881017 }
10891018
10901019 /*
1091− * Check to see if this CPU is in a non-context-switch quiescent state
1092− * (user mode or idle loop for rcu, non-softirq execution for rcu_bh).
1093− * Also schedule RCU core processing.
1094− *
1095− * This function must be called from hardirq context. It is normally
1096− * invoked from the scheduling-clock interrupt.
1020+ * Check to see if this CPU is in a non-context-switch quiescent state,
1021+ * namely user mode and idle loop.
10971022 */
1098−static void rcu_flavor_check_callbacks(int user)
1023+static void rcu_flavor_sched_clock_irq(int user)
10991024 {
11001025 if (user || rcu_is_cpu_rrupt_from_idle()) {
11011026
@@ -1115,22 +1040,6 @@ static void rcu_flavor_check_callbacks(int user) · 13 unmodified lines
11151040 }
11161041 }
11171042
1118−/* PREEMPT=n implementation of synchronize_rcu(). */
1119−void synchronize_rcu(void)
1120−{
1121− RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
1122− lock_is_held(&rcu_lock_map) ||
1123− lock_is_held(&rcu_sched_lock_map),
1124− "Illegal synchronize_rcu() in RCU read-side critical section");
1125− if (rcu_blocking_is_gp())
1126− return;
1127− if (rcu_gp_is_expedited())
1128− synchronize_rcu_expedited();
1129− else
1130− wait_rcu_gp(call_rcu);
1131−}
1132−EXPORT_SYMBOL_GPL(synchronize_rcu);
1133−
11341043 /*
11351044 * Because preemptible RCU does not exist, tasks cannot possibly exit
11361045 * while in preemptible RCU read-side critical sections.
@@ -1307,11 +1216,11 @@ static void invoke_rcu_callbacks_kthread(void) · 170 unmodified lines
13071216 unsigned long flags;
13081217
13091218 local_irq_save(flags);
1310− __this_cpu_write(rcu_cpu_has_work, 1);
1311− if (__this_cpu_read(rcu_cpu_kthread_task) != NULL &&
1312− current != __this_cpu_read(rcu_cpu_kthread_task)) {
1313− rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task),
1314− __this_cpu_read(rcu_cpu_kthread_status));
1219+ __this_cpu_write(rcu_data.rcu_cpu_has_work, 1);
1220+ if (__this_cpu_read(rcu_data.rcu_cpu_kthread_task) != NULL &&
1221+ current != __this_cpu_read(rcu_data.rcu_cpu_kthread_task)) {
1222+ rcu_wake_cond(__this_cpu_read(rcu_data.rcu_cpu_kthread_task),
1223+ __this_cpu_read(rcu_data.rcu_cpu_kthread_status));
13151224 }
13161225 local_irq_restore(flags);
13171226 }
@@ -1322,7 +1231,7 @@ static void invoke_rcu_callbacks_kthread(void) · 4 unmodified lines
13221231 */
13231232 static bool rcu_is_callbacks_kthread(void)
13241233 {
1325− return __this_cpu_read(rcu_cpu_kthread_task) == current;
1234+ return __this_cpu_read(rcu_data.rcu_cpu_kthread_task) == current;
13261235 }
13271236
13281237 #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
@@ -1369,11 +1278,6 @@ static int rcu_spawn_one_boost_kthread(struct rcu_node *rnp) · 40 unmodified lines
13691278 return 0;
13701279 }
13711280
1372−static void rcu_kthread_do_work(void)
1373−{
1374− rcu_do_batch(this_cpu_ptr(&rcu_data));
1375−}
1376−
13771281 static void rcu_cpu_kthread_setup(unsigned int cpu)
13781282 {
13791283 struct sched_param sp;
@@ -1384,12 +1288,12 @@ static void rcu_cpu_kthread_setup(unsigned int cpu) · 4 unmodified lines
13841288
13851289 static void rcu_cpu_kthread_park(unsigned int cpu)
13861290 {
1387− per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
1291+ per_cpu(rcu_data.rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
13881292 }
13891293
13901294 static int rcu_cpu_kthread_should_run(unsigned int cpu)
13911295 {
1392− return __this_cpu_read(rcu_cpu_has_work);
1296+ return __this_cpu_read(rcu_data.rcu_cpu_has_work);
13931297 }
13941298
13951299 /*
@@ -1399,21 +1303,20 @@ static int rcu_cpu_kthread_should_run(unsigned int cpu) · 3 unmodified lines
13991303 */
14001304 static void rcu_cpu_kthread(unsigned int cpu)
14011305 {
1402− unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
1403− char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
1306+ unsigned int *statusp = this_cpu_ptr(&rcu_data.rcu_cpu_kthread_status);
1307+ char work, *workp = this_cpu_ptr(&rcu_data.rcu_cpu_has_work);
14041308 int spincnt;
14051309
14061310 for (spincnt = 0; spincnt < 10; spincnt++) {
14071311 trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
14081312 local_bh_disable();
14091313 *statusp = RCU_KTHREAD_RUNNING;
1410− this_cpu_inc(rcu_cpu_kthread_loops);
14111314 local_irq_disable();
14121315 work = *workp;
14131316 *workp = 0;
14141317 local_irq_enable();
14151318 if (work)
1416− rcu_kthread_do_work();
1319+ rcu_do_batch(this_cpu_ptr(&rcu_data));
14171320 local_bh_enable();
14181321 if (*workp == 0) {
14191322 trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
@@ -1459,7 +1362,7 @@ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu) · 39 unmodified lines
14591362 }
14601363
14611364 static struct smp_hotplug_thread rcu_cpu_thread_spec = {
1462− .store = &rcu_cpu_kthread_task,
1365+ .store = &rcu_data.rcu_cpu_kthread_task,
14631366 .thread_should_run = rcu_cpu_kthread_should_run,
14641367 .thread_fn = rcu_cpu_kthread,
14651368 .thread_comm = "rcuc/%u",
@@ -1476,7 +1379,7 @@ static void __init rcu_spawn_boost_kthreads(void) · 10 unmodified lines
14761379 int cpu;
14771380
14781381 for_each_possible_cpu(cpu)
1479− per_cpu(rcu_cpu_has_work, cpu) = 0;
1382+ per_cpu(rcu_data.rcu_cpu_has_work, cpu) = 0;
14801383 if (WARN_ONCE(smpboot_register_percpu_thread(&rcu_cpu_thread_spec), "%s: Could not start rcub kthread, OOM is now expected behavior\n", __func__))
14811384 return;
14821385 rcu_for_each_leaf_node(rnp)
@@ -1543,7 +1446,7 @@ static void rcu_prepare_kthreads(int cpu) · 60 unmodified lines
15431446 int rcu_needs_cpu(u64 basemono, u64 *nextevt)
15441447 {
15451448 *nextevt = KTIME_MAX;
1546− return rcu_cpu_has_callbacks(NULL);
1449+ return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist);
15471450 }
15481451
15491452 /*
@@ -1562,14 +1465,6 @@ static void rcu_prepare_for_idle(void) · 12 unmodified lines
15621465 {
15631466 }
15641467
1565−/*
1566− * Don't bother keeping a running count of the number of RCU callbacks
1567− * posted because CONFIG_RCU_FAST_NO_HZ=n.
1568− */
1569−static void rcu_idle_count_callbacks_posted(void)
1570−{
1571−}
1572−
15731468 #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
15741469
15751470 /*
@@ -1652,11 +1547,8 @@ int rcu_needs_cpu(u64 basemono, u64 *nextevt) · 76 unmodified lines
16521547
16531548 lockdep_assert_irqs_disabled();
16541549
1655− /* Snapshot to detect later posting of non-lazy callback. */
1656− rdp->nonlazy_posted_snap = rdp->nonlazy_posted;
1657−
16581550 /* If no callbacks, RCU doesn't need the CPU. */
1659− if (!rcu_cpu_has_callbacks(&rdp->all_lazy)) {
1551+ if (rcu_segcblist_empty(&rdp->cblist)) {
16601552 *nextevt = KTIME_MAX;
16611553 return 0;
16621554 }
@@ -1670,11 +1562,12 @@ int rcu_needs_cpu(u64 basemono, u64 *nextevt) · 7 unmodified lines
16701562 rdp->last_accelerate = jiffies;
16711563
16721564 /* Request timer delay depending on laziness, and round. */
1673− if (!rdp->all_lazy) {
1565+ rdp->all_lazy = !rcu_segcblist_n_nonlazy_cbs(&rdp->cblist);
1566+ if (rdp->all_lazy) {
1567+ dj = round_jiffies(rcu_idle_lazy_gp_delay + jiffies) - jiffies;
1568+ } else {
16741569 dj = round_up(rcu_idle_gp_delay + jiffies,
16751570 rcu_idle_gp_delay) - jiffies;
1676− } else {
1677− dj = round_jiffies(rcu_idle_lazy_gp_delay + jiffies) - jiffies;
16781571 }
16791572 *nextevt = basemono + dj * TICK_NSEC;
16801573 return 0;
@@ -1704,7 +1597,7 @@ static void rcu_prepare_for_idle(void) · 23 unmodified lines
17041597 /* Handle nohz enablement switches conservatively. */
17051598 tne = READ_ONCE(tick_nohz_active);
17061599 if (tne != rdp->tick_nohz_enabled_snap) {
1707− if (rcu_cpu_has_callbacks(NULL))
1600+ if (!rcu_segcblist_empty(&rdp->cblist))
17081601 invoke_rcu_core(); /* force nohz to see update. */
17091602 rdp->tick_nohz_enabled_snap = tne;
17101603 return;
@@ -1717,10 +1610,8 @@ static void rcu_prepare_for_idle(void) · 6 unmodified lines
17171610 * callbacks, invoke RCU core for the side-effect of recalculating
17181611 * idle duration on re-entry to idle.
17191612 */
1720− if (rdp->all_lazy &&
1721− rdp->nonlazy_posted != rdp->nonlazy_posted_snap) {
1613+ if (rdp->all_lazy && rcu_segcblist_n_nonlazy_cbs(&rdp->cblist)) {
17221614 rdp->all_lazy = false;
1723− rdp->nonlazy_posted_snap = rdp->nonlazy_posted;
17241615 invoke_rcu_core();
17251616 return;
17261617 }
@@ -1756,19 +1647,6 @@ static void rcu_cleanup_after_idle(void) · 29 unmodified lines
17561647 invoke_rcu_core();
17571648 }
17581649
1759−/*
1760− * Keep a running count of the number of non-lazy callbacks posted
1761− * on this CPU. This running counter (which is never decremented) allows
1762− * rcu_prepare_for_idle() to detect when something out of the idle loop
1763− * posts a callback, even if an equal number of callbacks are invoked.
1764− * Of course, callbacks should only be posted from within a trace event
1765− * designed to be called from idle or from within RCU_NONIDLE().
1766− */
1767−static void rcu_idle_count_callbacks_posted(void)
1768−{
1769− __this_cpu_add(rcu_data.nonlazy_posted, 1);
1770−}
1771−
17721650 #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
17731651
17741652 #ifdef CONFIG_RCU_FAST_NO_HZ
@@ -1776,13 +1654,12 @@ static void rcu_idle_count_callbacks_posted(void) · 1 unmodified line
17761654 static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
17771655 {
17781656 struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
1779− unsigned long nlpd = rdp->nonlazy_posted - rdp->nonlazy_posted_snap;
17801657
1781− sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c",
1658+ sprintf(cp, "last_accelerate: %04lx/%04lx, Nonlazy posted: %c%c%c",
17821659 rdp->last_accelerate & 0xffff, jiffies & 0xffff,
1783− ulong2long(nlpd),
1784− rdp->all_lazy ? 'L' : '.',
1785− rdp->tick_nohz_enabled_snap ? '.' : 'D');
1660+ ".l"[rdp->all_lazy],
1661+ ".L"[!rcu_segcblist_n_nonlazy_cbs(&rdp->cblist)],
1662+ ".D"[!rdp->tick_nohz_enabled_snap]);
17861663 }
17871664
17881665 #else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
@@ -1868,22 +1745,24 @@ static void zero_cpu_stall_ticks(struct rcu_data *rdp) · 79 unmodified lines
18681745
18691746 /*
18701747 * Offload callback processing from the boot-time-specified set of CPUs
1871− * specified by rcu_nocb_mask. For each CPU in the set, there is a
1872− * kthread created that pulls the callbacks from the corresponding CPU,
1873− * waits for a grace period to elapse, and invokes the callbacks.
1874− * The no-CBs CPUs do a wake_up() on their kthread when they insert
1875− * a callback into any empty list, unless the rcu_nocb_poll boot parameter
1876− * has been specified, in which case each kthread actively polls its
1877− * CPU. (Which isn't so great for energy efficiency, but which does
1878− * reduce RCU's overhead on that CPU.)
1748+ * specified by rcu_nocb_mask. For the CPUs in the set, there are kthreads
1749+ * created that pull the callbacks from the corresponding CPU, wait for
1750+ * a grace period to elapse, and invoke the callbacks. These kthreads
1751+ * are organized into leaders, which manage incoming callbacks, wait for
1752+ * grace periods, and awaken followers, and the followers, which only
1753+ * invoke callbacks. Each leader is its own follower. The no-CBs CPUs
1754+ * do a wake_up() on their kthread when they insert a callback into any
1755+ * empty list, unless the rcu_nocb_poll boot parameter has been specified,
1756+ * in which case each kthread actively polls its CPU. (Which isn't so great
1757+ * for energy efficiency, but which does reduce RCU's overhead on that CPU.)
18791758 *
18801759 * This is intended to be used in conjunction with Frederic Weisbecker's
18811760 * adaptive-idle work, which would seriously reduce OS jitter on CPUs
18821761 * running CPU-bound user-mode computations.
18831762 *
1884− * Offloading of callback processing could also in theory be used as
1885− * an energy-efficiency measure because CPUs with no RCU callbacks
1886− * queued are more aggressive about entering dyntick-idle mode.
1763+ * Offloading of callbacks can also be used as an energy-efficiency
1764+ * measure because CPUs with no RCU callbacks queued are more aggressive
1765+ * about entering dyntick-idle mode.
18871766 */
18881767
18891768
@@ -1987,10 +1866,7 @@ static void wake_nocb_leader_defer(struct rcu_data *rdp, int waketype, · 97 unmodified lines
19871866 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
19881867 }
19891868
1990−/*
1991− * Does the specified CPU need an RCU callback for this invocation
1992− * of rcu_barrier()?
1993− */
1869+/* Does rcu_barrier need to queue an RCU callback on the specified CPU? */
19941870 static bool rcu_nocb_cpu_needs_barrier(int cpu)
19951871 {
19961872 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
@@ -2006,8 +1882,8 @@ static bool rcu_nocb_cpu_needs_barrier(int cpu) · 9 unmodified lines
20061882 * callbacks would be posted. In the worst case, the first
20071883 * barrier in rcu_barrier() suffices (but the caller cannot
20081884 * necessarily rely on this, not a substitute for the caller
2009− * getting the concurrency design right!). There must also be
2010− * a barrier between the following load an posting of a callback
1885+ * getting the concurrency design right!). There must also be a
1886+ * barrier between the following load and posting of a callback
20111887 * (if a callback is in fact needed). This is associated with an
20121888 * atomic_inc() in the caller.
20131889 */
@@ -2517,9 +2393,9 @@ static void rcu_spawn_one_nocb_kthread(int cpu) · 503 unmodified lines
25172393
25182394 /*
25192395 * If the specified CPU is a no-CBs CPU that does not already have its
2520− * rcuo kthreads, spawn them.
2396+ * rcuo kthread, spawn it.
25212397 */
2522−static void rcu_spawn_all_nocb_kthreads(int cpu)
2398+static void rcu_spawn_cpu_nocb_kthread(int cpu)
25232399 {
25242400 if (rcu_scheduler_fully_active)
25252401 rcu_spawn_one_nocb_kthread(cpu);
@@ -2536,7 +2412,7 @@ static void __init rcu_spawn_nocb_kthreads(void) · 10 unmodified lines
25362412 int cpu;
25372413
25382414 for_each_online_cpu(cpu)
2539− rcu_spawn_all_nocb_kthreads(cpu);
2415+ rcu_spawn_cpu_nocb_kthread(cpu);
25402416 }
25412417
25422418 /* How many follower CPU IDs per leader? Default of -1 for sqrt(nr_cpu_ids). */
@@ -2670,7 +2546,7 @@ static void do_nocb_deferred_wakeup(struct rcu_data *rdp) · 127 unmodified lines
26702546 {
26712547 }
26722548
2673−static void rcu_spawn_all_nocb_kthreads(int cpu)
2549+static void rcu_spawn_cpu_nocb_kthread(int cpu)
26742550 {
26752551 }
26762552
kernel/rcu/update.c+2−15
@@ -1,26 +1,13 @@
1+// SPDX-License-Identifier: GPL-2.0+
12 /*
23 * Read-Copy Update mechanism for mutual exclusion
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright IBM Corporation, 2001
196 *
207 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
218 * Manfred Spraul <manfred@colorfullife.com>
229 *
23− * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
10+ * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
2411 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
2512 * Papers:
2613 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
kernel/sched/cpufreq.c+2−2
@@ -48,8 +48,8 @@ EXPORT_SYMBOL_GPL(cpufreq_add_update_util_hook); · 47 unmodified lines
4848 *
4949 * Clear the update_util_data pointer for the given CPU.
5050 *
51− * Callers must use RCU-sched callbacks to free any memory that might be
52− * accessed via the old update_util_data pointer or invoke synchronize_sched()
51+ * Callers must use RCU callbacks to free any memory that might be
52+ * accessed via the old update_util_data pointer or invoke synchronize_rcu()
5353 * right after this function to avoid use-after-free.
5454 */
5555 void cpufreq_remove_update_util_hook(int cpu)
kernel/sched/cpufreq_schedutil.c+1−1
@@ -859,7 +859,7 @@ static void sugov_stop(struct cpufreq_policy *policy) · 858 unmodified lines
859859 for_each_cpu(cpu, policy->cpus)
860860 cpufreq_remove_update_util_hook(cpu);
861861
862− synchronize_sched();
862+ synchronize_rcu();
863863
864864 if (!policy->fast_switch_enabled) {
865865 irq_work_sync(&sg_policy->irq_work);
kernel/sched/sched.h+1−1
@@ -1260,7 +1260,7 @@ extern void sched_ttwu_pending(void); · 1259 unmodified lines
12601260
12611261 /*
12621262 * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
1263− * See detach_destroy_domains: synchronize_sched for details.
1263+ * See destroy_sched_domains: call_rcu for details.
12641264 *
12651265 * The domain tree of any CPU may only be accessed from within
12661266 * preempt-disabled sections.
kernel/sched/topology.c+2−2
@@ -442,7 +442,7 @@ void rq_attach_root(struct rq *rq, struct root_domain *rd) · 441 unmodified lines
442442 raw_spin_unlock_irqrestore(&rq->lock, flags);
443443
444444 if (old_rd)
445− call_rcu_sched(&old_rd->rcu, free_rootdomain);
445+ call_rcu(&old_rd->rcu, free_rootdomain);
446446 }
447447
448448 void sched_get_rd(struct root_domain *rd)
@@ -455,7 +455,7 @@ void sched_put_rd(struct root_domain *rd) · 6 unmodified lines
455455 if (!atomic_dec_and_test(&rd->refcount))
456456 return;
457457
458− call_rcu_sched(&rd->rcu, free_rootdomain);
458+ call_rcu(&rd->rcu, free_rootdomain);
459459 }
460460
461461 static int init_rootdomain(struct root_domain *rd)
kernel/time/timer.c+1−1
@@ -1632,7 +1632,7 @@ void update_process_times(int user_tick) · 1631 unmodified lines
16321632 /* Note: this timer irq context must be accounted for as well. */
16331633 account_process_tick(p, user_tick);
16341634 run_local_timers();
1635− rcu_check_callbacks(user_tick);
1635+ rcu_sched_clock_irq(user_tick);
16361636 #ifdef CONFIG_IRQ_WORK
16371637 if (in_irq())
16381638 irq_work_tick();
kernel/torture.c+8−17
@@ -1,23 +1,10 @@
1+// SPDX-License-Identifier: GPL-2.0+
12 /*
23 * Common functions for in-kernel torture tests.
34 *
4− * This program is free software; you can redistribute it and/or modify
5− * it under the terms of the GNU General Public License as published by
6− * the Free Software Foundation; either version 2 of the License, or
7− * (at your option) any later version.
8− *
9− * This program is distributed in the hope that it will be useful,
10− * but WITHOUT ANY WARRANTY; without even the implied warranty of
11− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12− * GNU General Public License for more details.
13− *
14− * You should have received a copy of the GNU General Public License
15− * along with this program; if not, you can access it online at
16− * http://www.gnu.org/licenses/gpl-2.0.html.
17− *
185 * Copyright (C) IBM Corporation, 2014
196 *
20− * Author: Paul E. McKenney <paulmck@us.ibm.com>
7+ * Author: Paul E. McKenney <paulmck@linux.ibm.com>
218 * Based on kernel/rcu/torture.c.
229 */
2310
@@ -53,7 +40,7 @@ · 29 unmodified lines
5340 #include "rcu/rcu.h"
5441
5542 MODULE_LICENSE("GPL");
56−MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
43+MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
5744
5845 static char *torture_type;
5946 static int verbose;
@@ -75,6 +62,7 @@ static DEFINE_MUTEX(fullstop_mutex); · 15 unmodified lines
7562 static struct task_struct *onoff_task;
7663 static long onoff_holdoff;
7764 static long onoff_interval;
65+static torture_ofl_func *onoff_f;
7866 static long n_offline_attempts;
7967 static long n_offline_successes;
8068 static unsigned long sum_offline;
@@ -118,6 +106,8 @@ bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes, · 37 unmodified lines
118106 pr_alert("%s" TORTURE_FLAG
119107 "torture_onoff task: offlined %d\n",
120108 torture_type, cpu);
109+ if (onoff_f)
110+ onoff_f();
121111 (*n_offl_successes)++;
122112 delta = jiffies - starttime;
123113 *sum_offl += delta;
@@ -243,11 +233,12 @@ stop: · 119 unmodified lines
243233 /*
244234 * Initiate online-offline handling.
245235 */
246−int torture_onoff_init(long ooholdoff, long oointerval)
236+int torture_onoff_init(long ooholdoff, long oointerval, torture_ofl_func *f)
247237 {
248238 #ifdef CONFIG_HOTPLUG_CPU
249239 onoff_holdoff = ooholdoff;
250240 onoff_interval = oointerval;
241+ onoff_f = f;
251242 if (onoff_interval <= 0)
252243 return 0;
253244 return torture_create_kthread(torture_onoff, NULL, onoff_task);
tools/include/nolibc/nolibc.hrenamed from tools/testing/selftests/rcutorture/bin/nolibc.h+92−26
@@ -3,7 +3,85 @@ · 2 unmodified lines
33 * Copyright (C) 2017-2018 Willy Tarreau <w@1wt.eu>
44 */
55
6−/* some archs (at least aarch64) don't expose the regular syscalls anymore by
6+/*
7+ * This file is designed to be used as a libc alternative for minimal programs
8+ * with very limited requirements. It consists of a small number of syscall and
9+ * type definitions, and the minimal startup code needed to call main().
10+ * All syscalls are declared as static functions so that they can be optimized
11+ * away by the compiler when not used.
12+ *
13+ * Syscalls are split into 3 levels:
14+ * - The lower level is the arch-specific syscall() definition, consisting in
15+ * assembly code in compound expressions. These are called my_syscall0() to
16+ * my_syscall6() depending on the number of arguments. The MIPS
17+ * implementation is limited to 5 arguments. All input arguments are cast
18+ * to a long stored in a register. These expressions always return the
19+ * syscall's return value as a signed long value which is often either a
20+ * pointer or the negated errno value.
21+ *
22+ * - The second level is mostly architecture-independent. It is made of
23+ * static functions called sys_<name>() which rely on my_syscallN()
24+ * depending on the syscall definition. These functions are responsible
25+ * for exposing the appropriate types for the syscall arguments (int,
26+ * pointers, etc) and for setting the appropriate return type (often int).
27+ * A few of them are architecture-specific because the syscalls are not all
28+ * mapped exactly the same among architectures. For example, some archs do
29+ * not implement select() and need pselect6() instead, so the sys_select()
30+ * function will have to abstract this.
31+ *
32+ * - The third level is the libc call definition. It exposes the lower raw
33+ * sys_<name>() calls in a way that looks like what a libc usually does,
34+ * takes care of specific input values, and of setting errno upon error.
35+ * There can be minor variations compared to standard libc calls. For
36+ * example the open() call always takes 3 args here.
37+ *
38+ * The errno variable is declared static and unused. This way it can be
39+ * optimized away if not used. However this means that a program made of
40+ * multiple C files may observe different errno values (one per C file). For
41+ * the type of programs this project targets it usually is not a problem. The
42+ * resulting program may even be reduced by defining the NOLIBC_IGNORE_ERRNO
43+ * macro, in which case the errno value will never be assigned.
44+ *
45+ * Some stdint-like integer types are defined. These are valid on all currently
46+ * supported architectures, because signs are enforced, ints are assumed to be
47+ * 32 bits, longs the size of a pointer and long long 64 bits. If more
48+ * architectures have to be supported, this may need to be adapted.
49+ *
50+ * Some macro definitions like the O_* values passed to open(), and some
51+ * structures like the sys_stat struct depend on the architecture.
52+ *
53+ * The definitions start with the architecture-specific parts, which are picked
54+ * based on what the compiler knows about the target architecture, and are
55+ * completed with the generic code. Since it is the compiler which sets the
56+ * target architecture, cross-compiling normally works out of the box without
57+ * having to specify anything.
58+ *
59+ * Finally some very common libc-level functions are provided. It is the case
60+ * for a few functions usually found in string.h, ctype.h, or stdlib.h. Nothing
61+ * is currently provided regarding stdio emulation.
62+ *
63+ * The macro NOLIBC is always defined, so that it is possible for a program to
64+ * check this macro to know if it is being built against and decide to disable
65+ * some features or simply not to include some standard libc files.
66+ *
67+ * Ideally this file should be split in multiple files for easier long term
68+ * maintenance, but provided as a single file as it is now, it's quite
69+ * convenient to use. Maybe some variations involving a set of includes at the
70+ * top could work.
71+ *
72+ * A simple static executable may be built this way :
73+ * $ gcc -fno-asynchronous-unwind-tables -fno-ident -s -Os -nostdlib \
74+ * -static -include nolibc.h -lgcc -o hello hello.c
75+ *
76+ * A very useful calling convention table may be found here :
77+ * http://man7.org/linux/man-pages/man2/syscall.2.html
78+ *
79+ * This doc is quite convenient though not necessarily up to date :
80+ * https://w3challs.com/syscalls/
81+ *
82+ */
83+
84+/* Some archs (at least aarch64) don't expose the regular syscalls anymore by
785 * default, either because they have an "_at" replacement, or because there are
886 * more modern alternatives. For now we'd rather still use them.
987 */
@@ -19,18 +97,6 @@ · 9 unmodified lines
1997
2098 #define NOLIBC
2199
22−/* Build a static executable this way :
23− * $ gcc -fno-asynchronous-unwind-tables -fno-ident -s -Os -nostdlib \
24− * -static -include nolibc.h -lgcc -o hello hello.c
25− *
26− * Useful calling convention table found here :
27− * http://man7.org/linux/man-pages/man2/syscall.2.html
28− *
29− * This doc is even better :
30− * https://w3challs.com/syscalls/
31− */
32−
33−
34100 /* this way it will be removed if unused */
35101 static int errno;
36102
@@ -81,9 +147,9 @@ typedef signed long time_t; · 44 unmodified lines
81147
82148 /* for poll() */
83149 struct pollfd {
84− int fd;
85− short int events;
86− short int revents;
150+ int fd;
151+ short int events;
152+ short int revents;
87153 };
88154
89155 /* for select() */
@@ -239,7 +305,7 @@ struct stat { · 149 unmodified lines
239305 "syscall\n" \
240306 : "=a" (_ret) \
241307 : "0"(_num) \
242− : "rcx", "r8", "r9", "r10", "r11", "memory", "cc" \
308+ : "rcx", "r8", "r9", "r10", "r11", "memory", "cc" \
243309 ); \
244310 _ret; \
245311 })
@@ -255,7 +321,7 @@ struct stat { · 9 unmodified lines
255321 : "=a" (_ret) \
256322 : "r"(_arg1), \
257323 "0"(_num) \
258− : "rcx", "r8", "r9", "r10", "r11", "memory", "cc" \
324+ : "rcx", "r8", "r9", "r10", "r11", "memory", "cc" \
259325 ); \
260326 _ret; \
261327 })
@@ -272,7 +338,7 @@ struct stat { · 10 unmodified lines
272338 : "=a" (_ret) \
273339 : "r"(_arg1), "r"(_arg2), \
274340 "0"(_num) \
275− : "rcx", "r8", "r9", "r10", "r11", "memory", "cc" \
341+ : "rcx", "r8", "r9", "r10", "r11", "memory", "cc" \
276342 ); \
277343 _ret; \
278344 })
@@ -290,7 +356,7 @@ struct stat { · 11 unmodified lines
290356 : "=a" (_ret) \
291357 : "r"(_arg1), "r"(_arg2), "r"(_arg3), \
292358 "0"(_num) \
293− : "rcx", "r8", "r9", "r10", "r11", "memory", "cc" \
359+ : "rcx", "r8", "r9", "r10", "r11", "memory", "cc" \
294360 ); \
295361 _ret; \
296362 })
@@ -1006,7 +1072,7 @@ struct sys_stat_struct { · 709 unmodified lines
10061072 : "=r"(_num), "=r"(_arg4) \
10071073 : "r"(_num) \
10081074 : "memory", "cc", "at", "v1", "hi", "lo", \
1009− \
1075+ "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9" \
10101076 ); \
10111077 _arg4 ? -_num : _num; \
10121078 })
@@ -1025,7 +1091,7 @@ struct sys_stat_struct { · 12 unmodified lines
10251091 : "0"(_num), \
10261092 "r"(_arg1) \
10271093 : "memory", "cc", "at", "v1", "hi", "lo", \
1028− \
1094+ "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9" \
10291095 ); \
10301096 _arg4 ? -_num : _num; \
10311097 })
@@ -1045,7 +1111,7 @@ struct sys_stat_struct { · 13 unmodified lines
10451111 : "0"(_num), \
10461112 "r"(_arg1), "r"(_arg2) \
10471113 : "memory", "cc", "at", "v1", "hi", "lo", \
1048− \
1114+ "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9" \
10491115 ); \
10501116 _arg4 ? -_num : _num; \
10511117 })
@@ -1066,7 +1132,7 @@ struct sys_stat_struct { · 14 unmodified lines
10661132 : "0"(_num), \
10671133 "r"(_arg1), "r"(_arg2), "r"(_arg3) \
10681134 : "memory", "cc", "at", "v1", "hi", "lo", \
1069− \
1135+ "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9" \
10701136 ); \
10711137 _arg4 ? -_num : _num; \
10721138 })
@@ -1087,7 +1153,7 @@ struct sys_stat_struct { · 14 unmodified lines
10871153 : "0"(_num), \
10881154 "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4) \
10891155 : "memory", "cc", "at", "v1", "hi", "lo", \
1090− \
1156+ "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9" \
10911157 ); \
10921158 _arg4 ? -_num : _num; \
10931159 })
@@ -1110,7 +1176,7 @@ struct sys_stat_struct { · 16 unmodified lines
11101176 : "0"(_num), \
11111177 "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5) \
11121178 : "memory", "cc", "at", "v1", "hi", "lo", \
1113− \
1179+ "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9" \
11141180 ); \
11151181 _arg4 ? -_num : _num; \
11161182 })
tools/memory-model/.gitignore+1−0
@@ -0,0 +1 @@
1+litmus
tools/memory-model/README+2−0
@@ -156,6 +156,8 @@ lock.cat · 155 unmodified lines
156156 README
157157 This file.
158158
159+scripts Various scripts, see scripts/README.
160+
159161
160162 ===========
161163 LIMITATIONS
tools/memory-model/linux-kernel.bell+2−1
@@ -29,7 +29,8 @@ enum Barriers = 'wmb (*smp_wmb*) || · 28 unmodified lines
2929 'sync-rcu (*synchronize_rcu*) ||
3030 'before-atomic (*smp_mb__before_atomic*) ||
3131 'after-atomic (*smp_mb__after_atomic*) ||
32− 'after-spinlock (*smp_mb__after_spinlock*)
32+ 'after-spinlock (*smp_mb__after_spinlock*) ||
33+ 'after-unlock-lock (*smp_mb__after_unlock_lock*)
3334 instructions F[Barriers]
3435
3536 (* Compute matching pairs of nested Rcu-lock and Rcu-unlock *)
tools/memory-model/linux-kernel.cat+3−1
@@ -30,7 +30,9 @@ let wmb = [W] ; fencerel(Wmb) ; [W] · 29 unmodified lines
3030 let mb = ([M] ; fencerel(Mb) ; [M]) |
3131 ([M] ; fencerel(Before-atomic) ; [RMW] ; po? ; [M]) |
3232 ([M] ; po? ; [RMW] ; fencerel(After-atomic) ; [M]) |
33− ([M] ; po? ; [LKW] ; fencerel(After-spinlock) ; [M])
33+ ([M] ; po? ; [LKW] ; fencerel(After-spinlock) ; [M]) |
34+ ([M] ; po ; [UL] ; (co | po) ; [LKW] ;
35+ fencerel(After-unlock-lock) ; [M])
3436 let gp = po ; [Sync-rcu] ; po?
3537
3638 let strong-fence = mb | gp
tools/memory-model/linux-kernel.def+1−0
@@ -23,6 +23,7 @@ smp_wmb() { __fence{wmb}; } · 22 unmodified lines
2323 smp_mb__before_atomic() { __fence{before-atomic}; }
2424 smp_mb__after_atomic() { __fence{after-atomic}; }
2525 smp_mb__after_spinlock() { __fence{after-spinlock}; }
26+smp_mb__after_unlock_lock() { __fence{after-unlock-lock}; }
2627
2728 // Exchange
2829 xchg(X,V) __xchg{mb}(X,V)
tools/memory-model/scripts/README+70−0
@@ -0,0 +1,70 @@
1+ ============
2+ LKMM SCRIPTS
3+ ============
4+
5+
6+These scripts are run from the tools/memory-model directory.
7+
8+checkalllitmus.sh
9+
10+ Run all litmus tests in the litmus-tests directory, checking
11+ the results against the expected results recorded in the
12+ "Result:" comment lines.
13+
14+checkghlitmus.sh
15+
16+ Run all litmus tests in the https://github.com/paulmckrcu/litmus
17+ archive that are C-language and that have "Result:" comment lines
18+ documenting expected results, comparing the actual results to
19+ those expected.
20+
21+checklitmushist.sh
22+
23+ Run all litmus tests having .litmus.out files from previous
24+ initlitmushist.sh or newlitmushist.sh runs, comparing the
25+ herd output to that of the original runs.
26+
27+checklitmus.sh
28+
29+ Check a single litmus test against its "Result:" expected result.
30+
31+cmplitmushist.sh
32+
33+ Compare output from two different runs of the same litmus tests,
34+ with the absolute pathnames of the tests to run provided one
35+ name per line on standard input. Not normally run manually,
36+ provided instead for use by other scripts.
37+
38+initlitmushist.sh
39+
40+ Run all litmus tests having no more than the specified number
41+ of processes given a specified timeout, recording the results
42+ in .litmus.out files.
43+
44+judgelitmus.sh
45+
46+ Given a .litmus file and its .litmus.out herd output, check the
47+ .litmus.out file against the .litmus file's "Result:" comment to
48+ judge whether the test ran correctly. Not normally run manually,
49+ provided instead for use by other scripts.
50+
51+newlitmushist.sh
52+
53+ For all new or updated litmus tests having no more than the
54+ specified number of processes given a specified timeout, run
55+ and record the results in .litmus.out files.
56+
57+parseargs.sh
58+
59+ Parse command-line arguments. Not normally run manually,
60+ provided instead for use by other scripts.
61+
62+runlitmushist.sh
63+
64+ Run the litmus tests whose absolute pathnames are provided one
65+ name per line on standard input. Not normally run manually,
66+ provided instead for use by other scripts.
67+
68+README
69+
70+ This file
tools/memory-model/scripts/checkalllitmus.sh+23−30
@@ -1,42 +1,27 @@
11 #!/bin/sh
2+# SPDX-License-Identifier: GPL-2.0+
23 #
3−# Run herd tests on all .litmus files in the specified directory (which
4−# defaults to litmus-tests) and check each file's result against a "Result:"
5−# comment within that litmus test. If the verification result does not
6−# match that specified in the litmus test, this script prints an error
7−# message prefixed with "^^^". It also outputs verification results to
8−# a file whose name is that of the specified litmus test, but with ".out"
9−# appended.
4+# Run herd tests on all .litmus files in the litmus-tests directory
5+# and check each file's result against a "Result:" comment within that
6+# litmus test. If the verification result does not match that specified
7+# in the litmus test, this script prints an error message prefixed with
8+# "^^^". It also outputs verification results to a file whose name is
9+# that of the specified litmus test, but with ".out" appended.
1010 #
1111 # Usage:
12−# checkalllitmus.sh [ directory ]
12+# checkalllitmus.sh
1313 #
14−# The LINUX_HERD_OPTIONS environment variable may be used to specify
15−# arguments to herd, whose default is defined by the checklitmus.sh script.
16−# Thus, one would normally run this in the directory containing the memory
17−# model, specifying the pathname of the litmus test to check.
14+# Run this in the directory containing the memory model.
1815 #
1916 # This script makes no attempt to run the litmus tests concurrently.
2017 #
21−# This program is free software; you can redistribute it and/or modify
22−# it under the terms of the GNU General Public License as published by
23−# the Free Software Foundation; either version 2 of the License, or
24−# (at your option) any later version.
25−#
26−# This program is distributed in the hope that it will be useful,
27−# but WITHOUT ANY WARRANTY; without even the implied warranty of
28−# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
29−# GNU General Public License for more details.
30−#
31−# You should have received a copy of the GNU General Public License
32−# along with this program; if not, you can access it online at
33−# http://www.gnu.org/licenses/gpl-2.0.html.
34−#
3518 # Copyright IBM Corporation, 2018
3619 #
3720 # Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
3821
39−litmusdir=${1-litmus-tests}
22+. scripts/parseargs.sh
23+
24+litmusdir=litmus-tests
4025 if test -d "$litmusdir" -a -r "$litmusdir" -a -x "$litmusdir"
4126 then
4227 :
@@ -45,6 +30,14 @@ else · 2 unmodified lines
4530 exit 255
4631 fi
4732
33+# Create any new directories that have appeared in the github litmus
34+# repo since the last run.
35+if test "$LKMM_DESTDIR" != "."
36+then
37+ find $litmusdir -type d -print |
38+ ( cd "$LKMM_DESTDIR"; sed -e 's/^/mkdir -p /' | sh )
39+fi
40+
4841 # Find the checklitmus script. If it is not where we expect it, then
4942 # assume that the caller has the PATH environment variable set
5043 # appropriately.
@@ -57,7 +50,7 @@ fi · 6 unmodified lines
5750
5851 # Run the script on all the litmus tests in the specified directory
5952 ret=0
60−for i in litmus-tests/*.litmus
53+for i in $litmusdir/*.litmus
6154 do
6255 if ! $clscript $i
6356 then
@@ -66,8 +59,8 @@ do · 2 unmodified lines
6659 done
6760 if test "$ret" -ne 0
6861 then
69− echo " ^^^ VERIFICATION MISMATCHES"
62+ echo " ^^^ VERIFICATION MISMATCHES" 1>&2
7063 else
71− echo All litmus tests verified as was expected.
64+ echo All litmus tests verified as was expected. 1>&2
7265 fi
7366 exit $ret
tools/memory-model/scripts/checkghlitmus.sh+65−0
@@ -0,0 +1,65 @@
1+#!/bin/sh
2+# SPDX-License-Identifier: GPL-2.0+
3+#
4+# Runs the C-language litmus tests having a maximum number of processes
5+# to run, defaults to 6.
6+#
7+# sh checkghlitmus.sh
8+#
9+# Run from the Linux kernel tools/memory-model directory. See the
10+# parseargs.sh scripts for arguments.
11+
12+. scripts/parseargs.sh
13+
14+T=/tmp/checkghlitmus.sh.$$
15+trap 'rm -rf $T' 0
16+mkdir $T
17+
18+# Clone the repository if it is not already present.
19+if test -d litmus
20+then
21+ :
22+else
23+ git clone https://github.com/paulmckrcu/litmus
24+ ( cd litmus; git checkout origin/master )
25+fi
26+
27+# Create any new directories that have appeared in the github litmus
28+# repo since the last run.
29+if test "$LKMM_DESTDIR" != "."
30+then
31+ find litmus -type d -print |
32+ ( cd "$LKMM_DESTDIR"; sed -e 's/^/mkdir -p /' | sh )
33+fi
34+
35+# Create a list of the C-language litmus tests previously run.
36+( cd $LKMM_DESTDIR; find litmus -name '*.litmus.out' -print ) |
37+ sed -e 's/\.out$//' |
38+ xargs -r egrep -l '^ \* Result: (Never|Sometimes|Always|DEADLOCK)' |
39+ xargs -r grep -L "^P${LKMM_PROCS}"> $T/list-C-already
40+
41+# Create a list of C-language litmus tests with "Result:" commands and
42+# no more than the specified number of processes.
43+find litmus -name '*.litmus' -exec grep -l -m 1 "^C " {} \; > $T/list-C
44+xargs < $T/list-C -r egrep -l '^ \* Result: (Never|Sometimes|Always|DEADLOCK)' > $T/list-C-result
45+xargs < $T/list-C-result -r grep -L "^P${LKMM_PROCS}" > $T/list-C-result-short
46+
47+# Form list of tests without corresponding .litmus.out files
48+sort $T/list-C-already $T/list-C-result-short | uniq -u > $T/list-C-needed
49+
50+# Run any needed tests.
51+if scripts/runlitmushist.sh < $T/list-C-needed > $T/run.stdout 2> $T/run.stderr
52+then
53+ errs=
54+else
55+ errs=1
56+fi
57+
58+sed < $T/list-C-result-short -e 's,^,scripts/judgelitmus.sh ,' |
59+ sh > $T/judge.stdout 2> $T/judge.stderr
60+
61+if test -n "$errs"
62+then
63+ cat $T/run.stderr 1>&2
64+fi
65+grep '!!!' $T/judge.stdout
tools/memory-model/scripts/checklitmus.sh+11−63
@@ -1,40 +1,24 @@
11 #!/bin/sh
2+# SPDX-License-Identifier: GPL-2.0+
23 #
3−# Run a herd test and check the result against a "Result:" comment within
4−# the litmus test. If the verification result does not match that specified
5−# in the litmus test, this script prints an error message prefixed with
6−# "^^^" and exits with a non-zero status. It also outputs verification
4+# Run a herd test and invokes judgelitmus.sh to check the result against
5+# a "Result:" comment within the litmus test. It also outputs verification
76 # results to a file whose name is that of the specified litmus test, but
87 # with ".out" appended.
98 #
109 # Usage:
1110 # checklitmus.sh file.litmus
1211 #
13−# The LINUX_HERD_OPTIONS environment variable may be used to specify
14−# arguments to herd, which default to "-conf linux-kernel.cfg". Thus,
15−# one would normally run this in the directory containing the memory model,
16−# specifying the pathname of the litmus test to check.
17−#
18−# This program is free software; you can redistribute it and/or modify
19−# it under the terms of the GNU General Public License as published by
20−# the Free Software Foundation; either version 2 of the License, or
21−# (at your option) any later version.
22−#
23−# This program is distributed in the hope that it will be useful,
24−# but WITHOUT ANY WARRANTY; without even the implied warranty of
25−# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26−# GNU General Public License for more details.
27−#
28−# You should have received a copy of the GNU General Public License
29−# along with this program; if not, you can access it online at
30−# http://www.gnu.org/licenses/gpl-2.0.html.
12+# Run this in the directory containing the memory model, specifying the
13+# pathname of the litmus test to check. The caller is expected to have
14+# properly set up the LKMM environment variables.
3115 #
3216 # Copyright IBM Corporation, 2018
3317 #
3418 # Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
3519
3620 litmus=$1
37−herdoptions=${LINUX_HERD_OPTIONS--conf linux-kernel.cfg}
21+herdoptions=${LKMM_HERD_OPTIONS--conf linux-kernel.cfg}
3822
3923 if test -f "$litmus" -a -r "$litmus"
4024 then
@@ -43,44 +27,8 @@ else · 2 unmodified lines
4327 echo ' --- ' error: \"$litmus\" is not a readable file
4428 exit 255
4529 fi
46−if grep -q '^ \* Result: ' $litmus
47−then
48− outcome=`grep -m 1 '^ \* Result: ' $litmus | awk '{ print $3 }'`
49−else
50− outcome=specified
51−fi
5230
53−echo Herd options: $herdoptions > $litmus.out
54−/usr/bin/time herd7 -o ~/tmp $herdoptions $litmus >> $litmus.out 2>&1
55−grep "Herd options:" $litmus.out
56−grep '^Observation' $litmus.out
57−if grep -q '^Observation' $litmus.out
58−then
59− :
60−else
61− cat $litmus.out
62− echo ' ^^^ Verification error'
63− echo ' ^^^ Verification error' >> $litmus.out 2>&1
64− exit 255
65−fi
66−if test "$outcome" = DEADLOCK
67−then
68− echo grep 3 and 4
69− if grep '^Observation' $litmus.out | grep -q 'Never 0 0$'
70− then
71− ret=0
72− else
73− echo " ^^^ Unexpected non-$outcome verification"
74− echo " ^^^ Unexpected non-$outcome verification" >> $litmus.out 2>&1
75− ret=1
76− fi
77−elif grep '^Observation' $litmus.out | grep -q $outcome || test "$outcome" = Maybe
78−then
79− ret=0
80−else
81− echo " ^^^ Unexpected non-$outcome verification"
82− echo " ^^^ Unexpected non-$outcome verification" >> $litmus.out 2>&1
83− ret=1
84−fi
85−tail -2 $litmus.out | head -1
86−exit $ret
31+echo Herd options: $herdoptions > $LKMM_DESTDIR/$litmus.out
32+/usr/bin/time $LKMM_TIMEOUT_CMD herd7 $herdoptions $litmus >> $LKMM_DESTDIR/$litmus.out 2>&1
33+
34+scripts/judgelitmus.sh $litmus
tools/memory-model/scripts/checklitmushist.sh+60−0
@@ -0,0 +1,60 @@
1+#!/bin/sh
2+# SPDX-License-Identifier: GPL-2.0+
3+#
4+# Reruns the C-language litmus tests previously run that match the
5+# specified criteria, and compares the result to that of the previous
6+# runs from initlitmushist.sh and/or newlitmushist.sh.
7+#
8+# sh checklitmushist.sh
9+#
10+# Run from the Linux kernel tools/memory-model directory.
11+# See scripts/parseargs.sh for list of arguments.
12+#
13+# Copyright IBM Corporation, 2018
14+#
15+# Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
16+
17+. scripts/parseargs.sh
18+
19+T=/tmp/checklitmushist.sh.$$
20+trap 'rm -rf $T' 0
21+mkdir $T
22+
23+if test -d litmus
24+then
25+ :
26+else
27+ echo Run scripts/initlitmushist.sh first, need litmus repo.
28+ exit 1
29+fi
30+
31+# Create the results directory and populate it with subdirectories.
32+# The initial output is created here to avoid clobbering the output
33+# generated earlier.
34+mkdir $T/results
35+find litmus -type d -print | ( cd $T/results; sed -e 's/^/mkdir -p /' | sh )
36+
37+# Create the list of litmus tests already run, then remove those that
38+# are excluded by this run's --procs argument.
39+( cd $LKMM_DESTDIR; find litmus -name '*.litmus.out' -print ) |
40+ sed -e 's/\.out$//' |
41+ xargs -r grep -L "^P${LKMM_PROCS}"> $T/list-C-already
42+xargs < $T/list-C-already -r grep -L "^P${LKMM_PROCS}" > $T/list-C-short
43+
44+# Redirect output, run tests, then restore destination directory.
45+destdir="$LKMM_DESTDIR"
46+LKMM_DESTDIR=$T/results; export LKMM_DESTDIR
47+scripts/runlitmushist.sh < $T/list-C-short > $T/runlitmushist.sh.out 2>&1
48+LKMM_DESTDIR="$destdir"; export LKMM_DESTDIR
49+
50+# Move the newly generated .litmus.out files to .litmus.out.new files
51+# in the destination directory.
52+cdir=`pwd`
53+ddir=`awk -v c="$cdir" -v d="$LKMM_DESTDIR" \
54+ 'END { if (d ~ /^\//) print d; else print c "/" d; }' < /dev/null`
55+( cd $T/results; find litmus -type f -name '*.litmus.out' -print |
56+ sed -e 's,^.*$,cp & '"$ddir"'/&.new,' | sh )
57+
58+sed < $T/list-C-short -e 's,^,'"$LKMM_DESTDIR/"',' |
59+ sh scripts/cmplitmushist.sh
60+exit $?
tools/memory-model/scripts/cmplitmushist.sh+87−0
@@ -0,0 +1,87 @@
1+#!/bin/sh
2+# SPDX-License-Identifier: GPL-2.0+
3+#
4+# Compares .out and .out.new files for each name on standard input,
5+# one full pathname per line. Outputs comparison results followed by
6+# a summary.
7+#
8+# sh cmplitmushist.sh
9+
10+T=/tmp/cmplitmushist.sh.$$
11+trap 'rm -rf $T' 0
12+mkdir $T
13+
14+# comparetest oldpath newpath
15+perfect=0
16+obsline=0
17+noobsline=0
18+obsresult=0
19+badcompare=0
20+comparetest () {
21+ grep -v 'maxresident)k\|minor)pagefaults\|^Time' $1 > $T/oldout
22+ grep -v 'maxresident)k\|minor)pagefaults\|^Time' $2 > $T/newout
23+ if cmp -s $T/oldout $T/newout && grep -q '^Observation' $1
24+ then
25+ echo Exact output match: $2
26+ perfect=`expr "$perfect" + 1`
27+ return 0
28+ fi
29+
30+ grep '^Observation' $1 > $T/oldout
31+ grep '^Observation' $2 > $T/newout
32+ if test -s $T/oldout -o -s $T/newout
33+ then
34+ if cmp -s $T/oldout $T/newout
35+ then
36+ echo Matching Observation result and counts: $2
37+ obsline=`expr "$obsline" + 1`
38+ return 0
39+ fi
40+ else
41+ echo Missing Observation line "(e.g., herd7 timeout)": $2
42+ noobsline=`expr "$noobsline" + 1`
43+ return 0
44+ fi
45+
46+ grep '^Observation' $1 | awk '{ print $3 }' > $T/oldout
47+ grep '^Observation' $2 | awk '{ print $3 }' > $T/newout
48+ if cmp -s $T/oldout $T/newout
49+ then
50+ echo Matching Observation Always/Sometimes/Never result: $2
51+ obsresult=`expr "$obsresult" + 1`
52+ return 0
53+ fi
54+ echo ' !!!' Result changed: $2
55+ badcompare=`expr "$badcompare" + 1`
56+ return 1
57+}
58+
59+sed -e 's/^.*$/comparetest &.out &.out.new/' > $T/cmpscript
60+. $T/cmpscript > $T/cmpscript.out
61+cat $T/cmpscript.out
62+
63+echo ' ---' Summary: 1>&2
64+grep '!!!' $T/cmpscript.out 1>&2
65+if test "$perfect" -ne 0
66+then
67+ echo Exact output matches: $perfect 1>&2
68+fi
69+if test "$obsline" -ne 0
70+then
71+ echo Matching Observation result and counts: $obsline 1>&2
72+fi
73+if test "$noobsline" -ne 0
74+then
75+ echo Missing Observation line "(e.g., herd7 timeout)": $noobsline 1>&2
76+fi
77+if test "$obsresult" -ne 0
78+then
79+ echo Matching Observation Always/Sometimes/Never result: $obsresult 1>&2
80+fi
81+if test "$badcompare" -ne 0
82+then
83+ echo "!!!" Result changed: $badcompare 1>&2
84+ exit 1
85+fi
86+
87+exit 0
tools/memory-model/scripts/initlitmushist.sh+68−0
@@ -0,0 +1,68 @@
1+#!/bin/sh
2+# SPDX-License-Identifier: GPL-2.0+
3+#
4+# Runs the C-language litmus tests matching the specified criteria.
5+# Generates the output for each .litmus file into a corresponding
6+# .litmus.out file, and does not judge the result.
7+#
8+# sh initlitmushist.sh
9+#
10+# Run from the Linux kernel tools/memory-model directory.
11+# See scripts/parseargs.sh for list of arguments.
12+#
13+# This script can consume significant wallclock time and CPU, especially as
14+# the value of --procs rises. On a four-core (eight hardware threads)
15+# 2.5GHz x86 with a one-minute per-run timeout:
16+#
17+# --procs wallclock CPU timeouts tests
18+# 1 0m11.241s 0m1.086s 0 19
19+# 2 1m12.598s 2m8.459s 2 393
20+# 3 1m30.007s 6m2.479s 4 2291
21+# 4 3m26.042s 18m5.139s 9 3217
22+# 5 4m26.661s 23m54.128s 13 3784
23+# 6 4m41.900s 26m4.721s 13 4352
24+# 7 5m51.463s 35m50.868s 13 4626
25+# 8 10m5.235s 68m43.672s 34 5117
26+# 9 15m57.80s 105m58.101s 69 5156
27+# 10 16m14.13s 103m35.009s 69 5165
28+# 20 27m48.55s 198m3.286s 156 5269
29+#
30+# Increasing the timeout on the 20-process run to five minutes increases
31+# the runtime to about 90 minutes with the CPU time rising to about
32+# 10 hours. On the other hand, it decreases the number of timeouts to 101.
33+#
34+# Note that there are historical tests for which herd7 will fail
35+# completely, for example, litmus/manual/atomic/C-unlock-wait-00.litmus
36+# contains a call to spin_unlock_wait(), which no longer exists in either
37+# the kernel or LKMM.
38+
39+. scripts/parseargs.sh
40+
41+T=/tmp/initlitmushist.sh.$$
42+trap 'rm -rf $T' 0
43+mkdir $T
44+
45+if test -d litmus
46+then
47+ :
48+else
49+ git clone https://github.com/paulmckrcu/litmus
50+ ( cd litmus; git checkout origin/master )
51+fi
52+
53+# Create any new directories that have appeared in the github litmus
54+# repo since the last run.
55+if test "$LKMM_DESTDIR" != "."
56+then
57+ find litmus -type d -print |
58+ ( cd "$LKMM_DESTDIR"; sed -e 's/^/mkdir -p /' | sh )
59+fi
60+
61+# Create a list of the C-language litmus tests with no more than the
62+# specified number of processes (per the --procs argument).
63+find litmus -name '*.litmus' -exec grep -l -m 1 "^C " {} \; > $T/list-C
64+xargs < $T/list-C -r grep -L "^P${LKMM_PROCS}" > $T/list-C-short
65+
66+scripts/runlitmushist.sh < $T/list-C-short
67+
68+exit 0
tools/memory-model/scripts/judgelitmus.sh+78−0
@@ -0,0 +1,78 @@
1+#!/bin/sh
2+# SPDX-License-Identifier: GPL-2.0+
3+#
4+# Given a .litmus test and the corresponding .litmus.out file, check
5+# the .litmus.out file against the "Result:" comment to judge whether
6+# the test ran correctly.
7+#
8+# Usage:
9+# judgelitmus.sh file.litmus
10+#
11+# Run this in the directory containing the memory model, specifying the
12+# pathname of the litmus test to check.
13+#
14+# Copyright IBM Corporation, 2018
15+#
16+# Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
17+
18+litmus=$1
19+
20+if test -f "$litmus" -a -r "$litmus"
21+then
22+ :
23+else
24+ echo ' --- ' error: \"$litmus\" is not a readable file
25+ exit 255
26+fi
27+if test -f "$LKMM_DESTDIR/$litmus".out -a -r "$LKMM_DESTDIR/$litmus".out
28+then
29+ :
30+else
31+ echo ' --- ' error: \"$LKMM_DESTDIR/$litmus\".out is not a readable file
32+ exit 255
33+fi
34+if grep -q '^ \* Result: ' $litmus
35+then
36+ outcome=`grep -m 1 '^ \* Result: ' $litmus | awk '{ print $3 }'`
37+else
38+ outcome=specified
39+fi
40+
41+grep '^Observation' $LKMM_DESTDIR/$litmus.out
42+if grep -q '^Observation' $LKMM_DESTDIR/$litmus.out
43+then
44+ :
45+else
46+ echo ' !!! Verification error' $litmus
47+ if ! grep -q '!!!' $LKMM_DESTDIR/$litmus.out
48+ then
49+ echo ' !!! Verification error' >> $LKMM_DESTDIR/$litmus.out 2>&1
50+ fi
51+ exit 255
52+fi
53+if test "$outcome" = DEADLOCK
54+then
55+ if grep '^Observation' $LKMM_DESTDIR/$litmus.out | grep -q 'Never 0 0$'
56+ then
57+ ret=0
58+ else
59+ echo " !!! Unexpected non-$outcome verification" $litmus
60+ if ! grep -q '!!!' $LKMM_DESTDIR/$litmus.out
61+ then
62+ echo " !!! Unexpected non-$outcome verification" >> $LKMM_DESTDIR/$litmus.out 2>&1
63+ fi
64+ ret=1
65+ fi
66+elif grep '^Observation' $LKMM_DESTDIR/$litmus.out | grep -q $outcome || test "$outcome" = Maybe
67+then
68+ ret=0
69+else
70+ echo " !!! Unexpected non-$outcome verification" $litmus
71+ if ! grep -q '!!!' $LKMM_DESTDIR/$litmus.out
72+ then
73+ echo " !!! Unexpected non-$outcome verification" >> $LKMM_DESTDIR/$litmus.out 2>&1
74+ fi
75+ ret=1
76+fi
77+tail -2 $LKMM_DESTDIR/$litmus.out | head -1
78+exit $ret
tools/memory-model/scripts/newlitmushist.sh+61−0
@@ -0,0 +1,61 @@
1+#!/bin/sh
2+# SPDX-License-Identifier: GPL-2.0+
3+#
4+# Runs the C-language litmus tests matching the specified criteria
5+# that do not already have a corresponding .litmus.out file, and does
6+# not judge the result.
7+#
8+# sh newlitmushist.sh
9+#
10+# Run from the Linux kernel tools/memory-model directory.
11+# See scripts/parseargs.sh for list of arguments.
12+#
13+# Copyright IBM Corporation, 2018
14+#
15+# Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
16+
17+. scripts/parseargs.sh
18+
19+T=/tmp/newlitmushist.sh.$$
20+trap 'rm -rf $T' 0
21+mkdir $T
22+
23+if test -d litmus
24+then
25+ :
26+else
27+ echo Run scripts/initlitmushist.sh first, need litmus repo.
28+ exit 1
29+fi
30+
31+# Create any new directories that have appeared in the github litmus
32+# repo since the last run.
33+if test "$LKMM_DESTDIR" != "."
34+then
35+ find litmus -type d -print |
36+ ( cd "$LKMM_DESTDIR"; sed -e 's/^/mkdir -p /' | sh )
37+fi
38+
39+# Create a list of the C-language litmus tests previously run.
40+( cd $LKMM_DESTDIR; find litmus -name '*.litmus.out' -print ) |
41+ sed -e 's/\.out$//' |
42+ xargs -r grep -L "^P${LKMM_PROCS}"> $T/list-C-already
43+
44+# Form full list of litmus tests with no more than the specified
45+# number of processes (per the --procs argument).
46+find litmus -name '*.litmus' -exec grep -l -m 1 "^C " {} \; > $T/list-C-all
47+xargs < $T/list-C-all -r grep -L "^P${LKMM_PROCS}" > $T/list-C-short
48+
49+# Form list of new tests. Note: This does not handle litmus-test deletion!
50+sort $T/list-C-already $T/list-C-short | uniq -u > $T/list-C-new
51+
52+# Form list of litmus tests that have changed since the last run.
53+sed < $T/list-C-short -e 's,^.*$,if test & -nt '"$LKMM_DESTDIR"'/&.out; then echo &; fi,' > $T/list-C-script
54+sh $T/list-C-script > $T/list-C-newer
55+
56+# Merge the list of new and of updated litmus tests: These must be (re)run.
57+sort -u $T/list-C-new $T/list-C-newer > $T/list-C-needed
58+
59+scripts/runlitmushist.sh < $T/list-C-needed
60+
61+exit 0
tools/memory-model/scripts/parseargs.sh+136−0
@@ -0,0 +1,136 @@
1+#!/bin/sh
2+# SPDX-License-Identifier: GPL-2.0+
3+#
4+# the corresponding .litmus.out file, and does not judge the result.
5+#
6+# . scripts/parseargs.sh
7+#
8+# Include into other Linux kernel tools/memory-model scripts.
9+#
10+# Copyright IBM Corporation, 2018
11+#
12+# Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
13+
14+T=/tmp/parseargs.sh.$$
15+mkdir $T
16+
17+# Initialize one parameter: initparam name default
18+initparam () {
19+ echo if test -z '"$'$1'"' > $T/s
20+ echo then >> $T/s
21+ echo $1='"'$2'"' >> $T/s
22+ echo export $1 >> $T/s
23+ echo fi >> $T/s
24+ echo $1_DEF='$'$1 >> $T/s
25+ . $T/s
26+}
27+
28+initparam LKMM_DESTDIR "."
29+initparam LKMM_HERD_OPTIONS "-conf linux-kernel.cfg"
30+initparam LKMM_JOBS `getconf _NPROCESSORS_ONLN`
31+initparam LKMM_PROCS "3"
32+initparam LKMM_TIMEOUT "1m"
33+
34+scriptname=$0
35+
36+usagehelp () {
37+ echo "Usage $scriptname [ arguments ]"
38+ echo " --destdir path (place for .litmus.out, default by .litmus)"
39+ echo " --herdopts -conf linux-kernel.cfg ..."
40+ echo " --jobs N (number of jobs, default one per CPU)"
41+ echo " --procs N (litmus tests with at most this many processes)"
42+ echo " --timeout N (herd7 timeout (e.g., 10s, 1m, 2hr, 1d, '')"
43+ echo "Defaults: --destdir '$LKMM_DESTDIR_DEF' --herdopts '$LKMM_HERD_OPTIONS_DEF' --jobs '$LKMM_JOBS_DEF' --procs '$LKMM_PROCS_DEF' --timeout '$LKMM_TIMEOUT_DEF'"
44+ exit 1
45+}
46+
47+usage () {
48+ usagehelp 1>&2
49+}
50+
51+# checkarg --argname argtype $# arg mustmatch cannotmatch
52+checkarg () {
53+ if test $3 -le 1
54+ then
55+ echo $1 needs argument $2 matching \"$5\"
56+ usage
57+ fi
58+ if echo "$4" | grep -q -e "$5"
59+ then
60+ :
61+ else
62+ echo $1 $2 \"$4\" must match \"$5\"
63+ usage
64+ fi
65+ if echo "$4" | grep -q -e "$6"
66+ then
67+ echo $1 $2 \"$4\" must not match \"$6\"
68+ usage
69+ fi
70+}
71+
72+while test $# -gt 0
73+do
74+ case "$1" in
75+ --destdir)
76+ checkarg --destdir "(path to directory)" "$#" "$2" '.\+' '^--'
77+ LKMM_DESTDIR="$2"
78+ mkdir $LKMM_DESTDIR > /dev/null 2>&1
79+ if ! test -e "$LKMM_DESTDIR"
80+ then
81+ echo "Cannot create directory --destdir '$LKMM_DESTDIR'"
82+ usage
83+ fi
84+ if test -d "$LKMM_DESTDIR" -a -w "$LKMM_DESTDIR" -a -x "$LKMM_DESTDIR"
85+ then
86+ :
87+ else
88+ echo "Directory --destdir '$LKMM_DESTDIR' insufficient permissions to create files"
89+ usage
90+ fi
91+ shift
92+ ;;
93+ --herdopts|--herdopt)
94+ checkarg --destdir "(herd options)" "$#" "$2" '.*' '^--'
95+ LKMM_HERD_OPTIONS="$2"
96+ shift
97+ ;;
98+ -j[1-9]*)
99+ njobs="`echo $1 | sed -e 's/^-j//'`"
100+ trailchars="`echo $njobs | sed -e 's/[0-9]\+\(.*\)$/\1/'`"
101+ if test -n "$trailchars"
102+ then
103+ echo $1 trailing characters "'$trailchars'"
104+ usagehelp
105+ fi
106+ LKMM_JOBS="`echo $njobs | sed -e 's/^\([0-9]\+\).*$/\1/'`"
107+ ;;
108+ --jobs|--job|-j)
109+ checkarg --jobs "(number)" "$#" "$2" '^[1-9][0-9]\+$' '^--'
110+ LKMM_JOBS="$2"
111+ shift
112+ ;;
113+ --procs|--proc)
114+ checkarg --procs "(number)" "$#" "$2" '^[0-9]\+$' '^--'
115+ LKMM_PROCS="$2"
116+ shift
117+ ;;
118+ --timeout)
119+ checkarg --timeout "(timeout spec)" "$#" "$2" '^\([0-9]\+[smhd]\?\|\)$' '^--'
120+ LKMM_TIMEOUT="$2"
121+ shift
122+ ;;
123+ *)
124+ echo Unknown argument $1
125+ usage
126+ ;;
127+ esac
128+ shift
129+done
130+if test -z "$LKMM_TIMEOUT"
131+then
132+ LKMM_TIMEOUT_CMD=""; export LKMM_TIMEOUT_CMD
133+else
134+ LKMM_TIMEOUT_CMD="timeout $LKMM_TIMEOUT"; export LKMM_TIMEOUT_CMD
135+fi
136+rm -rf $T
tools/memory-model/scripts/runlitmushist.sh+87−0
@@ -0,0 +1,87 @@
1+#!/bin/bash
2+# SPDX-License-Identifier: GPL-2.0+
3+#
4+# Runs the C-language litmus tests specified on standard input, using up
5+# to the specified number of CPUs (defaulting to all of them) and placing
6+# the results in the specified directory (defaulting to the same place
7+# the litmus test came from).
8+#
9+# sh runlitmushist.sh
10+#
11+# Run from the Linux kernel tools/memory-model directory.
12+# This script uses environment variables produced by parseargs.sh.
13+#
14+# Copyright IBM Corporation, 2018
15+#
16+# Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
17+
18+T=/tmp/runlitmushist.sh.$$
19+trap 'rm -rf $T' 0
20+mkdir $T
21+
22+if test -d litmus
23+then
24+ :
25+else
26+ echo Directory \"litmus\" missing, aborting run.
27+ exit 1
28+fi
29+
30+# Prefixes for per-CPU scripts
31+for ((i=0;i<$LKMM_JOBS;i++))
32+do
33+ echo dir="$LKMM_DESTDIR" > $T/$i.sh
34+ echo T=$T >> $T/$i.sh
35+ echo herdoptions=\"$LKMM_HERD_OPTIONS\" >> $T/$i.sh
36+ cat << '___EOF___' >> $T/$i.sh
37+ runtest () {
38+ echo ' ... ' /usr/bin/time $LKMM_TIMEOUT_CMD herd7 $herdoptions $1 '>' $dir/$1.out '2>&1'
39+ if /usr/bin/time $LKMM_TIMEOUT_CMD herd7 $herdoptions $1 > $dir/$1.out 2>&1
40+ then
41+ if ! grep -q '^Observation ' $dir/$1.out
42+ then
43+ echo ' !!! Herd failed, no Observation:' $1
44+ fi
45+ else
46+ exitcode=$?
47+ if test "$exitcode" -eq 124
48+ then
49+ exitmsg="timed out"
50+ else
51+ exitmsg="failed, exit code $exitcode"
52+ fi
53+ echo ' !!! Herd' ${exitmsg}: $1
54+ fi
55+ }
56+___EOF___
57+done
58+
59+awk -v q="'" -v b='\\' '
60+{
61+ print "echo `grep " q "^P[0-9]" b "+(" q " " $0 " | tail -1 | sed -e " q "s/^P" b "([0-9]" b "+" b ")(.*$/" b "1/" q "` " $0
62+}' | bash |
63+sort -k1n |
64+awk -v ncpu=$LKMM_JOBS -v t=$T '
65+{
66+ print "runtest " $2 >> t "/" NR % ncpu ".sh";
67+}
68+
69+END {
70+ for (i = 0; i < ncpu; i++) {
71+ print "sh " t "/" i ".sh > " t "/" i ".sh.out 2>&1 &";
72+ close(t "/" i ".sh");
73+ }
74+ print "wait";
75+}' | sh
76+cat $T/*.sh.out
77+if grep -q '!!!' $T/*.sh.out
78+then
79+ echo ' ---' Summary: 1>&2
80+ grep '!!!' $T/*.sh.out 1>&2
81+ nfail="`grep '!!!' $T/*.sh.out | wc -l`"
82+ echo 'Number of failed herd runs (e.g., timeout): ' $nfail 1>&2
83+ exit 1
84+else
85+ echo All runs completed successfully. 1>&2
86+ exit 0
87+fi
tools/testing/selftests/rcutorture/bin/mkinitrd.sh+17−10
@@ -40,17 +40,24 @@ mkdir $T · 39 unmodified lines
4040 cat > $T/init << '__EOF___'
4141 #!/bin/sh
4242 # Run in userspace a few milliseconds every second. This helps to
43−# exercise the NO_HZ_FULL portions of RCU.
43+# exercise the NO_HZ_FULL portions of RCU. The 192 instances of "a" was
44+# empirically shown to give a nice multi-millisecond burst of user-mode
45+# execution on a 2GHz CPU, as desired. Modern CPUs will vary from a
46+# couple of milliseconds up to perhaps 100 milliseconds, which is an
47+# acceptable range.
48+#
49+# Why not calibrate an exact delay? Because within this initrd, we
50+# are restricted to Bourne-shell builtins, which as far as I know do not
51+# provide any means of obtaining a fine-grained timestamp.
52+
53+a4="a a a a"
54+a16="$a4 $a4 $a4 $a4"
55+a64="$a16 $a16 $a16 $a16"
56+a192="$a64 $a64 $a64"
4457 while :
4558 do
4659 q=
47− for i in \
48− a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a \
49− a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a \
50− a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a \
51− a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a \
52− a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a \
53− a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a
60+ for i in $a192
5461 do
5562 q="$q $i"
5663 done
@@ -124,8 +131,8 @@ if echo -e "#if __x86_64__||__i386__||__i486__||__i586__||__i686__" \ · 67 unmodified lines
124131 | grep -q '^yes'; then
125132 # architecture supported by nolibc
126133 ${CROSS_COMPILE}gcc -fno-asynchronous-unwind-tables -fno-ident \
127− -nostdlib -include ../bin/nolibc.h -lgcc -s -static -Os \
128− -o init init.c
134+ -nostdlib -include ../../../../include/nolibc/nolibc.h \
135+ -lgcc -s -static -Os -o init init.c
129136 else
130137 ${CROSS_COMPILE}gcc -s -static -Os -o init init.c
131138 fi
virt/kvm/kvm_main.c+1−1
@@ -4084,7 +4084,7 @@ static int kvm_suspend(void) · 4083 unmodified lines
40844084 static void kvm_resume(void)
40854085 {
40864086 if (kvm_usage_count) {
4087− WARN_ON(raw_spin_is_locked(&kvm_count_lock));
4087+ lockdep_assert_held(&kvm_count_lock);
40884088 hardware_enable_nolock(NULL);
40894089 }
40904090 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t