History

Merge branches 'doc.2020.02.27a', 'fixes.2020.03.21a', 'kfree_rcu.2020.02.20a', 'locktorture.2020.02.20a', 'ovld.2020.02.20a', 'rcu-tasks.2020.02.20a', 'srcu.2020.02.20a' and 'torture.2020.02.20a' into HEAD

doc.2020.02.27a: Documentation updates. fixes.2020.03.21a: Miscellaneous fixes. kfree_rcu.2020.02.20a: Updates to kfree_rcu(). locktorture.2020.02.20a: Lock torture-test updates. ovld.2020.02.20a: Updates to callback-overload handling. rcu-tasks.2020.02.20a: RCU-tasks updates. srcu.2020.02.20a: SRCU updates. torture.2020.02.20a: Torture-test updates.

Paul E. McKenney committed 6 years ago

Commit aa93ec68 parents 8149b5c 127e298 6137079 28e09a2 b692dc4 90ba11b 7104260 a14493529 files · +644 −203Browse files

Showing the changes to kernel (16 of 29 files). Show all

Unified · Split

16 files
  1. kernel/locking/locktorture.c +8 −7
  2. kernel/locking/rtmutex.c +1 −1
  3. kernel/rcu/Makefile +4 −0
  4. kernel/rcu/rcu.h +21 −2
  5. kernel/rcu/rcu_segcblist.c +1 −3
  6. kernel/rcu/rcuperf.c +12 −2
  7. kernel/rcu/rcutorture.c +50 −17
  8. kernel/rcu/srcutree.c +10 −8
  9. kernel/rcu/tree.c +339 −113
  10. kernel/rcu/tree.h +4 −0
  11. kernel/rcu/tree_exp.h +9 −4
  12. kernel/rcu/tree_plugin.h +16 −9
  13. kernel/rcu/tree_stall.h +23 −18
  14. kernel/rcu/update.c +23 −5
  15. kernel/time/timer.c +4 −3
  16. kernel/torture.c +25 −4
kernel/locking/locktorture.c+8−7
@@ -618,7 +618,7 @@ static struct lock_torture_ops percpu_rwsem_lock_ops = { · 617 unmodified lines
618static int lock_torture_writer(void *arg)618static int lock_torture_writer(void *arg)
619{619{
620 struct lock_stress_stats *lwsp = arg;620 struct lock_stress_stats *lwsp = arg;
621 static DEFINE_TORTURE_RANDOM(rand);621 DEFINE_TORTURE_RANDOM(rand);
622 622
623 VERBOSE_TOROUT_STRING("lock_torture_writer task started");623 VERBOSE_TOROUT_STRING("lock_torture_writer task started");
624 set_user_nice(current, MAX_NICE);624 set_user_nice(current, MAX_NICE);
@@ -655,7 +655,7 @@ static int lock_torture_writer(void *arg) · 30 unmodified lines
655static int lock_torture_reader(void *arg)655static int lock_torture_reader(void *arg)
656{656{
657 struct lock_stress_stats *lrsp = arg;657 struct lock_stress_stats *lrsp = arg;
658 static DEFINE_TORTURE_RANDOM(rand);658 DEFINE_TORTURE_RANDOM(rand);
659 659
660 VERBOSE_TOROUT_STRING("lock_torture_reader task started");660 VERBOSE_TOROUT_STRING("lock_torture_reader task started");
661 set_user_nice(current, MAX_NICE);661 set_user_nice(current, MAX_NICE);
@@ -696,15 +696,16 @@ static void __torture_print_stats(char *page, · 34 unmodified lines
696 if (statp[i].n_lock_fail)696 if (statp[i].n_lock_fail)
697 fail = true;697 fail = true;
698 sum += statp[i].n_lock_acquired;698 sum += statp[i].n_lock_acquired;
699 if (max < statp[i].n_lock_fail)699 if (max < statp[i].n_lock_acquired)
700 max = statp[i].n_lock_fail;700 max = statp[i].n_lock_acquired;
701 if (min > statp[i].n_lock_fail)701 if (min > statp[i].n_lock_acquired)
702 min = statp[i].n_lock_fail;702 min = statp[i].n_lock_acquired;
703 }703 }
704 page += sprintf(page,704 page += sprintf(page,
705 "%s: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n",705 "%s: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n",
706 write ? "Writes" : "Reads ",706 write ? "Writes" : "Reads ",
707 sum, max, min, max / 2 > min ? "???" : "",707 sum, max, min,
708 !onoff_interval && max / 2 > min ? "???" : "",
708 fail, fail ? "!!!" : "");709 fail, fail ? "!!!" : "");
709 if (fail)710 if (fail)
710 atomic_inc(&cxt.n_lock_torture_errors);711 atomic_inc(&cxt.n_lock_torture_errors);
kernel/locking/rtmutex.c+1−1
@@ -57,7 +57,7 @@ rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner) · 56 unmodified lines
57 if (rt_mutex_has_waiters(lock))57 if (rt_mutex_has_waiters(lock))
58 val |= RT_MUTEX_HAS_WAITERS;58 val |= RT_MUTEX_HAS_WAITERS;
59 59
60 lock->owner = (struct task_struct *)val;60 WRITE_ONCE(lock->owner, (struct task_struct *)val);
61}61}
62 62
63static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)63static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
kernel/rcu/Makefile+4−0
@@ -3,6 +3,10 @@ · 2 unmodified lines
3# and is generally not a function of system call inputs.3# and is generally not a function of system call inputs.
4KCOV_INSTRUMENT := n4KCOV_INSTRUMENT := n
5 5
6ifeq ($(CONFIG_KCSAN),y)
7KBUILD_CFLAGS += -g -fno-omit-frame-pointer
8endif
9
6obj-y += update.o sync.o10obj-y += update.o sync.o
7obj-$(CONFIG_TREE_SRCU) += srcutree.o11obj-$(CONFIG_TREE_SRCU) += srcutree.o
8obj-$(CONFIG_TINY_SRCU) += srcutiny.o12obj-$(CONFIG_TINY_SRCU) += srcutiny.o
kernel/rcu/rcu.h+21−2
@@ -198,6 +198,13 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head) · 197 unmodified lines
198}198}
199#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */199#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
200 200
201extern int rcu_cpu_stall_suppress_at_boot;
202
203static inline bool rcu_stall_is_suppressed_at_boot(void)
204{
205 return rcu_cpu_stall_suppress_at_boot && !rcu_inkernel_boot_has_ended();
206}
207
201#ifdef CONFIG_RCU_STALL_COMMON208#ifdef CONFIG_RCU_STALL_COMMON
202 209
203extern int rcu_cpu_stall_ftrace_dump;210extern int rcu_cpu_stall_ftrace_dump;
@@ -205,6 +212,11 @@ extern int rcu_cpu_stall_suppress; · 1 unmodified line
205extern int rcu_cpu_stall_timeout;212extern int rcu_cpu_stall_timeout;
206int rcu_jiffies_till_stall_check(void);213int rcu_jiffies_till_stall_check(void);
207 214
215static inline bool rcu_stall_is_suppressed(void)
216{
217 return rcu_stall_is_suppressed_at_boot() || rcu_cpu_stall_suppress;
218}
219
208#define rcu_ftrace_dump_stall_suppress() \220#define rcu_ftrace_dump_stall_suppress() \
209do { \221do { \
210 if (!rcu_cpu_stall_suppress) \222 if (!rcu_cpu_stall_suppress) \
@@ -218,6 +230,11 @@ do { \ · 7 unmodified lines
218} while (0)230} while (0)
219 231
220#else /* #endif #ifdef CONFIG_RCU_STALL_COMMON */232#else /* #endif #ifdef CONFIG_RCU_STALL_COMMON */
233
234static inline bool rcu_stall_is_suppressed(void)
235{
236 return rcu_stall_is_suppressed_at_boot();
237}
221#define rcu_ftrace_dump_stall_suppress()238#define rcu_ftrace_dump_stall_suppress()
222#define rcu_ftrace_dump_stall_unsuppress()239#define rcu_ftrace_dump_stall_unsuppress()
223#endif /* #ifdef CONFIG_RCU_STALL_COMMON */240#endif /* #ifdef CONFIG_RCU_STALL_COMMON */
@@ -325,7 +342,8 @@ static inline void rcu_init_levelspread(int *levelspread, const int *levelcnt) · 101 unmodified lines
325 * Iterate over all possible CPUs in a leaf RCU node.342 * Iterate over all possible CPUs in a leaf RCU node.
326 */343 */
327#define for_each_leaf_node_possible_cpu(rnp, cpu) \344#define for_each_leaf_node_possible_cpu(rnp, cpu) \
328 for ((cpu) = cpumask_next((rnp)->grplo - 1, cpu_possible_mask); \345 for (WARN_ON_ONCE(!rcu_is_leaf_node(rnp)), \
346 (cpu) = cpumask_next((rnp)->grplo - 1, cpu_possible_mask); \
329 (cpu) <= rnp->grphi; \347 (cpu) <= rnp->grphi; \
330 (cpu) = cpumask_next((cpu), cpu_possible_mask))348 (cpu) = cpumask_next((cpu), cpu_possible_mask))
331 349
@@ -335,7 +353,8 @@ static inline void rcu_init_levelspread(int *levelspread, const int *levelcnt) · 3 unmodified lines
335#define rcu_find_next_bit(rnp, cpu, mask) \353#define rcu_find_next_bit(rnp, cpu, mask) \
336 ((rnp)->grplo + find_next_bit(&(mask), BITS_PER_LONG, (cpu)))354 ((rnp)->grplo + find_next_bit(&(mask), BITS_PER_LONG, (cpu)))
337#define for_each_leaf_node_cpu_mask(rnp, cpu, mask) \355#define for_each_leaf_node_cpu_mask(rnp, cpu, mask) \
338 for ((cpu) = rcu_find_next_bit((rnp), 0, (mask)); \356 for (WARN_ON_ONCE(!rcu_is_leaf_node(rnp)), \
357 (cpu) = rcu_find_next_bit((rnp), 0, (mask)); \
339 (cpu) <= rnp->grphi; \358 (cpu) <= rnp->grphi; \
340 (cpu) = rcu_find_next_bit((rnp), (cpu) + 1 - (rnp->grplo), (mask)))359 (cpu) = rcu_find_next_bit((rnp), (cpu) + 1 - (rnp->grplo), (mask)))
341 360
kernel/rcu/rcu_segcblist.c+1−3
@@ -182,7 +182,7 @@ void rcu_segcblist_offload(struct rcu_segcblist *rsclp) · 181 unmodified lines
182bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp)182bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp)
183{183{
184 return rcu_segcblist_is_enabled(rsclp) &&184 return rcu_segcblist_is_enabled(rsclp) &&
185 &rsclp->head != rsclp->tails[RCU_DONE_TAIL];185 &rsclp->head != READ_ONCE(rsclp->tails[RCU_DONE_TAIL]);
186}186}
187 187
188/*188/*
@@ -381,8 +381,6 @@ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, · 192 unmodified lines
381 return; /* Nothing to do. */381 return; /* Nothing to do. */
382 WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head);382 WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head);
383 WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail);383 WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail);
384 rclp->head = NULL;
385 rclp->tail = &rclp->head;
386}384}
387 385
388/*386/*
kernel/rcu/rcuperf.c+12−2
@@ -12,6 +12,7 @@ · 11 unmodified lines
12#include <linux/types.h>12#include <linux/types.h>
13#include <linux/kernel.h>13#include <linux/kernel.h>
14#include <linux/init.h>14#include <linux/init.h>
15#include <linux/mm.h>
15#include <linux/module.h>16#include <linux/module.h>
16#include <linux/kthread.h>17#include <linux/kthread.h>
17#include <linux/err.h>18#include <linux/err.h>
@@ -611,6 +612,7 @@ kfree_perf_thread(void *arg) · 593 unmodified lines
611 long me = (long)arg;612 long me = (long)arg;
612 struct kfree_obj *alloc_ptr;613 struct kfree_obj *alloc_ptr;
613 u64 start_time, end_time;614 u64 start_time, end_time;
615 long long mem_begin, mem_during = 0;
614 616
615 VERBOSE_PERFOUT_STRING("kfree_perf_thread task started");617 VERBOSE_PERFOUT_STRING("kfree_perf_thread task started");
616 set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));618 set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
@@ -626,6 +628,12 @@ kfree_perf_thread(void *arg) · 9 unmodified lines
626 }628 }
627 629
628 do {630 do {
631 if (!mem_during) {
632 mem_during = mem_begin = si_mem_available();
633 } else if (loop % (kfree_loops / 4) == 0) {
634 mem_during = (mem_during + si_mem_available()) / 2;
635 }
636
629 for (i = 0; i < kfree_alloc_num; i++) {637 for (i = 0; i < kfree_alloc_num; i++) {
630 alloc_ptr = kmalloc(sizeof(struct kfree_obj), GFP_KERNEL);638 alloc_ptr = kmalloc(sizeof(struct kfree_obj), GFP_KERNEL);
631 if (!alloc_ptr)639 if (!alloc_ptr)
@@ -645,9 +653,11 @@ kfree_perf_thread(void *arg) · 13 unmodified lines
645 else653 else
646 b_rcu_gp_test_finished = cur_ops->get_gp_seq();654 b_rcu_gp_test_finished = cur_ops->get_gp_seq();
647 655
648 pr_alert("Total time taken by all kfree'ers: %llu ns, loops: %d, batches: %ld\n",656 pr_alert("Total time taken by all kfree'ers: %llu ns, loops: %d, batches: %ld, memory footprint: %lldMB\n",
649 (unsigned long long)(end_time - start_time), kfree_loops,657 (unsigned long long)(end_time - start_time), kfree_loops,
650 rcuperf_seq_diff(b_rcu_gp_test_finished, b_rcu_gp_test_started));658 rcuperf_seq_diff(b_rcu_gp_test_finished, b_rcu_gp_test_started),
659 (mem_begin - mem_during) >> (20 - PAGE_SHIFT));
660
651 if (shutdown) {661 if (shutdown) {
652 smp_mb(); /* Assign before wake. */662 smp_mb(); /* Assign before wake. */
653 wake_up(&shutdown_wq);663 wake_up(&shutdown_wq);
kernel/rcu/rcutorture.c+50−17
@@ -339,7 +339,7 @@ rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) · 338 unmodified lines
339 * period, and we want a long delay occasionally to trigger339 * period, and we want a long delay occasionally to trigger
340 * force_quiescent_state. */340 * force_quiescent_state. */
341 341
342 if (!rcu_fwd_cb_nodelay &&342 if (!READ_ONCE(rcu_fwd_cb_nodelay) &&
343 !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) {343 !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) {
344 started = cur_ops->get_gp_seq();344 started = cur_ops->get_gp_seq();
345 ts = rcu_trace_clock_local();345 ts = rcu_trace_clock_local();
@@ -375,11 +375,12 @@ rcu_torture_pipe_update_one(struct rcu_torture *rp) · 29 unmodified lines
375{375{
376 int i;376 int i;
377 377
378 i = rp->rtort_pipe_count;378 i = READ_ONCE(rp->rtort_pipe_count);
379 if (i > RCU_TORTURE_PIPE_LEN)379 if (i > RCU_TORTURE_PIPE_LEN)
380 i = RCU_TORTURE_PIPE_LEN;380 i = RCU_TORTURE_PIPE_LEN;
381 atomic_inc(&rcu_torture_wcount[i]);381 atomic_inc(&rcu_torture_wcount[i]);
382 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {382 WRITE_ONCE(rp->rtort_pipe_count, i + 1);
383 if (rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
383 rp->rtort_mbtest = 0;384 rp->rtort_mbtest = 0;
384 return true;385 return true;
385 }386 }
@@ -1015,7 +1016,8 @@ rcu_torture_writer(void *arg) · 629 unmodified lines
1015 if (i > RCU_TORTURE_PIPE_LEN)1016 if (i > RCU_TORTURE_PIPE_LEN)
1016 i = RCU_TORTURE_PIPE_LEN;1017 i = RCU_TORTURE_PIPE_LEN;
1017 atomic_inc(&rcu_torture_wcount[i]);1018 atomic_inc(&rcu_torture_wcount[i]);
1018 old_rp->rtort_pipe_count++;1019 WRITE_ONCE(old_rp->rtort_pipe_count,
1020 old_rp->rtort_pipe_count + 1);
1019 switch (synctype[torture_random(&rand) % nsynctypes]) {1021 switch (synctype[torture_random(&rand) % nsynctypes]) {
1020 case RTWS_DEF_FREE:1022 case RTWS_DEF_FREE:
1021 rcu_torture_writer_state = RTWS_DEF_FREE;1023 rcu_torture_writer_state = RTWS_DEF_FREE;
@@ -1067,7 +1069,8 @@ rcu_torture_writer(void *arg) · 45 unmodified lines
1067 if (stutter_wait("rcu_torture_writer") &&1069 if (stutter_wait("rcu_torture_writer") &&
1068 !READ_ONCE(rcu_fwd_cb_nodelay) &&1070 !READ_ONCE(rcu_fwd_cb_nodelay) &&
1069 !cur_ops->slow_gps &&1071 !cur_ops->slow_gps &&
1070 !torture_must_stop())1072 !torture_must_stop() &&
1073 rcu_inkernel_boot_has_ended())
1071 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++)1074 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++)
1072 if (list_empty(&rcu_tortures[i].rtort_free) &&1075 if (list_empty(&rcu_tortures[i].rtort_free) &&
1073 rcu_access_pointer(rcu_torture_current) !=1076 rcu_access_pointer(rcu_torture_current) !=
@@ -1290,7 +1293,7 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp) · 216 unmodified lines
1290 atomic_inc(&n_rcu_torture_mberror);1293 atomic_inc(&n_rcu_torture_mberror);
1291 rtrsp = rcutorture_loop_extend(&readstate, trsp, rtrsp);1294 rtrsp = rcutorture_loop_extend(&readstate, trsp, rtrsp);
1292 preempt_disable();1295 preempt_disable();
1293 pipe_count = p->rtort_pipe_count;1296 pipe_count = READ_ONCE(p->rtort_pipe_count);
1294 if (pipe_count > RCU_TORTURE_PIPE_LEN) {1297 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
1295 /* Should not happen, but... */1298 /* Should not happen, but... */
1296 pipe_count = RCU_TORTURE_PIPE_LEN;1299 pipe_count = RCU_TORTURE_PIPE_LEN;
@@ -1404,14 +1407,15 @@ rcu_torture_stats_print(void) · 107 unmodified lines
1404 int i;1407 int i;
1405 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };1408 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1406 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };1409 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1410 struct rcu_torture *rtcp;
1407 static unsigned long rtcv_snap = ULONG_MAX;1411 static unsigned long rtcv_snap = ULONG_MAX;
1408 static bool splatted;1412 static bool splatted;
1409 struct task_struct *wtp;1413 struct task_struct *wtp;
1410 1414
1411 for_each_possible_cpu(cpu) {1415 for_each_possible_cpu(cpu) {
1412 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {1416 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1413 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];1417 pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]);
1414 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];1418 batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]);
1415 }1419 }
1416 }1420 }
1417 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {1421 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
@@ -1420,9 +1424,10 @@ rcu_torture_stats_print(void) · 2 unmodified lines
1420 }1424 }
1421 1425
1422 pr_alert("%s%s ", torture_type, TORTURE_FLAG);1426 pr_alert("%s%s ", torture_type, TORTURE_FLAG);
1427 rtcp = rcu_access_pointer(rcu_torture_current);
1423 pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",1428 pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
1424 rcu_torture_current,1429 rtcp,
1425 rcu_torture_current ? "ver" : "VER",1430 rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER",
1426 rcu_torture_current_version,1431 rcu_torture_current_version,
1427 list_empty(&rcu_torture_freelist),1432 list_empty(&rcu_torture_freelist),
1428 atomic_read(&n_rcu_torture_alloc),1433 atomic_read(&n_rcu_torture_alloc),
@@ -1478,7 +1483,8 @@ rcu_torture_stats_print(void) · 49 unmodified lines
1478 if (cur_ops->stats)1483 if (cur_ops->stats)
1479 cur_ops->stats();1484 cur_ops->stats();
1480 if (rtcv_snap == rcu_torture_current_version &&1485 if (rtcv_snap == rcu_torture_current_version &&
1481 rcu_torture_current != NULL) {1486 rcu_access_pointer(rcu_torture_current) &&
1487 !rcu_stall_is_suppressed()) {
1482 int __maybe_unused flags = 0;1488 int __maybe_unused flags = 0;
1483 unsigned long __maybe_unused gp_seq = 0;1489 unsigned long __maybe_unused gp_seq = 0;
1484 1490
@@ -1993,8 +1999,11 @@ static int rcu_torture_fwd_prog(void *args) · 508 unmodified lines
1993 schedule_timeout_interruptible(fwd_progress_holdoff * HZ);1999 schedule_timeout_interruptible(fwd_progress_holdoff * HZ);
1994 WRITE_ONCE(rcu_fwd_emergency_stop, false);2000 WRITE_ONCE(rcu_fwd_emergency_stop, false);
1995 register_oom_notifier(&rcutorture_oom_nb);2001 register_oom_notifier(&rcutorture_oom_nb);
1996 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);2002 if (!IS_ENABLED(CONFIG_TINY_RCU) ||
1997 rcu_torture_fwd_prog_cr(rfp);2003 rcu_inkernel_boot_has_ended())
2004 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);
2005 if (rcu_inkernel_boot_has_ended())
2006 rcu_torture_fwd_prog_cr(rfp);
1998 unregister_oom_notifier(&rcutorture_oom_nb);2007 unregister_oom_notifier(&rcutorture_oom_nb);
1999 2008
2000 /* Avoid slow periods, better to test when busy. */2009 /* Avoid slow periods, better to test when busy. */
@@ -2044,6 +2053,14 @@ static void rcu_torture_barrier_cbf(struct rcu_head *rcu) · 43 unmodified lines
2044 atomic_inc(&barrier_cbs_invoked);2053 atomic_inc(&barrier_cbs_invoked);
2045}2054}
2046 2055
2056/* IPI handler to get callback posted on desired CPU, if online. */
2057static void rcu_torture_barrier1cb(void *rcu_void)
2058{
2059 struct rcu_head *rhp = rcu_void;
2060
2061 cur_ops->call(rhp, rcu_torture_barrier_cbf);
2062}
2063
2047/* kthread function to register callbacks used to test RCU barriers. */2064/* kthread function to register callbacks used to test RCU barriers. */
2048static int rcu_torture_barrier_cbs(void *arg)2065static int rcu_torture_barrier_cbs(void *arg)
2049{2066{
@@ -2067,9 +2084,11 @@ static int rcu_torture_barrier_cbs(void *arg) · 17 unmodified lines
2067 * The above smp_load_acquire() ensures barrier_phase load2084 * The above smp_load_acquire() ensures barrier_phase load
2068 * is ordered before the following ->call().2085 * is ordered before the following ->call().
2069 */2086 */
2070 local_irq_disable(); /* Just to test no-irq call_rcu(). */2087 if (smp_call_function_single(myid, rcu_torture_barrier1cb,
2071 cur_ops->call(&rcu, rcu_torture_barrier_cbf);2088 &rcu, 1)) {
2072 local_irq_enable();2089 // IPI failed, so use direct call from current CPU.
2090 cur_ops->call(&rcu, rcu_torture_barrier_cbf);
2091 }
2073 if (atomic_dec_and_test(&barrier_cbs_count))2092 if (atomic_dec_and_test(&barrier_cbs_count))
2074 wake_up(&barrier_wq);2093 wake_up(&barrier_wq);
2075 } while (!torture_must_stop());2094 } while (!torture_must_stop());
@@ -2105,7 +2124,21 @@ static int rcu_torture_barrier(void *arg) · 29 unmodified lines
2105 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n",2124 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n",
2106 atomic_read(&barrier_cbs_invoked),2125 atomic_read(&barrier_cbs_invoked),
2107 n_barrier_cbs);2126 n_barrier_cbs);
2108 WARN_ON_ONCE(1);2127 WARN_ON(1);
2128 // Wait manually for the remaining callbacks
2129 i = 0;
2130 do {
2131 if (WARN_ON(i++ > HZ))
2132 i = INT_MIN;
2133 schedule_timeout_interruptible(1);
2134 cur_ops->cb_barrier();
2135 } while (atomic_read(&barrier_cbs_invoked) !=
2136 n_barrier_cbs &&
2137 !torture_must_stop());
2138 smp_mb(); // Can't trust ordering if broken.
2139 if (!torture_must_stop())
2140 pr_err("Recovered: barrier_cbs_invoked = %d\n",
2141 atomic_read(&barrier_cbs_invoked));
2109 } else {2142 } else {
2110 n_barrier_successes++;2143 n_barrier_successes++;
2111 }2144 }
kernel/rcu/srcutree.c+10−8
@@ -5,7 +5,7 @@ · 4 unmodified lines
5 * Copyright (C) IBM Corporation, 20065 * Copyright (C) IBM Corporation, 2006
6 * Copyright (C) Fujitsu, 20126 * Copyright (C) Fujitsu, 2012
7 *7 *
8 * Author: Paul McKenney <paulmck@linux.ibm.com>8 * Authors: Paul McKenney <paulmck@linux.ibm.com>
9 * Lai Jiangshan <laijs@cn.fujitsu.com>9 * Lai Jiangshan <laijs@cn.fujitsu.com>
10 *10 *
11 * For detailed explanation of Read-Copy Update mechanism see -11 * For detailed explanation of Read-Copy Update mechanism see -
@@ -450,7 +450,7 @@ static void srcu_gp_start(struct srcu_struct *ssp) · 438 unmodified lines
450 spin_unlock_rcu_node(sdp); /* Interrupts remain disabled. */450 spin_unlock_rcu_node(sdp); /* Interrupts remain disabled. */
451 smp_mb(); /* Order prior store to ->srcu_gp_seq_needed vs. GP start. */451 smp_mb(); /* Order prior store to ->srcu_gp_seq_needed vs. GP start. */
452 rcu_seq_start(&ssp->srcu_gp_seq);452 rcu_seq_start(&ssp->srcu_gp_seq);
453 state = rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq));453 state = rcu_seq_state(ssp->srcu_gp_seq);
454 WARN_ON_ONCE(state != SRCU_STATE_SCAN1);454 WARN_ON_ONCE(state != SRCU_STATE_SCAN1);
455}455}
456 456
@@ -534,7 +534,7 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 77 unmodified lines
534 rcu_seq_end(&ssp->srcu_gp_seq);534 rcu_seq_end(&ssp->srcu_gp_seq);
535 gpseq = rcu_seq_current(&ssp->srcu_gp_seq);535 gpseq = rcu_seq_current(&ssp->srcu_gp_seq);
536 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, gpseq))536 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, gpseq))
537 ssp->srcu_gp_seq_needed_exp = gpseq;537 WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, gpseq);
538 spin_unlock_irq_rcu_node(ssp);538 spin_unlock_irq_rcu_node(ssp);
539 mutex_unlock(&ssp->srcu_gp_mutex);539 mutex_unlock(&ssp->srcu_gp_mutex);
540 /* A new grace period can start at this point. But only one. */540 /* A new grace period can start at this point. But only one. */
@@ -550,7 +550,7 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 9 unmodified lines
550 snp->srcu_have_cbs[idx] = gpseq;550 snp->srcu_have_cbs[idx] = gpseq;
551 rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);551 rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);
552 if (ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, gpseq))552 if (ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, gpseq))
553 snp->srcu_gp_seq_needed_exp = gpseq;553 WRITE_ONCE(snp->srcu_gp_seq_needed_exp, gpseq);
554 mask = snp->srcu_data_have_cbs[idx];554 mask = snp->srcu_data_have_cbs[idx];
555 snp->srcu_data_have_cbs[idx] = 0;555 snp->srcu_data_have_cbs[idx] = 0;
556 spin_unlock_irq_rcu_node(snp);556 spin_unlock_irq_rcu_node(snp);
@@ -614,7 +614,7 @@ static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp · 57 unmodified lines
614 }614 }
615 spin_lock_irqsave_rcu_node(ssp, flags);615 spin_lock_irqsave_rcu_node(ssp, flags);
616 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))616 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))
617 ssp->srcu_gp_seq_needed_exp = s;617 WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s);
618 spin_unlock_irqrestore_rcu_node(ssp, flags);618 spin_unlock_irqrestore_rcu_node(ssp, flags);
619}619}
620 620
@@ -660,7 +660,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, · 39 unmodified lines
660 if (snp == sdp->mynode)660 if (snp == sdp->mynode)
661 snp->srcu_data_have_cbs[idx] |= sdp->grpmask;661 snp->srcu_data_have_cbs[idx] |= sdp->grpmask;
662 if (!do_norm && ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, s))662 if (!do_norm && ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, s))
663 snp->srcu_gp_seq_needed_exp = s;663 WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);
664 spin_unlock_irqrestore_rcu_node(snp, flags);664 spin_unlock_irqrestore_rcu_node(snp, flags);
665 }665 }
666 666
@@ -674,7 +674,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, · 7 unmodified lines
674 smp_store_release(&ssp->srcu_gp_seq_needed, s); /*^^^*/674 smp_store_release(&ssp->srcu_gp_seq_needed, s); /*^^^*/
675 }675 }
676 if (!do_norm && ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))676 if (!do_norm && ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))
677 ssp->srcu_gp_seq_needed_exp = s;677 WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s);
678 678
679 /* If grace period not already done and none in progress, start it. */679 /* If grace period not already done and none in progress, start it. */
680 if (!rcu_seq_done(&ssp->srcu_gp_seq, s) &&680 if (!rcu_seq_done(&ssp->srcu_gp_seq, s) &&
@@ -1079,7 +1079,7 @@ EXPORT_SYMBOL_GPL(srcu_barrier); · 398 unmodified lines
1079 */1079 */
1080unsigned long srcu_batches_completed(struct srcu_struct *ssp)1080unsigned long srcu_batches_completed(struct srcu_struct *ssp)
1081{1081{
1082 return ssp->srcu_idx;1082 return READ_ONCE(ssp->srcu_idx);
1083}1083}
1084EXPORT_SYMBOL_GPL(srcu_batches_completed);1084EXPORT_SYMBOL_GPL(srcu_batches_completed);
1085 1085
@@ -1130,7 +1130,9 @@ static void srcu_advance_state(struct srcu_struct *ssp) · 44 unmodified lines
1130 return; /* readers present, retry later. */1130 return; /* readers present, retry later. */
1131 }1131 }
1132 srcu_flip(ssp);1132 srcu_flip(ssp);
1133 spin_lock_irq_rcu_node(ssp);
1133 rcu_seq_set_state(&ssp->srcu_gp_seq, SRCU_STATE_SCAN2);1134 rcu_seq_set_state(&ssp->srcu_gp_seq, SRCU_STATE_SCAN2);
1135 spin_unlock_irq_rcu_node(ssp);
1134 }1136 }
1135 1137
1136 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN2) {1138 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN2) {
kernel/rcu/tree.c+339−113
@@ -1,12 +1,12 @@
1// SPDX-License-Identifier: GPL-2.0+1// SPDX-License-Identifier: GPL-2.0+
2/*2/*
3 * Read-Copy Update mechanism for mutual exclusion3 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
4 *4 *
5 * Copyright IBM Corporation, 20085 * Copyright IBM Corporation, 2008
6 *6 *
7 * Authors: Dipankar Sarma <dipankar@in.ibm.com>7 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
8 * Manfred Spraul <manfred@colorfullife.com>8 * Manfred Spraul <manfred@colorfullife.com>
9 * Paul E. McKenney <paulmck@linux.ibm.com> Hierarchical version9 * Paul E. McKenney <paulmck@linux.ibm.com>
10 *10 *
11 * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>11 * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
12 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.12 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
@@ -150,6 +150,7 @@ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu) · 137 unmodified lines
150static void invoke_rcu_core(void);150static void invoke_rcu_core(void);
151static void rcu_report_exp_rdp(struct rcu_data *rdp);151static void rcu_report_exp_rdp(struct rcu_data *rdp);
152static void sync_sched_exp_online_cleanup(int cpu);152static void sync_sched_exp_online_cleanup(int cpu);
153static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
153 154
154/* rcuc/rcub kthread realtime priority */155/* rcuc/rcub kthread realtime priority */
155static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;156static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
@@ -342,14 +343,17 @@ bool rcu_eqs_special_set(int cpu) · 186 unmodified lines
342{343{
343 int old;344 int old;
344 int new;345 int new;
346 int new_old;
345 struct rcu_data *rdp = &per_cpu(rcu_data, cpu);347 struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
346 348
349 new_old = atomic_read(&rdp->dynticks);
347 do {350 do {
348 old = atomic_read(&rdp->dynticks);351 old = new_old;
349 if (old & RCU_DYNTICK_CTRL_CTR)352 if (old & RCU_DYNTICK_CTRL_CTR)
350 return false;353 return false;
351 new = old | RCU_DYNTICK_CTRL_MASK;354 new = old | RCU_DYNTICK_CTRL_MASK;
352 } while (atomic_cmpxchg(&rdp->dynticks, old, new) != old);355 new_old = atomic_cmpxchg(&rdp->dynticks, old, new);
356 } while (new_old != old);
353 return true;357 return true;
354}358}
355 359
@@ -410,10 +414,15 @@ static long blimit = DEFAULT_RCU_BLIMIT; · 54 unmodified lines
410static long qhimark = DEFAULT_RCU_QHIMARK;414static long qhimark = DEFAULT_RCU_QHIMARK;
411#define DEFAULT_RCU_QLOMARK 100 /* Once only this many pending, use blimit. */415#define DEFAULT_RCU_QLOMARK 100 /* Once only this many pending, use blimit. */
412static long qlowmark = DEFAULT_RCU_QLOMARK;416static long qlowmark = DEFAULT_RCU_QLOMARK;
417#define DEFAULT_RCU_QOVLD_MULT 2
418#define DEFAULT_RCU_QOVLD (DEFAULT_RCU_QOVLD_MULT * DEFAULT_RCU_QHIMARK)
419static long qovld = DEFAULT_RCU_QOVLD; /* If this many pending, hammer QS. */
420static long qovld_calc = -1; /* No pre-initialization lock acquisitions! */
413 421
414module_param(blimit, long, 0444);422module_param(blimit, long, 0444);
415module_param(qhimark, long, 0444);423module_param(qhimark, long, 0444);
416module_param(qlowmark, long, 0444);424module_param(qlowmark, long, 0444);
425module_param(qovld, long, 0444);
417 426
418static ulong jiffies_till_first_fqs = ULONG_MAX;427static ulong jiffies_till_first_fqs = ULONG_MAX;
419static ulong jiffies_till_next_fqs = ULONG_MAX;428static ulong jiffies_till_next_fqs = ULONG_MAX;
@@ -818,11 +827,12 @@ static __always_inline void rcu_nmi_enter_common(bool irq) · 398 unmodified lines
818 incby = 1;827 incby = 1;
819 } else if (tick_nohz_full_cpu(rdp->cpu) &&828 } else if (tick_nohz_full_cpu(rdp->cpu) &&
820 rdp->dynticks_nmi_nesting == DYNTICK_IRQ_NONIDLE &&829 rdp->dynticks_nmi_nesting == DYNTICK_IRQ_NONIDLE &&
821 READ_ONCE(rdp->rcu_urgent_qs) && !rdp->rcu_forced_tick) {830 READ_ONCE(rdp->rcu_urgent_qs) &&
831 !READ_ONCE(rdp->rcu_forced_tick)) {
822 raw_spin_lock_rcu_node(rdp->mynode);832 raw_spin_lock_rcu_node(rdp->mynode);
823 // Recheck under lock.833 // Recheck under lock.
824 if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {834 if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {
825 rdp->rcu_forced_tick = true;835 WRITE_ONCE(rdp->rcu_forced_tick, true);
826 tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);836 tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
827 }837 }
828 raw_spin_unlock_rcu_node(rdp->mynode);838 raw_spin_unlock_rcu_node(rdp->mynode);
@@ -899,7 +909,7 @@ static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp) · 70 unmodified lines
899 WRITE_ONCE(rdp->rcu_need_heavy_qs, false);909 WRITE_ONCE(rdp->rcu_need_heavy_qs, false);
900 if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) {910 if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) {
901 tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU);911 tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
902 rdp->rcu_forced_tick = false;912 WRITE_ONCE(rdp->rcu_forced_tick, false);
903 }913 }
904}914}
905 915
@@ -1072,7 +1082,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) · 166 unmodified lines
1072 rnhqp = &per_cpu(rcu_data.rcu_need_heavy_qs, rdp->cpu);1082 rnhqp = &per_cpu(rcu_data.rcu_need_heavy_qs, rdp->cpu);
1073 if (!READ_ONCE(*rnhqp) &&1083 if (!READ_ONCE(*rnhqp) &&
1074 (time_after(jiffies, rcu_state.gp_start + jtsq * 2) ||1084 (time_after(jiffies, rcu_state.gp_start + jtsq * 2) ||
1075 time_after(jiffies, rcu_state.jiffies_resched))) {1085 time_after(jiffies, rcu_state.jiffies_resched) ||
1086 rcu_state.cbovld)) {
1076 WRITE_ONCE(*rnhqp, true);1087 WRITE_ONCE(*rnhqp, true);
1077 /* Store rcu_need_heavy_qs before rcu_urgent_qs. */1088 /* Store rcu_need_heavy_qs before rcu_urgent_qs. */
1078 smp_store_release(ruqp, true);1089 smp_store_release(ruqp, true);
@@ -1089,8 +1100,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) · 10 unmodified lines
1089 * So hit them over the head with the resched_cpu() hammer!1100 * So hit them over the head with the resched_cpu() hammer!
1090 */1101 */
1091 if (tick_nohz_full_cpu(rdp->cpu) &&1102 if (tick_nohz_full_cpu(rdp->cpu) &&
1092 time_after(jiffies,1103 (time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) ||
1093 READ_ONCE(rdp->last_fqs_resched) + jtsq * 3)) {1104 rcu_state.cbovld)) {
1094 WRITE_ONCE(*ruqp, true);1105 WRITE_ONCE(*ruqp, true);
1095 resched_cpu(rdp->cpu);1106 resched_cpu(rdp->cpu);
1096 WRITE_ONCE(rdp->last_fqs_resched, jiffies);1107 WRITE_ONCE(rdp->last_fqs_resched, jiffies);
@@ -1126,8 +1137,9 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) · 29 unmodified lines
1126static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp,1137static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp,
1127 unsigned long gp_seq_req, const char *s)1138 unsigned long gp_seq_req, const char *s)
1128{1139{
1129 trace_rcu_future_grace_period(rcu_state.name, rnp->gp_seq, gp_seq_req,1140 trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
1130 rnp->level, rnp->grplo, rnp->grphi, s);1141 gp_seq_req, rnp->level,
1142 rnp->grplo, rnp->grphi, s);
1131}1143}
1132 1144
1133/*1145/*
@@ -1174,7 +1186,7 @@ static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, · 40 unmodified lines
1174 TPS("Prestarted"));1186 TPS("Prestarted"));
1175 goto unlock_out;1187 goto unlock_out;
1176 }1188 }
1177 rnp->gp_seq_needed = gp_seq_req;1189 WRITE_ONCE(rnp->gp_seq_needed, gp_seq_req);
1178 if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {1190 if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {
1179 /*1191 /*
1180 * We just marked the leaf or internal node, and a1192 * We just marked the leaf or internal node, and a
@@ -1199,18 +1211,18 @@ static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, · 18 unmodified lines
1199 }1211 }
1200 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot"));1212 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot"));
1201 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT);1213 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT);
1202 rcu_state.gp_req_activity = jiffies;1214 WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
1203 if (!rcu_state.gp_kthread) {1215 if (!READ_ONCE(rcu_state.gp_kthread)) {
1204 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread"));1216 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread"));
1205 goto unlock_out;1217 goto unlock_out;
1206 }1218 }
1207 trace_rcu_grace_period(rcu_state.name, READ_ONCE(rcu_state.gp_seq), TPS("newreq"));1219 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("newreq"));
1208 ret = true; /* Caller must wake GP kthread. */1220 ret = true; /* Caller must wake GP kthread. */
1209unlock_out:1221unlock_out:
1210 /* Push furthest requested GP to leaf node and rcu_data structure. */1222 /* Push furthest requested GP to leaf node and rcu_data structure. */
1211 if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) {1223 if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) {
1212 rnp_start->gp_seq_needed = rnp->gp_seq_needed;1224 WRITE_ONCE(rnp_start->gp_seq_needed, rnp->gp_seq_needed);
1213 rdp->gp_seq_needed = rnp->gp_seq_needed;1225 WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
1214 }1226 }
1215 if (rnp != rnp_start)1227 if (rnp != rnp_start)
1216 raw_spin_unlock_rcu_node(rnp);1228 raw_spin_unlock_rcu_node(rnp);
@@ -1235,12 +1247,13 @@ static bool rcu_future_gp_cleanup(struct rcu_node *rnp) · 18 unmodified lines
1235}1247}
1236 1248
1237/*1249/*
1238 * Awaken the grace-period kthread. Don't do a self-awaken (unless in1250 * Awaken the grace-period kthread. Don't do a self-awaken (unless in an
1239 * an interrupt or softirq handler), and don't bother awakening when there1251 * interrupt or softirq handler, in which case we just might immediately
1240 * is nothing for the grace-period kthread to do (as in several CPUs raced1252 * sleep upon return, resulting in a grace-period hang), and don't bother
1241 * to awaken, and we lost), and finally don't try to awaken a kthread that1253 * awakening when there is nothing for the grace-period kthread to do
1242 * has not yet been created. If all those checks are passed, track some1254 * (as in several CPUs raced to awaken, we lost), and finally don't try
1243 * debug information and awaken.1255 * to awaken a kthread that has not yet been created. If all those checks
1256 * are passed, track some debug information and awaken.
1244 *1257 *
1245 * So why do the self-wakeup when in an interrupt or softirq handler1258 * So why do the self-wakeup when in an interrupt or softirq handler
1246 * in the grace-period kthread's context? Because the kthread might have1259 * in the grace-period kthread's context? Because the kthread might have
@@ -1250,10 +1263,10 @@ static bool rcu_future_gp_cleanup(struct rcu_node *rnp) · 3 unmodified lines
1250 */1263 */
1251static void rcu_gp_kthread_wake(void)1264static void rcu_gp_kthread_wake(void)
1252{1265{
1253 if ((current == rcu_state.gp_kthread &&1266 struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
1254 !in_irq() && !in_serving_softirq()) ||1267
1255 !READ_ONCE(rcu_state.gp_flags) ||1268 if ((current == t && !in_irq() && !in_serving_softirq()) ||
1256 !rcu_state.gp_kthread)1269 !READ_ONCE(rcu_state.gp_flags) || !t)
1257 return;1270 return;
1258 WRITE_ONCE(rcu_state.gp_wake_time, jiffies);1271 WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
1259 WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));1272 WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
@@ -1321,7 +1334,7 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, · 61 unmodified lines
1321 1334
1322 rcu_lockdep_assert_cblist_protected(rdp);1335 rcu_lockdep_assert_cblist_protected(rdp);
1323 c = rcu_seq_snap(&rcu_state.gp_seq);1336 c = rcu_seq_snap(&rcu_state.gp_seq);
1324 if (!rdp->gpwrap && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {1337 if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
1325 /* Old request still live, so mark recent callbacks. */1338 /* Old request still live, so mark recent callbacks. */
1326 (void)rcu_segcblist_accelerate(&rdp->cblist, c);1339 (void)rcu_segcblist_accelerate(&rdp->cblist, c);
1327 return;1340 return;
@@ -1386,7 +1399,7 @@ static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp, · 58 unmodified lines
1386static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp)1399static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp)
1387{1400{
1388 bool ret = false;1401 bool ret = false;
1389 bool need_gp;1402 bool need_qs;
1390 const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&1403 const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
1391 rcu_segcblist_is_offloaded(&rdp->cblist);1404 rcu_segcblist_is_offloaded(&rdp->cblist);
1392 1405
@@ -1400,10 +1413,13 @@ static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp) · 7 unmodified lines
1400 unlikely(READ_ONCE(rdp->gpwrap))) {1413 unlikely(READ_ONCE(rdp->gpwrap))) {
1401 if (!offloaded)1414 if (!offloaded)
1402 ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */1415 ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */
1416 rdp->core_needs_qs = false;
1403 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));1417 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));
1404 } else {1418 } else {
1405 if (!offloaded)1419 if (!offloaded)
1406 ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */1420 ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */
1421 if (rdp->core_needs_qs)
1422 rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask);
1407 }1423 }
1408 1424
1409 /* Now handle the beginnings of any new-to-this-CPU grace periods. */1425 /* Now handle the beginnings of any new-to-this-CPU grace periods. */
@@ -1415,14 +1431,14 @@ static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp) · 5 unmodified lines
1415 * go looking for one.1431 * go looking for one.
1416 */1432 */
1417 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));1433 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));
1418 need_gp = !!(rnp->qsmask & rdp->grpmask);1434 need_qs = !!(rnp->qsmask & rdp->grpmask);
1419 rdp->cpu_no_qs.b.norm = need_gp;1435 rdp->cpu_no_qs.b.norm = need_qs;
1420 rdp->core_needs_qs = need_gp;1436 rdp->core_needs_qs = need_qs;
1421 zero_cpu_stall_ticks(rdp);1437 zero_cpu_stall_ticks(rdp);
1422 }1438 }
1423 rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */1439 rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */
1424 if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) || rdp->gpwrap)1440 if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) || rdp->gpwrap)
1425 rdp->gp_seq_needed = rnp->gp_seq_needed;1441 WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
1426 WRITE_ONCE(rdp->gpwrap, false);1442 WRITE_ONCE(rdp->gpwrap, false);
1427 rcu_gpnum_ovf(rnp, rdp);1443 rcu_gpnum_ovf(rnp, rdp);
1428 return ret;1444 return ret;
@@ -1651,8 +1667,7 @@ static void rcu_gp_fqs_loop(void) · 222 unmodified lines
1651 WRITE_ONCE(rcu_state.jiffies_kick_kthreads,1667 WRITE_ONCE(rcu_state.jiffies_kick_kthreads,
1652 jiffies + (j ? 3 * j : 2));1668 jiffies + (j ? 3 * j : 2));
1653 }1669 }
1654 trace_rcu_grace_period(rcu_state.name,1670 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1655 READ_ONCE(rcu_state.gp_seq),
1656 TPS("fqswait"));1671 TPS("fqswait"));
1657 rcu_state.gp_state = RCU_GP_WAIT_FQS;1672 rcu_state.gp_state = RCU_GP_WAIT_FQS;
1658 ret = swait_event_idle_timeout_exclusive(1673 ret = swait_event_idle_timeout_exclusive(
@@ -1666,13 +1681,11 @@ static void rcu_gp_fqs_loop(void) · 7 unmodified lines
1666 /* If time for quiescent-state forcing, do it. */1681 /* If time for quiescent-state forcing, do it. */
1667 if (ULONG_CMP_GE(jiffies, rcu_state.jiffies_force_qs) ||1682 if (ULONG_CMP_GE(jiffies, rcu_state.jiffies_force_qs) ||
1668 (gf & RCU_GP_FLAG_FQS)) {1683 (gf & RCU_GP_FLAG_FQS)) {
1669 trace_rcu_grace_period(rcu_state.name,1684 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1670 READ_ONCE(rcu_state.gp_seq),
1671 TPS("fqsstart"));1685 TPS("fqsstart"));
1672 rcu_gp_fqs(first_gp_fqs);1686 rcu_gp_fqs(first_gp_fqs);
1673 first_gp_fqs = false;1687 first_gp_fqs = false;
1674 trace_rcu_grace_period(rcu_state.name,1688 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1675 READ_ONCE(rcu_state.gp_seq),
1676 TPS("fqsend"));1689 TPS("fqsend"));
1677 cond_resched_tasks_rcu_qs();1690 cond_resched_tasks_rcu_qs();
1678 WRITE_ONCE(rcu_state.gp_activity, jiffies);1691 WRITE_ONCE(rcu_state.gp_activity, jiffies);
@@ -1683,8 +1696,7 @@ static void rcu_gp_fqs_loop(void) · 4 unmodified lines
1683 cond_resched_tasks_rcu_qs();1696 cond_resched_tasks_rcu_qs();
1684 WRITE_ONCE(rcu_state.gp_activity, jiffies);1697 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1685 WARN_ON(signal_pending(current));1698 WARN_ON(signal_pending(current));
1686 trace_rcu_grace_period(rcu_state.name,1699 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1687 READ_ONCE(rcu_state.gp_seq),
1688 TPS("fqswaitsig"));1700 TPS("fqswaitsig"));
1689 ret = 1; /* Keep old FQS timing. */1701 ret = 1; /* Keep old FQS timing. */
1690 j = jiffies;1702 j = jiffies;
@@ -1701,8 +1713,9 @@ static void rcu_gp_fqs_loop(void) · 10 unmodified lines
1701 */1713 */
1702static void rcu_gp_cleanup(void)1714static void rcu_gp_cleanup(void)
1703{1715{
1704 unsigned long gp_duration;1716 int cpu;
1705 bool needgp = false;1717 bool needgp = false;
1718 unsigned long gp_duration;
1706 unsigned long new_gp_seq;1719 unsigned long new_gp_seq;
1707 bool offloaded;1720 bool offloaded;
1708 struct rcu_data *rdp;1721 struct rcu_data *rdp;
@@ -1748,6 +1761,12 @@ static void rcu_gp_cleanup(void) · 39 unmodified lines
1748 needgp = __note_gp_changes(rnp, rdp) || needgp;1761 needgp = __note_gp_changes(rnp, rdp) || needgp;
1749 /* smp_mb() provided by prior unlock-lock pair. */1762 /* smp_mb() provided by prior unlock-lock pair. */
1750 needgp = rcu_future_gp_cleanup(rnp) || needgp;1763 needgp = rcu_future_gp_cleanup(rnp) || needgp;
1764 // Reset overload indication for CPUs no longer overloaded
1765 if (rcu_is_leaf_node(rnp))
1766 for_each_leaf_node_cpu_mask(rnp, cpu, rnp->cbovldmask) {
1767 rdp = per_cpu_ptr(&rcu_data, cpu);
1768 check_cb_ovld_locked(rdp, rnp);
1769 }
1751 sq = rcu_nocb_gp_get(rnp);1770 sq = rcu_nocb_gp_get(rnp);
1752 raw_spin_unlock_irq_rcu_node(rnp);1771 raw_spin_unlock_irq_rcu_node(rnp);
1753 rcu_nocb_gp_cleanup(sq);1772 rcu_nocb_gp_cleanup(sq);
@@ -1774,9 +1793,9 @@ static void rcu_gp_cleanup(void) · 20 unmodified lines
1774 rcu_segcblist_is_offloaded(&rdp->cblist);1793 rcu_segcblist_is_offloaded(&rdp->cblist);
1775 if ((offloaded || !rcu_accelerate_cbs(rnp, rdp)) && needgp) {1794 if ((offloaded || !rcu_accelerate_cbs(rnp, rdp)) && needgp) {
1776 WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);1795 WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);
1777 rcu_state.gp_req_activity = jiffies;1796 WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
1778 trace_rcu_grace_period(rcu_state.name,1797 trace_rcu_grace_period(rcu_state.name,
1779 READ_ONCE(rcu_state.gp_seq),1798 rcu_state.gp_seq,
1780 TPS("newreq"));1799 TPS("newreq"));
1781 } else {1800 } else {
1782 WRITE_ONCE(rcu_state.gp_flags,1801 WRITE_ONCE(rcu_state.gp_flags,
@@ -1795,8 +1814,7 @@ static int __noreturn rcu_gp_kthread(void *unused) · 12 unmodified lines
1795 1814
1796 /* Handle grace-period start. */1815 /* Handle grace-period start. */
1797 for (;;) {1816 for (;;) {
1798 trace_rcu_grace_period(rcu_state.name,1817 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1799 READ_ONCE(rcu_state.gp_seq),
1800 TPS("reqwait"));1818 TPS("reqwait"));
1801 rcu_state.gp_state = RCU_GP_WAIT_GPS;1819 rcu_state.gp_state = RCU_GP_WAIT_GPS;
1802 swait_event_idle_exclusive(rcu_state.gp_wq,1820 swait_event_idle_exclusive(rcu_state.gp_wq,
@@ -1809,8 +1827,7 @@ static int __noreturn rcu_gp_kthread(void *unused) · 6 unmodified lines
1809 cond_resched_tasks_rcu_qs();1827 cond_resched_tasks_rcu_qs();
1810 WRITE_ONCE(rcu_state.gp_activity, jiffies);1828 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1811 WARN_ON(signal_pending(current));1829 WARN_ON(signal_pending(current));
1812 trace_rcu_grace_period(rcu_state.name,1830 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1813 READ_ONCE(rcu_state.gp_seq),
1814 TPS("reqwaitsig"));1831 TPS("reqwaitsig"));
1815 }1832 }
1816 1833
@@ -1881,7 +1898,7 @@ static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp, · 64 unmodified lines
1881 WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */1898 WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */
1882 WARN_ON_ONCE(!rcu_is_leaf_node(rnp) &&1899 WARN_ON_ONCE(!rcu_is_leaf_node(rnp) &&
1883 rcu_preempt_blocked_readers_cgp(rnp));1900 rcu_preempt_blocked_readers_cgp(rnp));
1884 rnp->qsmask &= ~mask;1901 WRITE_ONCE(rnp->qsmask, rnp->qsmask & ~mask);
1885 trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,1902 trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,
1886 mask, rnp->qsmask, rnp->level,1903 mask, rnp->qsmask, rnp->level,
1887 rnp->grplo, rnp->grphi,1904 rnp->grplo, rnp->grphi,
@@ -1904,7 +1921,7 @@ static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp, · 16 unmodified lines
1904 rnp_c = rnp;1921 rnp_c = rnp;
1905 rnp = rnp->parent;1922 rnp = rnp->parent;
1906 raw_spin_lock_irqsave_rcu_node(rnp, flags);1923 raw_spin_lock_irqsave_rcu_node(rnp, flags);
1907 oldmask = rnp_c->qsmask;1924 oldmask = READ_ONCE(rnp_c->qsmask);
1908 }1925 }
1909 1926
1910 /*1927 /*
@@ -1987,6 +2004,8 @@ rcu_report_qs_rdp(int cpu, struct rcu_data *rdp) · 76 unmodified lines
1987 return;2004 return;
1988 }2005 }
1989 mask = rdp->grpmask;2006 mask = rdp->grpmask;
2007 if (rdp->cpu == smp_processor_id())
2008 rdp->core_needs_qs = false;
1990 if ((rnp->qsmask & mask) == 0) {2009 if ((rnp->qsmask & mask) == 0) {
1991 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);2010 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1992 } else {2011 } else {
@@ -2052,7 +2071,7 @@ int rcutree_dying_cpu(unsigned int cpu) · 59 unmodified lines
2052 return 0;2071 return 0;
2053 2072
2054 blkd = !!(rnp->qsmask & rdp->grpmask);2073 blkd = !!(rnp->qsmask & rdp->grpmask);
2055 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq,2074 trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
2056 blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));2075 blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));
2057 return 0;2076 return 0;
2058}2077}
@@ -2294,10 +2313,13 @@ static void force_qs_rnp(int (*f)(struct rcu_data *rdp)) · 235 unmodified lines
2294 struct rcu_data *rdp;2313 struct rcu_data *rdp;
2295 struct rcu_node *rnp;2314 struct rcu_node *rnp;
2296 2315
2316 rcu_state.cbovld = rcu_state.cbovldnext;
2317 rcu_state.cbovldnext = false;
2297 rcu_for_each_leaf_node(rnp) {2318 rcu_for_each_leaf_node(rnp) {
2298 cond_resched_tasks_rcu_qs();2319 cond_resched_tasks_rcu_qs();
2299 mask = 0;2320 mask = 0;
2300 raw_spin_lock_irqsave_rcu_node(rnp, flags);2321 raw_spin_lock_irqsave_rcu_node(rnp, flags);
2322 rcu_state.cbovldnext |= !!rnp->cbovldmask;
2301 if (rnp->qsmask == 0) {2323 if (rnp->qsmask == 0) {
2302 if (!IS_ENABLED(CONFIG_PREEMPT_RCU) ||2324 if (!IS_ENABLED(CONFIG_PREEMPT_RCU) ||
2303 rcu_preempt_blocked_readers_cgp(rnp)) {2325 rcu_preempt_blocked_readers_cgp(rnp)) {
@@ -2579,11 +2601,48 @@ static void rcu_leak_callback(struct rcu_head *rhp) · 275 unmodified lines
2579}2601}
2580 2602
2581/*2603/*
2582 * Helper function for call_rcu() and friends. The cpu argument will2604 * Check and if necessary update the leaf rcu_node structure's
2583 * normally be -1, indicating "currently running CPU". It may specify2605 * ->cbovldmask bit corresponding to the current CPU based on that CPU's
2584 * a CPU only if that CPU is a no-CBs CPU. Currently, only rcu_barrier()2606 * number of queued RCU callbacks. The caller must hold the leaf rcu_node
2585 * is expected to specify a CPU.2607 * structure's ->lock.
2586 */2608 */
2609static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp)
2610{
2611 raw_lockdep_assert_held_rcu_node(rnp);
2612 if (qovld_calc <= 0)
2613 return; // Early boot and wildcard value set.
2614 if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc)
2615 WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask | rdp->grpmask);
2616 else
2617 WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask & ~rdp->grpmask);
2618}
2619
2620/*
2621 * Check and if necessary update the leaf rcu_node structure's
2622 * ->cbovldmask bit corresponding to the current CPU based on that CPU's
2623 * number of queued RCU callbacks. No locks need be held, but the
2624 * caller must have disabled interrupts.
2625 *
2626 * Note that this function ignores the possibility that there are a lot
2627 * of callbacks all of which have already seen the end of their respective
2628 * grace periods. This omission is due to the need for no-CBs CPUs to
2629 * be holding ->nocb_lock to do this check, which is too heavy for a
2630 * common-case operation.
2631 */
2632static void check_cb_ovld(struct rcu_data *rdp)
2633{
2634 struct rcu_node *const rnp = rdp->mynode;
2635
2636 if (qovld_calc <= 0 ||
2637 ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) ==
2638 !!(READ_ONCE(rnp->cbovldmask) & rdp->grpmask)))
2639 return; // Early boot wildcard value or already set correctly.
2640 raw_spin_lock_rcu_node(rnp);
2641 check_cb_ovld_locked(rdp, rnp);
2642 raw_spin_unlock_rcu_node(rnp);
2643}
2644
2645/* Helper function for call_rcu() and friends. */
2587static void2646static void
2588__call_rcu(struct rcu_head *head, rcu_callback_t func)2647__call_rcu(struct rcu_head *head, rcu_callback_t func)
2589{2648{
@@ -2621,9 +2680,10 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func) · 31 unmodified lines
2621 rcu_segcblist_init(&rdp->cblist);2680 rcu_segcblist_init(&rdp->cblist);
2622 }2681 }
2623 2682
2683 check_cb_ovld(rdp);
2624 if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags))2684 if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags))
2625 return; // Enqueued onto ->nocb_bypass, so just leave.2685 return; // Enqueued onto ->nocb_bypass, so just leave.
2626 /* If we get here, rcu_nocb_try_bypass() acquired ->nocb_lock. */2686 // If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock.
2627 rcu_segcblist_enqueue(&rdp->cblist, head);2687 rcu_segcblist_enqueue(&rdp->cblist, head);
2628 if (__is_kfree_rcu_offset((unsigned long)func))2688 if (__is_kfree_rcu_offset((unsigned long)func))
2629 trace_rcu_kfree_callback(rcu_state.name, head,2689 trace_rcu_kfree_callback(rcu_state.name, head,
@@ -2689,22 +2749,47 @@ EXPORT_SYMBOL_GPL(call_rcu); · 59 unmodified lines
2689#define KFREE_DRAIN_JIFFIES (HZ / 50)2749#define KFREE_DRAIN_JIFFIES (HZ / 50)
2690#define KFREE_N_BATCHES 22750#define KFREE_N_BATCHES 2
2691 2751
2752/*
2753 * This macro defines how many entries the "records" array
2754 * will contain. It is based on the fact that the size of
2755 * kfree_rcu_bulk_data structure becomes exactly one page.
2756 */
2757#define KFREE_BULK_MAX_ENTR ((PAGE_SIZE / sizeof(void *)) - 3)
2758
2759/**
2760 * struct kfree_rcu_bulk_data - single block to store kfree_rcu() pointers
2761 * @nr_records: Number of active pointers in the array
2762 * @records: Array of the kfree_rcu() pointers
2763 * @next: Next bulk object in the block chain
2764 * @head_free_debug: For debug, when CONFIG_DEBUG_OBJECTS_RCU_HEAD is set
2765 */
2766struct kfree_rcu_bulk_data {
2767 unsigned long nr_records;
2768 void *records[KFREE_BULK_MAX_ENTR];
2769 struct kfree_rcu_bulk_data *next;
2770 struct rcu_head *head_free_debug;
2771};
2772
2692/**2773/**
2693 * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests2774 * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests
2694 * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period2775 * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period
2695 * @head_free: List of kfree_rcu() objects waiting for a grace period2776 * @head_free: List of kfree_rcu() objects waiting for a grace period
2777 * @bhead_free: Bulk-List of kfree_rcu() objects waiting for a grace period
2696 * @krcp: Pointer to @kfree_rcu_cpu structure2778 * @krcp: Pointer to @kfree_rcu_cpu structure
2697 */2779 */
2698 2780
2699struct kfree_rcu_cpu_work {2781struct kfree_rcu_cpu_work {
2700 struct rcu_work rcu_work;2782 struct rcu_work rcu_work;
2701 struct rcu_head *head_free;2783 struct rcu_head *head_free;
2784 struct kfree_rcu_bulk_data *bhead_free;
2702 struct kfree_rcu_cpu *krcp;2785 struct kfree_rcu_cpu *krcp;
2703};2786};
2704 2787
2705/**2788/**
2706 * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period2789 * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period
2707 * @head: List of kfree_rcu() objects not yet waiting for a grace period2790 * @head: List of kfree_rcu() objects not yet waiting for a grace period
2791 * @bhead: Bulk-List of kfree_rcu() objects not yet waiting for a grace period
2792 * @bcached: Keeps at most one object for later reuse when build chain blocks
2708 * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period2793 * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period
2709 * @lock: Synchronize access to this structure2794 * @lock: Synchronize access to this structure
2710 * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES2795 * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES
@@ -2718,6 +2803,8 @@ struct kfree_rcu_cpu_work { · 7 unmodified lines
2718 */2803 */
2719struct kfree_rcu_cpu {2804struct kfree_rcu_cpu {
2720 struct rcu_head *head;2805 struct rcu_head *head;
2806 struct kfree_rcu_bulk_data *bhead;
2807 struct kfree_rcu_bulk_data *bcached;
2721 struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES];2808 struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES];
2722 spinlock_t lock;2809 spinlock_t lock;
2723 struct delayed_work monitor_work;2810 struct delayed_work monitor_work;
@@ -2727,14 +2814,24 @@ struct kfree_rcu_cpu { · 3 unmodified lines
2727 2814
2728static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc);2815static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc);
2729 2816
2817static __always_inline void
2818debug_rcu_head_unqueue_bulk(struct rcu_head *head)
2819{
2820#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
2821 for (; head; head = head->next)
2822 debug_rcu_head_unqueue(head);
2823#endif
2824}
2825
2730/*2826/*
2731 * This function is invoked in workqueue context after a grace period.2827 * This function is invoked in workqueue context after a grace period.
2732 * It frees all the objects queued on ->head_free.2828 * It frees all the objects queued on ->bhead_free or ->head_free.
2733 */2829 */
2734static void kfree_rcu_work(struct work_struct *work)2830static void kfree_rcu_work(struct work_struct *work)
2735{2831{
2736 unsigned long flags;2832 unsigned long flags;
2737 struct rcu_head *head, *next;2833 struct rcu_head *head, *next;
2834 struct kfree_rcu_bulk_data *bhead, *bnext;
2738 struct kfree_rcu_cpu *krcp;2835 struct kfree_rcu_cpu *krcp;
2739 struct kfree_rcu_cpu_work *krwp;2836 struct kfree_rcu_cpu_work *krwp;
2740 2837
@@ -2744,22 +2841,44 @@ static void kfree_rcu_work(struct work_struct *work) · 3 unmodified lines
2744 spin_lock_irqsave(&krcp->lock, flags);2841 spin_lock_irqsave(&krcp->lock, flags);
2745 head = krwp->head_free;2842 head = krwp->head_free;
2746 krwp->head_free = NULL;2843 krwp->head_free = NULL;
2844 bhead = krwp->bhead_free;
2845 krwp->bhead_free = NULL;
2747 spin_unlock_irqrestore(&krcp->lock, flags);2846 spin_unlock_irqrestore(&krcp->lock, flags);
2748 2847
2749 // List "head" is now private, so traverse locklessly.2848 /* "bhead" is now private, so traverse locklessly. */
2849 for (; bhead; bhead = bnext) {
2850 bnext = bhead->next;
2851
2852 debug_rcu_head_unqueue_bulk(bhead->head_free_debug);
2853
2854 rcu_lock_acquire(&rcu_callback_map);
2855 trace_rcu_invoke_kfree_bulk_callback(rcu_state.name,
2856 bhead->nr_records, bhead->records);
2857
2858 kfree_bulk(bhead->nr_records, bhead->records);
2859 rcu_lock_release(&rcu_callback_map);
2860
2861 if (cmpxchg(&krcp->bcached, NULL, bhead))
2862 free_page((unsigned long) bhead);
2863
2864 cond_resched_tasks_rcu_qs();
2865 }
2866
2867 /*
2868 * Emergency case only. It can happen under low memory
2869 * condition when an allocation gets failed, so the "bulk"
2870 * path can not be temporary maintained.
2871 */
2750 for (; head; head = next) {2872 for (; head; head = next) {
2751 unsigned long offset = (unsigned long)head->func;2873 unsigned long offset = (unsigned long)head->func;
2752 2874
2753 next = head->next;2875 next = head->next;
2754 // Potentially optimize with kfree_bulk in future.
2755 debug_rcu_head_unqueue(head);2876 debug_rcu_head_unqueue(head);
2756 rcu_lock_acquire(&rcu_callback_map);2877 rcu_lock_acquire(&rcu_callback_map);
2757 trace_rcu_invoke_kfree_callback(rcu_state.name, head, offset);2878 trace_rcu_invoke_kfree_callback(rcu_state.name, head, offset);
2758 2879
2759 if (!WARN_ON_ONCE(!__is_kfree_rcu_offset(offset))) {2880 if (!WARN_ON_ONCE(!__is_kfree_rcu_offset(offset)))
2760 /* Could be optimized with kfree_bulk() in future. */
2761 kfree((void *)head - offset);2881 kfree((void *)head - offset);
2762 }
2763 2882
2764 rcu_lock_release(&rcu_callback_map);2883 rcu_lock_release(&rcu_callback_map);
2765 cond_resched_tasks_rcu_qs();2884 cond_resched_tasks_rcu_qs();
@@ -2774,26 +2893,48 @@ static void kfree_rcu_work(struct work_struct *work) · 8 unmodified lines
2774 */2893 */
2775static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp)2894static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp)
2776{2895{
2896 struct kfree_rcu_cpu_work *krwp;
2897 bool queued = false;
2777 int i;2898 int i;
2778 struct kfree_rcu_cpu_work *krwp = NULL;
2779 2899
2780 lockdep_assert_held(&krcp->lock);2900 lockdep_assert_held(&krcp->lock);
2781 for (i = 0; i < KFREE_N_BATCHES; i++)
2782 if (!krcp->krw_arr[i].head_free) {
2783 krwp = &(krcp->krw_arr[i]);
2784 break;
2785 }
2786 2901
2787 // If a previous RCU batch is in progress, we cannot immediately2902 for (i = 0; i < KFREE_N_BATCHES; i++) {
2788 // queue another one, so return false to tell caller to retry.2903 krwp = &(krcp->krw_arr[i]);
2789 if (!krwp)
2790 return false;
2791 2904
2792 krwp->head_free = krcp->head;2905 /*
2793 krcp->head = NULL;2906 * Try to detach bhead or head and attach it over any
2794 INIT_RCU_WORK(&krwp->rcu_work, kfree_rcu_work);2907 * available corresponding free channel. It can be that
2795 queue_rcu_work(system_wq, &krwp->rcu_work);2908 * a previous RCU batch is in progress, it means that
2796 return true;2909 * immediately to queue another one is not possible so
2910 * return false to tell caller to retry.
2911 */
2912 if ((krcp->bhead && !krwp->bhead_free) ||
2913 (krcp->head && !krwp->head_free)) {
2914 /* Channel 1. */
2915 if (!krwp->bhead_free) {
2916 krwp->bhead_free = krcp->bhead;
2917 krcp->bhead = NULL;
2918 }
2919
2920 /* Channel 2. */
2921 if (!krwp->head_free) {
2922 krwp->head_free = krcp->head;
2923 krcp->head = NULL;
2924 }
2925
2926 /*
2927 * One work is per one batch, so there are two "free channels",
2928 * "bhead_free" and "head_free" the batch can handle. It can be
2929 * that the work is in the pending state when two channels have
2930 * been detached following each other, one by one.
2931 */
2932 queue_rcu_work(system_wq, &krwp->rcu_work);
2933 queued = true;
2934 }
2935 }
2936
2937 return queued;
2797}2938}
2798 2939
2799static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu *krcp,2940static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu *krcp,
@@ -2830,19 +2971,65 @@ static void kfree_rcu_monitor(struct work_struct *work) · 30 unmodified lines
2830 spin_unlock_irqrestore(&krcp->lock, flags);2971 spin_unlock_irqrestore(&krcp->lock, flags);
2831}2972}
2832 2973
2974static inline bool
2975kfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp,
2976 struct rcu_head *head, rcu_callback_t func)
2977{
2978 struct kfree_rcu_bulk_data *bnode;
2979
2980 if (unlikely(!krcp->initialized))
2981 return false;
2982
2983 lockdep_assert_held(&krcp->lock);
2984
2985 /* Check if a new block is required. */
2986 if (!krcp->bhead ||
2987 krcp->bhead->nr_records == KFREE_BULK_MAX_ENTR) {
2988 bnode = xchg(&krcp->bcached, NULL);
2989 if (!bnode) {
2990 WARN_ON_ONCE(sizeof(struct kfree_rcu_bulk_data) > PAGE_SIZE);
2991
2992 bnode = (struct kfree_rcu_bulk_data *)
2993 __get_free_page(GFP_NOWAIT | __GFP_NOWARN);
2994 }
2995
2996 /* Switch to emergency path. */
2997 if (unlikely(!bnode))
2998 return false;
2999
3000 /* Initialize the new block. */
3001 bnode->nr_records = 0;
3002 bnode->next = krcp->bhead;
3003 bnode->head_free_debug = NULL;
3004
3005 /* Attach it to the head. */
3006 krcp->bhead = bnode;
3007 }
3008
3009#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
3010 head->func = func;
3011 head->next = krcp->bhead->head_free_debug;
3012 krcp->bhead->head_free_debug = head;
3013#endif
3014
3015 /* Finally insert. */
3016 krcp->bhead->records[krcp->bhead->nr_records++] =
3017 (void *) head - (unsigned long) func;
3018
3019 return true;
3020}
3021
2833/*3022/*
2834 * Queue a request for lazy invocation of kfree() after a grace period.3023 * Queue a request for lazy invocation of kfree_bulk()/kfree() after a grace
3024 * period. Please note there are two paths are maintained, one is the main one
3025 * that uses kfree_bulk() interface and second one is emergency one, that is
3026 * used only when the main path can not be maintained temporary, due to memory
3027 * pressure.
2835 *3028 *
2836 * Each kfree_call_rcu() request is added to a batch. The batch will be drained3029 * Each kfree_call_rcu() request is added to a batch. The batch will be drained
2837 * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch3030 * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch will
2838 * will be kfree'd in workqueue context. This allows us to:3031 * be free'd in workqueue context. This allows us to: batch requests together to
2839 *3032 * reduce the number of grace periods during heavy kfree_rcu() load.
2840 * 1. Batch requests together to reduce the number of grace periods during
2841 * heavy kfree_rcu() load.
2842 *
2843 * 2. It makes it possible to use kfree_bulk() on a large number of
2844 * kfree_rcu() requests thus reducing cache misses and the per-object
2845 * overhead of kfree().
2846 */3033 */
2847void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)3034void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
2848{3035{
@@ -2861,9 +3048,16 @@ void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func) · 12 unmodified lines
2861 __func__, head);3048 __func__, head);
2862 goto unlock_return;3049 goto unlock_return;
2863 }3050 }
2864 head->func = func;3051
2865 head->next = krcp->head;3052 /*
2866 krcp->head = head;3053 * Under high memory pressure GFP_NOWAIT can fail,
3054 * in that case the emergency path is maintained.
3055 */
3056 if (unlikely(!kfree_call_rcu_add_ptr_to_bulk(krcp, head, func))) {
3057 head->func = func;
3058 head->next = krcp->head;
3059 krcp->head = head;
3060 }
2867 3061
2868 // Set timer to drain after KFREE_DRAIN_JIFFIES.3062 // Set timer to drain after KFREE_DRAIN_JIFFIES.
2869 if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&3063 if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
@@ -3075,24 +3269,32 @@ static void rcu_barrier_trace(const char *s, int cpu, unsigned long done) · 205 unmodified lines
3075/*3269/*
3076 * RCU callback function for rcu_barrier(). If we are last, wake3270 * RCU callback function for rcu_barrier(). If we are last, wake
3077 * up the task executing rcu_barrier().3271 * up the task executing rcu_barrier().
3272 *
3273 * Note that the value of rcu_state.barrier_sequence must be captured
3274 * before the atomic_dec_and_test(). Otherwise, if this CPU is not last,
3275 * other CPUs might count the value down to zero before this CPU gets
3276 * around to invoking rcu_barrier_trace(), which might result in bogus
3277 * data from the next instance of rcu_barrier().
3078 */3278 */
3079static void rcu_barrier_callback(struct rcu_head *rhp)3279static void rcu_barrier_callback(struct rcu_head *rhp)
3080{3280{
3281 unsigned long __maybe_unused s = rcu_state.barrier_sequence;
3282
3081 if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {3283 if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {
3082 rcu_barrier_trace(TPS("LastCB"), -1,3284 rcu_barrier_trace(TPS("LastCB"), -1, s);
3083 rcu_state.barrier_sequence);
3084 complete(&rcu_state.barrier_completion);3285 complete(&rcu_state.barrier_completion);
3085 } else {3286 } else {
3086 rcu_barrier_trace(TPS("CB"), -1, rcu_state.barrier_sequence);3287 rcu_barrier_trace(TPS("CB"), -1, s);
3087 }3288 }
3088}3289}
3089 3290
3090/*3291/*
3091 * Called with preemption disabled, and from cross-cpu IRQ context.3292 * Called with preemption disabled, and from cross-cpu IRQ context.
3092 */3293 */
3093static void rcu_barrier_func(void *unused)3294static void rcu_barrier_func(void *cpu_in)
3094{3295{
3095 struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);3296 uintptr_t cpu = (uintptr_t)cpu_in;
3297 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3096 3298
3097 rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);3299 rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);
3098 rdp->barrier_head.func = rcu_barrier_callback;3300 rdp->barrier_head.func = rcu_barrier_callback;
@@ -3119,7 +3321,7 @@ static void rcu_barrier_func(void *unused) · 20 unmodified lines
3119 */3321 */
3120void rcu_barrier(void)3322void rcu_barrier(void)
3121{3323{
3122 int cpu;3324 uintptr_t cpu;
3123 struct rcu_data *rdp;3325 struct rcu_data *rdp;
3124 unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);3326 unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);
3125 3327
@@ -3142,13 +3344,14 @@ void rcu_barrier(void) · 16 unmodified lines
3142 rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);3344 rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);
3143 3345
3144 /*3346 /*
3145 * Initialize the count to one rather than to zero in order to3347 * Initialize the count to two rather than to zero in order
3146 * avoid a too-soon return to zero in case of a short grace period3348 * to avoid a too-soon return to zero in case of an immediate
3147 * (or preemption of this task). Exclude CPU-hotplug operations3349 * invocation of the just-enqueued callback (or preemption of
3148 * to ensure that no offline CPU has callbacks queued.3350 * this task). Exclude CPU-hotplug operations to ensure that no
3351 * offline non-offloaded CPU has callbacks queued.
3149 */3352 */
3150 init_completion(&rcu_state.barrier_completion);3353 init_completion(&rcu_state.barrier_completion);
3151 atomic_set(&rcu_state.barrier_cpu_count, 1);3354 atomic_set(&rcu_state.barrier_cpu_count, 2);
3152 get_online_cpus();3355 get_online_cpus();
3153 3356
3154 /*3357 /*
@@ -3158,13 +3361,23 @@ void rcu_barrier(void) · 3 unmodified lines
3158 */3361 */
3159 for_each_possible_cpu(cpu) {3362 for_each_possible_cpu(cpu) {
3160 rdp = per_cpu_ptr(&rcu_data, cpu);3363 rdp = per_cpu_ptr(&rcu_data, cpu);
3161 if (!cpu_online(cpu) &&3364 if (cpu_is_offline(cpu) &&
3162 !rcu_segcblist_is_offloaded(&rdp->cblist))3365 !rcu_segcblist_is_offloaded(&rdp->cblist))
3163 continue;3366 continue;
3164 if (rcu_segcblist_n_cbs(&rdp->cblist)) {3367 if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) {
3165 rcu_barrier_trace(TPS("OnlineQ"), cpu,3368 rcu_barrier_trace(TPS("OnlineQ"), cpu,
3166 rcu_state.barrier_sequence);3369 rcu_state.barrier_sequence);
3167 smp_call_function_single(cpu, rcu_barrier_func, NULL, 1);3370 smp_call_function_single(cpu, rcu_barrier_func, (void *)cpu, 1);
3371 } else if (rcu_segcblist_n_cbs(&rdp->cblist) &&
3372 cpu_is_offline(cpu)) {
3373 rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu,
3374 rcu_state.barrier_sequence);
3375 local_irq_disable();
3376 rcu_barrier_func((void *)cpu);
3377 local_irq_enable();
3378 } else if (cpu_is_offline(cpu)) {
3379 rcu_barrier_trace(TPS("OfflineNoCBNoQ"), cpu,
3380 rcu_state.barrier_sequence);
3168 } else {3381 } else {
3169 rcu_barrier_trace(TPS("OnlineNQ"), cpu,3382 rcu_barrier_trace(TPS("OnlineNQ"), cpu,
3170 rcu_state.barrier_sequence);3383 rcu_state.barrier_sequence);
@@ -3176,7 +3389,7 @@ void rcu_barrier(void) · 5 unmodified lines
3176 * Now that we have an rcu_barrier_callback() callback on each3389 * Now that we have an rcu_barrier_callback() callback on each
3177 * CPU, and thus each counted, remove the initial count.3390 * CPU, and thus each counted, remove the initial count.
3178 */3391 */
3179 if (atomic_dec_and_test(&rcu_state.barrier_cpu_count))3392 if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count))
3180 complete(&rcu_state.barrier_completion);3393 complete(&rcu_state.barrier_completion);
3181 3394
3182 /* Wait for all rcu_barrier_callback() callbacks to be invoked. */3395 /* Wait for all rcu_barrier_callback() callbacks to be invoked. */
@@ -3275,12 +3488,12 @@ int rcutree_prepare_cpu(unsigned int cpu) · 92 unmodified lines
3275 rnp = rdp->mynode;3488 rnp = rdp->mynode;
3276 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */3489 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
3277 rdp->beenonline = true; /* We have now been online. */3490 rdp->beenonline = true; /* We have now been online. */
3278 rdp->gp_seq = rnp->gp_seq;3491 rdp->gp_seq = READ_ONCE(rnp->gp_seq);
3279 rdp->gp_seq_needed = rnp->gp_seq;3492 rdp->gp_seq_needed = rdp->gp_seq;
3280 rdp->cpu_no_qs.b.norm = true;3493 rdp->cpu_no_qs.b.norm = true;
3281 rdp->core_needs_qs = false;3494 rdp->core_needs_qs = false;
3282 rdp->rcu_iw_pending = false;3495 rdp->rcu_iw_pending = false;
3283 rdp->rcu_iw_gp_seq = rnp->gp_seq - 1;3496 rdp->rcu_iw_gp_seq = rdp->gp_seq - 1;
3284 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));3497 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
3285 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);3498 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3286 rcu_prepare_kthreads(cpu);3499 rcu_prepare_kthreads(cpu);
@@ -3378,7 +3591,7 @@ void rcu_cpu_starting(unsigned int cpu) · 91 unmodified lines
3378 rnp = rdp->mynode;3591 rnp = rdp->mynode;
3379 mask = rdp->grpmask;3592 mask = rdp->grpmask;
3380 raw_spin_lock_irqsave_rcu_node(rnp, flags);3593 raw_spin_lock_irqsave_rcu_node(rnp, flags);
3381 rnp->qsmaskinitnext |= mask;3594 WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext | mask);
3382 oldmask = rnp->expmaskinitnext;3595 oldmask = rnp->expmaskinitnext;
3383 rnp->expmaskinitnext |= mask;3596 rnp->expmaskinitnext |= mask;
3384 oldmask ^= rnp->expmaskinitnext;3597 oldmask ^= rnp->expmaskinitnext;
@@ -3431,7 +3644,7 @@ void rcu_report_dead(unsigned int cpu) · 46 unmodified lines
3431 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);3644 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
3432 raw_spin_lock_irqsave_rcu_node(rnp, flags);3645 raw_spin_lock_irqsave_rcu_node(rnp, flags);
3433 }3646 }
3434 rnp->qsmaskinitnext &= ~mask;3647 WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext & ~mask);
3435 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);3648 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3436 raw_spin_unlock(&rcu_state.ofl_lock);3649 raw_spin_unlock(&rcu_state.ofl_lock);
3437 3650
@@ -3545,7 +3758,10 @@ static int __init rcu_spawn_gp_kthread(void) · 107 unmodified lines
3545 }3758 }
3546 rnp = rcu_get_root();3759 rnp = rcu_get_root();
3547 raw_spin_lock_irqsave_rcu_node(rnp, flags);3760 raw_spin_lock_irqsave_rcu_node(rnp, flags);
3548 rcu_state.gp_kthread = t;3761 WRITE_ONCE(rcu_state.gp_activity, jiffies);
3762 WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
3763 // Reset .gp_activity and .gp_req_activity before setting .gp_kthread.
3764 smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */
3549 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);3765 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3550 wake_up_process(t);3766 wake_up_process(t);
3551 rcu_spawn_nocb_kthreads();3767 rcu_spawn_nocb_kthreads();
@@ -3769,8 +3985,11 @@ static void __init kfree_rcu_batch_init(void) · 217 unmodified lines
3769 struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);3985 struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
3770 3986
3771 spin_lock_init(&krcp->lock);3987 spin_lock_init(&krcp->lock);
3772 for (i = 0; i < KFREE_N_BATCHES; i++)3988 for (i = 0; i < KFREE_N_BATCHES; i++) {
3989 INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
3773 krcp->krw_arr[i].krcp = krcp;3990 krcp->krw_arr[i].krcp = krcp;
3991 }
3992
3774 INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);3993 INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
3775 krcp->initialized = true;3994 krcp->initialized = true;
3776 }3995 }
@@ -3809,6 +4028,13 @@ void __init rcu_init(void) · 32 unmodified lines
3809 rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);4028 rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);
3810 WARN_ON(!rcu_par_gp_wq);4029 WARN_ON(!rcu_par_gp_wq);
3811 srcu_init();4030 srcu_init();
4031
4032 /* Fill in default value for rcutree.qovld boot parameter. */
4033 /* -After- the rcu_node ->lock fields are initialized! */
4034 if (qovld < 0)
4035 qovld_calc = DEFAULT_RCU_QOVLD_MULT * qhimark;
4036 else
4037 qovld_calc = qovld;
3812}4038}
3813 4039
3814#include "tree_stall.h"4040#include "tree_stall.h"
kernel/rcu/tree.h+4−0
@@ -68,6 +68,8 @@ struct rcu_node { · 67 unmodified lines
68 /* Online CPUs for next expedited GP. */68 /* Online CPUs for next expedited GP. */
69 /* Any CPU that has ever been online will */69 /* Any CPU that has ever been online will */
70 /* have its bit set. */70 /* have its bit set. */
71 unsigned long cbovldmask;
72 /* CPUs experiencing callback overload. */
71 unsigned long ffmask; /* Fully functional CPUs. */73 unsigned long ffmask; /* Fully functional CPUs. */
72 unsigned long grpmask; /* Mask to apply to parent qsmask. */74 unsigned long grpmask; /* Mask to apply to parent qsmask. */
73 /* Only one bit will be set in this mask. */75 /* Only one bit will be set in this mask. */
@@ -321,6 +323,8 @@ struct rcu_state { · 247 unmodified lines
321 atomic_t expedited_need_qs; /* # CPUs left to check in. */323 atomic_t expedited_need_qs; /* # CPUs left to check in. */
322 struct swait_queue_head expedited_wq; /* Wait for check-ins. */324 struct swait_queue_head expedited_wq; /* Wait for check-ins. */
323 int ncpus_snap; /* # CPUs seen last time. */325 int ncpus_snap; /* # CPUs seen last time. */
326 u8 cbovld; /* Callback overload now? */
327 u8 cbovldnext; /* ^ ^ next time? */
324 328
325 unsigned long jiffies_force_qs; /* Time at which to invoke */329 unsigned long jiffies_force_qs; /* Time at which to invoke */
326 /* force_quiescent_state(). */330 /* force_quiescent_state(). */
kernel/rcu/tree_exp.h+9−4
@@ -314,7 +314,7 @@ static bool exp_funnel_lock(unsigned long s) · 313 unmodified lines
314 sync_exp_work_done(s));314 sync_exp_work_done(s));
315 return true;315 return true;
316 }316 }
317 rnp->exp_seq_rq = s; /* Followers can wait on us. */317 WRITE_ONCE(rnp->exp_seq_rq, s); /* Followers can wait on us. */
318 spin_unlock(&rnp->exp_lock);318 spin_unlock(&rnp->exp_lock);
319 trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level,319 trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level,
320 rnp->grplo, rnp->grphi, TPS("nxtlvl"));320 rnp->grplo, rnp->grphi, TPS("nxtlvl"));
@@ -485,6 +485,7 @@ static bool synchronize_rcu_expedited_wait_once(long tlimit) · 164 unmodified lines
485static void synchronize_rcu_expedited_wait(void)485static void synchronize_rcu_expedited_wait(void)
486{486{
487 int cpu;487 int cpu;
488 unsigned long j;
488 unsigned long jiffies_stall;489 unsigned long jiffies_stall;
489 unsigned long jiffies_start;490 unsigned long jiffies_start;
490 unsigned long mask;491 unsigned long mask;
@@ -496,7 +497,7 @@ static void synchronize_rcu_expedited_wait(void) · 5 unmodified lines
496 trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("startwait"));497 trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("startwait"));
497 jiffies_stall = rcu_jiffies_till_stall_check();498 jiffies_stall = rcu_jiffies_till_stall_check();
498 jiffies_start = jiffies;499 jiffies_start = jiffies;
499 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {500 if (tick_nohz_full_enabled() && rcu_inkernel_boot_has_ended()) {
500 if (synchronize_rcu_expedited_wait_once(1))501 if (synchronize_rcu_expedited_wait_once(1))
501 return;502 return;
502 rcu_for_each_leaf_node(rnp) {503 rcu_for_each_leaf_node(rnp) {
@@ -508,12 +509,16 @@ static void synchronize_rcu_expedited_wait(void) · 5 unmodified lines
508 tick_dep_set_cpu(cpu, TICK_DEP_BIT_RCU_EXP);509 tick_dep_set_cpu(cpu, TICK_DEP_BIT_RCU_EXP);
509 }510 }
510 }511 }
512 j = READ_ONCE(jiffies_till_first_fqs);
513 if (synchronize_rcu_expedited_wait_once(j + HZ))
514 return;
515 WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT));
511 }516 }
512 517
513 for (;;) {518 for (;;) {
514 if (synchronize_rcu_expedited_wait_once(jiffies_stall))519 if (synchronize_rcu_expedited_wait_once(jiffies_stall))
515 return;520 return;
516 if (rcu_cpu_stall_suppress)521 if (rcu_stall_is_suppressed())
517 continue;522 continue;
518 panic_on_rcu_stall();523 panic_on_rcu_stall();
519 pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {",524 pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {",
@@ -589,7 +594,7 @@ static void rcu_exp_wait_wake(unsigned long s) · 69 unmodified lines
589 spin_lock(&rnp->exp_lock);594 spin_lock(&rnp->exp_lock);
590 /* Recheck, avoid hang in case someone just arrived. */595 /* Recheck, avoid hang in case someone just arrived. */
591 if (ULONG_CMP_LT(rnp->exp_seq_rq, s))596 if (ULONG_CMP_LT(rnp->exp_seq_rq, s))
592 rnp->exp_seq_rq = s;597 WRITE_ONCE(rnp->exp_seq_rq, s);
593 spin_unlock(&rnp->exp_lock);598 spin_unlock(&rnp->exp_lock);
594 }599 }
595 smp_mb(); /* All above changes before wakeup. */600 smp_mb(); /* All above changes before wakeup. */
kernel/rcu/tree_plugin.h+16−9
@@ -56,6 +56,8 @@ static void __init rcu_bootup_announce_oddness(void) · 55 unmodified lines
56 pr_info("\tBoot-time adjustment of callback high-water mark to %ld.\n", qhimark);56 pr_info("\tBoot-time adjustment of callback high-water mark to %ld.\n", qhimark);
57 if (qlowmark != DEFAULT_RCU_QLOMARK)57 if (qlowmark != DEFAULT_RCU_QLOMARK)
58 pr_info("\tBoot-time adjustment of callback low-water mark to %ld.\n", qlowmark);58 pr_info("\tBoot-time adjustment of callback low-water mark to %ld.\n", qlowmark);
59 if (qovld != DEFAULT_RCU_QOVLD)
60 pr_info("\tBoot-time adjustment of callback overload level to %ld.\n", qovld);
59 if (jiffies_till_first_fqs != ULONG_MAX)61 if (jiffies_till_first_fqs != ULONG_MAX)
60 pr_info("\tBoot-time adjustment of first FQS scan delay to %ld jiffies.\n", jiffies_till_first_fqs);62 pr_info("\tBoot-time adjustment of first FQS scan delay to %ld jiffies.\n", jiffies_till_first_fqs);
61 if (jiffies_till_next_fqs != ULONG_MAX)63 if (jiffies_till_next_fqs != ULONG_MAX)
@@ -753,7 +755,7 @@ dump_blkd_tasks(struct rcu_node *rnp, int ncheck) · 691 unmodified lines
753 raw_lockdep_assert_held_rcu_node(rnp);755 raw_lockdep_assert_held_rcu_node(rnp);
754 pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",756 pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
755 __func__, rnp->grplo, rnp->grphi, rnp->level,757 __func__, rnp->grplo, rnp->grphi, rnp->level,
756 (long)rnp->gp_seq, (long)rnp->completedqs);758 (long)READ_ONCE(rnp->gp_seq), (long)rnp->completedqs);
757 for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent)759 for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent)
758 pr_info("%s: %d:%d ->qsmask %#lx ->qsmaskinit %#lx ->qsmaskinitnext %#lx\n",760 pr_info("%s: %d:%d ->qsmask %#lx ->qsmaskinit %#lx ->qsmaskinitnext %#lx\n",
759 __func__, rnp1->grplo, rnp1->grphi, rnp1->qsmask, rnp1->qsmaskinit, rnp1->qsmaskinitnext);761 __func__, rnp1->grplo, rnp1->grphi, rnp1->qsmask, rnp1->qsmaskinit, rnp1->qsmaskinitnext);
@@ -1032,18 +1034,18 @@ static int rcu_boost_kthread(void *arg) · 272 unmodified lines
1032 1034
1033 trace_rcu_utilization(TPS("Start boost kthread@init"));1035 trace_rcu_utilization(TPS("Start boost kthread@init"));
1034 for (;;) {1036 for (;;) {
1035 rnp->boost_kthread_status = RCU_KTHREAD_WAITING;1037 WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_WAITING);
1036 trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));1038 trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
1037 rcu_wait(rnp->boost_tasks || rnp->exp_tasks);1039 rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
1038 trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));1040 trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
1039 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;1041 WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_RUNNING);
1040 more2boost = rcu_boost(rnp);1042 more2boost = rcu_boost(rnp);
1041 if (more2boost)1043 if (more2boost)
1042 spincnt++;1044 spincnt++;
1043 else1045 else
1044 spincnt = 0;1046 spincnt = 0;
1045 if (spincnt > 10) {1047 if (spincnt > 10) {
1046 rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;1048 WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_YIELDING);
1047 trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));1049 trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
1048 schedule_timeout_interruptible(2);1050 schedule_timeout_interruptible(2);
1049 trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));1051 trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
@@ -1077,12 +1079,12 @@ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) · 27 unmodified lines
1077 (rnp->gp_tasks != NULL &&1079 (rnp->gp_tasks != NULL &&
1078 rnp->boost_tasks == NULL &&1080 rnp->boost_tasks == NULL &&
1079 rnp->qsmask == 0 &&1081 rnp->qsmask == 0 &&
1080 ULONG_CMP_GE(jiffies, rnp->boost_time))) {1082 (ULONG_CMP_GE(jiffies, rnp->boost_time) || rcu_state.cbovld))) {
1081 if (rnp->exp_tasks == NULL)1083 if (rnp->exp_tasks == NULL)
1082 rnp->boost_tasks = rnp->gp_tasks;1084 rnp->boost_tasks = rnp->gp_tasks;
1083 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);1085 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1084 rcu_wake_cond(rnp->boost_kthread_task,1086 rcu_wake_cond(rnp->boost_kthread_task,
1085 rnp->boost_kthread_status);1087 READ_ONCE(rnp->boost_kthread_status));
1086 } else {1088 } else {
1087 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);1089 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1088 }1090 }
@@ -1486,6 +1488,7 @@ module_param(nocb_nobypass_lim_per_jiffy, int, 0); · 397 unmodified lines
1486 * flag the contention.1488 * flag the contention.
1487 */1489 */
1488static void rcu_nocb_bypass_lock(struct rcu_data *rdp)1490static void rcu_nocb_bypass_lock(struct rcu_data *rdp)
1491 __acquires(&rdp->nocb_bypass_lock)
1489{1492{
1490 lockdep_assert_irqs_disabled();1493 lockdep_assert_irqs_disabled();
1491 if (raw_spin_trylock(&rdp->nocb_bypass_lock))1494 if (raw_spin_trylock(&rdp->nocb_bypass_lock))
@@ -1529,6 +1532,7 @@ static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp) · 37 unmodified lines
1529 * Release the specified rcu_data structure's ->nocb_bypass_lock.1532 * Release the specified rcu_data structure's ->nocb_bypass_lock.
1530 */1533 */
1531static void rcu_nocb_bypass_unlock(struct rcu_data *rdp)1534static void rcu_nocb_bypass_unlock(struct rcu_data *rdp)
1535 __releases(&rdp->nocb_bypass_lock)
1532{1536{
1533 lockdep_assert_irqs_disabled();1537 lockdep_assert_irqs_disabled();
1534 raw_spin_unlock(&rdp->nocb_bypass_lock);1538 raw_spin_unlock(&rdp->nocb_bypass_lock);
@@ -1577,8 +1581,7 @@ static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, · 42 unmodified lines
1577static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)1581static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)
1578{1582{
1579 lockdep_assert_irqs_disabled();1583 lockdep_assert_irqs_disabled();
1580 if (rcu_segcblist_is_offloaded(&rdp->cblist) &&1584 if (rcu_segcblist_is_offloaded(&rdp->cblist))
1581 cpu_online(rdp->cpu))
1582 lockdep_assert_held(&rdp->nocb_lock);1585 lockdep_assert_held(&rdp->nocb_lock);
1583}1586}
1584 1587
@@ -1930,6 +1933,7 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) · 345 unmodified lines
1930 struct rcu_data *rdp;1933 struct rcu_data *rdp;
1931 struct rcu_node *rnp;1934 struct rcu_node *rnp;
1932 unsigned long wait_gp_seq = 0; // Suppress "use uninitialized" warning.1935 unsigned long wait_gp_seq = 0; // Suppress "use uninitialized" warning.
1936 bool wasempty = false;
1933 1937
1934 /*1938 /*
1935 * Each pass through the following loop checks for CBs and for the1939 * Each pass through the following loop checks for CBs and for the
@@ -1969,10 +1973,13 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) · 33 unmodified lines
1969 rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) {1973 rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) {
1970 raw_spin_lock_rcu_node(rnp); /* irqs disabled. */1974 raw_spin_lock_rcu_node(rnp); /* irqs disabled. */
1971 needwake_gp = rcu_advance_cbs(rnp, rdp);1975 needwake_gp = rcu_advance_cbs(rnp, rdp);
1976 wasempty = rcu_segcblist_restempty(&rdp->cblist,
1977 RCU_NEXT_READY_TAIL);
1972 raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */1978 raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */
1973 }1979 }
1974 // Need to wait on some grace period?1980 // Need to wait on some grace period?
1975 WARN_ON_ONCE(!rcu_segcblist_restempty(&rdp->cblist,1981 WARN_ON_ONCE(wasempty &&
1982 !rcu_segcblist_restempty(&rdp->cblist,
1976 RCU_NEXT_READY_TAIL));1983 RCU_NEXT_READY_TAIL));
1977 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) {1984 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) {
1978 if (!needwait_gp ||1985 if (!needwait_gp ||
kernel/rcu/tree_stall.h+23−18
@@ -102,7 +102,7 @@ static void record_gp_stall_check_time(void) · 101 unmodified lines
102 unsigned long j = jiffies;102 unsigned long j = jiffies;
103 unsigned long j1;103 unsigned long j1;
104 104
105 rcu_state.gp_start = j;105 WRITE_ONCE(rcu_state.gp_start, j);
106 j1 = rcu_jiffies_till_stall_check();106 j1 = rcu_jiffies_till_stall_check();
107 /* Record ->gp_start before ->jiffies_stall. */107 /* Record ->gp_start before ->jiffies_stall. */
108 smp_store_release(&rcu_state.jiffies_stall, j + j1); /* ^^^ */108 smp_store_release(&rcu_state.jiffies_stall, j + j1); /* ^^^ */
@@ -383,7 +383,7 @@ static void print_other_cpu_stall(unsigned long gp_seq) · 274 unmodified lines
383 383
384 /* Kick and suppress, if so configured. */384 /* Kick and suppress, if so configured. */
385 rcu_stall_kick_kthreads();385 rcu_stall_kick_kthreads();
386 if (rcu_cpu_stall_suppress)386 if (rcu_stall_is_suppressed())
387 return;387 return;
388 388
389 /*389 /*
@@ -452,7 +452,7 @@ static void print_cpu_stall(void) · 62 unmodified lines
452 452
453 /* Kick and suppress, if so configured. */453 /* Kick and suppress, if so configured. */
454 rcu_stall_kick_kthreads();454 rcu_stall_kick_kthreads();
455 if (rcu_cpu_stall_suppress)455 if (rcu_stall_is_suppressed())
456 return;456 return;
457 457
458 /*458 /*
@@ -504,7 +504,7 @@ static void check_cpu_stall(struct rcu_data *rdp) · 45 unmodified lines
504 unsigned long js;504 unsigned long js;
505 struct rcu_node *rnp;505 struct rcu_node *rnp;
506 506
507 if ((rcu_cpu_stall_suppress && !rcu_kick_kthreads) ||507 if ((rcu_stall_is_suppressed() && !rcu_kick_kthreads) ||
508 !rcu_gp_in_progress())508 !rcu_gp_in_progress())
509 return;509 return;
510 rcu_stall_kick_kthreads();510 rcu_stall_kick_kthreads();
@@ -578,6 +578,7 @@ void show_rcu_gp_kthreads(void) · 67 unmodified lines
578 unsigned long jw;578 unsigned long jw;
579 struct rcu_data *rdp;579 struct rcu_data *rdp;
580 struct rcu_node *rnp;580 struct rcu_node *rnp;
581 struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
581 582
582 j = jiffies;583 j = jiffies;
583 ja = j - READ_ONCE(rcu_state.gp_activity);584 ja = j - READ_ONCE(rcu_state.gp_activity);
@@ -585,28 +586,28 @@ void show_rcu_gp_kthreads(void) · 1 unmodified line
585 jw = j - READ_ONCE(rcu_state.gp_wake_time);586 jw = j - READ_ONCE(rcu_state.gp_wake_time);
586 pr_info("%s: wait state: %s(%d) ->state: %#lx delta ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_flags %#x\n",587 pr_info("%s: wait state: %s(%d) ->state: %#lx delta ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_flags %#x\n",
587 rcu_state.name, gp_state_getname(rcu_state.gp_state),588 rcu_state.name, gp_state_getname(rcu_state.gp_state),
588 rcu_state.gp_state,589 rcu_state.gp_state, t ? t->state : 0x1ffffL,
589 rcu_state.gp_kthread ? rcu_state.gp_kthread->state : 0x1ffffL,
590 ja, jr, jw, (long)READ_ONCE(rcu_state.gp_wake_seq),590 ja, jr, jw, (long)READ_ONCE(rcu_state.gp_wake_seq),
591 (long)READ_ONCE(rcu_state.gp_seq),591 (long)READ_ONCE(rcu_state.gp_seq),
592 (long)READ_ONCE(rcu_get_root()->gp_seq_needed),592 (long)READ_ONCE(rcu_get_root()->gp_seq_needed),
593 READ_ONCE(rcu_state.gp_flags));593 READ_ONCE(rcu_state.gp_flags));
594 rcu_for_each_node_breadth_first(rnp) {594 rcu_for_each_node_breadth_first(rnp) {
595 if (ULONG_CMP_GE(rcu_state.gp_seq, rnp->gp_seq_needed))595 if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq),
596 READ_ONCE(rnp->gp_seq_needed)))
596 continue;597 continue;
597 pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld\n",598 pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld\n",
598 rnp->grplo, rnp->grphi, (long)rnp->gp_seq,599 rnp->grplo, rnp->grphi, (long)READ_ONCE(rnp->gp_seq),
599 (long)rnp->gp_seq_needed);600 (long)READ_ONCE(rnp->gp_seq_needed));
600 if (!rcu_is_leaf_node(rnp))601 if (!rcu_is_leaf_node(rnp))
601 continue;602 continue;
602 for_each_leaf_node_possible_cpu(rnp, cpu) {603 for_each_leaf_node_possible_cpu(rnp, cpu) {
603 rdp = per_cpu_ptr(&rcu_data, cpu);604 rdp = per_cpu_ptr(&rcu_data, cpu);
604 if (rdp->gpwrap ||605 if (READ_ONCE(rdp->gpwrap) ||
605 ULONG_CMP_GE(rcu_state.gp_seq,606 ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq),
606 rdp->gp_seq_needed))607 READ_ONCE(rdp->gp_seq_needed)))
607 continue;608 continue;
608 pr_info("\tcpu %d ->gp_seq_needed %ld\n",609 pr_info("\tcpu %d ->gp_seq_needed %ld\n",
609 cpu, (long)rdp->gp_seq_needed);610 cpu, (long)READ_ONCE(rdp->gp_seq_needed));
610 }611 }
611 }612 }
612 for_each_possible_cpu(cpu) {613 for_each_possible_cpu(cpu) {
@@ -631,7 +632,9 @@ static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, · 18 unmodified lines
631 static atomic_t warned = ATOMIC_INIT(0);632 static atomic_t warned = ATOMIC_INIT(0);
632 633
633 if (!IS_ENABLED(CONFIG_PROVE_RCU) || rcu_gp_in_progress() ||634 if (!IS_ENABLED(CONFIG_PROVE_RCU) || rcu_gp_in_progress() ||
634 ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed))635 ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
636 READ_ONCE(rnp_root->gp_seq_needed)) ||
637 !smp_load_acquire(&rcu_state.gp_kthread)) // Get stable kthread.
635 return;638 return;
636 j = jiffies; /* Expensive access, and in common case don't get here. */639 j = jiffies; /* Expensive access, and in common case don't get here. */
637 if (time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||640 if (time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
@@ -642,7 +645,8 @@ static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, · 4 unmodified lines
642 raw_spin_lock_irqsave_rcu_node(rnp, flags);645 raw_spin_lock_irqsave_rcu_node(rnp, flags);
643 j = jiffies;646 j = jiffies;
644 if (rcu_gp_in_progress() ||647 if (rcu_gp_in_progress() ||
645 ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) ||648 ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
649 READ_ONCE(rnp_root->gp_seq_needed)) ||
646 time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||650 time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
647 time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||651 time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
648 atomic_read(&warned)) {652 atomic_read(&warned)) {
@@ -655,9 +659,10 @@ static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, · 6 unmodified lines
655 raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */659 raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
656 j = jiffies;660 j = jiffies;
657 if (rcu_gp_in_progress() ||661 if (rcu_gp_in_progress() ||
658 ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) ||662 ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
659 time_before(j, rcu_state.gp_req_activity + gpssdelay) ||663 READ_ONCE(rnp_root->gp_seq_needed)) ||
660 time_before(j, rcu_state.gp_activity + gpssdelay) ||664 time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
665 time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
661 atomic_xchg(&warned, 1)) {666 atomic_xchg(&warned, 1)) {
662 if (rnp_root != rnp)667 if (rnp_root != rnp)
663 /* irqs remain disabled. */668 /* irqs remain disabled. */
kernel/rcu/update.c+23−5
@@ -183,6 +183,8 @@ void rcu_unexpedite_gp(void) · 182 unmodified lines
183}183}
184EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);184EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);
185 185
186static bool rcu_boot_ended __read_mostly;
187
186/*188/*
187 * Inform RCU of the end of the in-kernel boot sequence.189 * Inform RCU of the end of the in-kernel boot sequence.
188 */190 */
@@ -191,8 +193,18 @@ void rcu_end_inkernel_boot(void) · 2 unmodified lines
191 rcu_unexpedite_gp();193 rcu_unexpedite_gp();
192 if (rcu_normal_after_boot)194 if (rcu_normal_after_boot)
193 WRITE_ONCE(rcu_normal, 1);195 WRITE_ONCE(rcu_normal, 1);
196 rcu_boot_ended = 1;
194}197}
195 198
199/*
200 * Let rcutorture know when it is OK to turn it up to eleven.
201 */
202bool rcu_inkernel_boot_has_ended(void)
203{
204 return rcu_boot_ended;
205}
206EXPORT_SYMBOL_GPL(rcu_inkernel_boot_has_ended);
207
196#endif /* #ifndef CONFIG_TINY_RCU */208#endif /* #ifndef CONFIG_TINY_RCU */
197 209
198/*210/*
@@ -464,13 +476,19 @@ EXPORT_SYMBOL_GPL(rcutorture_sched_setaffinity); · 265 unmodified lines
464#ifdef CONFIG_RCU_STALL_COMMON476#ifdef CONFIG_RCU_STALL_COMMON
465int rcu_cpu_stall_ftrace_dump __read_mostly;477int rcu_cpu_stall_ftrace_dump __read_mostly;
466module_param(rcu_cpu_stall_ftrace_dump, int, 0644);478module_param(rcu_cpu_stall_ftrace_dump, int, 0644);
467int rcu_cpu_stall_suppress __read_mostly; /* 1 = suppress stall warnings. */479int rcu_cpu_stall_suppress __read_mostly; // !0 = suppress stall warnings.
468EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress);480EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress);
469module_param(rcu_cpu_stall_suppress, int, 0644);481module_param(rcu_cpu_stall_suppress, int, 0644);
470int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;482int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
471module_param(rcu_cpu_stall_timeout, int, 0644);483module_param(rcu_cpu_stall_timeout, int, 0644);
472#endif /* #ifdef CONFIG_RCU_STALL_COMMON */484#endif /* #ifdef CONFIG_RCU_STALL_COMMON */
473 485
486// Suppress boot-time RCU CPU stall warnings and rcutorture writer stall
487// warnings. Also used by rcutorture even if stall warnings are excluded.
488int rcu_cpu_stall_suppress_at_boot __read_mostly; // !0 = suppress boot stalls.
489EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress_at_boot);
490module_param(rcu_cpu_stall_suppress_at_boot, int, 0444);
491
474#ifdef CONFIG_TASKS_RCU492#ifdef CONFIG_TASKS_RCU
475 493
476/*494/*
@@ -528,7 +546,7 @@ void call_rcu_tasks(struct rcu_head *rhp, rcu_callback_t func) · 51 unmodified lines
528 rhp->func = func;546 rhp->func = func;
529 raw_spin_lock_irqsave(&rcu_tasks_cbs_lock, flags);547 raw_spin_lock_irqsave(&rcu_tasks_cbs_lock, flags);
530 needwake = !rcu_tasks_cbs_head;548 needwake = !rcu_tasks_cbs_head;
531 *rcu_tasks_cbs_tail = rhp;549 WRITE_ONCE(*rcu_tasks_cbs_tail, rhp);
532 rcu_tasks_cbs_tail = &rhp->next;550 rcu_tasks_cbs_tail = &rhp->next;
533 raw_spin_unlock_irqrestore(&rcu_tasks_cbs_lock, flags);551 raw_spin_unlock_irqrestore(&rcu_tasks_cbs_lock, flags);
534 /* We can't create the thread unless interrupts are enabled. */552 /* We can't create the thread unless interrupts are enabled. */
@@ -658,7 +676,7 @@ static int __noreturn rcu_tasks_kthread(void *arg) · 123 unmodified lines
658 /* If there were none, wait a bit and start over. */676 /* If there were none, wait a bit and start over. */
659 if (!list) {677 if (!list) {
660 wait_event_interruptible(rcu_tasks_cbs_wq,678 wait_event_interruptible(rcu_tasks_cbs_wq,
661 rcu_tasks_cbs_head);679 READ_ONCE(rcu_tasks_cbs_head));
662 if (!rcu_tasks_cbs_head) {680 if (!rcu_tasks_cbs_head) {
663 WARN_ON(signal_pending(current));681 WARN_ON(signal_pending(current));
664 schedule_timeout_interruptible(HZ/10);682 schedule_timeout_interruptible(HZ/10);
@@ -801,7 +819,7 @@ static int __init rcu_spawn_tasks_kthread(void) · 136 unmodified lines
801core_initcall(rcu_spawn_tasks_kthread);819core_initcall(rcu_spawn_tasks_kthread);
802 820
803/* Do the srcu_read_lock() for the above synchronize_srcu(). */821/* Do the srcu_read_lock() for the above synchronize_srcu(). */
804void exit_tasks_rcu_start(void)822void exit_tasks_rcu_start(void) __acquires(&tasks_rcu_exit_srcu)
805{823{
806 preempt_disable();824 preempt_disable();
807 current->rcu_tasks_idx = __srcu_read_lock(&tasks_rcu_exit_srcu);825 current->rcu_tasks_idx = __srcu_read_lock(&tasks_rcu_exit_srcu);
@@ -809,7 +827,7 @@ void exit_tasks_rcu_start(void) · 1 unmodified line
809}827}
810 828
811/* Do the srcu_read_unlock() for the above synchronize_srcu(). */829/* Do the srcu_read_unlock() for the above synchronize_srcu(). */
812void exit_tasks_rcu_finish(void)830void exit_tasks_rcu_finish(void) __releases(&tasks_rcu_exit_srcu)
813{831{
814 preempt_disable();832 preempt_disable();
815 __srcu_read_unlock(&tasks_rcu_exit_srcu, current->rcu_tasks_idx);833 __srcu_read_unlock(&tasks_rcu_exit_srcu, current->rcu_tasks_idx);
kernel/time/timer.c+4−3
@@ -944,6 +944,7 @@ static struct timer_base *lock_timer_base(struct timer_list *timer, · 943 unmodified lines
944 944
945#define MOD_TIMER_PENDING_ONLY 0x01945#define MOD_TIMER_PENDING_ONLY 0x01
946#define MOD_TIMER_REDUCE 0x02946#define MOD_TIMER_REDUCE 0x02
947#define MOD_TIMER_NOTPENDING 0x04
947 948
948static inline int949static inline int
949__mod_timer(struct timer_list *timer, unsigned long expires, unsigned int options)950__mod_timer(struct timer_list *timer, unsigned long expires, unsigned int options)
@@ -960,7 +961,7 @@ __mod_timer(struct timer_list *timer, unsigned long expires, unsigned int option · 10 unmodified lines
960 * the timer is re-modified to have the same timeout or ends up in the961 * the timer is re-modified to have the same timeout or ends up in the
961 * same array bucket then just return:962 * same array bucket then just return:
962 */963 */
963 if (timer_pending(timer)) {964 if (!(options & MOD_TIMER_NOTPENDING) && timer_pending(timer)) {
964 /*965 /*
965 * The downside of this optimization is that it can result in966 * The downside of this optimization is that it can result in
966 * larger granularity than you would get from adding a new967 * larger granularity than you would get from adding a new
@@ -1133,7 +1134,7 @@ EXPORT_SYMBOL(timer_reduce); · 166 unmodified lines
1133void add_timer(struct timer_list *timer)1134void add_timer(struct timer_list *timer)
1134{1135{
1135 BUG_ON(timer_pending(timer));1136 BUG_ON(timer_pending(timer));
1136 mod_timer(timer, timer->expires);1137 __mod_timer(timer, timer->expires, MOD_TIMER_NOTPENDING);
1137}1138}
1138EXPORT_SYMBOL(add_timer);1139EXPORT_SYMBOL(add_timer);
1139 1140
@@ -1891,7 +1892,7 @@ signed long __sched schedule_timeout(signed long timeout) · 751 unmodified lines
1891 1892
1892 timer.task = current;1893 timer.task = current;
1893 timer_setup_on_stack(&timer.timer, process_timeout, 0);1894 timer_setup_on_stack(&timer.timer, process_timeout, 0);
1894 __mod_timer(&timer.timer, expire, 0);1895 __mod_timer(&timer.timer, expire, MOD_TIMER_NOTPENDING);
1895 schedule();1896 schedule();
1896 del_singleshot_timer_sync(&timer.timer);1897 del_singleshot_timer_sync(&timer.timer);
1897 1898
kernel/torture.c+25−4
@@ -42,6 +42,9 @@ · 41 unmodified lines
42MODULE_LICENSE("GPL");42MODULE_LICENSE("GPL");
43MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");43MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
44 44
45static bool disable_onoff_at_boot;
46module_param(disable_onoff_at_boot, bool, 0444);
47
45static char *torture_type;48static char *torture_type;
46static int verbose;49static int verbose;
47 50
@@ -84,6 +87,7 @@ bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes, · 36 unmodified lines
84{87{
85 unsigned long delta;88 unsigned long delta;
86 int ret;89 int ret;
90 char *s;
87 unsigned long starttime;91 unsigned long starttime;
88 92
89 if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu))93 if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
@@ -99,10 +103,16 @@ bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes, · 9 unmodified lines
99 (*n_offl_attempts)++;103 (*n_offl_attempts)++;
100 ret = cpu_down(cpu);104 ret = cpu_down(cpu);
101 if (ret) {105 if (ret) {
106 s = "";
107 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
108 // PCI probe frequently disables hotplug during boot.
109 (*n_offl_attempts)--;
110 s = " (-EBUSY forgiven during boot)";
111 }
102 if (verbose)112 if (verbose)
103 pr_alert("%s" TORTURE_FLAG113 pr_alert("%s" TORTURE_FLAG
104 "torture_onoff task: offline %d failed: errno %d\n",114 "torture_onoff task: offline %d failed%s: errno %d\n",
105 torture_type, cpu, ret);115 torture_type, cpu, s, ret);
106 } else {116 } else {
107 if (verbose > 1)117 if (verbose > 1)
108 pr_alert("%s" TORTURE_FLAG118 pr_alert("%s" TORTURE_FLAG
@@ -137,6 +147,7 @@ bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes, · 28 unmodified lines
137{147{
138 unsigned long delta;148 unsigned long delta;
139 int ret;149 int ret;
150 char *s;
140 unsigned long starttime;151 unsigned long starttime;
141 152
142 if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu))153 if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
@@ -150,10 +161,16 @@ bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes, · 7 unmodified lines
150 (*n_onl_attempts)++;161 (*n_onl_attempts)++;
151 ret = cpu_up(cpu);162 ret = cpu_up(cpu);
152 if (ret) {163 if (ret) {
164 s = "";
165 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
166 // PCI probe frequently disables hotplug during boot.
167 (*n_onl_attempts)--;
168 s = " (-EBUSY forgiven during boot)";
169 }
153 if (verbose)170 if (verbose)
154 pr_alert("%s" TORTURE_FLAG171 pr_alert("%s" TORTURE_FLAG
155 "torture_onoff task: online %d failed: errno %d\n",172 "torture_onoff task: online %d failed%s: errno %d\n",
156 torture_type, cpu, ret);173 torture_type, cpu, s, ret);
157 } else {174 } else {
158 if (verbose > 1)175 if (verbose > 1)
159 pr_alert("%s" TORTURE_FLAG176 pr_alert("%s" TORTURE_FLAG
@@ -215,6 +232,10 @@ torture_onoff(void *arg) · 55 unmodified lines
215 VERBOSE_TOROUT_STRING("torture_onoff end holdoff");232 VERBOSE_TOROUT_STRING("torture_onoff end holdoff");
216 }233 }
217 while (!torture_must_stop()) {234 while (!torture_must_stop()) {
235 if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) {
236 schedule_timeout_interruptible(HZ / 10);
237 continue;
238 }
218 cpu = (torture_random(&rand) >> 4) % (maxcpu + 1);239 cpu = (torture_random(&rand) >> 4) % (maxcpu + 1);
219 if (!torture_offline(cpu,240 if (!torture_offline(cpu,
220 &n_offline_attempts, &n_offline_successes,241 &n_offline_attempts, &n_offline_successes,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t