Unified · Split
| @@ -2129,6 +2129,8 @@ Some of the relevant points of interest are as follows: · 2128 unmodified lines | ||
| 2129 | 2129 | <li> <a href="#Hotplug CPU">Hotplug CPU</a>. |
| 2130 | 2130 | <li> <a href="#Scheduler and RCU">Scheduler and RCU</a>. |
| 2131 | 2131 | <li> <a href="#Tracing and RCU">Tracing and RCU</a>. |
| 2132 | +<li> <a href="#Accesses to User Memory and RCU"> | |
| 2133 | +Accesses to User Memory and RCU</a>. | |
| 2132 | 2134 | <li> <a href="#Energy Efficiency">Energy Efficiency</a>. |
| 2133 | 2135 | <li> <a href="#Scheduling-Clock Interrupts and RCU"> |
| 2134 | 2136 | Scheduling-Clock Interrupts and RCU</a>. |
| @@ -2512,7 +2514,7 @@ disabled across the entire RCU read-side critical section. · 377 unmodified lines | ||
| 2512 | 2514 | <p> |
| 2513 | 2515 | It is possible to use tracing on RCU code, but tracing itself |
| 2514 | 2516 | uses RCU. |
| 2515 | −For this reason, <tt>rcu_dereference_raw_notrace()</tt> | |
| 2517 | +For this reason, <tt>rcu_dereference_raw_check()</tt> | |
| 2516 | 2518 | is provided for use by tracing, which avoids the destructive |
| 2517 | 2519 | recursion that could otherwise ensue. |
| 2518 | 2520 | This API is also used by virtualization in some architectures, |
| @@ -2521,6 +2523,75 @@ cannot be used. · 2 unmodified lines | ||
| 2521 | 2523 | The tracing folks both located the requirement and provided the |
| 2522 | 2524 | needed fix, so this surprise requirement was relatively painless. |
| 2523 | 2525 | |
| 2526 | +<h3><a name="Accesses to User Memory and RCU"> | |
| 2527 | +Accesses to User Memory and RCU</a></h3> | |
| 2528 | + | |
| 2529 | +<p> | |
| 2530 | +The kernel needs to access user-space memory, for example, to access | |
| 2531 | +data referenced by system-call parameters. | |
| 2532 | +The <tt>get_user()</tt> macro does this job. | |
| 2533 | + | |
| 2534 | +<p> | |
| 2535 | +However, user-space memory might well be paged out, which means | |
| 2536 | +that <tt>get_user()</tt> might well page-fault and thus block while | |
| 2537 | +waiting for the resulting I/O to complete. | |
| 2538 | +It would be a very bad thing for the compiler to reorder | |
| 2539 | +a <tt>get_user()</tt> invocation into an RCU read-side critical | |
| 2540 | +section. | |
| 2541 | +For example, suppose that the source code looked like this: | |
| 2542 | + | |
| 2543 | +<blockquote> | |
| 2544 | +<pre> | |
| 2545 | + 1 rcu_read_lock(); | |
| 2546 | + 2 p = rcu_dereference(gp); | |
| 2547 | + 3 v = p->value; | |
| 2548 | + 4 rcu_read_unlock(); | |
| 2549 | + 5 get_user(user_v, user_p); | |
| 2550 | + 6 do_something_with(v, user_v); | |
| 2551 | +</pre> | |
| 2552 | +</blockquote> | |
| 2553 | + | |
| 2554 | +<p> | |
| 2555 | +The compiler must not be permitted to transform this source code into | |
| 2556 | +the following: | |
| 2557 | + | |
| 2558 | +<blockquote> | |
| 2559 | +<pre> | |
| 2560 | + 1 rcu_read_lock(); | |
| 2561 | + 2 p = rcu_dereference(gp); | |
| 2562 | + 3 get_user(user_v, user_p); // BUG: POSSIBLE PAGE FAULT!!! | |
| 2563 | + 4 v = p->value; | |
| 2564 | + 5 rcu_read_unlock(); | |
| 2565 | + 6 do_something_with(v, user_v); | |
| 2566 | +</pre> | |
| 2567 | +</blockquote> | |
| 2568 | + | |
| 2569 | +<p> | |
| 2570 | +If the compiler did make this transformation in a | |
| 2571 | +<tt>CONFIG_PREEMPT=n</tt> kernel build, and if <tt>get_user()</tt> did | |
| 2572 | +page fault, the result would be a quiescent state in the middle | |
| 2573 | +of an RCU read-side critical section. | |
| 2574 | +This misplaced quiescent state could result in line 4 being | |
| 2575 | +a use-after-free access, which could be bad for your kernel's | |
| 2576 | +actuarial statistics. | |
| 2577 | +Similar examples can be constructed with the call to <tt>get_user()</tt> | |
| 2578 | +preceding the <tt>rcu_read_lock()</tt>. | |
| 2579 | + | |
| 2580 | +<p> | |
| 2581 | +Unfortunately, <tt>get_user()</tt> doesn't have any particular | |
| 2582 | +ordering properties, and in some architectures the underlying <tt>asm</tt> | |
| 2583 | +isn't even marked <tt>volatile</tt>. | |
| 2584 | +And even if it was marked <tt>volatile</tt>, the above access to | |
| 2585 | +<tt>p->value</tt> is not volatile, so the compiler would not have any | |
| 2586 | +reason to keep those two accesses in order. | |
| 2587 | + | |
| 2588 | +<p> | |
| 2589 | +Therefore, the Linux-kernel definitions of <tt>rcu_read_lock()</tt> | |
| 2590 | +and <tt>rcu_read_unlock()</tt> must act as compiler barriers, | |
| 2591 | +at least for outermost instances of <tt>rcu_read_lock()</tt> and | |
| 2592 | +<tt>rcu_read_unlock()</tt> within a nested set of RCU read-side critical | |
| 2593 | +sections. | |
| 2594 | + | |
| 2524 | 2595 | <h3><a name="Energy Efficiency">Energy Efficiency</a></h3> |
| 2525 | 2596 | |
| 2526 | 2597 | <p> |
| @@ -57,6 +57,12 @@ o A CPU-bound real-time task in a CONFIG_PREEMPT_RT kernel that · 56 unmodified lines | ||
| 57 | 57 | CONFIG_PREEMPT_RCU case, you might see stall-warning |
| 58 | 58 | messages. |
| 59 | 59 | |
| 60 | + You can use the rcutree.kthread_prio kernel boot parameter to | |
| 61 | + increase the scheduling priority of RCU's kthreads, which can | |
| 62 | + help avoid this problem. However, please note that doing this | |
| 63 | + can increase your system's context-switch rate and thus degrade | |
| 64 | + performance. | |
| 65 | + | |
| 60 | 66 | o A periodic interrupt whose handler takes longer than the time |
| 61 | 67 | interval between successive pairs of interrupts. This can |
| 62 | 68 | prevent RCU's kthreads and softirq handlers from running. |
| @@ -3837,12 +3837,13 @@ · 3836 unmodified lines | ||
| 3837 | 3837 | RCU_BOOST is not set, valid values are 0-99 and |
| 3838 | 3838 | the default is zero (non-realtime operation). |
| 3839 | 3839 | |
| 3840 | − rcutree.rcu_nocb_leader_stride= [KNL] | |
| 3841 | − Set the number of NOCB kthread groups, which | |
| 3842 | − defaults to the square root of the number of | |
| 3843 | − CPUs. Larger numbers reduces the wakeup overhead | |
| 3844 | − on the per-CPU grace-period kthreads, but increases | |
| 3845 | − that same overhead on each group's leader. | |
| 3840 | + rcutree.rcu_nocb_gp_stride= [KNL] | |
| 3841 | + Set the number of NOCB callback kthreads in | |
| 3842 | + each group, which defaults to the square root | |
| 3843 | + of the number of CPUs. Larger numbers reduce | |
| 3844 | + the wakeup overhead on the global grace-period | |
| 3845 | + kthread, but increases that same overhead on | |
| 3846 | + each group's NOCB grace-period kthread. | |
| 3846 | 3847 | |
| 3847 | 3848 | rcutree.qhimark= [KNL] |
| 3848 | 3849 | Set threshold of queued RCU callbacks beyond which |
| @@ -4047,6 +4048,10 @@ · 198 unmodified lines | ||
| 4047 | 4048 | rcutorture.verbose= [KNL] |
| 4048 | 4049 | Enable additional printk() statements. |
| 4049 | 4050 | |
| 4051 | + rcupdate.rcu_cpu_stall_ftrace_dump= [KNL] | |
| 4052 | + Dump ftrace buffer after reporting RCU CPU | |
| 4053 | + stall warning. | |
| 4054 | + | |
| 4050 | 4055 | rcupdate.rcu_cpu_stall_suppress= [KNL] |
| 4051 | 4056 | Suppress RCU CPU stall warning messages. |
| 4052 | 4057 | |
| @@ -9326,7 +9326,7 @@ F: drivers/misc/lkdtm/* · 9325 unmodified lines | ||
| 9326 | 9326 | |
| 9327 | 9327 | LINUX KERNEL MEMORY CONSISTENCY MODEL (LKMM) |
| 9328 | 9328 | M: Alan Stern <stern@rowland.harvard.edu> |
| 9329 | −M: Andrea Parri <andrea.parri@amarulasolutions.com> | |
| 9329 | +M: Andrea Parri <parri.andrea@gmail.com> | |
| 9330 | 9330 | M: Will Deacon <will@kernel.org> |
| 9331 | 9331 | M: Peter Zijlstra <peterz@infradead.org> |
| 9332 | 9332 | M: Boqun Feng <boqun.feng@gmail.com> |
| @@ -264,15 +264,13 @@ int __cpu_disable(void) · 263 unmodified lines | ||
| 264 | 264 | return 0; |
| 265 | 265 | } |
| 266 | 266 | |
| 267 | −static DECLARE_COMPLETION(cpu_died); | |
| 268 | − | |
| 269 | 267 | /* |
| 270 | 268 | * called on the thread which is asking for a CPU to be shutdown - |
| 271 | 269 | * waits until shutdown has completed, or it is timed out. |
| 272 | 270 | */ |
| 273 | 271 | void __cpu_die(unsigned int cpu) |
| 274 | 272 | { |
| 275 | − if (!wait_for_completion_timeout(&cpu_died, msecs_to_jiffies(5000))) { | |
| 273 | + if (!cpu_wait_death(cpu, 5)) { | |
| 276 | 274 | pr_err("CPU%u: cpu didn't die\n", cpu); |
| 277 | 275 | return; |
| 278 | 276 | } |
| @@ -319,7 +317,7 @@ void arch_cpu_idle_dead(void) · 40 unmodified lines | ||
| 319 | 317 | * this returns, power and/or clocks can be removed at any point |
| 320 | 318 | * from this CPU and its cache by platform_cpu_kill(). |
| 321 | 319 | */ |
| 322 | − complete(&cpu_died); | |
| 320 | + (void)cpu_report_death(); | |
| 323 | 321 | |
| 324 | 322 | /* |
| 325 | 323 | * Ensure that the cache lines associated with that completion are |
| @@ -535,7 +535,7 @@ static inline void note_hpte_modification(struct kvm *kvm, · 534 unmodified lines | ||
| 535 | 535 | */ |
| 536 | 536 | static inline struct kvm_memslots *kvm_memslots_raw(struct kvm *kvm) |
| 537 | 537 | { |
| 538 | − return rcu_dereference_raw_notrace(kvm->memslots[0]); | |
| 538 | + return rcu_dereference_raw_check(kvm->memslots[0]); | |
| 539 | 539 | } |
| 540 | 540 | |
| 541 | 541 | extern void kvmppc_mmu_debugfs_init(struct kvm *kvm); |
| @@ -29,6 +29,7 @@ · 28 unmodified lines | ||
| 29 | 29 | static bool pci_mmcfg_running_state; |
| 30 | 30 | static bool pci_mmcfg_arch_init_failed; |
| 31 | 31 | static DEFINE_MUTEX(pci_mmcfg_lock); |
| 32 | +#define pci_mmcfg_lock_held() lock_is_held(&(pci_mmcfg_lock).dep_map) | |
| 32 | 33 | |
| 33 | 34 | LIST_HEAD(pci_mmcfg_list); |
| 34 | 35 | |
| @@ -54,7 +55,7 @@ static void list_add_sorted(struct pci_mmcfg_region *new) · 19 unmodified lines | ||
| 54 | 55 | struct pci_mmcfg_region *cfg; |
| 55 | 56 | |
| 56 | 57 | /* keep list sorted by segment and starting bus number */ |
| 57 | − list_for_each_entry_rcu(cfg, &pci_mmcfg_list, list) { | |
| 58 | + list_for_each_entry_rcu(cfg, &pci_mmcfg_list, list, pci_mmcfg_lock_held()) { | |
| 58 | 59 | if (cfg->segment > new->segment || |
| 59 | 60 | (cfg->segment == new->segment && |
| 60 | 61 | cfg->start_bus >= new->start_bus)) { |
| @@ -118,7 +119,7 @@ struct pci_mmcfg_region *pci_mmconfig_lookup(int segment, int bus) · 57 unmodified lines | ||
| 118 | 119 | { |
| 119 | 120 | struct pci_mmcfg_region *cfg; |
| 120 | 121 | |
| 121 | − list_for_each_entry_rcu(cfg, &pci_mmcfg_list, list) | |
| 122 | + list_for_each_entry_rcu(cfg, &pci_mmcfg_list, list, pci_mmcfg_lock_held()) | |
| 122 | 123 | if (cfg->segment == segment && |
| 123 | 124 | cfg->start_bus <= bus && bus <= cfg->end_bus) |
| 124 | 125 | return cfg; |
| @@ -14,6 +14,7 @@ · 13 unmodified lines | ||
| 14 | 14 | #include <linux/slab.h> |
| 15 | 15 | #include <linux/mm.h> |
| 16 | 16 | #include <linux/highmem.h> |
| 17 | +#include <linux/lockdep.h> | |
| 17 | 18 | #include <linux/pci.h> |
| 18 | 19 | #include <linux/interrupt.h> |
| 19 | 20 | #include <linux/kmod.h> |
| @@ -80,6 +81,7 @@ struct acpi_ioremap { · 60 unmodified lines | ||
| 80 | 81 | |
| 81 | 82 | static LIST_HEAD(acpi_ioremaps); |
| 82 | 83 | static DEFINE_MUTEX(acpi_ioremap_lock); |
| 84 | +#define acpi_ioremap_lock_held() lock_is_held(&acpi_ioremap_lock.dep_map) | |
| 83 | 85 | |
| 84 | 86 | static void __init acpi_request_region (struct acpi_generic_address *gas, |
| 85 | 87 | unsigned int length, char *desc) |
| @@ -206,7 +208,7 @@ acpi_map_lookup(acpi_physical_address phys, acpi_size size) · 120 unmodified lines | ||
| 206 | 208 | { |
| 207 | 209 | struct acpi_ioremap *map; |
| 208 | 210 | |
| 209 | − list_for_each_entry_rcu(map, &acpi_ioremaps, list) | |
| 211 | + list_for_each_entry_rcu(map, &acpi_ioremaps, list, acpi_ioremap_lock_held()) | |
| 210 | 212 | if (map->phys <= phys && |
| 211 | 213 | phys + size <= map->phys + map->size) |
| 212 | 214 | return map; |
| @@ -249,7 +251,7 @@ acpi_map_lookup_virt(void __iomem *virt, acpi_size size) · 36 unmodified lines | ||
| 249 | 251 | { |
| 250 | 252 | struct acpi_ioremap *map; |
| 251 | 253 | |
| 252 | − list_for_each_entry_rcu(map, &acpi_ioremaps, list) | |
| 254 | + list_for_each_entry_rcu(map, &acpi_ioremaps, list, acpi_ioremap_lock_held()) | |
| 253 | 255 | if (map->virt <= virt && |
| 254 | 256 | virt + size <= map->virt + map->size) |
| 255 | 257 | return map; |
| @@ -165,6 +165,7 @@ static inline int devtmpfs_init(void) { return 0; } · 164 unmodified lines | ||
| 165 | 165 | /* Device links support */ |
| 166 | 166 | extern int device_links_read_lock(void); |
| 167 | 167 | extern void device_links_read_unlock(int idx); |
| 168 | +extern int device_links_read_lock_held(void); | |
| 168 | 169 | extern int device_links_check_suppliers(struct device *dev); |
| 169 | 170 | extern void device_links_driver_bound(struct device *dev); |
| 170 | 171 | extern void device_links_driver_cleanup(struct device *dev); |
| @@ -68,6 +68,11 @@ void device_links_read_unlock(int idx) · 67 unmodified lines | ||
| 68 | 68 | { |
| 69 | 69 | srcu_read_unlock(&device_links_srcu, idx); |
| 70 | 70 | } |
| 71 | + | |
| 72 | +int device_links_read_lock_held(void) | |
| 73 | +{ | |
| 74 | + return srcu_read_lock_held(&device_links_srcu); | |
| 75 | +} | |
| 71 | 76 | #else /* !CONFIG_SRCU */ |
| 72 | 77 | static DECLARE_RWSEM(device_links_lock); |
| 73 | 78 | |
| @@ -91,6 +96,13 @@ void device_links_read_unlock(int not_used) · 17 unmodified lines | ||
| 91 | 96 | { |
| 92 | 97 | up_read(&device_links_lock); |
| 93 | 98 | } |
| 99 | + | |
| 100 | +#ifdef CONFIG_DEBUG_LOCK_ALLOC | |
| 101 | +int device_links_read_lock_held(void) | |
| 102 | +{ | |
| 103 | + return lockdep_is_held(&device_links_lock); | |
| 104 | +} | |
| 105 | +#endif | |
| 94 | 106 | #endif /* !CONFIG_SRCU */ |
| 95 | 107 | |
| 96 | 108 | /** |
| @@ -287,7 +287,8 @@ static int rpm_get_suppliers(struct device *dev) · 286 unmodified lines | ||
| 287 | 287 | { |
| 288 | 288 | struct device_link *link; |
| 289 | 289 | |
| 290 | − list_for_each_entry_rcu(link, &dev->links.suppliers, c_node) { | |
| 290 | + list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, | |
| 291 | + device_links_read_lock_held()) { | |
| 291 | 292 | int retval; |
| 292 | 293 | |
| 293 | 294 | if (!(link->flags & DL_FLAG_PM_RUNTIME) || |
| @@ -309,7 +310,8 @@ static void rpm_put_suppliers(struct device *dev) · 15 unmodified lines | ||
| 309 | 310 | { |
| 310 | 311 | struct device_link *link; |
| 311 | 312 | |
| 312 | − list_for_each_entry_rcu(link, &dev->links.suppliers, c_node) { | |
| 313 | + list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, | |
| 314 | + device_links_read_lock_held()) { | |
| 313 | 315 | if (READ_ONCE(link->status) == DL_STATE_SUPPLIER_UNBIND) |
| 314 | 316 | continue; |
| 315 | 317 | |
| @@ -1640,7 +1642,8 @@ void pm_runtime_clean_up_links(struct device *dev) · 1324 unmodified lines | ||
| 1640 | 1642 | |
| 1641 | 1643 | idx = device_links_read_lock(); |
| 1642 | 1644 | |
| 1643 | − list_for_each_entry_rcu(link, &dev->links.consumers, s_node) { | |
| 1645 | + list_for_each_entry_rcu(link, &dev->links.consumers, s_node, | |
| 1646 | + device_links_read_lock_held()) { | |
| 1644 | 1647 | if (link->flags & DL_FLAG_STATELESS) |
| 1645 | 1648 | continue; |
| 1646 | 1649 | |
| @@ -1662,7 +1665,8 @@ void pm_runtime_get_suppliers(struct device *dev) · 15 unmodified lines | ||
| 1662 | 1665 | |
| 1663 | 1666 | idx = device_links_read_lock(); |
| 1664 | 1667 | |
| 1665 | − list_for_each_entry_rcu(link, &dev->links.suppliers, c_node) | |
| 1668 | + list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, | |
| 1669 | + device_links_read_lock_held()) | |
| 1666 | 1670 | if (link->flags & DL_FLAG_PM_RUNTIME) { |
| 1667 | 1671 | link->supplier_preactivated = true; |
| 1668 | 1672 | refcount_inc(&link->rpm_active); |
| @@ -1683,7 +1687,8 @@ void pm_runtime_put_suppliers(struct device *dev) · 14 unmodified lines | ||
| 1683 | 1687 | |
| 1684 | 1688 | idx = device_links_read_lock(); |
| 1685 | 1689 | |
| 1686 | − list_for_each_entry_rcu(link, &dev->links.suppliers, c_node) | |
| 1690 | + list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, | |
| 1691 | + device_links_read_lock_held()) | |
| 1687 | 1692 | if (link->supplier_preactivated) { |
| 1688 | 1693 | link->supplier_preactivated = false; |
| 1689 | 1694 | if (refcount_dec_not_one(&link->rpm_active)) |
| @@ -14,6 +14,9 @@ · 13 unmodified lines | ||
| 14 | 14 | #ifndef __INCLUDE_LINUX_RCU_SEGCBLIST_H |
| 15 | 15 | #define __INCLUDE_LINUX_RCU_SEGCBLIST_H |
| 16 | 16 | |
| 17 | +#include <linux/types.h> | |
| 18 | +#include <linux/atomic.h> | |
| 19 | + | |
| 17 | 20 | /* Simple unsegmented callback lists. */ |
| 18 | 21 | struct rcu_cblist { |
| 19 | 22 | struct rcu_head *head; |
| @@ -65,8 +68,14 @@ struct rcu_segcblist { · 45 unmodified lines | ||
| 65 | 68 | struct rcu_head *head; |
| 66 | 69 | struct rcu_head **tails[RCU_CBLIST_NSEGS]; |
| 67 | 70 | unsigned long gp_seq[RCU_CBLIST_NSEGS]; |
| 71 | +#ifdef CONFIG_RCU_NOCB_CPU | |
| 72 | + atomic_long_t len; | |
| 73 | +#else | |
| 68 | 74 | long len; |
| 75 | +#endif | |
| 69 | 76 | long len_lazy; |
| 77 | + u8 enabled; | |
| 78 | + u8 offloaded; | |
| 70 | 79 | }; |
| 71 | 80 | |
| 72 | 81 | #define RCU_SEGCBLIST_INITIALIZER(n) \ |
| @@ -31,9 +31,7 @@ struct rcu_sync { · 30 unmodified lines | ||
| 31 | 31 | */ |
| 32 | 32 | static inline bool rcu_sync_is_idle(struct rcu_sync *rsp) |
| 33 | 33 | { |
| 34 | − RCU_LOCKDEP_WARN(!rcu_read_lock_held() && | |
| 35 | − !rcu_read_lock_bh_held() && | |
| 36 | − !rcu_read_lock_sched_held(), | |
| 34 | + RCU_LOCKDEP_WARN(!rcu_read_lock_any_held(), | |
| 37 | 35 | "suspicious rcu_sync_is_idle() usage"); |
| 38 | 36 | return !READ_ONCE(rsp->gp_state); /* GP_IDLE */ |
| 39 | 37 | } |
| @@ -40,6 +40,24 @@ static inline void INIT_LIST_HEAD_RCU(struct list_head *list) · 39 unmodified lines | ||
| 40 | 40 | */ |
| 41 | 41 | #define list_next_rcu(list) (*((struct list_head __rcu **)(&(list)->next))) |
| 42 | 42 | |
| 43 | +/* | |
| 44 | + * Check during list traversal that we are within an RCU reader | |
| 45 | + */ | |
| 46 | + | |
| 47 | +#define check_arg_count_one(dummy) | |
| 48 | + | |
| 49 | +#ifdef CONFIG_PROVE_RCU_LIST | |
| 50 | +#define __list_check_rcu(dummy, cond, extra...) \ | |
| 51 | + ({ \ | |
| 52 | + check_arg_count_one(extra); \ | |
| 53 | + RCU_LOCKDEP_WARN(!cond && !rcu_read_lock_any_held(), \ | |
| 54 | + "RCU-list traversed in non-reader section!"); \ | |
| 55 | + }) | |
| 56 | +#else | |
| 57 | +#define __list_check_rcu(dummy, cond, extra...) \ | |
| 58 | + ({ check_arg_count_one(extra); }) | |
| 59 | +#endif | |
| 60 | + | |
| 43 | 61 | /* |
| 44 | 62 | * Insert a new entry between two known consecutive entries. |
| 45 | 63 | * |
| @@ -343,14 +361,16 @@ static inline void list_splice_tail_init_rcu(struct list_head *list, · 297 unmodified lines | ||
| 343 | 361 | * @pos: the type * to use as a loop cursor. |
| 344 | 362 | * @head: the head for your list. |
| 345 | 363 | * @member: the name of the list_head within the struct. |
| 364 | + * @cond: optional lockdep expression if called from non-RCU protection. | |
| 346 | 365 | * |
| 347 | 366 | * This list-traversal primitive may safely run concurrently with |
| 348 | 367 | * the _rcu list-mutation primitives such as list_add_rcu() |
| 349 | 368 | * as long as the traversal is guarded by rcu_read_lock(). |
| 350 | 369 | */ |
| 351 | −#define list_for_each_entry_rcu(pos, head, member) \ | |
| 352 | − for (pos = list_entry_rcu((head)->next, typeof(*pos), member); \ | |
| 353 | − &pos->member != (head); \ | |
| 370 | +#define list_for_each_entry_rcu(pos, head, member, cond...) \ | |
| 371 | + for (__list_check_rcu(dummy, ## cond, 0), \ | |
| 372 | + pos = list_entry_rcu((head)->next, typeof(*pos), member); \ | |
| 373 | + &pos->member != (head); \ | |
| 354 | 374 | pos = list_entry_rcu(pos->member.next, typeof(*pos), member)) |
| 355 | 375 | |
| 356 | 376 | /** |
| @@ -616,13 +636,15 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n, · 259 unmodified lines | ||
| 616 | 636 | * @pos: the type * to use as a loop cursor. |
| 617 | 637 | * @head: the head for your list. |
| 618 | 638 | * @member: the name of the hlist_node within the struct. |
| 639 | + * @cond: optional lockdep expression if called from non-RCU protection. | |
| 619 | 640 | * |
| 620 | 641 | * This list-traversal primitive may safely run concurrently with |
| 621 | 642 | * the _rcu list-mutation primitives such as hlist_add_head_rcu() |
| 622 | 643 | * as long as the traversal is guarded by rcu_read_lock(). |
| 623 | 644 | */ |
| 624 | −#define hlist_for_each_entry_rcu(pos, head, member) \ | |
| 625 | − for (pos = hlist_entry_safe (rcu_dereference_raw(hlist_first_rcu(head)),\ | |
| 645 | +#define hlist_for_each_entry_rcu(pos, head, member, cond...) \ | |
| 646 | + for (__list_check_rcu(dummy, ## cond, 0), \ | |
| 647 | + pos = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),\ | |
| 626 | 648 | typeof(*(pos)), member); \ |
| 627 | 649 | pos; \ |
| 628 | 650 | pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\ |
| @@ -642,10 +664,10 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n, · 13 unmodified lines | ||
| 642 | 664 | * not do any RCU debugging or tracing. |
| 643 | 665 | */ |
| 644 | 666 | #define hlist_for_each_entry_rcu_notrace(pos, head, member) \ |
| 645 | − for (pos = hlist_entry_safe (rcu_dereference_raw_notrace(hlist_first_rcu(head)),\ | |
| 667 | + for (pos = hlist_entry_safe(rcu_dereference_raw_check(hlist_first_rcu(head)),\ | |
| 646 | 668 | typeof(*(pos)), member); \ |
| 647 | 669 | pos; \ |
| 648 | − pos = hlist_entry_safe(rcu_dereference_raw_notrace(hlist_next_rcu(\ | |
| 670 | + pos = hlist_entry_safe(rcu_dereference_raw_check(hlist_next_rcu(\ | |
| 649 | 671 | &(pos)->member)), typeof(*(pos)), member)) |
| 650 | 672 | |
| 651 | 673 | /** |
| @@ -221,6 +221,7 @@ int debug_lockdep_rcu_enabled(void); · 220 unmodified lines | ||
| 221 | 221 | int rcu_read_lock_held(void); |
| 222 | 222 | int rcu_read_lock_bh_held(void); |
| 223 | 223 | int rcu_read_lock_sched_held(void); |
| 224 | +int rcu_read_lock_any_held(void); | |
| 224 | 225 | |
| 225 | 226 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
| 226 | 227 | |
| @@ -241,6 +242,12 @@ static inline int rcu_read_lock_sched_held(void) · 14 unmodified lines | ||
| 241 | 242 | { |
| 242 | 243 | return !preemptible(); |
| 243 | 244 | } |
| 245 | + | |
| 246 | +static inline int rcu_read_lock_any_held(void) | |
| 247 | +{ | |
| 248 | + return !preemptible(); | |
| 249 | +} | |
| 250 | + | |
| 244 | 251 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
| 245 | 252 | |
| 246 | 253 | #ifdef CONFIG_PROVE_RCU |
| @@ -476,7 +483,7 @@ do { \ · 229 unmodified lines | ||
| 476 | 483 | * The no-tracing version of rcu_dereference_raw() must not call |
| 477 | 484 | * rcu_read_lock_held(). |
| 478 | 485 | */ |
| 479 | −#define rcu_dereference_raw_notrace(p) __rcu_dereference_check((p), 1, __rcu) | |
| 486 | +#define rcu_dereference_raw_check(p) __rcu_dereference_check((p), 1, __rcu) | |
| 480 | 487 | |
| 481 | 488 | /** |
| 482 | 489 | * rcu_dereference_protected() - fetch RCU pointer when updates prevented |
| @@ -100,7 +100,6 @@ TRACE_EVENT_RCU(rcu_grace_period, · 99 unmodified lines | ||
| 100 | 100 | * "Startedroot": Requested a nocb grace period based on root-node data. |
| 101 | 101 | * "NoGPkthread": The RCU grace-period kthread has not yet started. |
| 102 | 102 | * "StartWait": Start waiting for the requested grace period. |
| 103 | − * "ResumeWait": Resume waiting after signal. | |
| 104 | 103 | * "EndWait": Complete wait. |
| 105 | 104 | * "Cleanup": Clean up rcu_node structure after previous GP. |
| 106 | 105 | * "CleanupMore": Clean up, and another GP is needed. |
| @@ -267,7 +266,8 @@ TRACE_EVENT_RCU(rcu_exp_funnel_lock, · 160 unmodified lines | ||
| 267 | 266 | * "WakeNotPoll": Don't wake rcuo kthread because it is polling. |
| 268 | 267 | * "DeferredWake": Carried out the "IsDeferred" wakeup. |
| 269 | 268 | * "Poll": Start of new polling cycle for rcu_nocb_poll. |
| 270 | − * "Sleep": Sleep waiting for CBs for !rcu_nocb_poll. | |
| 269 | + * "Sleep": Sleep waiting for GP for !rcu_nocb_poll. | |
| 270 | + * "CBSleep": Sleep waiting for CBs for !rcu_nocb_poll. | |
| 271 | 271 | * "WokeEmpty": rcuo kthread woke to find empty list. |
| 272 | 272 | * "WokeNonEmpty": rcuo kthread woke to find non-empty list. |
| 273 | 273 | * "WaitQueue": Enqueue partially done, timed wait for it to complete. |
| @@ -620,7 +620,7 @@ static void print_lock(struct held_lock *hlock) · 619 unmodified lines | ||
| 620 | 620 | return; |
| 621 | 621 | } |
| 622 | 622 | |
| 623 | − printk(KERN_CONT "%p", hlock->instance); | |
| 623 | + printk(KERN_CONT "%px", hlock->instance); | |
| 624 | 624 | print_lock_name(lock); |
| 625 | 625 | printk(KERN_CONT ", at: %pS\n", (void *)hlock->acquire_ip); |
| 626 | 626 | } |
| @@ -8,6 +8,17 @@ menu "RCU Debugging" · 7 unmodified lines | ||
| 8 | 8 | config PROVE_RCU |
| 9 | 9 | def_bool PROVE_LOCKING |
| 10 | 10 | |
| 11 | +config PROVE_RCU_LIST | |
| 12 | + bool "RCU list lockdep debugging" | |
| 13 | + depends on PROVE_RCU && RCU_EXPERT | |
| 14 | + default n | |
| 15 | + help | |
| 16 | + Enable RCU lockdep checking for list usages. By default it is | |
| 17 | + turned off since there are several list RCU users that still | |
| 18 | + need to be converted to pass a lockdep expression. To prevent | |
| 19 | + false-positive splats, we keep it default disabled but once all | |
| 20 | + users are converted, we can remove this config option. | |
| 21 | + | |
| 11 | 22 | config TORTURE_TEST |
| 12 | 23 | tristate |
| 13 | 24 | default n |
| @@ -227,6 +227,7 @@ static inline bool __rcu_reclaim(const char *rn, struct rcu_head *head) · 226 unmodified lines | ||
| 227 | 227 | |
| 228 | 228 | #ifdef CONFIG_RCU_STALL_COMMON |
| 229 | 229 | |
| 230 | +extern int rcu_cpu_stall_ftrace_dump; | |
| 230 | 231 | extern int rcu_cpu_stall_suppress; |
| 231 | 232 | extern int rcu_cpu_stall_timeout; |
| 232 | 233 | int rcu_jiffies_till_stall_check(void); |
| @@ -23,6 +23,49 @@ void rcu_cblist_init(struct rcu_cblist *rclp) · 22 unmodified lines | ||
| 23 | 23 | rclp->len_lazy = 0; |
| 24 | 24 | } |
| 25 | 25 | |
| 26 | +/* | |
| 27 | + * Enqueue an rcu_head structure onto the specified callback list. | |
| 28 | + * This function assumes that the callback is non-lazy because it | |
| 29 | + * is intended for use by no-CBs CPUs, which do not distinguish | |
| 30 | + * between lazy and non-lazy RCU callbacks. | |
| 31 | + */ | |
| 32 | +void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp) | |
| 33 | +{ | |
| 34 | + *rclp->tail = rhp; | |
| 35 | + rclp->tail = &rhp->next; | |
| 36 | + WRITE_ONCE(rclp->len, rclp->len + 1); | |
| 37 | +} | |
| 38 | + | |
| 39 | +/* | |
| 40 | + * Flush the second rcu_cblist structure onto the first one, obliterating | |
| 41 | + * any contents of the first. If rhp is non-NULL, enqueue it as the sole | |
| 42 | + * element of the second rcu_cblist structure, but ensuring that the second | |
| 43 | + * rcu_cblist structure, if initially non-empty, always appears non-empty | |
| 44 | + * throughout the process. If rdp is NULL, the second rcu_cblist structure | |
| 45 | + * is instead initialized to empty. | |
| 46 | + */ | |
| 47 | +void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, | |
| 48 | + struct rcu_cblist *srclp, | |
| 49 | + struct rcu_head *rhp) | |
| 50 | +{ | |
| 51 | + drclp->head = srclp->head; | |
| 52 | + if (drclp->head) | |
| 53 | + drclp->tail = srclp->tail; | |
| 54 | + else | |
| 55 | + drclp->tail = &drclp->head; | |
| 56 | + drclp->len = srclp->len; | |
| 57 | + drclp->len_lazy = srclp->len_lazy; | |
| 58 | + if (!rhp) { | |
| 59 | + rcu_cblist_init(srclp); | |
| 60 | + } else { | |
| 61 | + rhp->next = NULL; | |
| 62 | + srclp->head = rhp; | |
| 63 | + srclp->tail = &rhp->next; | |
| 64 | + WRITE_ONCE(srclp->len, 1); | |
| 65 | + srclp->len_lazy = 0; | |
| 66 | + } | |
| 67 | +} | |
| 68 | + | |
| 26 | 69 | /* |
| 27 | 70 | * Dequeue the oldest rcu_head structure from the specified callback |
| 28 | 71 | * list. This function assumes that the callback is non-lazy, but |
| @@ -44,6 +87,67 @@ struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp) · 15 unmodified lines | ||
| 44 | 87 | return rhp; |
| 45 | 88 | } |
| 46 | 89 | |
| 90 | +/* Set the length of an rcu_segcblist structure. */ | |
| 91 | +void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v) | |
| 92 | +{ | |
| 93 | +#ifdef CONFIG_RCU_NOCB_CPU | |
| 94 | + atomic_long_set(&rsclp->len, v); | |
| 95 | +#else | |
| 96 | + WRITE_ONCE(rsclp->len, v); | |
| 97 | +#endif | |
| 98 | +} | |
| 99 | + | |
| 100 | +/* | |
| 101 | + * Increase the numeric length of an rcu_segcblist structure by the | |
| 102 | + * specified amount, which can be negative. This can cause the ->len | |
| 103 | + * field to disagree with the actual number of callbacks on the structure. | |
| 104 | + * This increase is fully ordered with respect to the callers accesses | |
| 105 | + * both before and after. | |
| 106 | + */ | |
| 107 | +void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v) | |
| 108 | +{ | |
| 109 | +#ifdef CONFIG_RCU_NOCB_CPU | |
| 110 | + smp_mb__before_atomic(); /* Up to the caller! */ | |
| 111 | + atomic_long_add(v, &rsclp->len); | |
| 112 | + smp_mb__after_atomic(); /* Up to the caller! */ | |
| 113 | +#else | |
| 114 | + smp_mb(); /* Up to the caller! */ | |
| 115 | + WRITE_ONCE(rsclp->len, rsclp->len + v); | |
| 116 | + smp_mb(); /* Up to the caller! */ | |
| 117 | +#endif | |
| 118 | +} | |
| 119 | + | |
| 120 | +/* | |
| 121 | + * Increase the numeric length of an rcu_segcblist structure by one. | |
| 122 | + * This can cause the ->len field to disagree with the actual number of | |
| 123 | + * callbacks on the structure. This increase is fully ordered with respect | |
| 124 | + * to the callers accesses both before and after. | |
| 125 | + */ | |
| 126 | +void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp) | |
| 127 | +{ | |
| 128 | + rcu_segcblist_add_len(rsclp, 1); | |
| 129 | +} | |
| 130 | + | |
| 131 | +/* | |
| 132 | + * Exchange the numeric length of the specified rcu_segcblist structure | |
| 133 | + * with the specified value. This can cause the ->len field to disagree | |
| 134 | + * with the actual number of callbacks on the structure. This exchange is | |
| 135 | + * fully ordered with respect to the callers accesses both before and after. | |
| 136 | + */ | |
| 137 | +long rcu_segcblist_xchg_len(struct rcu_segcblist *rsclp, long v) | |
| 138 | +{ | |
| 139 | +#ifdef CONFIG_RCU_NOCB_CPU | |
| 140 | + return atomic_long_xchg(&rsclp->len, v); | |
| 141 | +#else | |
| 142 | + long ret = rsclp->len; | |
| 143 | + | |
| 144 | + smp_mb(); /* Up to the caller! */ | |
| 145 | + WRITE_ONCE(rsclp->len, v); | |
| 146 | + smp_mb(); /* Up to the caller! */ | |
| 147 | + return ret; | |
| 148 | +#endif | |
| 149 | +} | |
| 150 | + | |
| 47 | 151 | /* |
| 48 | 152 | * Initialize an rcu_segcblist structure. |
| 49 | 153 | */ |
| @@ -56,8 +160,9 @@ void rcu_segcblist_init(struct rcu_segcblist *rsclp) · 6 unmodified lines | ||
| 56 | 160 | rsclp->head = NULL; |
| 57 | 161 | for (i = 0; i < RCU_CBLIST_NSEGS; i++) |
| 58 | 162 | rsclp->tails[i] = &rsclp->head; |
| 59 | − rsclp->len = 0; | |
| 163 | + rcu_segcblist_set_len(rsclp, 0); | |
| 60 | 164 | rsclp->len_lazy = 0; |
| 165 | + rsclp->enabled = 1; | |
| 61 | 166 | } |
| 62 | 167 | |
| 63 | 168 | /* |
| @@ -69,7 +174,16 @@ void rcu_segcblist_disable(struct rcu_segcblist *rsclp) · 5 unmodified lines | ||
| 69 | 174 | WARN_ON_ONCE(!rcu_segcblist_empty(rsclp)); |
| 70 | 175 | WARN_ON_ONCE(rcu_segcblist_n_cbs(rsclp)); |
| 71 | 176 | WARN_ON_ONCE(rcu_segcblist_n_lazy_cbs(rsclp)); |
| 72 | − rsclp->tails[RCU_NEXT_TAIL] = NULL; | |
| 177 | + rsclp->enabled = 0; | |
| 178 | +} | |
| 179 | + | |
| 180 | +/* | |
| 181 | + * Mark the specified rcu_segcblist structure as offloaded. This | |
| 182 | + * structure must be empty. | |
| 183 | + */ | |
| 184 | +void rcu_segcblist_offload(struct rcu_segcblist *rsclp) | |
| 185 | +{ | |
| 186 | + rsclp->offloaded = 1; | |
| 73 | 187 | } |
| 74 | 188 | |
| 75 | 189 | /* |
| @@ -117,6 +231,18 @@ struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp) · 41 unmodified lines | ||
| 117 | 231 | return NULL; |
| 118 | 232 | } |
| 119 | 233 | |
| 234 | +/* | |
| 235 | + * Return false if there are no CBs awaiting grace periods, otherwise, | |
| 236 | + * return true and store the nearest waited-upon grace period into *lp. | |
| 237 | + */ | |
| 238 | +bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp) | |
| 239 | +{ | |
| 240 | + if (!rcu_segcblist_pend_cbs(rsclp)) | |
| 241 | + return false; | |
| 242 | + *lp = rsclp->gp_seq[RCU_WAIT_TAIL]; | |
| 243 | + return true; | |
| 244 | +} | |
| 245 | + | |
| 120 | 246 | /* |
| 121 | 247 | * Enqueue the specified callback onto the specified rcu_segcblist |
| 122 | 248 | * structure, updating accounting as needed. Note that the ->len |
| @@ -129,13 +255,13 @@ struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp) · 6 unmodified lines | ||
| 129 | 255 | void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, |
| 130 | 256 | struct rcu_head *rhp, bool lazy) |
| 131 | 257 | { |
| 132 | − WRITE_ONCE(rsclp->len, rsclp->len + 1); /* ->len sampled locklessly. */ | |
| 258 | + rcu_segcblist_inc_len(rsclp); | |
| 133 | 259 | if (lazy) |
| 134 | 260 | rsclp->len_lazy++; |
| 135 | 261 | smp_mb(); /* Ensure counts are updated before callback is enqueued. */ |
| 136 | 262 | rhp->next = NULL; |
| 137 | − *rsclp->tails[RCU_NEXT_TAIL] = rhp; | |
| 138 | − rsclp->tails[RCU_NEXT_TAIL] = &rhp->next; | |
| 263 | + WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rhp); | |
| 264 | + WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], &rhp->next); | |
| 139 | 265 | } |
| 140 | 266 | |
| 141 | 267 | /* |
| @@ -155,7 +281,7 @@ bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, · 13 unmodified lines | ||
| 155 | 281 | |
| 156 | 282 | if (rcu_segcblist_n_cbs(rsclp) == 0) |
| 157 | 283 | return false; |
| 158 | − WRITE_ONCE(rsclp->len, rsclp->len + 1); | |
| 284 | + rcu_segcblist_inc_len(rsclp); | |
| 159 | 285 | if (lazy) |
| 160 | 286 | rsclp->len_lazy++; |
| 161 | 287 | smp_mb(); /* Ensure counts are updated before callback is entrained. */ |
| @@ -163,9 +289,9 @@ bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, · 1 unmodified line | ||
| 163 | 289 | for (i = RCU_NEXT_TAIL; i > RCU_DONE_TAIL; i--) |
| 164 | 290 | if (rsclp->tails[i] != rsclp->tails[i - 1]) |
| 165 | 291 | break; |
| 166 | − *rsclp->tails[i] = rhp; | |
| 292 | + WRITE_ONCE(*rsclp->tails[i], rhp); | |
| 167 | 293 | for (; i <= RCU_NEXT_TAIL; i++) |
| 168 | − rsclp->tails[i] = &rhp->next; | |
| 294 | + WRITE_ONCE(rsclp->tails[i], &rhp->next); | |
| 169 | 295 | return true; |
| 170 | 296 | } |
| 171 | 297 | |
| @@ -182,9 +308,8 @@ void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp, · 10 unmodified lines | ||
| 182 | 308 | struct rcu_cblist *rclp) |
| 183 | 309 | { |
| 184 | 310 | rclp->len_lazy += rsclp->len_lazy; |
| 185 | − rclp->len += rsclp->len; | |
| 186 | 311 | rsclp->len_lazy = 0; |
| 187 | − WRITE_ONCE(rsclp->len, 0); /* ->len sampled locklessly. */ | |
| 312 | + rclp->len = rcu_segcblist_xchg_len(rsclp, 0); | |
| 188 | 313 | } |
| 189 | 314 | |
| 190 | 315 | /* |
| @@ -200,12 +325,12 @@ void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, · 9 unmodified lines | ||
| 200 | 325 | if (!rcu_segcblist_ready_cbs(rsclp)) |
| 201 | 326 | return; /* Nothing to do. */ |
| 202 | 327 | *rclp->tail = rsclp->head; |
| 203 | − rsclp->head = *rsclp->tails[RCU_DONE_TAIL]; | |
| 204 | − *rsclp->tails[RCU_DONE_TAIL] = NULL; | |
| 328 | + WRITE_ONCE(rsclp->head, *rsclp->tails[RCU_DONE_TAIL]); | |
| 329 | + WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL); | |
| 205 | 330 | rclp->tail = rsclp->tails[RCU_DONE_TAIL]; |
| 206 | 331 | for (i = RCU_CBLIST_NSEGS - 1; i >= RCU_DONE_TAIL; i--) |
| 207 | 332 | if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL]) |
| 208 | − rsclp->tails[i] = &rsclp->head; | |
| 333 | + WRITE_ONCE(rsclp->tails[i], &rsclp->head); | |
| 209 | 334 | } |
| 210 | 335 | |
| 211 | 336 | /* |
| @@ -224,9 +349,9 @@ void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, · 12 unmodified lines | ||
| 224 | 349 | return; /* Nothing to do. */ |
| 225 | 350 | *rclp->tail = *rsclp->tails[RCU_DONE_TAIL]; |
| 226 | 351 | rclp->tail = rsclp->tails[RCU_NEXT_TAIL]; |
| 227 | − *rsclp->tails[RCU_DONE_TAIL] = NULL; | |
| 352 | + WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL); | |
| 228 | 353 | for (i = RCU_DONE_TAIL + 1; i < RCU_CBLIST_NSEGS; i++) |
| 229 | − rsclp->tails[i] = rsclp->tails[RCU_DONE_TAIL]; | |
| 354 | + WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_DONE_TAIL]); | |
| 230 | 355 | } |
| 231 | 356 | |
| 232 | 357 | /* |
| @@ -237,8 +362,7 @@ void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp, · 4 unmodified lines | ||
| 237 | 362 | struct rcu_cblist *rclp) |
| 238 | 363 | { |
| 239 | 364 | rsclp->len_lazy += rclp->len_lazy; |
| 240 | − /* ->len sampled locklessly. */ | |
| 241 | − WRITE_ONCE(rsclp->len, rsclp->len + rclp->len); | |
| 365 | + rcu_segcblist_add_len(rsclp, rclp->len); | |
| 242 | 366 | rclp->len_lazy = 0; |
| 243 | 367 | rclp->len = 0; |
| 244 | 368 | } |
| @@ -255,10 +379,10 @@ void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, · 10 unmodified lines | ||
| 255 | 379 | if (!rclp->head) |
| 256 | 380 | return; /* No callbacks to move. */ |
| 257 | 381 | *rclp->tail = rsclp->head; |
| 258 | − rsclp->head = rclp->head; | |
| 382 | + WRITE_ONCE(rsclp->head, rclp->head); | |
| 259 | 383 | for (i = RCU_DONE_TAIL; i < RCU_CBLIST_NSEGS; i++) |
| 260 | 384 | if (&rsclp->head == rsclp->tails[i]) |
| 261 | − rsclp->tails[i] = rclp->tail; | |
| 385 | + WRITE_ONCE(rsclp->tails[i], rclp->tail); | |
| 262 | 386 | else |
| 263 | 387 | break; |
| 264 | 388 | rclp->head = NULL; |
| @@ -274,8 +398,8 @@ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, · 9 unmodified lines | ||
| 274 | 398 | { |
| 275 | 399 | if (!rclp->head) |
| 276 | 400 | return; /* Nothing to do. */ |
| 277 | − *rsclp->tails[RCU_NEXT_TAIL] = rclp->head; | |
| 278 | − rsclp->tails[RCU_NEXT_TAIL] = rclp->tail; | |
| 401 | + WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head); | |
| 402 | + WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail); | |
| 279 | 403 | rclp->head = NULL; |
| 280 | 404 | rclp->tail = &rclp->head; |
| 281 | 405 | } |
| @@ -299,7 +423,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) · 17 unmodified lines | ||
| 299 | 423 | for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { |
| 300 | 424 | if (ULONG_CMP_LT(seq, rsclp->gp_seq[i])) |
| 301 | 425 | break; |
| 302 | − rsclp->tails[RCU_DONE_TAIL] = rsclp->tails[i]; | |
| 426 | + WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]); | |
| 303 | 427 | } |
| 304 | 428 | |
| 305 | 429 | /* If no callbacks moved, nothing more need be done. */ |
| @@ -308,7 +432,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) · 2 unmodified lines | ||
| 308 | 432 | |
| 309 | 433 | /* Clean up tail pointers that might have been misordered above. */ |
| 310 | 434 | for (j = RCU_WAIT_TAIL; j < i; j++) |
| 311 | − rsclp->tails[j] = rsclp->tails[RCU_DONE_TAIL]; | |
| 435 | + WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]); | |
| 312 | 436 | |
| 313 | 437 | /* |
| 314 | 438 | * Callbacks moved, so clean up the misordered ->tails[] pointers |
| @@ -319,7 +443,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) · 4 unmodified lines | ||
| 319 | 443 | for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) { |
| 320 | 444 | if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL]) |
| 321 | 445 | break; /* No more callbacks. */ |
| 322 | − rsclp->tails[j] = rsclp->tails[i]; | |
| 446 | + WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]); | |
| 323 | 447 | rsclp->gp_seq[j] = rsclp->gp_seq[i]; |
| 324 | 448 | } |
| 325 | 449 | } |
| @@ -384,7 +508,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) · 58 unmodified lines | ||
| 384 | 508 | * structure other than in the RCU_NEXT_TAIL segment. |
| 385 | 509 | */ |
| 386 | 510 | for (; i < RCU_NEXT_TAIL; i++) { |
| 387 | − rsclp->tails[i] = rsclp->tails[RCU_NEXT_TAIL]; | |
| 511 | + WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]); | |
| 388 | 512 | rsclp->gp_seq[i] = seq; |
| 389 | 513 | } |
| 390 | 514 | return true; |
| @@ -9,6 +9,12 @@ · 8 unmodified lines | ||
| 9 | 9 | |
| 10 | 10 | #include <linux/rcu_segcblist.h> |
| 11 | 11 | |
| 12 | +/* Return number of callbacks in the specified callback list. */ | |
| 13 | +static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp) | |
| 14 | +{ | |
| 15 | + return READ_ONCE(rclp->len); | |
| 16 | +} | |
| 17 | + | |
| 12 | 18 | /* |
| 13 | 19 | * Account for the fact that a previously dequeued callback turned out |
| 14 | 20 | * to be marked as lazy. |
| @@ -19,6 +25,10 @@ static inline void rcu_cblist_dequeued_lazy(struct rcu_cblist *rclp) · 4 unmodified lines | ||
| 19 | 25 | } |
| 20 | 26 | |
| 21 | 27 | void rcu_cblist_init(struct rcu_cblist *rclp); |
| 28 | +void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp); | |
| 29 | +void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, | |
| 30 | + struct rcu_cblist *srclp, | |
| 31 | + struct rcu_head *rhp); | |
| 22 | 32 | struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp); |
| 23 | 33 | |
| 24 | 34 | /* |
| @@ -36,13 +46,17 @@ struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp); · 11 unmodified lines | ||
| 36 | 46 | */ |
| 37 | 47 | static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp) |
| 38 | 48 | { |
| 39 | − return !rsclp->head; | |
| 49 | + return !READ_ONCE(rsclp->head); | |
| 40 | 50 | } |
| 41 | 51 | |
| 42 | 52 | /* Return number of callbacks in segmented callback list. */ |
| 43 | 53 | static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp) |
| 44 | 54 | { |
| 55 | +#ifdef CONFIG_RCU_NOCB_CPU | |
| 56 | + return atomic_long_read(&rsclp->len); | |
| 57 | +#else | |
| 45 | 58 | return READ_ONCE(rsclp->len); |
| 59 | +#endif | |
| 46 | 60 | } |
| 47 | 61 | |
| 48 | 62 | /* Return number of lazy callbacks in segmented callback list. */ |
| @@ -54,16 +68,22 @@ static inline long rcu_segcblist_n_lazy_cbs(struct rcu_segcblist *rsclp) · 5 unmodified lines | ||
| 54 | 68 | /* Return number of lazy callbacks in segmented callback list. */ |
| 55 | 69 | static inline long rcu_segcblist_n_nonlazy_cbs(struct rcu_segcblist *rsclp) |
| 56 | 70 | { |
| 57 | − return rsclp->len - rsclp->len_lazy; | |
| 71 | + return rcu_segcblist_n_cbs(rsclp) - rsclp->len_lazy; | |
| 58 | 72 | } |
| 59 | 73 | |
| 60 | 74 | /* |
| 61 | 75 | * Is the specified rcu_segcblist enabled, for example, not corresponding |
| 62 | − * to an offline or callback-offloaded CPU? | |
| 76 | + * to an offline CPU? | |
| 63 | 77 | */ |
| 64 | 78 | static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) |
| 65 | 79 | { |
| 66 | − return !!rsclp->tails[RCU_NEXT_TAIL]; | |
| 80 | + return rsclp->enabled; | |
| 81 | +} | |
| 82 | + | |
| 83 | +/* Is the specified rcu_segcblist offloaded? */ | |
| 84 | +static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) | |
| 85 | +{ | |
| 86 | + return rsclp->offloaded; | |
| 67 | 87 | } |
| 68 | 88 | |
| 69 | 89 | /* |
| @@ -73,36 +93,18 @@ static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) · 3 unmodified lines | ||
| 73 | 93 | */ |
| 74 | 94 | static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg) |
| 75 | 95 | { |
| 76 | − return !*rsclp->tails[seg]; | |
| 77 | −} | |
| 78 | − | |
| 79 | −/* | |
| 80 | − * Interim function to return rcu_segcblist head pointer. Longer term, the | |
| 81 | − * rcu_segcblist will be used more pervasively, removing the need for this | |
| 82 | − * function. | |
| 83 | − */ | |
| 84 | −static inline struct rcu_head *rcu_segcblist_head(struct rcu_segcblist *rsclp) | |
| 85 | −{ | |
| 86 | − return rsclp->head; | |
| 87 | −} | |
| 88 | − | |
| 89 | −/* | |
| 90 | − * Interim function to return rcu_segcblist head pointer. Longer term, the | |
| 91 | − * rcu_segcblist will be used more pervasively, removing the need for this | |
| 92 | − * function. | |
| 93 | − */ | |
| 94 | −static inline struct rcu_head **rcu_segcblist_tail(struct rcu_segcblist *rsclp) | |
| 95 | −{ | |
| 96 | − WARN_ON_ONCE(rcu_segcblist_empty(rsclp)); | |
| 97 | − return rsclp->tails[RCU_NEXT_TAIL]; | |
| 96 | + return !READ_ONCE(*READ_ONCE(rsclp->tails[seg])); | |
| 98 | 97 | } |
| 99 | 98 | |
| 99 | +void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp); | |
| 100 | 100 | void rcu_segcblist_init(struct rcu_segcblist *rsclp); |
| 101 | 101 | void rcu_segcblist_disable(struct rcu_segcblist *rsclp); |
| 102 | +void rcu_segcblist_offload(struct rcu_segcblist *rsclp); | |
| 102 | 103 | bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp); |
| 103 | 104 | bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp); |
| 104 | 105 | struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp); |
| 105 | 106 | struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp); |
| 107 | +bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp); | |
| 106 | 108 | void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, |
| 107 | 109 | struct rcu_head *rhp, bool lazy); |
| 108 | 110 | bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, |
| @@ -89,7 +89,7 @@ torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable · 88 unmodified lines | ||
| 89 | 89 | |
| 90 | 90 | static char *perf_type = "rcu"; |
| 91 | 91 | module_param(perf_type, charp, 0444); |
| 92 | −MODULE_PARM_DESC(perf_type, "Type of RCU to performance-test (rcu, rcu_bh, ...)"); | |
| 92 | +MODULE_PARM_DESC(perf_type, "Type of RCU to performance-test (rcu, srcu, ...)"); | |
| 93 | 93 | |
| 94 | 94 | static int nrealreaders; |
| 95 | 95 | static int nrealwriters; |
| @@ -375,6 +375,14 @@ rcu_perf_writer(void *arg) · 279 unmodified lines | ||
| 375 | 375 | if (holdoff) |
| 376 | 376 | schedule_timeout_uninterruptible(holdoff * HZ); |
| 377 | 377 | |
| 378 | + /* | |
| 379 | + * Wait until rcu_end_inkernel_boot() is called for normal GP tests | |
| 380 | + * so that RCU is not always expedited for normal GP tests. | |
| 381 | + * The system_state test is approximate, but works well in practice. | |
| 382 | + */ | |
| 383 | + while (!gp_exp && system_state != SYSTEM_RUNNING) | |
| 384 | + schedule_timeout_uninterruptible(1); | |
| 385 | + | |
| 378 | 386 | t = ktime_get_mono_fast_ns(); |
| 379 | 387 | if (atomic_inc_return(&n_rcu_perf_writer_started) >= nrealwriters) { |
| 380 | 388 | t_rcu_perf_writer_started = t; |
| @@ -161,6 +161,7 @@ static atomic_long_t n_rcu_torture_timers; · 160 unmodified lines | ||
| 161 | 161 | static long n_barrier_attempts; |
| 162 | 162 | static long n_barrier_successes; /* did rcu_barrier test succeed? */ |
| 163 | 163 | static struct list_head rcu_torture_removed; |
| 164 | +static unsigned long shutdown_jiffies; | |
| 164 | 165 | |
| 165 | 166 | static int rcu_torture_writer_state; |
| 166 | 167 | #define RTWS_FIXED_DELAY 0 |
| @@ -228,6 +229,15 @@ static u64 notrace rcu_trace_clock_local(void) · 61 unmodified lines | ||
| 228 | 229 | } |
| 229 | 230 | #endif /* #else #ifdef CONFIG_RCU_TRACE */ |
| 230 | 231 | |
| 232 | +/* | |
| 233 | + * Stop aggressive CPU-hog tests a bit before the end of the test in order | |
| 234 | + * to avoid interfering with test shutdown. | |
| 235 | + */ | |
| 236 | +static bool shutdown_time_arrived(void) | |
| 237 | +{ | |
| 238 | + return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ); | |
| 239 | +} | |
| 240 | + | |
| 231 | 241 | static unsigned long boost_starttime; /* jiffies of next boost test start. */ |
| 232 | 242 | static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ |
| 233 | 243 | /* and boost task create/destroy. */ |
| @@ -1713,12 +1723,14 @@ static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp) · 1479 unmodified lines | ||
| 1713 | 1723 | } |
| 1714 | 1724 | |
| 1715 | 1725 | // Give the scheduler a chance, even on nohz_full CPUs. |
| 1716 | −static void rcu_torture_fwd_prog_cond_resched(void) | |
| 1726 | +static void rcu_torture_fwd_prog_cond_resched(unsigned long iter) | |
| 1717 | 1727 | { |
| 1718 | 1728 | if (IS_ENABLED(CONFIG_PREEMPT) && IS_ENABLED(CONFIG_NO_HZ_FULL)) { |
| 1719 | − if (need_resched()) | |
| 1729 | + // Real call_rcu() floods hit userspace, so emulate that. | |
| 1730 | + if (need_resched() || (iter & 0xfff)) | |
| 1720 | 1731 | schedule(); |
| 1721 | 1732 | } else { |
| 1733 | + // No userspace emulation: CB invocation throttles call_rcu() | |
| 1722 | 1734 | cond_resched(); |
| 1723 | 1735 | } |
| 1724 | 1736 | } |
| @@ -1746,7 +1758,7 @@ static unsigned long rcu_torture_fwd_prog_cbfree(void) · 21 unmodified lines | ||
| 1746 | 1758 | spin_unlock_irqrestore(&rcu_fwd_lock, flags); |
| 1747 | 1759 | kfree(rfcp); |
| 1748 | 1760 | freed++; |
| 1749 | − rcu_torture_fwd_prog_cond_resched(); | |
| 1761 | + rcu_torture_fwd_prog_cond_resched(freed); | |
| 1750 | 1762 | } |
| 1751 | 1763 | return freed; |
| 1752 | 1764 | } |
| @@ -1785,15 +1797,17 @@ static void rcu_torture_fwd_prog_nr(int *tested, int *tested_tries) · 32 unmodified lines | ||
| 1785 | 1797 | WRITE_ONCE(rcu_fwd_startat, jiffies); |
| 1786 | 1798 | stopat = rcu_fwd_startat + dur; |
| 1787 | 1799 | while (time_before(jiffies, stopat) && |
| 1800 | + !shutdown_time_arrived() && | |
| 1788 | 1801 | !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { |
| 1789 | 1802 | idx = cur_ops->readlock(); |
| 1790 | 1803 | udelay(10); |
| 1791 | 1804 | cur_ops->readunlock(idx); |
| 1792 | 1805 | if (!fwd_progress_need_resched || need_resched()) |
| 1793 | − rcu_torture_fwd_prog_cond_resched(); | |
| 1806 | + rcu_torture_fwd_prog_cond_resched(1); | |
| 1794 | 1807 | } |
| 1795 | 1808 | (*tested_tries)++; |
| 1796 | 1809 | if (!time_before(jiffies, stopat) && |
| 1810 | + !shutdown_time_arrived() && | |
| 1797 | 1811 | !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { |
| 1798 | 1812 | (*tested)++; |
| 1799 | 1813 | cver = READ_ONCE(rcu_torture_current_version) - cver; |
| @@ -1852,6 +1866,7 @@ static void rcu_torture_fwd_prog_cr(void) · 52 unmodified lines | ||
| 1852 | 1866 | gps = cur_ops->get_gp_seq(); |
| 1853 | 1867 | rcu_launder_gp_seq_start = gps; |
| 1854 | 1868 | while (time_before(jiffies, stopat) && |
| 1869 | + !shutdown_time_arrived() && | |
| 1855 | 1870 | !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { |
| 1856 | 1871 | rfcp = READ_ONCE(rcu_fwd_cb_head); |
| 1857 | 1872 | rfcpn = NULL; |
| @@ -1875,7 +1890,7 @@ static void rcu_torture_fwd_prog_cr(void) · 17 unmodified lines | ||
| 1875 | 1890 | rfcp->rfc_gps = 0; |
| 1876 | 1891 | } |
| 1877 | 1892 | cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr); |
| 1878 | − rcu_torture_fwd_prog_cond_resched(); | |
| 1893 | + rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs); | |
| 1879 | 1894 | } |
| 1880 | 1895 | stoppedat = jiffies; |
| 1881 | 1896 | n_launders_cb_snap = READ_ONCE(n_launders_cb); |
| @@ -1884,7 +1899,8 @@ static void rcu_torture_fwd_prog_cr(void) · 2 unmodified lines | ||
| 1884 | 1899 | cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */ |
| 1885 | 1900 | (void)rcu_torture_fwd_prog_cbfree(); |
| 1886 | 1901 | |
| 1887 | − if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop)) { | |
| 1902 | + if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) && | |
| 1903 | + !shutdown_time_arrived()) { | |
| 1888 | 1904 | WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED); |
| 1889 | 1905 | pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld\n", |
| 1890 | 1906 | __func__, |
| @@ -2160,6 +2176,7 @@ rcu_torture_cleanup(void) · 269 unmodified lines | ||
| 2160 | 2176 | return; |
| 2161 | 2177 | } |
| 2162 | 2178 | |
| 2179 | + show_rcu_gp_kthreads(); | |
| 2163 | 2180 | rcu_torture_barrier_cleanup(); |
| 2164 | 2181 | torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_task); |
| 2165 | 2182 | torture_stop_kthread(rcu_torture_stall, stall_task); |
| @@ -2465,6 +2482,7 @@ rcu_torture_init(void) · 299 unmodified lines | ||
| 2465 | 2482 | goto unwind; |
| 2466 | 2483 | rcutor_hp = firsterr; |
| 2467 | 2484 | } |
| 2485 | + shutdown_jiffies = jiffies + shutdown_secs * HZ; | |
| 2468 | 2486 | firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); |
| 2469 | 2487 | if (firsterr) |
| 2470 | 2488 | goto unwind; |
| @@ -1279,8 +1279,9 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf) · 1278 unmodified lines | ||
| 1279 | 1279 | |
| 1280 | 1280 | c0 = l0 - u0; |
| 1281 | 1281 | c1 = l1 - u1; |
| 1282 | − pr_cont(" %d(%ld,%ld %1p)", | |
| 1283 | − cpu, c0, c1, rcu_segcblist_head(&sdp->srcu_cblist)); | |
| 1282 | + pr_cont(" %d(%ld,%ld %c)", | |
| 1283 | + cpu, c0, c1, | |
| 1284 | + "C."[rcu_segcblist_empty(&sdp->srcu_cblist)]); | |
| 1284 | 1285 | s0 += c0; |
| 1285 | 1286 | s1 += c1; |
| 1286 | 1287 | } |
| @@ -56,6 +56,7 @@ · 55 unmodified lines | ||
| 56 | 56 | #include <linux/smpboot.h> |
| 57 | 57 | #include <linux/jiffies.h> |
| 58 | 58 | #include <linux/sched/isolation.h> |
| 59 | +#include <linux/sched/clock.h> | |
| 59 | 60 | #include "../time/tick-internal.h" |
| 60 | 61 | |
| 61 | 62 | #include "tree.h" |
| @@ -210,9 +211,9 @@ static long rcu_get_n_cbs_cpu(int cpu) · 148 unmodified lines | ||
| 210 | 211 | { |
| 211 | 212 | struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); |
| 212 | 213 | |
| 213 | − if (rcu_segcblist_is_enabled(&rdp->cblist)) /* Online normal CPU? */ | |
| 214 | + if (rcu_segcblist_is_enabled(&rdp->cblist)) | |
| 214 | 215 | return rcu_segcblist_n_cbs(&rdp->cblist); |
| 215 | − return rcu_get_n_cbs_nocb_cpu(rdp); /* Works for offline, too. */ | |
| 216 | + return 0; | |
| 216 | 217 | } |
| 217 | 218 | |
| 218 | 219 | void rcu_softirq_qs(void) |
| @@ -416,6 +417,12 @@ module_param(qlowmark, long, 0444); · 197 unmodified lines | ||
| 416 | 417 | static ulong jiffies_till_first_fqs = ULONG_MAX; |
| 417 | 418 | static ulong jiffies_till_next_fqs = ULONG_MAX; |
| 418 | 419 | static bool rcu_kick_kthreads; |
| 420 | +static int rcu_divisor = 7; | |
| 421 | +module_param(rcu_divisor, int, 0644); | |
| 422 | + | |
| 423 | +/* Force an exit from rcu_do_batch() after 3 milliseconds. */ | |
| 424 | +static long rcu_resched_ns = 3 * NSEC_PER_MSEC; | |
| 425 | +module_param(rcu_resched_ns, long, 0644); | |
| 419 | 426 | |
| 420 | 427 | /* |
| 421 | 428 | * How long the grace period must be before we start recruiting |
| @@ -1251,6 +1258,7 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) · 829 unmodified lines | ||
| 1251 | 1258 | unsigned long gp_seq_req; |
| 1252 | 1259 | bool ret = false; |
| 1253 | 1260 | |
| 1261 | + rcu_lockdep_assert_cblist_protected(rdp); | |
| 1254 | 1262 | raw_lockdep_assert_held_rcu_node(rnp); |
| 1255 | 1263 | |
| 1256 | 1264 | /* If no pending (not yet ready to invoke) callbacks, nothing to do. */ |
| @@ -1292,7 +1300,7 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, · 35 unmodified lines | ||
| 1292 | 1300 | unsigned long c; |
| 1293 | 1301 | bool needwake; |
| 1294 | 1302 | |
| 1295 | − lockdep_assert_irqs_disabled(); | |
| 1303 | + rcu_lockdep_assert_cblist_protected(rdp); | |
| 1296 | 1304 | c = rcu_seq_snap(&rcu_state.gp_seq); |
| 1297 | 1305 | if (!rdp->gpwrap && ULONG_CMP_GE(rdp->gp_seq_needed, c)) { |
| 1298 | 1306 | /* Old request still live, so mark recent callbacks. */ |
| @@ -1318,6 +1326,7 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, · 19 unmodified lines | ||
| 1318 | 1326 | */ |
| 1319 | 1327 | static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) |
| 1320 | 1328 | { |
| 1329 | + rcu_lockdep_assert_cblist_protected(rdp); | |
| 1321 | 1330 | raw_lockdep_assert_held_rcu_node(rnp); |
| 1322 | 1331 | |
| 1323 | 1332 | /* If no pending (not yet ready to invoke) callbacks, nothing to do. */ |
| @@ -1334,6 +1343,21 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) · 10 unmodified lines | ||
| 1334 | 1343 | return rcu_accelerate_cbs(rnp, rdp); |
| 1335 | 1344 | } |
| 1336 | 1345 | |
| 1346 | +/* | |
| 1347 | + * Move and classify callbacks, but only if doing so won't require | |
| 1348 | + * that the RCU grace-period kthread be awakened. | |
| 1349 | + */ | |
| 1350 | +static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp, | |
| 1351 | + struct rcu_data *rdp) | |
| 1352 | +{ | |
| 1353 | + rcu_lockdep_assert_cblist_protected(rdp); | |
| 1354 | + if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) || | |
| 1355 | + !raw_spin_trylock_rcu_node(rnp)) | |
| 1356 | + return; | |
| 1357 | + WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp)); | |
| 1358 | + raw_spin_unlock_rcu_node(rnp); | |
| 1359 | +} | |
| 1360 | + | |
| 1337 | 1361 | /* |
| 1338 | 1362 | * Update CPU-local rcu_data state to record the beginnings and ends of |
| 1339 | 1363 | * grace periods. The caller must hold the ->lock of the leaf rcu_node |
| @@ -1342,8 +1366,10 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) · 2 unmodified lines | ||
| 1342 | 1366 | */ |
| 1343 | 1367 | static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp) |
| 1344 | 1368 | { |
| 1345 | − bool ret; | |
| 1369 | + bool ret = false; | |
| 1346 | 1370 | bool need_gp; |
| 1371 | + const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) && | |
| 1372 | + rcu_segcblist_is_offloaded(&rdp->cblist); | |
| 1347 | 1373 | |
| 1348 | 1374 | raw_lockdep_assert_held_rcu_node(rnp); |
| 1349 | 1375 | |
| @@ -1353,10 +1379,12 @@ static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp) · 3 unmodified lines | ||
| 1353 | 1379 | /* Handle the ends of any preceding grace periods first. */ |
| 1354 | 1380 | if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) || |
| 1355 | 1381 | unlikely(READ_ONCE(rdp->gpwrap))) { |
| 1356 | − ret = rcu_advance_cbs(rnp, rdp); /* Advance callbacks. */ | |
| 1382 | + if (!offloaded) | |
| 1383 | + ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */ | |
| 1357 | 1384 | trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend")); |
| 1358 | 1385 | } else { |
| 1359 | − ret = rcu_accelerate_cbs(rnp, rdp); /* Recent callbacks. */ | |
| 1386 | + if (!offloaded) | |
| 1387 | + ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */ | |
| 1360 | 1388 | } |
| 1361 | 1389 | |
| 1362 | 1390 | /* Now handle the beginnings of any new-to-this-CPU grace periods. */ |
| @@ -1657,6 +1685,7 @@ static void rcu_gp_cleanup(void) · 294 unmodified lines | ||
| 1657 | 1685 | unsigned long gp_duration; |
| 1658 | 1686 | bool needgp = false; |
| 1659 | 1687 | unsigned long new_gp_seq; |
| 1688 | + bool offloaded; | |
| 1660 | 1689 | struct rcu_data *rdp; |
| 1661 | 1690 | struct rcu_node *rnp = rcu_get_root(); |
| 1662 | 1691 | struct swait_queue_head *sq; |
| @@ -1722,7 +1751,9 @@ static void rcu_gp_cleanup(void) · 59 unmodified lines | ||
| 1722 | 1751 | needgp = true; |
| 1723 | 1752 | } |
| 1724 | 1753 | /* Advance CBs to reduce false positives below. */ |
| 1725 | − if (!rcu_accelerate_cbs(rnp, rdp) && needgp) { | |
| 1754 | + offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) && | |
| 1755 | + rcu_segcblist_is_offloaded(&rdp->cblist); | |
| 1756 | + if ((offloaded || !rcu_accelerate_cbs(rnp, rdp)) && needgp) { | |
| 1726 | 1757 | WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT); |
| 1727 | 1758 | rcu_state.gp_req_activity = jiffies; |
| 1728 | 1759 | trace_rcu_grace_period(rcu_state.name, |
| @@ -1916,7 +1947,9 @@ rcu_report_qs_rdp(int cpu, struct rcu_data *rdp) · 187 unmodified lines | ||
| 1916 | 1947 | { |
| 1917 | 1948 | unsigned long flags; |
| 1918 | 1949 | unsigned long mask; |
| 1919 | − bool needwake; | |
| 1950 | + bool needwake = false; | |
| 1951 | + const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) && | |
| 1952 | + rcu_segcblist_is_offloaded(&rdp->cblist); | |
| 1920 | 1953 | struct rcu_node *rnp; |
| 1921 | 1954 | |
| 1922 | 1955 | rnp = rdp->mynode; |
| @@ -1943,7 +1976,8 @@ rcu_report_qs_rdp(int cpu, struct rcu_data *rdp) · 20 unmodified lines | ||
| 1943 | 1976 | * This GP can't end until cpu checks in, so all of our |
| 1944 | 1977 | * callbacks can be processed during the next GP. |
| 1945 | 1978 | */ |
| 1946 | − needwake = rcu_accelerate_cbs(rnp, rdp); | |
| 1979 | + if (!offloaded) | |
| 1980 | + needwake = rcu_accelerate_cbs(rnp, rdp); | |
| 1947 | 1981 | |
| 1948 | 1982 | rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags); |
| 1949 | 1983 | /* ^^^ Released rnp->lock */ |
| @@ -2077,9 +2111,12 @@ int rcutree_dead_cpu(unsigned int cpu) · 127 unmodified lines | ||
| 2077 | 2111 | static void rcu_do_batch(struct rcu_data *rdp) |
| 2078 | 2112 | { |
| 2079 | 2113 | unsigned long flags; |
| 2114 | + const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) && | |
| 2115 | + rcu_segcblist_is_offloaded(&rdp->cblist); | |
| 2080 | 2116 | struct rcu_head *rhp; |
| 2081 | 2117 | struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl); |
| 2082 | 2118 | long bl, count; |
| 2119 | + long pending, tlimit = 0; | |
| 2083 | 2120 | |
| 2084 | 2121 | /* If no callbacks are ready, just return. */ |
| 2085 | 2122 | if (!rcu_segcblist_ready_cbs(&rdp->cblist)) { |
| @@ -2099,13 +2136,19 @@ static void rcu_do_batch(struct rcu_data *rdp) · 13 unmodified lines | ||
| 2099 | 2136 | * callback counts, as rcu_barrier() needs to be conservative. |
| 2100 | 2137 | */ |
| 2101 | 2138 | local_irq_save(flags); |
| 2139 | + rcu_nocb_lock(rdp); | |
| 2102 | 2140 | WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); |
| 2103 | − bl = rdp->blimit; | |
| 2141 | + pending = rcu_segcblist_n_cbs(&rdp->cblist); | |
| 2142 | + bl = max(rdp->blimit, pending >> rcu_divisor); | |
| 2143 | + if (unlikely(bl > 100)) | |
| 2144 | + tlimit = local_clock() + rcu_resched_ns; | |
| 2104 | 2145 | trace_rcu_batch_start(rcu_state.name, |
| 2105 | 2146 | rcu_segcblist_n_lazy_cbs(&rdp->cblist), |
| 2106 | 2147 | rcu_segcblist_n_cbs(&rdp->cblist), bl); |
| 2107 | 2148 | rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl); |
| 2108 | − local_irq_restore(flags); | |
| 2149 | + if (offloaded) | |
| 2150 | + rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); | |
| 2151 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 2109 | 2152 | |
| 2110 | 2153 | /* Invoke callbacks. */ |
| 2111 | 2154 | rhp = rcu_cblist_dequeue(&rcl); |
| @@ -2117,13 +2160,29 @@ static void rcu_do_batch(struct rcu_data *rdp) · 5 unmodified lines | ||
| 2117 | 2160 | * Stop only if limit reached and CPU has something to do. |
| 2118 | 2161 | * Note: The rcl structure counts down from zero. |
| 2119 | 2162 | */ |
| 2120 | − if (-rcl.len >= bl && | |
| 2163 | + if (-rcl.len >= bl && !offloaded && | |
| 2121 | 2164 | (need_resched() || |
| 2122 | 2165 | (!is_idle_task(current) && !rcu_is_callbacks_kthread()))) |
| 2123 | 2166 | break; |
| 2167 | + if (unlikely(tlimit)) { | |
| 2168 | + /* only call local_clock() every 32 callbacks */ | |
| 2169 | + if (likely((-rcl.len & 31) || local_clock() < tlimit)) | |
| 2170 | + continue; | |
| 2171 | + /* Exceeded the time limit, so leave. */ | |
| 2172 | + break; | |
| 2173 | + } | |
| 2174 | + if (offloaded) { | |
| 2175 | + WARN_ON_ONCE(in_serving_softirq()); | |
| 2176 | + local_bh_enable(); | |
| 2177 | + lockdep_assert_irqs_enabled(); | |
| 2178 | + cond_resched_tasks_rcu_qs(); | |
| 2179 | + lockdep_assert_irqs_enabled(); | |
| 2180 | + local_bh_disable(); | |
| 2181 | + } | |
| 2124 | 2182 | } |
| 2125 | 2183 | |
| 2126 | 2184 | local_irq_save(flags); |
| 2185 | + rcu_nocb_lock(rdp); | |
| 2127 | 2186 | count = -rcl.len; |
| 2128 | 2187 | trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(), |
| 2129 | 2188 | is_idle_task(current), rcu_is_callbacks_kthread()); |
| @@ -2149,12 +2208,14 @@ static void rcu_do_batch(struct rcu_data *rdp) · 19 unmodified lines | ||
| 2149 | 2208 | * The following usually indicates a double call_rcu(). To track |
| 2150 | 2209 | * this down, try building with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. |
| 2151 | 2210 | */ |
| 2152 | − WARN_ON_ONCE(rcu_segcblist_empty(&rdp->cblist) != (count == 0)); | |
| 2211 | + WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist)); | |
| 2212 | + WARN_ON_ONCE(!IS_ENABLED(CONFIG_RCU_NOCB_CPU) && | |
| 2213 | + count != 0 && rcu_segcblist_empty(&rdp->cblist)); | |
| 2153 | 2214 | |
| 2154 | − local_irq_restore(flags); | |
| 2215 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 2155 | 2216 | |
| 2156 | 2217 | /* Re-invoke RCU core processing if there are callbacks remaining. */ |
| 2157 | − if (rcu_segcblist_ready_cbs(&rdp->cblist)) | |
| 2218 | + if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist)) | |
| 2158 | 2219 | invoke_rcu_core(); |
| 2159 | 2220 | } |
| 2160 | 2221 | |
| @@ -2280,6 +2341,8 @@ static __latent_entropy void rcu_core(void) · 119 unmodified lines | ||
| 2280 | 2341 | unsigned long flags; |
| 2281 | 2342 | struct rcu_data *rdp = raw_cpu_ptr(&rcu_data); |
| 2282 | 2343 | struct rcu_node *rnp = rdp->mynode; |
| 2344 | + const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) && | |
| 2345 | + rcu_segcblist_is_offloaded(&rdp->cblist); | |
| 2283 | 2346 | |
| 2284 | 2347 | if (cpu_is_offline(smp_processor_id())) |
| 2285 | 2348 | return; |
| @@ -2299,7 +2362,7 @@ static __latent_entropy void rcu_core(void) · 13 unmodified lines | ||
| 2299 | 2362 | |
| 2300 | 2363 | /* No grace period and unregistered callbacks? */ |
| 2301 | 2364 | if (!rcu_gp_in_progress() && |
| 2302 | − rcu_segcblist_is_enabled(&rdp->cblist)) { | |
| 2365 | + rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) { | |
| 2303 | 2366 | local_irq_save(flags); |
| 2304 | 2367 | if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) |
| 2305 | 2368 | rcu_accelerate_cbs_unlocked(rnp, rdp); |
| @@ -2309,7 +2372,7 @@ static __latent_entropy void rcu_core(void) · 3 unmodified lines | ||
| 2309 | 2372 | rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check()); |
| 2310 | 2373 | |
| 2311 | 2374 | /* If there are callbacks ready, invoke them. */ |
| 2312 | − if (rcu_segcblist_ready_cbs(&rdp->cblist) && | |
| 2375 | + if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) && | |
| 2313 | 2376 | likely(READ_ONCE(rcu_scheduler_fully_active))) |
| 2314 | 2377 | rcu_do_batch(rdp); |
| 2315 | 2378 | |
| @@ -2489,10 +2552,11 @@ static void rcu_leak_callback(struct rcu_head *rhp) · 173 unmodified lines | ||
| 2489 | 2552 | * is expected to specify a CPU. |
| 2490 | 2553 | */ |
| 2491 | 2554 | static void |
| 2492 | −__call_rcu(struct rcu_head *head, rcu_callback_t func, int cpu, bool lazy) | |
| 2555 | +__call_rcu(struct rcu_head *head, rcu_callback_t func, bool lazy) | |
| 2493 | 2556 | { |
| 2494 | 2557 | unsigned long flags; |
| 2495 | 2558 | struct rcu_data *rdp; |
| 2559 | + bool was_alldone; | |
| 2496 | 2560 | |
| 2497 | 2561 | /* Misaligned rcu_head! */ |
| 2498 | 2562 | WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1)); |
| @@ -2514,28 +2578,18 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func, int cpu, bool lazy) · 15 unmodified lines | ||
| 2514 | 2578 | rdp = this_cpu_ptr(&rcu_data); |
| 2515 | 2579 | |
| 2516 | 2580 | /* Add the callback to our list. */ |
| 2517 | − if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist)) || cpu != -1) { | |
| 2518 | − int offline; | |
| 2519 | − | |
| 2520 | − if (cpu != -1) | |
| 2521 | − rdp = per_cpu_ptr(&rcu_data, cpu); | |
| 2522 | − if (likely(rdp->mynode)) { | |
| 2523 | − /* Post-boot, so this should be for a no-CBs CPU. */ | |
| 2524 | − offline = !__call_rcu_nocb(rdp, head, lazy, flags); | |
| 2525 | − WARN_ON_ONCE(offline); | |
| 2526 | − /* Offline CPU, _call_rcu() illegal, leak callback. */ | |
| 2527 | − local_irq_restore(flags); | |
| 2528 | − return; | |
| 2529 | − } | |
| 2530 | − /* | |
| 2531 | − * Very early boot, before rcu_init(). Initialize if needed | |
| 2532 | − * and then drop through to queue the callback. | |
| 2533 | − */ | |
| 2534 | − WARN_ON_ONCE(cpu != -1); | |
| 2581 | + if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) { | |
| 2582 | + // This can trigger due to call_rcu() from offline CPU: | |
| 2583 | + WARN_ON_ONCE(rcu_scheduler_active != RCU_SCHEDULER_INACTIVE); | |
| 2535 | 2584 | WARN_ON_ONCE(!rcu_is_watching()); |
| 2585 | + // Very early boot, before rcu_init(). Initialize if needed | |
| 2586 | + // and then drop through to queue the callback. | |
| 2536 | 2587 | if (rcu_segcblist_empty(&rdp->cblist)) |
| 2537 | 2588 | rcu_segcblist_init(&rdp->cblist); |
| 2538 | 2589 | } |
| 2590 | + if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags)) | |
| 2591 | + return; // Enqueued onto ->nocb_bypass, so just leave. | |
| 2592 | + /* If we get here, rcu_nocb_try_bypass() acquired ->nocb_lock. */ | |
| 2539 | 2593 | rcu_segcblist_enqueue(&rdp->cblist, head, lazy); |
| 2540 | 2594 | if (__is_kfree_rcu_offset((unsigned long)func)) |
| 2541 | 2595 | trace_rcu_kfree_callback(rcu_state.name, head, |
| @@ -2548,8 +2602,13 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func, int cpu, bool lazy) · 6 unmodified lines | ||
| 2548 | 2602 | rcu_segcblist_n_cbs(&rdp->cblist)); |
| 2549 | 2603 | |
| 2550 | 2604 | /* Go handle any RCU core processing required. */ |
| 2551 | − __call_rcu_core(rdp, head, flags); | |
| 2552 | − local_irq_restore(flags); | |
| 2605 | + if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) && | |
| 2606 | + unlikely(rcu_segcblist_is_offloaded(&rdp->cblist))) { | |
| 2607 | + __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */ | |
| 2608 | + } else { | |
| 2609 | + __call_rcu_core(rdp, head, flags); | |
| 2610 | + local_irq_restore(flags); | |
| 2611 | + } | |
| 2553 | 2612 | } |
| 2554 | 2613 | |
| 2555 | 2614 | /** |
| @@ -2589,7 +2648,7 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func, int cpu, bool lazy) · 33 unmodified lines | ||
| 2589 | 2648 | */ |
| 2590 | 2649 | void call_rcu(struct rcu_head *head, rcu_callback_t func) |
| 2591 | 2650 | { |
| 2592 | − __call_rcu(head, func, -1, 0); | |
| 2651 | + __call_rcu(head, func, 0); | |
| 2593 | 2652 | } |
| 2594 | 2653 | EXPORT_SYMBOL_GPL(call_rcu); |
| 2595 | 2654 | |
| @@ -2602,7 +2661,7 @@ EXPORT_SYMBOL_GPL(call_rcu); · 6 unmodified lines | ||
| 2602 | 2661 | */ |
| 2603 | 2662 | void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func) |
| 2604 | 2663 | { |
| 2605 | − __call_rcu(head, func, -1, 1); | |
| 2664 | + __call_rcu(head, func, 1); | |
| 2606 | 2665 | } |
| 2607 | 2666 | EXPORT_SYMBOL_GPL(kfree_call_rcu); |
| 2608 | 2667 | |
| @@ -2735,6 +2794,10 @@ static int rcu_pending(void) · 126 unmodified lines | ||
| 2735 | 2794 | /* Check for CPU stalls, if enabled. */ |
| 2736 | 2795 | check_cpu_stall(rdp); |
| 2737 | 2796 | |
| 2797 | + /* Does this CPU need a deferred NOCB wakeup? */ | |
| 2798 | + if (rcu_nocb_need_deferred_wakeup(rdp)) | |
| 2799 | + return 1; | |
| 2800 | + | |
| 2738 | 2801 | /* Is this CPU a NO_HZ_FULL CPU that should ignore RCU? */ |
| 2739 | 2802 | if (rcu_nohz_full_cpu()) |
| 2740 | 2803 | return 0; |
| @@ -2750,6 +2813,8 @@ static int rcu_pending(void) · 9 unmodified lines | ||
| 2750 | 2813 | /* Has RCU gone idle with this CPU needing another grace period? */ |
| 2751 | 2814 | if (!rcu_gp_in_progress() && |
| 2752 | 2815 | rcu_segcblist_is_enabled(&rdp->cblist) && |
| 2816 | + (!IS_ENABLED(CONFIG_RCU_NOCB_CPU) || | |
| 2817 | + !rcu_segcblist_is_offloaded(&rdp->cblist)) && | |
| 2753 | 2818 | !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) |
| 2754 | 2819 | return 1; |
| 2755 | 2820 | |
| @@ -2758,10 +2823,6 @@ static int rcu_pending(void) · 2 unmodified lines | ||
| 2758 | 2823 | unlikely(READ_ONCE(rdp->gpwrap))) /* outside lock */ |
| 2759 | 2824 | return 1; |
| 2760 | 2825 | |
| 2761 | − /* Does this CPU need a deferred NOCB wakeup? */ | |
| 2762 | − if (rcu_nocb_need_deferred_wakeup(rdp)) | |
| 2763 | − return 1; | |
| 2764 | − | |
| 2765 | 2826 | /* nothing to do */ |
| 2766 | 2827 | return 0; |
| 2767 | 2828 | } |
| @@ -2801,6 +2862,8 @@ static void rcu_barrier_func(void *unused) · 33 unmodified lines | ||
| 2801 | 2862 | rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence); |
| 2802 | 2863 | rdp->barrier_head.func = rcu_barrier_callback; |
| 2803 | 2864 | debug_rcu_head_queue(&rdp->barrier_head); |
| 2865 | + rcu_nocb_lock(rdp); | |
| 2866 | + WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies)); | |
| 2804 | 2867 | if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head, 0)) { |
| 2805 | 2868 | atomic_inc(&rcu_state.barrier_cpu_count); |
| 2806 | 2869 | } else { |
| @@ -2808,6 +2871,7 @@ static void rcu_barrier_func(void *unused) · 1 unmodified line | ||
| 2808 | 2871 | rcu_barrier_trace(TPS("IRQNQ"), -1, |
| 2809 | 2872 | rcu_state.barrier_sequence); |
| 2810 | 2873 | } |
| 2874 | + rcu_nocb_unlock(rdp); | |
| 2811 | 2875 | } |
| 2812 | 2876 | |
| 2813 | 2877 | /** |
| @@ -2858,22 +2922,11 @@ void rcu_barrier(void) · 44 unmodified lines | ||
| 2858 | 2922 | * corresponding CPU's preceding callbacks have been invoked. |
| 2859 | 2923 | */ |
| 2860 | 2924 | for_each_possible_cpu(cpu) { |
| 2861 | − if (!cpu_online(cpu) && !rcu_is_nocb_cpu(cpu)) | |
| 2862 | − continue; | |
| 2863 | 2925 | rdp = per_cpu_ptr(&rcu_data, cpu); |
| 2864 | − if (rcu_is_nocb_cpu(cpu)) { | |
| 2865 | − if (!rcu_nocb_cpu_needs_barrier(cpu)) { | |
| 2866 | − rcu_barrier_trace(TPS("OfflineNoCB"), cpu, | |
| 2867 | − rcu_state.barrier_sequence); | |
| 2868 | − } else { | |
| 2869 | − rcu_barrier_trace(TPS("OnlineNoCB"), cpu, | |
| 2870 | − rcu_state.barrier_sequence); | |
| 2871 | − smp_mb__before_atomic(); | |
| 2872 | − atomic_inc(&rcu_state.barrier_cpu_count); | |
| 2873 | − __call_rcu(&rdp->barrier_head, | |
| 2874 | − rcu_barrier_callback, cpu, 0); | |
| 2875 | − } | |
| 2876 | − } else if (rcu_segcblist_n_cbs(&rdp->cblist)) { | |
| 2926 | + if (!cpu_online(cpu) && | |
| 2927 | + !rcu_segcblist_is_offloaded(&rdp->cblist)) | |
| 2928 | + continue; | |
| 2929 | + if (rcu_segcblist_n_cbs(&rdp->cblist)) { | |
| 2877 | 2930 | rcu_barrier_trace(TPS("OnlineQ"), cpu, |
| 2878 | 2931 | rcu_state.barrier_sequence); |
| 2879 | 2932 | smp_call_function_single(cpu, rcu_barrier_func, NULL, 1); |
| @@ -2958,7 +3011,8 @@ rcu_boot_init_percpu_data(int cpu) · 78 unmodified lines | ||
| 2958 | 3011 | * Initializes a CPU's per-CPU RCU data. Note that only one online or |
| 2959 | 3012 | * offline event can be happening at a given time. Note also that we can |
| 2960 | 3013 | * accept some slop in the rsp->gp_seq access due to the fact that this |
| 2961 | − * CPU cannot possibly have any RCU callbacks in flight yet. | |
| 3014 | + * CPU cannot possibly have any non-offloaded RCU callbacks in flight yet. | |
| 3015 | + * And any offloaded callbacks are being numbered elsewhere. | |
| 2962 | 3016 | */ |
| 2963 | 3017 | int rcutree_prepare_cpu(unsigned int cpu) |
| 2964 | 3018 | { |
| @@ -2972,7 +3026,7 @@ int rcutree_prepare_cpu(unsigned int cpu) · 7 unmodified lines | ||
| 2972 | 3026 | rdp->n_force_qs_snap = rcu_state.n_force_qs; |
| 2973 | 3027 | rdp->blimit = blimit; |
| 2974 | 3028 | if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */ |
| 2975 | − !init_nocb_callback_list(rdp)) | |
| 3029 | + !rcu_segcblist_is_offloaded(&rdp->cblist)) | |
| 2976 | 3030 | rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */ |
| 2977 | 3031 | rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */ |
| 2978 | 3032 | rcu_dynticks_eqs_online(); |
| @@ -3151,29 +3205,38 @@ void rcutree_migrate_callbacks(int cpu) · 172 unmodified lines | ||
| 3151 | 3205 | { |
| 3152 | 3206 | unsigned long flags; |
| 3153 | 3207 | struct rcu_data *my_rdp; |
| 3208 | + struct rcu_node *my_rnp; | |
| 3154 | 3209 | struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); |
| 3155 | − struct rcu_node *rnp_root = rcu_get_root(); | |
| 3156 | 3210 | bool needwake; |
| 3157 | 3211 | |
| 3158 | − if (rcu_is_nocb_cpu(cpu) || rcu_segcblist_empty(&rdp->cblist)) | |
| 3212 | + if (rcu_segcblist_is_offloaded(&rdp->cblist) || | |
| 3213 | + rcu_segcblist_empty(&rdp->cblist)) | |
| 3159 | 3214 | return; /* No callbacks to migrate. */ |
| 3160 | 3215 | |
| 3161 | 3216 | local_irq_save(flags); |
| 3162 | 3217 | my_rdp = this_cpu_ptr(&rcu_data); |
| 3163 | − if (rcu_nocb_adopt_orphan_cbs(my_rdp, rdp, flags)) { | |
| 3164 | − local_irq_restore(flags); | |
| 3165 | − return; | |
| 3166 | − } | |
| 3167 | − raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */ | |
| 3218 | + my_rnp = my_rdp->mynode; | |
| 3219 | + rcu_nocb_lock(my_rdp); /* irqs already disabled. */ | |
| 3220 | + WARN_ON_ONCE(!rcu_nocb_flush_bypass(my_rdp, NULL, jiffies)); | |
| 3221 | + raw_spin_lock_rcu_node(my_rnp); /* irqs already disabled. */ | |
| 3168 | 3222 | /* Leverage recent GPs and set GP for new callbacks. */ |
| 3169 | − needwake = rcu_advance_cbs(rnp_root, rdp) || | |
| 3170 | − rcu_advance_cbs(rnp_root, my_rdp); | |
| 3223 | + needwake = rcu_advance_cbs(my_rnp, rdp) || | |
| 3224 | + rcu_advance_cbs(my_rnp, my_rdp); | |
| 3171 | 3225 | rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist); |
| 3226 | + needwake = needwake || rcu_advance_cbs(my_rnp, my_rdp); | |
| 3227 | + rcu_segcblist_disable(&rdp->cblist); | |
| 3172 | 3228 | WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) != |
| 3173 | 3229 | !rcu_segcblist_n_cbs(&my_rdp->cblist)); |
| 3174 | − raw_spin_unlock_irqrestore_rcu_node(rnp_root, flags); | |
| 3230 | + if (rcu_segcblist_is_offloaded(&my_rdp->cblist)) { | |
| 3231 | + raw_spin_unlock_rcu_node(my_rnp); /* irqs remain disabled. */ | |
| 3232 | + __call_rcu_nocb_wake(my_rdp, true, flags); | |
| 3233 | + } else { | |
| 3234 | + rcu_nocb_unlock(my_rdp); /* irqs remain disabled. */ | |
| 3235 | + raw_spin_unlock_irqrestore_rcu_node(my_rnp, flags); | |
| 3236 | + } | |
| 3175 | 3237 | if (needwake) |
| 3176 | 3238 | rcu_gp_kthread_wake(); |
| 3239 | + lockdep_assert_irqs_enabled(); | |
| 3177 | 3240 | WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 || |
| 3178 | 3241 | !rcu_segcblist_empty(&rdp->cblist), |
| 3179 | 3242 | "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n", |
| @@ -194,29 +194,38 @@ struct rcu_data { · 193 unmodified lines | ||
| 194 | 194 | |
| 195 | 195 | /* 5) Callback offloading. */ |
| 196 | 196 | #ifdef CONFIG_RCU_NOCB_CPU |
| 197 | − struct rcu_head *nocb_head; /* CBs waiting for kthread. */ | |
| 198 | − struct rcu_head **nocb_tail; | |
| 199 | − atomic_long_t nocb_q_count; /* # CBs waiting for nocb */ | |
| 200 | − atomic_long_t nocb_q_count_lazy; /* invocation (all stages). */ | |
| 201 | − struct rcu_head *nocb_follower_head; /* CBs ready to invoke. */ | |
| 202 | − struct rcu_head **nocb_follower_tail; | |
| 203 | − struct swait_queue_head nocb_wq; /* For nocb kthreads to sleep on. */ | |
| 204 | − struct task_struct *nocb_kthread; | |
| 197 | + struct swait_queue_head nocb_cb_wq; /* For nocb kthreads to sleep on. */ | |
| 198 | + struct task_struct *nocb_gp_kthread; | |
| 205 | 199 | raw_spinlock_t nocb_lock; /* Guard following pair of fields. */ |
| 200 | + atomic_t nocb_lock_contended; /* Contention experienced. */ | |
| 206 | 201 | int nocb_defer_wakeup; /* Defer wakeup of nocb_kthread. */ |
| 207 | 202 | struct timer_list nocb_timer; /* Enforce finite deferral. */ |
| 208 | − | |
| 209 | − /* The following fields are used by the leader, hence own cacheline. */ | |
| 210 | − struct rcu_head *nocb_gp_head ____cacheline_internodealigned_in_smp; | |
| 211 | − /* CBs waiting for GP. */ | |
| 212 | − struct rcu_head **nocb_gp_tail; | |
| 213 | − bool nocb_leader_sleep; /* Is the nocb leader thread asleep? */ | |
| 214 | − struct rcu_data *nocb_next_follower; | |
| 215 | − /* Next follower in wakeup chain. */ | |
| 216 | − | |
| 217 | − /* The following fields are used by the follower, hence new cachline. */ | |
| 218 | − struct rcu_data *nocb_leader ____cacheline_internodealigned_in_smp; | |
| 219 | − /* Leader CPU takes GP-end wakeups. */ | |
| 203 | + unsigned long nocb_gp_adv_time; /* Last call_rcu() CB adv (jiffies). */ | |
| 204 | + | |
| 205 | + /* The following fields are used by call_rcu, hence own cacheline. */ | |
| 206 | + raw_spinlock_t nocb_bypass_lock ____cacheline_internodealigned_in_smp; | |
| 207 | + struct rcu_cblist nocb_bypass; /* Lock-contention-bypass CB list. */ | |
| 208 | + unsigned long nocb_bypass_first; /* Time (jiffies) of first enqueue. */ | |
| 209 | + unsigned long nocb_nobypass_last; /* Last ->cblist enqueue (jiffies). */ | |
| 210 | + int nocb_nobypass_count; /* # ->cblist enqueues at ^^^ time. */ | |
| 211 | + | |
| 212 | + /* The following fields are used by GP kthread, hence own cacheline. */ | |
| 213 | + raw_spinlock_t nocb_gp_lock ____cacheline_internodealigned_in_smp; | |
| 214 | + struct timer_list nocb_bypass_timer; /* Force nocb_bypass flush. */ | |
| 215 | + u8 nocb_gp_sleep; /* Is the nocb GP thread asleep? */ | |
| 216 | + u8 nocb_gp_bypass; /* Found a bypass on last scan? */ | |
| 217 | + u8 nocb_gp_gp; /* GP to wait for on last scan? */ | |
| 218 | + unsigned long nocb_gp_seq; /* If so, ->gp_seq to wait for. */ | |
| 219 | + unsigned long nocb_gp_loops; /* # passes through wait code. */ | |
| 220 | + struct swait_queue_head nocb_gp_wq; /* For nocb kthreads to sleep on. */ | |
| 221 | + bool nocb_cb_sleep; /* Is the nocb CB thread asleep? */ | |
| 222 | + struct task_struct *nocb_cb_kthread; | |
| 223 | + struct rcu_data *nocb_next_cb_rdp; | |
| 224 | + /* Next rcu_data in wakeup chain. */ | |
| 225 | + | |
| 226 | + /* The following fields are used by CB kthread, hence new cacheline. */ | |
| 227 | + struct rcu_data *nocb_gp_rdp ____cacheline_internodealigned_in_smp; | |
| 228 | + /* GP rdp takes GP-end wakeups. */ | |
| 220 | 229 | #endif /* #ifdef CONFIG_RCU_NOCB_CPU */ |
| 221 | 230 | |
| 222 | 231 | /* 6) RCU priority boosting. */ |
| @@ -419,25 +428,39 @@ static bool rcu_preempt_has_tasks(struct rcu_node *rnp); · 196 unmodified lines | ||
| 419 | 428 | static bool rcu_preempt_need_deferred_qs(struct task_struct *t); |
| 420 | 429 | static void rcu_preempt_deferred_qs(struct task_struct *t); |
| 421 | 430 | static void zero_cpu_stall_ticks(struct rcu_data *rdp); |
| 422 | −static bool rcu_nocb_cpu_needs_barrier(int cpu); | |
| 423 | 431 | static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp); |
| 424 | 432 | static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq); |
| 425 | 433 | static void rcu_init_one_nocb(struct rcu_node *rnp); |
| 426 | −static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp, | |
| 427 | − bool lazy, unsigned long flags); | |
| 428 | −static bool rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp, | |
| 429 | − struct rcu_data *rdp, | |
| 430 | − unsigned long flags); | |
| 434 | +static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, | |
| 435 | + unsigned long j); | |
| 436 | +static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, | |
| 437 | + bool *was_alldone, unsigned long flags); | |
| 438 | +static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, | |
| 439 | + unsigned long flags); | |
| 431 | 440 | static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp); |
| 432 | 441 | static void do_nocb_deferred_wakeup(struct rcu_data *rdp); |
| 433 | 442 | static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp); |
| 434 | 443 | static void rcu_spawn_cpu_nocb_kthread(int cpu); |
| 435 | 444 | static void __init rcu_spawn_nocb_kthreads(void); |
| 445 | +static void show_rcu_nocb_state(struct rcu_data *rdp); | |
| 446 | +static void rcu_nocb_lock(struct rcu_data *rdp); | |
| 447 | +static void rcu_nocb_unlock(struct rcu_data *rdp); | |
| 448 | +static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, | |
| 449 | + unsigned long flags); | |
| 450 | +static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp); | |
| 436 | 451 | #ifdef CONFIG_RCU_NOCB_CPU |
| 437 | 452 | static void __init rcu_organize_nocb_kthreads(void); |
| 438 | −#endif /* #ifdef CONFIG_RCU_NOCB_CPU */ | |
| 439 | −static bool init_nocb_callback_list(struct rcu_data *rdp); | |
| 440 | −static unsigned long rcu_get_n_cbs_nocb_cpu(struct rcu_data *rdp); | |
| 453 | +#define rcu_nocb_lock_irqsave(rdp, flags) \ | |
| 454 | +do { \ | |
| 455 | + if (!rcu_segcblist_is_offloaded(&(rdp)->cblist)) \ | |
| 456 | + local_irq_save(flags); \ | |
| 457 | + else \ | |
| 458 | + raw_spin_lock_irqsave(&(rdp)->nocb_lock, (flags)); \ | |
| 459 | +} while (0) | |
| 460 | +#else /* #ifdef CONFIG_RCU_NOCB_CPU */ | |
| 461 | +#define rcu_nocb_lock_irqsave(rdp, flags) local_irq_save(flags) | |
| 462 | +#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ | |
| 463 | + | |
| 441 | 464 | static void rcu_bind_gp_kthread(void); |
| 442 | 465 | static bool rcu_nohz_full_cpu(void); |
| 443 | 466 | static void rcu_dynticks_task_enter(void); |
| @@ -781,7 +781,7 @@ static int rcu_print_task_exp_stall(struct rcu_node *rnp) · 780 unmodified lines | ||
| 781 | 781 | * other hand, if the CPU is not in an RCU read-side critical section, |
| 782 | 782 | * the IPI handler reports the quiescent state immediately. |
| 783 | 783 | * |
| 784 | − * Although this is a greate improvement over previous expedited | |
| 784 | + * Although this is a great improvement over previous expedited | |
| 785 | 785 | * implementations, it is still unfriendly to real-time workloads, so is |
| 786 | 786 | * thus not recommended for any sort of common-case code. In fact, if |
| 787 | 787 | * you are using synchronize_rcu_expedited() in a loop, please restructure |
| @@ -792,6 +792,7 @@ static int rcu_print_task_exp_stall(struct rcu_node *rnp) · 4 unmodified lines | ||
| 792 | 792 | */ |
| 793 | 793 | void synchronize_rcu_expedited(void) |
| 794 | 794 | { |
| 795 | + bool boottime = (rcu_scheduler_active == RCU_SCHEDULER_INIT); | |
| 795 | 796 | struct rcu_exp_work rew; |
| 796 | 797 | struct rcu_node *rnp; |
| 797 | 798 | unsigned long s; |
| @@ -817,7 +818,7 @@ void synchronize_rcu_expedited(void) · 19 unmodified lines | ||
| 817 | 818 | return; /* Someone else did our work for us. */ |
| 818 | 819 | |
| 819 | 820 | /* Ensure that load happens before action based on it. */ |
| 820 | − if (unlikely(rcu_scheduler_active == RCU_SCHEDULER_INIT)) { | |
| 821 | + if (unlikely(boottime)) { | |
| 821 | 822 | /* Direct call during scheduler init and early_initcalls(). */ |
| 822 | 823 | rcu_exp_sel_wait_wake(s); |
| 823 | 824 | } else { |
| @@ -835,5 +836,8 @@ void synchronize_rcu_expedited(void) · 11 unmodified lines | ||
| 835 | 836 | |
| 836 | 837 | /* Let the next expedited grace period start. */ |
| 837 | 838 | mutex_unlock(&rcu_state.exp_mutex); |
| 839 | + | |
| 840 | + if (likely(!boottime)) | |
| 841 | + destroy_work_on_stack(&rew.rew_work); | |
| 838 | 842 | } |
| 839 | 843 | EXPORT_SYMBOL_GPL(synchronize_rcu_expedited); |
| @@ -288,7 +288,6 @@ void rcu_note_context_switch(bool preempt) · 287 unmodified lines | ||
| 288 | 288 | struct rcu_data *rdp = this_cpu_ptr(&rcu_data); |
| 289 | 289 | struct rcu_node *rnp; |
| 290 | 290 | |
| 291 | − barrier(); /* Avoid RCU read-side critical sections leaking down. */ | |
| 292 | 291 | trace_rcu_utilization(TPS("Start context switch")); |
| 293 | 292 | lockdep_assert_irqs_disabled(); |
| 294 | 293 | WARN_ON_ONCE(!preempt && t->rcu_read_lock_nesting > 0); |
| @@ -314,15 +313,6 @@ void rcu_note_context_switch(bool preempt) · 19 unmodified lines | ||
| 314 | 313 | ? rnp->gp_seq |
| 315 | 314 | : rcu_seq_snap(&rnp->gp_seq)); |
| 316 | 315 | rcu_preempt_ctxt_queue(rnp, rdp); |
| 317 | − } else if (t->rcu_read_lock_nesting < 0 && | |
| 318 | − t->rcu_read_unlock_special.s) { | |
| 319 | − | |
| 320 | − /* | |
| 321 | − * Complete exit from RCU read-side critical section on | |
| 322 | − * behalf of preempted instance of __rcu_read_unlock(). | |
| 323 | − */ | |
| 324 | − rcu_read_unlock_special(t); | |
| 325 | − rcu_preempt_deferred_qs(t); | |
| 326 | 316 | } else { |
| 327 | 317 | rcu_preempt_deferred_qs(t); |
| 328 | 318 | } |
| @@ -340,7 +330,6 @@ void rcu_note_context_switch(bool preempt) · 11 unmodified lines | ||
| 340 | 330 | if (rdp->exp_deferred_qs) |
| 341 | 331 | rcu_report_exp_rdp(rdp); |
| 342 | 332 | trace_rcu_utilization(TPS("End context switch")); |
| 343 | − barrier(); /* Avoid RCU read-side critical sections leaking up. */ | |
| 344 | 333 | } |
| 345 | 334 | EXPORT_SYMBOL_GPL(rcu_note_context_switch); |
| 346 | 335 | |
| @@ -626,22 +615,18 @@ static void rcu_read_unlock_special(struct task_struct *t) · 279 unmodified lines | ||
| 626 | 615 | (rdp->grpmask & rnp->expmask) || |
| 627 | 616 | tick_nohz_full_cpu(rdp->cpu); |
| 628 | 617 | // Need to defer quiescent state until everything is enabled. |
| 629 | − if ((exp || in_irq()) && irqs_were_disabled && use_softirq && | |
| 630 | − (in_irq() || !t->rcu_read_unlock_special.b.deferred_qs)) { | |
| 618 | + if (irqs_were_disabled && use_softirq && | |
| 619 | + (in_interrupt() || | |
| 620 | + (exp && !t->rcu_read_unlock_special.b.deferred_qs))) { | |
| 631 | 621 | // Using softirq, safe to awaken, and we get |
| 632 | 622 | // no help from enabling irqs, unlike bh/preempt. |
| 633 | 623 | raise_softirq_irqoff(RCU_SOFTIRQ); |
| 634 | − } else if (exp && irqs_were_disabled && !use_softirq && | |
| 635 | − !t->rcu_read_unlock_special.b.deferred_qs) { | |
| 636 | − // Safe to awaken and we get no help from enabling | |
| 637 | − // irqs, unlike bh/preempt. | |
| 638 | − invoke_rcu_core(); | |
| 639 | 624 | } else { |
| 640 | 625 | // Enabling BH or preempt does reschedule, so... |
| 641 | 626 | // Also if no expediting or NO_HZ_FULL, slow is OK. |
| 642 | 627 | set_tsk_need_resched(current); |
| 643 | 628 | set_preempt_need_resched(); |
| 644 | − if (IS_ENABLED(CONFIG_IRQ_WORK) && | |
| 629 | + if (IS_ENABLED(CONFIG_IRQ_WORK) && irqs_were_disabled && | |
| 645 | 630 | !rdp->defer_qs_iw_pending && exp) { |
| 646 | 631 | // Get scheduler to re-evaluate and call hooks. |
| 647 | 632 | // If !IRQ_WORK, FQS scan will eventually IPI. |
| @@ -828,11 +813,6 @@ static void rcu_qs(void) · 180 unmodified lines | ||
| 828 | 813 | * dyntick-idle quiescent state visible to other CPUs, which will in |
| 829 | 814 | * some cases serve for expedited as well as normal grace periods. |
| 830 | 815 | * Either way, register a lightweight quiescent state. |
| 831 | − * | |
| 832 | − * The barrier() calls are redundant in the common case when this is | |
| 833 | − * called externally, but just in case this is called from within this | |
| 834 | − * file. | |
| 835 | − * | |
| 836 | 816 | */ |
| 837 | 817 | void rcu_all_qs(void) |
| 838 | 818 | { |
| @@ -847,14 +827,12 @@ void rcu_all_qs(void) · 8 unmodified lines | ||
| 847 | 827 | return; |
| 848 | 828 | } |
| 849 | 829 | this_cpu_write(rcu_data.rcu_urgent_qs, false); |
| 850 | − barrier(); /* Avoid RCU read-side critical sections leaking down. */ | |
| 851 | 830 | if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) { |
| 852 | 831 | local_irq_save(flags); |
| 853 | 832 | rcu_momentary_dyntick_idle(); |
| 854 | 833 | local_irq_restore(flags); |
| 855 | 834 | } |
| 856 | 835 | rcu_qs(); |
| 857 | − barrier(); /* Avoid RCU read-side critical sections leaking up. */ | |
| 858 | 836 | preempt_enable(); |
| 859 | 837 | } |
| 860 | 838 | EXPORT_SYMBOL_GPL(rcu_all_qs); |
| @@ -864,7 +842,6 @@ EXPORT_SYMBOL_GPL(rcu_all_qs); · 3 unmodified lines | ||
| 864 | 842 | */ |
| 865 | 843 | void rcu_note_context_switch(bool preempt) |
| 866 | 844 | { |
| 867 | − barrier(); /* Avoid RCU read-side critical sections leaking down. */ | |
| 868 | 845 | trace_rcu_utilization(TPS("Start context switch")); |
| 869 | 846 | rcu_qs(); |
| 870 | 847 | /* Load rcu_urgent_qs before other flags. */ |
| @@ -877,7 +854,6 @@ void rcu_note_context_switch(bool preempt) · 6 unmodified lines | ||
| 877 | 854 | rcu_tasks_qs(current); |
| 878 | 855 | out: |
| 879 | 856 | trace_rcu_utilization(TPS("End context switch")); |
| 880 | − barrier(); /* Avoid RCU read-side critical sections leaking up. */ | |
| 881 | 857 | } |
| 882 | 858 | EXPORT_SYMBOL_GPL(rcu_note_context_switch); |
| 883 | 859 | |
| @@ -1134,7 +1110,7 @@ static void rcu_preempt_boost_start_gp(struct rcu_node *rnp) · 250 unmodified lines | ||
| 1134 | 1110 | * already exist. We only create this kthread for preemptible RCU. |
| 1135 | 1111 | * Returns zero if all is well, a negated errno otherwise. |
| 1136 | 1112 | */ |
| 1137 | −static int rcu_spawn_one_boost_kthread(struct rcu_node *rnp) | |
| 1113 | +static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) | |
| 1138 | 1114 | { |
| 1139 | 1115 | int rnp_index = rnp - rcu_get_root(); |
| 1140 | 1116 | unsigned long flags; |
| @@ -1142,25 +1118,27 @@ static int rcu_spawn_one_boost_kthread(struct rcu_node *rnp) · 1 unmodified line | ||
| 1142 | 1118 | struct task_struct *t; |
| 1143 | 1119 | |
| 1144 | 1120 | if (!IS_ENABLED(CONFIG_PREEMPT_RCU)) |
| 1145 | − return 0; | |
| 1121 | + return; | |
| 1146 | 1122 | |
| 1147 | 1123 | if (!rcu_scheduler_fully_active || rcu_rnp_online_cpus(rnp) == 0) |
| 1148 | − return 0; | |
| 1124 | + return; | |
| 1149 | 1125 | |
| 1150 | 1126 | rcu_state.boost = 1; |
| 1127 | + | |
| 1151 | 1128 | if (rnp->boost_kthread_task != NULL) |
| 1152 | − return 0; | |
| 1129 | + return; | |
| 1130 | + | |
| 1153 | 1131 | t = kthread_create(rcu_boost_kthread, (void *)rnp, |
| 1154 | 1132 | "rcub/%d", rnp_index); |
| 1155 | − if (IS_ERR(t)) | |
| 1156 | − return PTR_ERR(t); | |
| 1133 | + if (WARN_ON_ONCE(IS_ERR(t))) | |
| 1134 | + return; | |
| 1135 | + | |
| 1157 | 1136 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
| 1158 | 1137 | rnp->boost_kthread_task = t; |
| 1159 | 1138 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
| 1160 | 1139 | sp.sched_priority = kthread_prio; |
| 1161 | 1140 | sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); |
| 1162 | 1141 | wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */ |
| 1163 | − return 0; | |
| 1164 | 1142 | } |
| 1165 | 1143 | |
| 1166 | 1144 | /* |
| @@ -1201,7 +1179,7 @@ static void __init rcu_spawn_boost_kthreads(void) · 34 unmodified lines | ||
| 1201 | 1179 | struct rcu_node *rnp; |
| 1202 | 1180 | |
| 1203 | 1181 | rcu_for_each_leaf_node(rnp) |
| 1204 | − (void)rcu_spawn_one_boost_kthread(rnp); | |
| 1182 | + rcu_spawn_one_boost_kthread(rnp); | |
| 1205 | 1183 | } |
| 1206 | 1184 | |
| 1207 | 1185 | static void rcu_prepare_kthreads(int cpu) |
| @@ -1211,7 +1189,7 @@ static void rcu_prepare_kthreads(int cpu) · 3 unmodified lines | ||
| 1211 | 1189 | |
| 1212 | 1190 | /* Fire up the incoming CPU's kthread and leaf rcu_node kthread. */ |
| 1213 | 1191 | if (rcu_scheduler_fully_active) |
| 1214 | − (void)rcu_spawn_one_boost_kthread(rnp); | |
| 1192 | + rcu_spawn_one_boost_kthread(rnp); | |
| 1215 | 1193 | } |
| 1216 | 1194 | |
| 1217 | 1195 | #else /* #ifdef CONFIG_RCU_BOOST */ |
| @@ -1248,10 +1226,10 @@ static void rcu_prepare_kthreads(int cpu) · 30 unmodified lines | ||
| 1248 | 1226 | #if !defined(CONFIG_RCU_FAST_NO_HZ) |
| 1249 | 1227 | |
| 1250 | 1228 | /* |
| 1251 | − * Check to see if any future RCU-related work will need to be done | |
| 1252 | − * by the current CPU, even if none need be done immediately, returning | |
| 1253 | − * 1 if so. This function is part of the RCU implementation; it is -not- | |
| 1254 | − * an exported member of the RCU API. | |
| 1229 | + * Check to see if any future non-offloaded RCU-related work will need | |
| 1230 | + * to be done by the current CPU, even if none need be done immediately, | |
| 1231 | + * returning 1 if so. This function is part of the RCU implementation; | |
| 1232 | + * it is -not- an exported member of the RCU API. | |
| 1255 | 1233 | * |
| 1256 | 1234 | * Because we not have RCU_FAST_NO_HZ, just check whether or not this |
| 1257 | 1235 | * CPU has RCU callbacks queued. |
| @@ -1259,7 +1237,8 @@ static void rcu_prepare_kthreads(int cpu) · 1 unmodified line | ||
| 1259 | 1237 | int rcu_needs_cpu(u64 basemono, u64 *nextevt) |
| 1260 | 1238 | { |
| 1261 | 1239 | *nextevt = KTIME_MAX; |
| 1262 | − return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist); | |
| 1240 | + return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) && | |
| 1241 | + !rcu_segcblist_is_offloaded(&this_cpu_ptr(&rcu_data)->cblist); | |
| 1263 | 1242 | } |
| 1264 | 1243 | |
| 1265 | 1244 | /* |
| @@ -1360,8 +1339,9 @@ int rcu_needs_cpu(u64 basemono, u64 *nextevt) · 94 unmodified lines | ||
| 1360 | 1339 | |
| 1361 | 1340 | lockdep_assert_irqs_disabled(); |
| 1362 | 1341 | |
| 1363 | − /* If no callbacks, RCU doesn't need the CPU. */ | |
| 1364 | − if (rcu_segcblist_empty(&rdp->cblist)) { | |
| 1342 | + /* If no non-offloaded callbacks, RCU doesn't need the CPU. */ | |
| 1343 | + if (rcu_segcblist_empty(&rdp->cblist) || | |
| 1344 | + rcu_segcblist_is_offloaded(&this_cpu_ptr(&rcu_data)->cblist)) { | |
| 1365 | 1345 | *nextevt = KTIME_MAX; |
| 1366 | 1346 | return 0; |
| 1367 | 1347 | } |
| @@ -1404,7 +1384,7 @@ static void rcu_prepare_for_idle(void) · 36 unmodified lines | ||
| 1404 | 1384 | int tne; |
| 1405 | 1385 | |
| 1406 | 1386 | lockdep_assert_irqs_disabled(); |
| 1407 | − if (rcu_is_nocb_cpu(smp_processor_id())) | |
| 1387 | + if (rcu_segcblist_is_offloaded(&rdp->cblist)) | |
| 1408 | 1388 | return; |
| 1409 | 1389 | |
| 1410 | 1390 | /* Handle nohz enablement switches conservatively. */ |
| @@ -1453,8 +1433,10 @@ static void rcu_prepare_for_idle(void) · 42 unmodified lines | ||
| 1453 | 1433 | */ |
| 1454 | 1434 | static void rcu_cleanup_after_idle(void) |
| 1455 | 1435 | { |
| 1436 | + struct rcu_data *rdp = this_cpu_ptr(&rcu_data); | |
| 1437 | + | |
| 1456 | 1438 | lockdep_assert_irqs_disabled(); |
| 1457 | − if (rcu_is_nocb_cpu(smp_processor_id())) | |
| 1439 | + if (rcu_segcblist_is_offloaded(&rdp->cblist)) | |
| 1458 | 1440 | return; |
| 1459 | 1441 | if (rcu_try_advance_all_cbs()) |
| 1460 | 1442 | invoke_rcu_core(); |
| @@ -1469,10 +1451,10 @@ static void rcu_cleanup_after_idle(void) · 8 unmodified lines | ||
| 1469 | 1451 | * specified by rcu_nocb_mask. For the CPUs in the set, there are kthreads |
| 1470 | 1452 | * created that pull the callbacks from the corresponding CPU, wait for |
| 1471 | 1453 | * a grace period to elapse, and invoke the callbacks. These kthreads |
| 1472 | − * are organized into leaders, which manage incoming callbacks, wait for | |
| 1473 | − * grace periods, and awaken followers, and the followers, which only | |
| 1474 | − * invoke callbacks. Each leader is its own follower. The no-CBs CPUs | |
| 1475 | − * do a wake_up() on their kthread when they insert a callback into any | |
| 1454 | + * are organized into GP kthreads, which manage incoming callbacks, wait for | |
| 1455 | + * grace periods, and awaken CB kthreads, and the CB kthreads, which only | |
| 1456 | + * invoke callbacks. Each GP kthread invokes its own CBs. The no-CBs CPUs | |
| 1457 | + * do a wake_up() on their GP kthread when they insert a callback into any | |
| 1476 | 1458 | * empty list, unless the rcu_nocb_poll boot parameter has been specified, |
| 1477 | 1459 | * in which case each kthread actively polls its CPU. (Which isn't so great |
| 1478 | 1460 | * for energy efficiency, but which does reduce RCU's overhead on that CPU.) |
| @@ -1514,6 +1496,116 @@ static int __init parse_rcu_nocb_poll(char *arg) · 35 unmodified lines | ||
| 1514 | 1496 | } |
| 1515 | 1497 | early_param("rcu_nocb_poll", parse_rcu_nocb_poll); |
| 1516 | 1498 | |
| 1499 | +/* | |
| 1500 | + * Don't bother bypassing ->cblist if the call_rcu() rate is low. | |
| 1501 | + * After all, the main point of bypassing is to avoid lock contention | |
| 1502 | + * on ->nocb_lock, which only can happen at high call_rcu() rates. | |
| 1503 | + */ | |
| 1504 | +int nocb_nobypass_lim_per_jiffy = 16 * 1000 / HZ; | |
| 1505 | +module_param(nocb_nobypass_lim_per_jiffy, int, 0); | |
| 1506 | + | |
| 1507 | +/* | |
| 1508 | + * Acquire the specified rcu_data structure's ->nocb_bypass_lock. If the | |
| 1509 | + * lock isn't immediately available, increment ->nocb_lock_contended to | |
| 1510 | + * flag the contention. | |
| 1511 | + */ | |
| 1512 | +static void rcu_nocb_bypass_lock(struct rcu_data *rdp) | |
| 1513 | +{ | |
| 1514 | + lockdep_assert_irqs_disabled(); | |
| 1515 | + if (raw_spin_trylock(&rdp->nocb_bypass_lock)) | |
| 1516 | + return; | |
| 1517 | + atomic_inc(&rdp->nocb_lock_contended); | |
| 1518 | + WARN_ON_ONCE(smp_processor_id() != rdp->cpu); | |
| 1519 | + smp_mb__after_atomic(); /* atomic_inc() before lock. */ | |
| 1520 | + raw_spin_lock(&rdp->nocb_bypass_lock); | |
| 1521 | + smp_mb__before_atomic(); /* atomic_dec() after lock. */ | |
| 1522 | + atomic_dec(&rdp->nocb_lock_contended); | |
| 1523 | +} | |
| 1524 | + | |
| 1525 | +/* | |
| 1526 | + * Spinwait until the specified rcu_data structure's ->nocb_lock is | |
| 1527 | + * not contended. Please note that this is extremely special-purpose, | |
| 1528 | + * relying on the fact that at most two kthreads and one CPU contend for | |
| 1529 | + * this lock, and also that the two kthreads are guaranteed to have frequent | |
| 1530 | + * grace-period-duration time intervals between successive acquisitions | |
| 1531 | + * of the lock. This allows us to use an extremely simple throttling | |
| 1532 | + * mechanism, and further to apply it only to the CPU doing floods of | |
| 1533 | + * call_rcu() invocations. Don't try this at home! | |
| 1534 | + */ | |
| 1535 | +static void rcu_nocb_wait_contended(struct rcu_data *rdp) | |
| 1536 | +{ | |
| 1537 | + WARN_ON_ONCE(smp_processor_id() != rdp->cpu); | |
| 1538 | + while (WARN_ON_ONCE(atomic_read(&rdp->nocb_lock_contended))) | |
| 1539 | + cpu_relax(); | |
| 1540 | +} | |
| 1541 | + | |
| 1542 | +/* | |
| 1543 | + * Conditionally acquire the specified rcu_data structure's | |
| 1544 | + * ->nocb_bypass_lock. | |
| 1545 | + */ | |
| 1546 | +static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp) | |
| 1547 | +{ | |
| 1548 | + lockdep_assert_irqs_disabled(); | |
| 1549 | + return raw_spin_trylock(&rdp->nocb_bypass_lock); | |
| 1550 | +} | |
| 1551 | + | |
| 1552 | +/* | |
| 1553 | + * Release the specified rcu_data structure's ->nocb_bypass_lock. | |
| 1554 | + */ | |
| 1555 | +static void rcu_nocb_bypass_unlock(struct rcu_data *rdp) | |
| 1556 | +{ | |
| 1557 | + lockdep_assert_irqs_disabled(); | |
| 1558 | + raw_spin_unlock(&rdp->nocb_bypass_lock); | |
| 1559 | +} | |
| 1560 | + | |
| 1561 | +/* | |
| 1562 | + * Acquire the specified rcu_data structure's ->nocb_lock, but only | |
| 1563 | + * if it corresponds to a no-CBs CPU. | |
| 1564 | + */ | |
| 1565 | +static void rcu_nocb_lock(struct rcu_data *rdp) | |
| 1566 | +{ | |
| 1567 | + lockdep_assert_irqs_disabled(); | |
| 1568 | + if (!rcu_segcblist_is_offloaded(&rdp->cblist)) | |
| 1569 | + return; | |
| 1570 | + raw_spin_lock(&rdp->nocb_lock); | |
| 1571 | +} | |
| 1572 | + | |
| 1573 | +/* | |
| 1574 | + * Release the specified rcu_data structure's ->nocb_lock, but only | |
| 1575 | + * if it corresponds to a no-CBs CPU. | |
| 1576 | + */ | |
| 1577 | +static void rcu_nocb_unlock(struct rcu_data *rdp) | |
| 1578 | +{ | |
| 1579 | + if (rcu_segcblist_is_offloaded(&rdp->cblist)) { | |
| 1580 | + lockdep_assert_irqs_disabled(); | |
| 1581 | + raw_spin_unlock(&rdp->nocb_lock); | |
| 1582 | + } | |
| 1583 | +} | |
| 1584 | + | |
| 1585 | +/* | |
| 1586 | + * Release the specified rcu_data structure's ->nocb_lock and restore | |
| 1587 | + * interrupts, but only if it corresponds to a no-CBs CPU. | |
| 1588 | + */ | |
| 1589 | +static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, | |
| 1590 | + unsigned long flags) | |
| 1591 | +{ | |
| 1592 | + if (rcu_segcblist_is_offloaded(&rdp->cblist)) { | |
| 1593 | + lockdep_assert_irqs_disabled(); | |
| 1594 | + raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); | |
| 1595 | + } else { | |
| 1596 | + local_irq_restore(flags); | |
| 1597 | + } | |
| 1598 | +} | |
| 1599 | + | |
| 1600 | +/* Lockdep check that ->cblist may be safely accessed. */ | |
| 1601 | +static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) | |
| 1602 | +{ | |
| 1603 | + lockdep_assert_irqs_disabled(); | |
| 1604 | + if (rcu_segcblist_is_offloaded(&rdp->cblist) && | |
| 1605 | + cpu_online(rdp->cpu)) | |
| 1606 | + lockdep_assert_held(&rdp->nocb_lock); | |
| 1607 | +} | |
| 1608 | + | |
| 1517 | 1609 | /* |
| 1518 | 1610 | * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended |
| 1519 | 1611 | * grace period. |
| @@ -1543,440 +1635,514 @@ bool rcu_is_nocb_cpu(int cpu) · 23 unmodified lines | ||
| 1543 | 1635 | } |
| 1544 | 1636 | |
| 1545 | 1637 | /* |
| 1546 | − * Kick the leader kthread for this NOCB group. Caller holds ->nocb_lock | |
| 1638 | + * Kick the GP kthread for this NOCB group. Caller holds ->nocb_lock | |
| 1547 | 1639 | * and this function releases it. |
| 1548 | 1640 | */ |
| 1549 | −static void __wake_nocb_leader(struct rcu_data *rdp, bool force, | |
| 1550 | − unsigned long flags) | |
| 1641 | +static void wake_nocb_gp(struct rcu_data *rdp, bool force, | |
| 1642 | + unsigned long flags) | |
| 1551 | 1643 | __releases(rdp->nocb_lock) |
| 1552 | 1644 | { |
| 1553 | − struct rcu_data *rdp_leader = rdp->nocb_leader; | |
| 1645 | + bool needwake = false; | |
| 1646 | + struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; | |
| 1554 | 1647 | |
| 1555 | 1648 | lockdep_assert_held(&rdp->nocb_lock); |
| 1556 | − if (!READ_ONCE(rdp_leader->nocb_kthread)) { | |
| 1557 | − raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); | |
| 1649 | + if (!READ_ONCE(rdp_gp->nocb_gp_kthread)) { | |
| 1650 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, | |
| 1651 | + TPS("AlreadyAwake")); | |
| 1652 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 1558 | 1653 | return; |
| 1559 | 1654 | } |
| 1560 | − if (rdp_leader->nocb_leader_sleep || force) { | |
| 1561 | − /* Prior smp_mb__after_atomic() orders against prior enqueue. */ | |
| 1562 | − WRITE_ONCE(rdp_leader->nocb_leader_sleep, false); | |
| 1563 | − del_timer(&rdp->nocb_timer); | |
| 1564 | − raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); | |
| 1565 | − smp_mb(); /* ->nocb_leader_sleep before swake_up_one(). */ | |
| 1566 | − swake_up_one(&rdp_leader->nocb_wq); | |
| 1567 | − } else { | |
| 1568 | − raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); | |
| 1655 | + del_timer(&rdp->nocb_timer); | |
| 1656 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 1657 | + raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags); | |
| 1658 | + if (force || READ_ONCE(rdp_gp->nocb_gp_sleep)) { | |
| 1659 | + WRITE_ONCE(rdp_gp->nocb_gp_sleep, false); | |
| 1660 | + needwake = true; | |
| 1661 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake")); | |
| 1569 | 1662 | } |
| 1663 | + raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags); | |
| 1664 | + if (needwake) | |
| 1665 | + wake_up_process(rdp_gp->nocb_gp_kthread); | |
| 1570 | 1666 | } |
| 1571 | 1667 | |
| 1572 | 1668 | /* |
| 1573 | − * Kick the leader kthread for this NOCB group, but caller has not | |
| 1574 | − * acquired locks. | |
| 1669 | + * Arrange to wake the GP kthread for this NOCB group at some future | |
| 1670 | + * time when it is safe to do so. | |
| 1575 | 1671 | */ |
| 1576 | −static void wake_nocb_leader(struct rcu_data *rdp, bool force) | |
| 1672 | +static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, | |
| 1673 | + const char *reason) | |
| 1577 | 1674 | { |
| 1578 | − unsigned long flags; | |
| 1675 | + if (rdp->nocb_defer_wakeup == RCU_NOCB_WAKE_NOT) | |
| 1676 | + mod_timer(&rdp->nocb_timer, jiffies + 1); | |
| 1677 | + if (rdp->nocb_defer_wakeup < waketype) | |
| 1678 | + WRITE_ONCE(rdp->nocb_defer_wakeup, waketype); | |
| 1679 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason); | |
| 1680 | +} | |
| 1681 | + | |
| 1682 | +/* | |
| 1683 | + * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL. | |
| 1684 | + * However, if there is a callback to be enqueued and if ->nocb_bypass | |
| 1685 | + * proves to be initially empty, just return false because the no-CB GP | |
| 1686 | + * kthread may need to be awakened in this case. | |
| 1687 | + * | |
| 1688 | + * Note that this function always returns true if rhp is NULL. | |
| 1689 | + */ | |
| 1690 | +static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, | |
| 1691 | + unsigned long j) | |
| 1692 | +{ | |
| 1693 | + struct rcu_cblist rcl; | |
| 1579 | 1694 | |
| 1580 | − raw_spin_lock_irqsave(&rdp->nocb_lock, flags); | |
| 1581 | − __wake_nocb_leader(rdp, force, flags); | |
| 1695 | + WARN_ON_ONCE(!rcu_segcblist_is_offloaded(&rdp->cblist)); | |
| 1696 | + rcu_lockdep_assert_cblist_protected(rdp); | |
| 1697 | + lockdep_assert_held(&rdp->nocb_bypass_lock); | |
| 1698 | + if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) { | |
| 1699 | + raw_spin_unlock(&rdp->nocb_bypass_lock); | |
| 1700 | + return false; | |
| 1701 | + } | |
| 1702 | + /* Note: ->cblist.len already accounts for ->nocb_bypass contents. */ | |
| 1703 | + if (rhp) | |
| 1704 | + rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ | |
| 1705 | + rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); | |
| 1706 | + rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl); | |
| 1707 | + WRITE_ONCE(rdp->nocb_bypass_first, j); | |
| 1708 | + rcu_nocb_bypass_unlock(rdp); | |
| 1709 | + return true; | |
| 1582 | 1710 | } |
| 1583 | 1711 | |
| 1584 | 1712 | /* |
| 1585 | − * Arrange to wake the leader kthread for this NOCB group at some | |
| 1586 | − * future time when it is safe to do so. | |
| 1713 | + * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL. | |
| 1714 | + * However, if there is a callback to be enqueued and if ->nocb_bypass | |
| 1715 | + * proves to be initially empty, just return false because the no-CB GP | |
| 1716 | + * kthread may need to be awakened in this case. | |
| 1717 | + * | |
| 1718 | + * Note that this function always returns true if rhp is NULL. | |
| 1587 | 1719 | */ |
| 1588 | −static void wake_nocb_leader_defer(struct rcu_data *rdp, int waketype, | |
| 1589 | − const char *reason) | |
| 1720 | +static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, | |
| 1721 | + unsigned long j) | |
| 1590 | 1722 | { |
| 1591 | − unsigned long flags; | |
| 1723 | + if (!rcu_segcblist_is_offloaded(&rdp->cblist)) | |
| 1724 | + return true; | |
| 1725 | + rcu_lockdep_assert_cblist_protected(rdp); | |
| 1726 | + rcu_nocb_bypass_lock(rdp); | |
| 1727 | + return rcu_nocb_do_flush_bypass(rdp, rhp, j); | |
| 1728 | +} | |
| 1592 | 1729 | |
| 1593 | − raw_spin_lock_irqsave(&rdp->nocb_lock, flags); | |
| 1594 | − if (rdp->nocb_defer_wakeup == RCU_NOCB_WAKE_NOT) | |
| 1595 | − mod_timer(&rdp->nocb_timer, jiffies + 1); | |
| 1596 | − WRITE_ONCE(rdp->nocb_defer_wakeup, waketype); | |
| 1597 | − trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason); | |
| 1598 | − raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); | |
| 1730 | +/* | |
| 1731 | + * If the ->nocb_bypass_lock is immediately available, flush the | |
| 1732 | + * ->nocb_bypass queue into ->cblist. | |
| 1733 | + */ | |
| 1734 | +static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j) | |
| 1735 | +{ | |
| 1736 | + rcu_lockdep_assert_cblist_protected(rdp); | |
| 1737 | + if (!rcu_segcblist_is_offloaded(&rdp->cblist) || | |
| 1738 | + !rcu_nocb_bypass_trylock(rdp)) | |
| 1739 | + return; | |
| 1740 | + WARN_ON_ONCE(!rcu_nocb_do_flush_bypass(rdp, NULL, j)); | |
| 1599 | 1741 | } |
| 1600 | 1742 | |
| 1601 | −/* Does rcu_barrier need to queue an RCU callback on the specified CPU? */ | |
| 1602 | −static bool rcu_nocb_cpu_needs_barrier(int cpu) | |
| 1743 | +/* | |
| 1744 | + * See whether it is appropriate to use the ->nocb_bypass list in order | |
| 1745 | + * to control contention on ->nocb_lock. A limited number of direct | |
| 1746 | + * enqueues are permitted into ->cblist per jiffy. If ->nocb_bypass | |
| 1747 | + * is non-empty, further callbacks must be placed into ->nocb_bypass, | |
| 1748 | + * otherwise rcu_barrier() breaks. Use rcu_nocb_flush_bypass() to switch | |
| 1749 | + * back to direct use of ->cblist. However, ->nocb_bypass should not be | |
| 1750 | + * used if ->cblist is empty, because otherwise callbacks can be stranded | |
| 1751 | + * on ->nocb_bypass because we cannot count on the current CPU ever again | |
| 1752 | + * invoking call_rcu(). The general rule is that if ->nocb_bypass is | |
| 1753 | + * non-empty, the corresponding no-CBs grace-period kthread must not be | |
| 1754 | + * in an indefinite sleep state. | |
| 1755 | + * | |
| 1756 | + * Finally, it is not permitted to use the bypass during early boot, | |
| 1757 | + * as doing so would confuse the auto-initialization code. Besides | |
| 1758 | + * which, there is no point in worrying about lock contention while | |
| 1759 | + * there is only one CPU in operation. | |
| 1760 | + */ | |
| 1761 | +static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, | |
| 1762 | + bool *was_alldone, unsigned long flags) | |
| 1603 | 1763 | { |
| 1604 | − struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); | |
| 1605 | − unsigned long ret; | |
| 1606 | −#ifdef CONFIG_PROVE_RCU | |
| 1607 | − struct rcu_head *rhp; | |
| 1608 | −#endif /* #ifdef CONFIG_PROVE_RCU */ | |
| 1764 | + unsigned long c; | |
| 1765 | + unsigned long cur_gp_seq; | |
| 1766 | + unsigned long j = jiffies; | |
| 1767 | + long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); | |
| 1609 | 1768 | |
| 1610 | − /* | |
| 1611 | − * Check count of all no-CBs callbacks awaiting invocation. | |
| 1612 | − * There needs to be a barrier before this function is called, | |
| 1613 | − * but associated with a prior determination that no more | |
| 1614 | − * callbacks would be posted. In the worst case, the first | |
| 1615 | − * barrier in rcu_barrier() suffices (but the caller cannot | |
| 1616 | − * necessarily rely on this, not a substitute for the caller | |
| 1617 | − * getting the concurrency design right!). There must also be a | |
| 1618 | − * barrier between the following load and posting of a callback | |
| 1619 | − * (if a callback is in fact needed). This is associated with an | |
| 1620 | − * atomic_inc() in the caller. | |
| 1621 | − */ | |
| 1622 | − ret = rcu_get_n_cbs_nocb_cpu(rdp); | |
| 1623 | − | |
| 1624 | −#ifdef CONFIG_PROVE_RCU | |
| 1625 | − rhp = READ_ONCE(rdp->nocb_head); | |
| 1626 | − if (!rhp) | |
| 1627 | − rhp = READ_ONCE(rdp->nocb_gp_head); | |
| 1628 | − if (!rhp) | |
| 1629 | − rhp = READ_ONCE(rdp->nocb_follower_head); | |
| 1630 | − | |
| 1631 | − /* Having no rcuo kthread but CBs after scheduler starts is bad! */ | |
| 1632 | − if (!READ_ONCE(rdp->nocb_kthread) && rhp && | |
| 1633 | − rcu_scheduler_fully_active) { | |
| 1634 | − /* RCU callback enqueued before CPU first came online??? */ | |
| 1635 | − pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n", | |
| 1636 | − cpu, rhp->func); | |
| 1637 | − WARN_ON_ONCE(1); | |
| 1769 | + if (!rcu_segcblist_is_offloaded(&rdp->cblist)) { | |
| 1770 | + *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); | |
| 1771 | + return false; /* Not offloaded, no bypassing. */ | |
| 1772 | + } | |
| 1773 | + lockdep_assert_irqs_disabled(); | |
| 1774 | + | |
| 1775 | + // Don't use ->nocb_bypass during early boot. | |
| 1776 | + if (rcu_scheduler_active != RCU_SCHEDULER_RUNNING) { | |
| 1777 | + rcu_nocb_lock(rdp); | |
| 1778 | + WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); | |
| 1779 | + *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); | |
| 1780 | + return false; | |
| 1781 | + } | |
| 1782 | + | |
| 1783 | + // If we have advanced to a new jiffy, reset counts to allow | |
| 1784 | + // moving back from ->nocb_bypass to ->cblist. | |
| 1785 | + if (j == rdp->nocb_nobypass_last) { | |
| 1786 | + c = rdp->nocb_nobypass_count + 1; | |
| 1787 | + } else { | |
| 1788 | + WRITE_ONCE(rdp->nocb_nobypass_last, j); | |
| 1789 | + c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy; | |
| 1790 | + if (ULONG_CMP_LT(rdp->nocb_nobypass_count, | |
| 1791 | + nocb_nobypass_lim_per_jiffy)) | |
| 1792 | + c = 0; | |
| 1793 | + else if (c > nocb_nobypass_lim_per_jiffy) | |
| 1794 | + c = nocb_nobypass_lim_per_jiffy; | |
| 1795 | + } | |
| 1796 | + WRITE_ONCE(rdp->nocb_nobypass_count, c); | |
| 1797 | + | |
| 1798 | + // If there hasn't yet been all that many ->cblist enqueues | |
| 1799 | + // this jiffy, tell the caller to enqueue onto ->cblist. But flush | |
| 1800 | + // ->nocb_bypass first. | |
| 1801 | + if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy) { | |
| 1802 | + rcu_nocb_lock(rdp); | |
| 1803 | + *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); | |
| 1804 | + if (*was_alldone) | |
| 1805 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, | |
| 1806 | + TPS("FirstQ")); | |
| 1807 | + WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, j)); | |
| 1808 | + WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); | |
| 1809 | + return false; // Caller must enqueue the callback. | |
| 1810 | + } | |
| 1811 | + | |
| 1812 | + // If ->nocb_bypass has been used too long or is too full, | |
| 1813 | + // flush ->nocb_bypass to ->cblist. | |
| 1814 | + if ((ncbs && j != READ_ONCE(rdp->nocb_bypass_first)) || | |
| 1815 | + ncbs >= qhimark) { | |
| 1816 | + rcu_nocb_lock(rdp); | |
| 1817 | + if (!rcu_nocb_flush_bypass(rdp, rhp, j)) { | |
| 1818 | + *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); | |
| 1819 | + if (*was_alldone) | |
| 1820 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, | |
| 1821 | + TPS("FirstQ")); | |
| 1822 | + WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); | |
| 1823 | + return false; // Caller must enqueue the callback. | |
| 1824 | + } | |
| 1825 | + if (j != rdp->nocb_gp_adv_time && | |
| 1826 | + rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && | |
| 1827 | + rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { | |
| 1828 | + rcu_advance_cbs_nowake(rdp->mynode, rdp); | |
| 1829 | + rdp->nocb_gp_adv_time = j; | |
| 1830 | + } | |
| 1831 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 1832 | + return true; // Callback already enqueued. | |
| 1638 | 1833 | } |
| 1639 | −#endif /* #ifdef CONFIG_PROVE_RCU */ | |
| 1640 | 1834 | |
| 1641 | − return !!ret; | |
| 1835 | + // We need to use the bypass. | |
| 1836 | + rcu_nocb_wait_contended(rdp); | |
| 1837 | + rcu_nocb_bypass_lock(rdp); | |
| 1838 | + ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); | |
| 1839 | + rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ | |
| 1840 | + rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); | |
| 1841 | + if (!ncbs) { | |
| 1842 | + WRITE_ONCE(rdp->nocb_bypass_first, j); | |
| 1843 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ")); | |
| 1844 | + } | |
| 1845 | + rcu_nocb_bypass_unlock(rdp); | |
| 1846 | + smp_mb(); /* Order enqueue before wake. */ | |
| 1847 | + if (ncbs) { | |
| 1848 | + local_irq_restore(flags); | |
| 1849 | + } else { | |
| 1850 | + // No-CBs GP kthread might be indefinitely asleep, if so, wake. | |
| 1851 | + rcu_nocb_lock(rdp); // Rare during call_rcu() flood. | |
| 1852 | + if (!rcu_segcblist_pend_cbs(&rdp->cblist)) { | |
| 1853 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, | |
| 1854 | + TPS("FirstBQwake")); | |
| 1855 | + __call_rcu_nocb_wake(rdp, true, flags); | |
| 1856 | + } else { | |
| 1857 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, | |
| 1858 | + TPS("FirstBQnoWake")); | |
| 1859 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 1860 | + } | |
| 1861 | + } | |
| 1862 | + return true; // Callback already enqueued. | |
| 1642 | 1863 | } |
| 1643 | 1864 | |
| 1644 | 1865 | /* |
| 1645 | − * Enqueue the specified string of rcu_head structures onto the specified | |
| 1646 | − * CPU's no-CBs lists. The CPU is specified by rdp, the head of the | |
| 1647 | − * string by rhp, and the tail of the string by rhtp. The non-lazy/lazy | |
| 1648 | − * counts are supplied by rhcount and rhcount_lazy. | |
| 1866 | + * Awaken the no-CBs grace-period kthead if needed, either due to it | |
| 1867 | + * legitimately being asleep or due to overload conditions. | |
| 1649 | 1868 | * |
| 1650 | 1869 | * If warranted, also wake up the kthread servicing this CPUs queues. |
| 1651 | 1870 | */ |
| 1652 | −static void __call_rcu_nocb_enqueue(struct rcu_data *rdp, | |
| 1653 | − struct rcu_head *rhp, | |
| 1654 | − struct rcu_head **rhtp, | |
| 1655 | − int rhcount, int rhcount_lazy, | |
| 1656 | − unsigned long flags) | |
| 1871 | +static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, | |
| 1872 | + unsigned long flags) | |
| 1873 | + __releases(rdp->nocb_lock) | |
| 1657 | 1874 | { |
| 1658 | − int len; | |
| 1659 | − struct rcu_head **old_rhpp; | |
| 1875 | + unsigned long cur_gp_seq; | |
| 1876 | + unsigned long j; | |
| 1877 | + long len; | |
| 1660 | 1878 | struct task_struct *t; |
| 1661 | 1879 | |
| 1662 | − /* Enqueue the callback on the nocb list and update counts. */ | |
| 1663 | − atomic_long_add(rhcount, &rdp->nocb_q_count); | |
| 1664 | − /* rcu_barrier() relies on ->nocb_q_count add before xchg. */ | |
| 1665 | − old_rhpp = xchg(&rdp->nocb_tail, rhtp); | |
| 1666 | − WRITE_ONCE(*old_rhpp, rhp); | |
| 1667 | − atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy); | |
| 1668 | − smp_mb__after_atomic(); /* Store *old_rhpp before _wake test. */ | |
| 1669 | − | |
| 1670 | − /* If we are not being polled and there is a kthread, awaken it ... */ | |
| 1671 | − t = READ_ONCE(rdp->nocb_kthread); | |
| 1880 | + // If we are being polled or there is no kthread, just leave. | |
| 1881 | + t = READ_ONCE(rdp->nocb_gp_kthread); | |
| 1672 | 1882 | if (rcu_nocb_poll || !t) { |
| 1673 | 1883 | trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, |
| 1674 | 1884 | TPS("WakeNotPoll")); |
| 1885 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 1675 | 1886 | return; |
| 1676 | 1887 | } |
| 1677 | − len = rcu_get_n_cbs_nocb_cpu(rdp); | |
| 1678 | − if (old_rhpp == &rdp->nocb_head) { | |
| 1888 | + // Need to actually to a wakeup. | |
| 1889 | + len = rcu_segcblist_n_cbs(&rdp->cblist); | |
| 1890 | + if (was_alldone) { | |
| 1891 | + rdp->qlen_last_fqs_check = len; | |
| 1679 | 1892 | if (!irqs_disabled_flags(flags)) { |
| 1680 | 1893 | /* ... if queue was empty ... */ |
| 1681 | − wake_nocb_leader(rdp, false); | |
| 1894 | + wake_nocb_gp(rdp, false, flags); | |
| 1682 | 1895 | trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, |
| 1683 | 1896 | TPS("WakeEmpty")); |
| 1684 | 1897 | } else { |
| 1685 | − wake_nocb_leader_defer(rdp, RCU_NOCB_WAKE, | |
| 1686 | − TPS("WakeEmptyIsDeferred")); | |
| 1898 | + wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE, | |
| 1899 | + TPS("WakeEmptyIsDeferred")); | |
| 1900 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 1687 | 1901 | } |
| 1688 | − rdp->qlen_last_fqs_check = 0; | |
| 1689 | 1902 | } else if (len > rdp->qlen_last_fqs_check + qhimark) { |
| 1690 | 1903 | /* ... or if many callbacks queued. */ |
| 1691 | − if (!irqs_disabled_flags(flags)) { | |
| 1692 | − wake_nocb_leader(rdp, true); | |
| 1693 | − trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, | |
| 1694 | − TPS("WakeOvf")); | |
| 1695 | − } else { | |
| 1696 | − wake_nocb_leader_defer(rdp, RCU_NOCB_WAKE_FORCE, | |
| 1697 | − TPS("WakeOvfIsDeferred")); | |
| 1904 | + rdp->qlen_last_fqs_check = len; | |
| 1905 | + j = jiffies; | |
| 1906 | + if (j != rdp->nocb_gp_adv_time && | |
| 1907 | + rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && | |
| 1908 | + rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { | |
| 1909 | + rcu_advance_cbs_nowake(rdp->mynode, rdp); | |
| 1910 | + rdp->nocb_gp_adv_time = j; | |
| 1698 | 1911 | } |
| 1699 | − rdp->qlen_last_fqs_check = LONG_MAX / 2; | |
| 1912 | + smp_mb(); /* Enqueue before timer_pending(). */ | |
| 1913 | + if ((rdp->nocb_cb_sleep || | |
| 1914 | + !rcu_segcblist_ready_cbs(&rdp->cblist)) && | |
| 1915 | + !timer_pending(&rdp->nocb_bypass_timer)) | |
| 1916 | + wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE, | |
| 1917 | + TPS("WakeOvfIsDeferred")); | |
| 1918 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 1700 | 1919 | } else { |
| 1701 | 1920 | trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); |
| 1921 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 1702 | 1922 | } |
| 1703 | 1923 | return; |
| 1704 | 1924 | } |
| 1705 | 1925 | |
| 1706 | −/* | |
| 1707 | − * This is a helper for __call_rcu(), which invokes this when the normal | |
| 1708 | − * callback queue is inoperable. If this is not a no-CBs CPU, this | |
| 1709 | − * function returns failure back to __call_rcu(), which can complain | |
| 1710 | − * appropriately. | |
| 1711 | − * | |
| 1712 | − * Otherwise, this function queues the callback where the corresponding | |
| 1713 | − * "rcuo" kthread can find it. | |
| 1714 | − */ | |
| 1715 | −static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp, | |
| 1716 | − bool lazy, unsigned long flags) | |
| 1926 | +/* Wake up the no-CBs GP kthread to flush ->nocb_bypass. */ | |
| 1927 | +static void do_nocb_bypass_wakeup_timer(struct timer_list *t) | |
| 1717 | 1928 | { |
| 1929 | + unsigned long flags; | |
| 1930 | + struct rcu_data *rdp = from_timer(rdp, t, nocb_bypass_timer); | |
| 1718 | 1931 | |
| 1719 | − if (!rcu_is_nocb_cpu(rdp->cpu)) | |
| 1720 | − return false; | |
| 1721 | − __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy, flags); | |
| 1722 | − if (__is_kfree_rcu_offset((unsigned long)rhp->func)) | |
| 1723 | − trace_rcu_kfree_callback(rcu_state.name, rhp, | |
| 1724 | − (unsigned long)rhp->func, | |
| 1725 | − -atomic_long_read(&rdp->nocb_q_count_lazy), | |
| 1726 | − -rcu_get_n_cbs_nocb_cpu(rdp)); | |
| 1727 | − else | |
| 1728 | − trace_rcu_callback(rcu_state.name, rhp, | |
| 1729 | − -atomic_long_read(&rdp->nocb_q_count_lazy), | |
| 1730 | − -rcu_get_n_cbs_nocb_cpu(rdp)); | |
| 1731 | − | |
| 1732 | − /* | |
| 1733 | − * If called from an extended quiescent state with interrupts | |
| 1734 | − * disabled, invoke the RCU core in order to allow the idle-entry | |
| 1735 | − * deferred-wakeup check to function. | |
| 1736 | − */ | |
| 1737 | − if (irqs_disabled_flags(flags) && | |
| 1738 | − !rcu_is_watching() && | |
| 1739 | − cpu_online(smp_processor_id())) | |
| 1740 | − invoke_rcu_core(); | |
| 1741 | − | |
| 1742 | − return true; | |
| 1743 | −} | |
| 1744 | − | |
| 1745 | −/* | |
| 1746 | − * Adopt orphaned callbacks on a no-CBs CPU, or return 0 if this is | |
| 1747 | − * not a no-CBs CPU. | |
| 1748 | − */ | |
| 1749 | −static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp, | |
| 1750 | − struct rcu_data *rdp, | |
| 1751 | − unsigned long flags) | |
| 1752 | −{ | |
| 1753 | − lockdep_assert_irqs_disabled(); | |
| 1754 | − if (!rcu_is_nocb_cpu(smp_processor_id())) | |
| 1755 | − return false; /* Not NOCBs CPU, caller must migrate CBs. */ | |
| 1756 | − __call_rcu_nocb_enqueue(my_rdp, rcu_segcblist_head(&rdp->cblist), | |
| 1757 | − rcu_segcblist_tail(&rdp->cblist), | |
| 1758 | − rcu_segcblist_n_cbs(&rdp->cblist), | |
| 1759 | − rcu_segcblist_n_lazy_cbs(&rdp->cblist), flags); | |
| 1760 | − rcu_segcblist_init(&rdp->cblist); | |
| 1761 | − rcu_segcblist_disable(&rdp->cblist); | |
| 1762 | − return true; | |
| 1932 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer")); | |
| 1933 | + rcu_nocb_lock_irqsave(rdp, flags); | |
| 1934 | + smp_mb__after_spinlock(); /* Timer expire before wakeup. */ | |
| 1935 | + __call_rcu_nocb_wake(rdp, true, flags); | |
| 1763 | 1936 | } |
| 1764 | 1937 | |
| 1765 | 1938 | /* |
| 1766 | − * If necessary, kick off a new grace period, and either way wait | |
| 1767 | − * for a subsequent grace period to complete. | |
| 1939 | + * No-CBs GP kthreads come here to wait for additional callbacks to show up | |
| 1940 | + * or for grace periods to end. | |
| 1768 | 1941 | */ |
| 1769 | −static void rcu_nocb_wait_gp(struct rcu_data *rdp) | |
| 1942 | +static void nocb_gp_wait(struct rcu_data *my_rdp) | |
| 1770 | 1943 | { |
| 1771 | − unsigned long c; | |
| 1772 | − bool d; | |
| 1944 | + bool bypass = false; | |
| 1945 | + long bypass_ncbs; | |
| 1946 | + int __maybe_unused cpu = my_rdp->cpu; | |
| 1947 | + unsigned long cur_gp_seq; | |
| 1773 | 1948 | unsigned long flags; |
| 1949 | + bool gotcbs; | |
| 1950 | + unsigned long j = jiffies; | |
| 1951 | + bool needwait_gp = false; // This prevents actual uninitialized use. | |
| 1774 | 1952 | bool needwake; |
| 1775 | − struct rcu_node *rnp = rdp->mynode; | |
| 1953 | + bool needwake_gp; | |
| 1954 | + struct rcu_data *rdp; | |
| 1955 | + struct rcu_node *rnp; | |
| 1956 | + unsigned long wait_gp_seq = 0; // Suppress "use uninitialized" warning. | |
| 1776 | 1957 | |
| 1777 | − local_irq_save(flags); | |
| 1778 | − c = rcu_seq_snap(&rcu_state.gp_seq); | |
| 1779 | − if (!rdp->gpwrap && ULONG_CMP_GE(rdp->gp_seq_needed, c)) { | |
| 1780 | − local_irq_restore(flags); | |
| 1781 | − } else { | |
| 1782 | − raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ | |
| 1783 | − needwake = rcu_start_this_gp(rnp, rdp, c); | |
| 1784 | − raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | |
| 1785 | − if (needwake) | |
| 1958 | + /* | |
| 1959 | + * Each pass through the following loop checks for CBs and for the | |
| 1960 | + * nearest grace period (if any) to wait for next. The CB kthreads | |
| 1961 | + * and the global grace-period kthread are awakened if needed. | |
| 1962 | + */ | |
| 1963 | + for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_cb_rdp) { | |
| 1964 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check")); | |
| 1965 | + rcu_nocb_lock_irqsave(rdp, flags); | |
| 1966 | + bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); | |
| 1967 | + if (bypass_ncbs && | |
| 1968 | + (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) || | |
| 1969 | + bypass_ncbs > 2 * qhimark)) { | |
| 1970 | + // Bypass full or old, so flush it. | |
| 1971 | + (void)rcu_nocb_try_flush_bypass(rdp, j); | |
| 1972 | + bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); | |
| 1973 | + } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { | |
| 1974 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 1975 | + continue; /* No callbacks here, try next. */ | |
| 1976 | + } | |
| 1977 | + if (bypass_ncbs) { | |
| 1978 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, | |
| 1979 | + TPS("Bypass")); | |
| 1980 | + bypass = true; | |
| 1981 | + } | |
| 1982 | + rnp = rdp->mynode; | |
| 1983 | + if (bypass) { // Avoid race with first bypass CB. | |
| 1984 | + WRITE_ONCE(my_rdp->nocb_defer_wakeup, | |
| 1985 | + RCU_NOCB_WAKE_NOT); | |
| 1986 | + del_timer(&my_rdp->nocb_timer); | |
| 1987 | + } | |
| 1988 | + // Advance callbacks if helpful and low contention. | |
| 1989 | + needwake_gp = false; | |
| 1990 | + if (!rcu_segcblist_restempty(&rdp->cblist, | |
| 1991 | + RCU_NEXT_READY_TAIL) || | |
| 1992 | + (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && | |
| 1993 | + rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) { | |
| 1994 | + raw_spin_lock_rcu_node(rnp); /* irqs disabled. */ | |
| 1995 | + needwake_gp = rcu_advance_cbs(rnp, rdp); | |
| 1996 | + raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */ | |
| 1997 | + } | |
| 1998 | + // Need to wait on some grace period? | |
| 1999 | + WARN_ON_ONCE(!rcu_segcblist_restempty(&rdp->cblist, | |
| 2000 | + RCU_NEXT_READY_TAIL)); | |
| 2001 | + if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { | |
| 2002 | + if (!needwait_gp || | |
| 2003 | + ULONG_CMP_LT(cur_gp_seq, wait_gp_seq)) | |
| 2004 | + wait_gp_seq = cur_gp_seq; | |
| 2005 | + needwait_gp = true; | |
| 2006 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, | |
| 2007 | + TPS("NeedWaitGP")); | |
| 2008 | + } | |
| 2009 | + if (rcu_segcblist_ready_cbs(&rdp->cblist)) { | |
| 2010 | + needwake = rdp->nocb_cb_sleep; | |
| 2011 | + WRITE_ONCE(rdp->nocb_cb_sleep, false); | |
| 2012 | + smp_mb(); /* CB invocation -after- GP end. */ | |
| 2013 | + } else { | |
| 2014 | + needwake = false; | |
| 2015 | + } | |
| 2016 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 2017 | + if (needwake) { | |
| 2018 | + swake_up_one(&rdp->nocb_cb_wq); | |
| 2019 | + gotcbs = true; | |
| 2020 | + } | |
| 2021 | + if (needwake_gp) | |
| 1786 | 2022 | rcu_gp_kthread_wake(); |
| 1787 | 2023 | } |
| 1788 | 2024 | |
| 1789 | − /* | |
| 1790 | − * Wait for the grace period. Do so interruptibly to avoid messing | |
| 1791 | − * up the load average. | |
| 1792 | − */ | |
| 1793 | − trace_rcu_this_gp(rnp, rdp, c, TPS("StartWait")); | |
| 1794 | − for (;;) { | |
| 2025 | + my_rdp->nocb_gp_bypass = bypass; | |
| 2026 | + my_rdp->nocb_gp_gp = needwait_gp; | |
| 2027 | + my_rdp->nocb_gp_seq = needwait_gp ? wait_gp_seq : 0; | |
| 2028 | + if (bypass && !rcu_nocb_poll) { | |
| 2029 | + // At least one child with non-empty ->nocb_bypass, so set | |
| 2030 | + // timer in order to avoid stranding its callbacks. | |
| 2031 | + raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags); | |
| 2032 | + mod_timer(&my_rdp->nocb_bypass_timer, j + 2); | |
| 2033 | + raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags); | |
| 2034 | + } | |
| 2035 | + if (rcu_nocb_poll) { | |
| 2036 | + /* Polling, so trace if first poll in the series. */ | |
| 2037 | + if (gotcbs) | |
| 2038 | + trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll")); | |
| 2039 | + schedule_timeout_interruptible(1); | |
| 2040 | + } else if (!needwait_gp) { | |
| 2041 | + /* Wait for callbacks to appear. */ | |
| 2042 | + trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep")); | |
| 2043 | + swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq, | |
| 2044 | + !READ_ONCE(my_rdp->nocb_gp_sleep)); | |
| 2045 | + trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep")); | |
| 2046 | + } else { | |
| 2047 | + rnp = my_rdp->mynode; | |
| 2048 | + trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("StartWait")); | |
| 1795 | 2049 | swait_event_interruptible_exclusive( |
| 1796 | − rnp->nocb_gp_wq[rcu_seq_ctr(c) & 0x1], | |
| 1797 | − (d = rcu_seq_done(&rnp->gp_seq, c))); | |
| 1798 | − if (likely(d)) | |
| 1799 | − break; | |
| 1800 | − WARN_ON(signal_pending(current)); | |
| 1801 | − trace_rcu_this_gp(rnp, rdp, c, TPS("ResumeWait")); | |
| 2050 | + rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq) & 0x1], | |
| 2051 | + rcu_seq_done(&rnp->gp_seq, wait_gp_seq) || | |
| 2052 | + !READ_ONCE(my_rdp->nocb_gp_sleep)); | |
| 2053 | + trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("EndWait")); | |
| 1802 | 2054 | } |
| 1803 | − trace_rcu_this_gp(rnp, rdp, c, TPS("EndWait")); | |
| 1804 | − smp_mb(); /* Ensure that CB invocation happens after GP end. */ | |
| 2055 | + if (!rcu_nocb_poll) { | |
| 2056 | + raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags); | |
| 2057 | + if (bypass) | |
| 2058 | + del_timer(&my_rdp->nocb_bypass_timer); | |
| 2059 | + WRITE_ONCE(my_rdp->nocb_gp_sleep, true); | |
| 2060 | + raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags); | |
| 2061 | + } | |
| 2062 | + my_rdp->nocb_gp_seq = -1; | |
| 2063 | + WARN_ON(signal_pending(current)); | |
| 1805 | 2064 | } |
| 1806 | 2065 | |
| 1807 | 2066 | /* |
| 1808 | − * Leaders come here to wait for additional callbacks to show up. | |
| 1809 | − * This function does not return until callbacks appear. | |
| 2067 | + * No-CBs grace-period-wait kthread. There is one of these per group | |
| 2068 | + * of CPUs, but only once at least one CPU in that group has come online | |
| 2069 | + * at least once since boot. This kthread checks for newly posted | |
| 2070 | + * callbacks from any of the CPUs it is responsible for, waits for a | |
| 2071 | + * grace period, then awakens all of the rcu_nocb_cb_kthread() instances | |
| 2072 | + * that then have callback-invocation work to do. | |
| 1810 | 2073 | */ |
| 1811 | −static void nocb_leader_wait(struct rcu_data *my_rdp) | |
| 2074 | +static int rcu_nocb_gp_kthread(void *arg) | |
| 1812 | 2075 | { |
| 1813 | − bool firsttime = true; | |
| 1814 | − unsigned long flags; | |
| 1815 | − bool gotcbs; | |
| 1816 | − struct rcu_data *rdp; | |
| 1817 | − struct rcu_head **tail; | |
| 1818 | − | |
| 1819 | −wait_again: | |
| 1820 | − | |
| 1821 | − /* Wait for callbacks to appear. */ | |
| 1822 | − if (!rcu_nocb_poll) { | |
| 1823 | − trace_rcu_nocb_wake(rcu_state.name, my_rdp->cpu, TPS("Sleep")); | |
| 1824 | − swait_event_interruptible_exclusive(my_rdp->nocb_wq, | |
| 1825 | − !READ_ONCE(my_rdp->nocb_leader_sleep)); | |
| 1826 | − raw_spin_lock_irqsave(&my_rdp->nocb_lock, flags); | |
| 1827 | − my_rdp->nocb_leader_sleep = true; | |
| 1828 | − WRITE_ONCE(my_rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT); | |
| 1829 | − del_timer(&my_rdp->nocb_timer); | |
| 1830 | − raw_spin_unlock_irqrestore(&my_rdp->nocb_lock, flags); | |
| 1831 | − } else if (firsttime) { | |
| 1832 | − firsttime = false; /* Don't drown trace log with "Poll"! */ | |
| 1833 | − trace_rcu_nocb_wake(rcu_state.name, my_rdp->cpu, TPS("Poll")); | |
| 1834 | − } | |
| 1835 | − | |
| 1836 | − /* | |
| 1837 | − * Each pass through the following loop checks a follower for CBs. | |
| 1838 | − * We are our own first follower. Any CBs found are moved to | |
| 1839 | − * nocb_gp_head, where they await a grace period. | |
| 1840 | − */ | |
| 1841 | − gotcbs = false; | |
| 1842 | − smp_mb(); /* wakeup and _sleep before ->nocb_head reads. */ | |
| 1843 | − for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) { | |
| 1844 | − rdp->nocb_gp_head = READ_ONCE(rdp->nocb_head); | |
| 1845 | − if (!rdp->nocb_gp_head) | |
| 1846 | − continue; /* No CBs here, try next follower. */ | |
| 1847 | − | |
| 1848 | − /* Move callbacks to wait-for-GP list, which is empty. */ | |
| 1849 | − WRITE_ONCE(rdp->nocb_head, NULL); | |
| 1850 | − rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head); | |
| 1851 | − gotcbs = true; | |
| 1852 | − } | |
| 1853 | − | |
| 1854 | − /* No callbacks? Sleep a bit if polling, and go retry. */ | |
| 1855 | − if (unlikely(!gotcbs)) { | |
| 1856 | − WARN_ON(signal_pending(current)); | |
| 1857 | − if (rcu_nocb_poll) { | |
| 1858 | − schedule_timeout_interruptible(1); | |
| 1859 | − } else { | |
| 1860 | − trace_rcu_nocb_wake(rcu_state.name, my_rdp->cpu, | |
| 1861 | − TPS("WokeEmpty")); | |
| 1862 | − } | |
| 1863 | − goto wait_again; | |
| 1864 | − } | |
| 2076 | + struct rcu_data *rdp = arg; | |
| 1865 | 2077 | |
| 1866 | − /* Wait for one grace period. */ | |
| 1867 | − rcu_nocb_wait_gp(my_rdp); | |
| 1868 | − | |
| 1869 | − /* Each pass through the following loop wakes a follower, if needed. */ | |
| 1870 | − for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) { | |
| 1871 | − if (!rcu_nocb_poll && | |
| 1872 | − READ_ONCE(rdp->nocb_head) && | |
| 1873 | − READ_ONCE(my_rdp->nocb_leader_sleep)) { | |
| 1874 | − raw_spin_lock_irqsave(&my_rdp->nocb_lock, flags); | |
| 1875 | − my_rdp->nocb_leader_sleep = false;/* No need to sleep.*/ | |
| 1876 | − raw_spin_unlock_irqrestore(&my_rdp->nocb_lock, flags); | |
| 1877 | − } | |
| 1878 | − if (!rdp->nocb_gp_head) | |
| 1879 | − continue; /* No CBs, so no need to wake follower. */ | |
| 1880 | − | |
| 1881 | − /* Append callbacks to follower's "done" list. */ | |
| 1882 | − raw_spin_lock_irqsave(&rdp->nocb_lock, flags); | |
| 1883 | − tail = rdp->nocb_follower_tail; | |
| 1884 | − rdp->nocb_follower_tail = rdp->nocb_gp_tail; | |
| 1885 | − *tail = rdp->nocb_gp_head; | |
| 1886 | − raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); | |
| 1887 | − if (rdp != my_rdp && tail == &rdp->nocb_follower_head) { | |
| 1888 | − /* List was empty, so wake up the follower. */ | |
| 1889 | − swake_up_one(&rdp->nocb_wq); | |
| 1890 | − } | |
| 2078 | + for (;;) { | |
| 2079 | + WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1); | |
| 2080 | + nocb_gp_wait(rdp); | |
| 2081 | + cond_resched_tasks_rcu_qs(); | |
| 1891 | 2082 | } |
| 1892 | − | |
| 1893 | − /* If we (the leader) don't have CBs, go wait some more. */ | |
| 1894 | − if (!my_rdp->nocb_follower_head) | |
| 1895 | − goto wait_again; | |
| 2083 | + return 0; | |
| 1896 | 2084 | } |
| 1897 | 2085 | |
| 1898 | 2086 | /* |
| 1899 | − * Followers come here to wait for additional callbacks to show up. | |
| 1900 | − * This function does not return until callbacks appear. | |
| 2087 | + * Invoke any ready callbacks from the corresponding no-CBs CPU, | |
| 2088 | + * then, if there are no more, wait for more to appear. | |
| 1901 | 2089 | */ |
| 1902 | −static void nocb_follower_wait(struct rcu_data *rdp) | |
| 2090 | +static void nocb_cb_wait(struct rcu_data *rdp) | |
| 1903 | 2091 | { |
| 1904 | − for (;;) { | |
| 1905 | − trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FollowerSleep")); | |
| 1906 | − swait_event_interruptible_exclusive(rdp->nocb_wq, | |
| 1907 | − READ_ONCE(rdp->nocb_follower_head)); | |
| 1908 | − if (smp_load_acquire(&rdp->nocb_follower_head)) { | |
| 1909 | − /* ^^^ Ensure CB invocation follows _head test. */ | |
| 1910 | − return; | |
| 1911 | − } | |
| 1912 | − WARN_ON(signal_pending(current)); | |
| 1913 | − trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty")); | |
| 2092 | + unsigned long cur_gp_seq; | |
| 2093 | + unsigned long flags; | |
| 2094 | + bool needwake_gp = false; | |
| 2095 | + struct rcu_node *rnp = rdp->mynode; | |
| 2096 | + | |
| 2097 | + local_irq_save(flags); | |
| 2098 | + rcu_momentary_dyntick_idle(); | |
| 2099 | + local_irq_restore(flags); | |
| 2100 | + local_bh_disable(); | |
| 2101 | + rcu_do_batch(rdp); | |
| 2102 | + local_bh_enable(); | |
| 2103 | + lockdep_assert_irqs_enabled(); | |
| 2104 | + rcu_nocb_lock_irqsave(rdp, flags); | |
| 2105 | + if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && | |
| 2106 | + rcu_seq_done(&rnp->gp_seq, cur_gp_seq) && | |
| 2107 | + raw_spin_trylock_rcu_node(rnp)) { /* irqs already disabled. */ | |
| 2108 | + needwake_gp = rcu_advance_cbs(rdp->mynode, rdp); | |
| 2109 | + raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ | |
| 2110 | + } | |
| 2111 | + if (rcu_segcblist_ready_cbs(&rdp->cblist)) { | |
| 2112 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 2113 | + if (needwake_gp) | |
| 2114 | + rcu_gp_kthread_wake(); | |
| 2115 | + return; | |
| 2116 | + } | |
| 2117 | + | |
| 2118 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep")); | |
| 2119 | + WRITE_ONCE(rdp->nocb_cb_sleep, true); | |
| 2120 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 2121 | + if (needwake_gp) | |
| 2122 | + rcu_gp_kthread_wake(); | |
| 2123 | + swait_event_interruptible_exclusive(rdp->nocb_cb_wq, | |
| 2124 | + !READ_ONCE(rdp->nocb_cb_sleep)); | |
| 2125 | + if (!smp_load_acquire(&rdp->nocb_cb_sleep)) { /* VVV */ | |
| 2126 | + /* ^^^ Ensure CB invocation follows _sleep test. */ | |
| 2127 | + return; | |
| 1914 | 2128 | } |
| 2129 | + WARN_ON(signal_pending(current)); | |
| 2130 | + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty")); | |
| 1915 | 2131 | } |
| 1916 | 2132 | |
| 1917 | 2133 | /* |
| 1918 | − * Per-rcu_data kthread, but only for no-CBs CPUs. Each kthread invokes | |
| 1919 | − * callbacks queued by the corresponding no-CBs CPU, however, there is | |
| 1920 | − * an optional leader-follower relationship so that the grace-period | |
| 1921 | − * kthreads don't have to do quite so many wakeups. | |
| 2134 | + * Per-rcu_data kthread, but only for no-CBs CPUs. Repeatedly invoke | |
| 2135 | + * nocb_cb_wait() to do the dirty work. | |
| 1922 | 2136 | */ |
| 1923 | −static int rcu_nocb_kthread(void *arg) | |
| 2137 | +static int rcu_nocb_cb_kthread(void *arg) | |
| 1924 | 2138 | { |
| 1925 | − int c, cl; | |
| 1926 | − unsigned long flags; | |
| 1927 | − struct rcu_head *list; | |
| 1928 | − struct rcu_head *next; | |
| 1929 | − struct rcu_head **tail; | |
| 1930 | 2139 | struct rcu_data *rdp = arg; |
| 1931 | 2140 | |
| 1932 | − /* Each pass through this loop invokes one batch of callbacks */ | |
| 2141 | + // Each pass through this loop does one callback batch, and, | |
| 2142 | + // if there are no more ready callbacks, waits for them. | |
| 1933 | 2143 | for (;;) { |
| 1934 | − /* Wait for callbacks. */ | |
| 1935 | − if (rdp->nocb_leader == rdp) | |
| 1936 | − nocb_leader_wait(rdp); | |
| 1937 | − else | |
| 1938 | − nocb_follower_wait(rdp); | |
| 1939 | − | |
| 1940 | − /* Pull the ready-to-invoke callbacks onto local list. */ | |
| 1941 | − raw_spin_lock_irqsave(&rdp->nocb_lock, flags); | |
| 1942 | − list = rdp->nocb_follower_head; | |
| 1943 | − rdp->nocb_follower_head = NULL; | |
| 1944 | − tail = rdp->nocb_follower_tail; | |
| 1945 | − rdp->nocb_follower_tail = &rdp->nocb_follower_head; | |
| 1946 | − raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); | |
| 1947 | − if (WARN_ON_ONCE(!list)) | |
| 1948 | − continue; | |
| 1949 | − trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeNonEmpty")); | |
| 1950 | − | |
| 1951 | − /* Each pass through the following loop invokes a callback. */ | |
| 1952 | − trace_rcu_batch_start(rcu_state.name, | |
| 1953 | − atomic_long_read(&rdp->nocb_q_count_lazy), | |
| 1954 | − rcu_get_n_cbs_nocb_cpu(rdp), -1); | |
| 1955 | − c = cl = 0; | |
| 1956 | − while (list) { | |
| 1957 | − next = list->next; | |
| 1958 | − /* Wait for enqueuing to complete, if needed. */ | |
| 1959 | − while (next == NULL && &list->next != tail) { | |
| 1960 | − trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, | |
| 1961 | − TPS("WaitQueue")); | |
| 1962 | − schedule_timeout_interruptible(1); | |
| 1963 | − trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, | |
| 1964 | − TPS("WokeQueue")); | |
| 1965 | − next = list->next; | |
| 1966 | − } | |
| 1967 | − debug_rcu_head_unqueue(list); | |
| 1968 | − local_bh_disable(); | |
| 1969 | − if (__rcu_reclaim(rcu_state.name, list)) | |
| 1970 | − cl++; | |
| 1971 | − c++; | |
| 1972 | − local_bh_enable(); | |
| 1973 | − cond_resched_tasks_rcu_qs(); | |
| 1974 | − list = next; | |
| 1975 | − } | |
| 1976 | − trace_rcu_batch_end(rcu_state.name, c, !!list, 0, 0, 1); | |
| 1977 | − smp_mb__before_atomic(); /* _add after CB invocation. */ | |
| 1978 | − atomic_long_add(-c, &rdp->nocb_q_count); | |
| 1979 | − atomic_long_add(-cl, &rdp->nocb_q_count_lazy); | |
| 2144 | + nocb_cb_wait(rdp); | |
| 2145 | + cond_resched_tasks_rcu_qs(); | |
| 1980 | 2146 | } |
| 1981 | 2147 | return 0; |
| 1982 | 2148 | } |
| @@ -1993,14 +2159,14 @@ static void do_nocb_deferred_wakeup_common(struct rcu_data *rdp) · 10 unmodified lines | ||
| 1993 | 2159 | unsigned long flags; |
| 1994 | 2160 | int ndw; |
| 1995 | 2161 | |
| 1996 | − raw_spin_lock_irqsave(&rdp->nocb_lock, flags); | |
| 2162 | + rcu_nocb_lock_irqsave(rdp, flags); | |
| 1997 | 2163 | if (!rcu_nocb_need_deferred_wakeup(rdp)) { |
| 1998 | − raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); | |
| 2164 | + rcu_nocb_unlock_irqrestore(rdp, flags); | |
| 1999 | 2165 | return; |
| 2000 | 2166 | } |
| 2001 | 2167 | ndw = READ_ONCE(rdp->nocb_defer_wakeup); |
| 2002 | 2168 | WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT); |
| 2003 | − __wake_nocb_leader(rdp, ndw == RCU_NOCB_WAKE_FORCE, flags); | |
| 2169 | + wake_nocb_gp(rdp, ndw == RCU_NOCB_WAKE_FORCE, flags); | |
| 2004 | 2170 | trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake")); |
| 2005 | 2171 | } |
| 2006 | 2172 | |
| @@ -2027,6 +2193,7 @@ void __init rcu_init_nohz(void) · 20 unmodified lines | ||
| 2027 | 2193 | { |
| 2028 | 2194 | int cpu; |
| 2029 | 2195 | bool need_rcu_nocb_mask = false; |
| 2196 | + struct rcu_data *rdp; | |
| 2030 | 2197 | |
| 2031 | 2198 | #if defined(CONFIG_NO_HZ_FULL) |
| 2032 | 2199 | if (tick_nohz_full_running && cpumask_weight(tick_nohz_full_mask)) |
| @@ -2060,67 +2227,63 @@ void __init rcu_init_nohz(void) · 27 unmodified lines | ||
| 2060 | 2227 | if (rcu_nocb_poll) |
| 2061 | 2228 | pr_info("\tPoll for callbacks from no-CBs CPUs.\n"); |
| 2062 | 2229 | |
| 2063 | − for_each_cpu(cpu, rcu_nocb_mask) | |
| 2064 | − init_nocb_callback_list(per_cpu_ptr(&rcu_data, cpu)); | |
| 2230 | + for_each_cpu(cpu, rcu_nocb_mask) { | |
| 2231 | + rdp = per_cpu_ptr(&rcu_data, cpu); | |
| 2232 | + if (rcu_segcblist_empty(&rdp->cblist)) | |
| 2233 | + rcu_segcblist_init(&rdp->cblist); | |
| 2234 | + rcu_segcblist_offload(&rdp->cblist); | |
| 2235 | + } | |
| 2065 | 2236 | rcu_organize_nocb_kthreads(); |
| 2066 | 2237 | } |
| 2067 | 2238 | |
| 2068 | 2239 | /* Initialize per-rcu_data variables for no-CBs CPUs. */ |
| 2069 | 2240 | static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) |
| 2070 | 2241 | { |
| 2071 | − rdp->nocb_tail = &rdp->nocb_head; | |
| 2072 | − init_swait_queue_head(&rdp->nocb_wq); | |
| 2073 | − rdp->nocb_follower_tail = &rdp->nocb_follower_head; | |
| 2242 | + init_swait_queue_head(&rdp->nocb_cb_wq); | |
| 2243 | + init_swait_queue_head(&rdp->nocb_gp_wq); | |
| 2074 | 2244 | raw_spin_lock_init(&rdp->nocb_lock); |
| 2245 | + raw_spin_lock_init(&rdp->nocb_bypass_lock); | |
| 2246 | + raw_spin_lock_init(&rdp->nocb_gp_lock); | |
| 2075 | 2247 | timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0); |
| 2248 | + timer_setup(&rdp->nocb_bypass_timer, do_nocb_bypass_wakeup_timer, 0); | |
| 2249 | + rcu_cblist_init(&rdp->nocb_bypass); | |
| 2076 | 2250 | } |
| 2077 | 2251 | |
| 2078 | 2252 | /* |
| 2079 | 2253 | * If the specified CPU is a no-CBs CPU that does not already have its |
| 2080 | − * rcuo kthread, spawn it. If the CPUs are brought online out of order, | |
| 2081 | − * this can require re-organizing the leader-follower relationships. | |
| 2254 | + * rcuo CB kthread, spawn it. Additionally, if the rcuo GP kthread | |
| 2255 | + * for this CPU's group has not yet been created, spawn it as well. | |
| 2082 | 2256 | */ |
| 2083 | 2257 | static void rcu_spawn_one_nocb_kthread(int cpu) |
| 2084 | 2258 | { |
| 2085 | − struct rcu_data *rdp; | |
| 2086 | − struct rcu_data *rdp_last; | |
| 2087 | − struct rcu_data *rdp_old_leader; | |
| 2088 | − struct rcu_data *rdp_spawn = per_cpu_ptr(&rcu_data, cpu); | |
| 2259 | + struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); | |
| 2260 | + struct rcu_data *rdp_gp; | |
| 2089 | 2261 | struct task_struct *t; |
| 2090 | 2262 | |
| 2091 | 2263 | /* |
| 2092 | 2264 | * If this isn't a no-CBs CPU or if it already has an rcuo kthread, |
| 2093 | 2265 | * then nothing to do. |
| 2094 | 2266 | */ |
| 2095 | − if (!rcu_is_nocb_cpu(cpu) || rdp_spawn->nocb_kthread) | |
| 2267 | + if (!rcu_is_nocb_cpu(cpu) || rdp->nocb_cb_kthread) | |
| 2096 | 2268 | return; |
| 2097 | 2269 | |
| 2098 | − /* If we didn't spawn the leader first, reorganize! */ | |
| 2099 | − rdp_old_leader = rdp_spawn->nocb_leader; | |
| 2100 | − if (rdp_old_leader != rdp_spawn && !rdp_old_leader->nocb_kthread) { | |
| 2101 | − rdp_last = NULL; | |
| 2102 | − rdp = rdp_old_leader; | |
| 2103 | − do { | |
| 2104 | − rdp->nocb_leader = rdp_spawn; | |
| 2105 | − if (rdp_last && rdp != rdp_spawn) | |
| 2106 | − rdp_last->nocb_next_follower = rdp; | |
| 2107 | − if (rdp == rdp_spawn) { | |
| 2108 | − rdp = rdp->nocb_next_follower; | |
| 2109 | − } else { | |
| 2110 | − rdp_last = rdp; | |
| 2111 | − rdp = rdp->nocb_next_follower; | |
| 2112 | − rdp_last->nocb_next_follower = NULL; | |
| 2113 | − } | |
| 2114 | − } while (rdp); | |
| 2115 | − rdp_spawn->nocb_next_follower = rdp_old_leader; | |
| 2270 | + /* If we didn't spawn the GP kthread first, reorganize! */ | |
| 2271 | + rdp_gp = rdp->nocb_gp_rdp; | |
| 2272 | + if (!rdp_gp->nocb_gp_kthread) { | |
| 2273 | + t = kthread_run(rcu_nocb_gp_kthread, rdp_gp, | |
| 2274 | + "rcuog/%d", rdp_gp->cpu); | |
| 2275 | + if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo GP kthread, OOM is now expected behavior\n", __func__)) | |
| 2276 | + return; | |
| 2277 | + WRITE_ONCE(rdp_gp->nocb_gp_kthread, t); | |
| 2116 | 2278 | } |
| 2117 | 2279 | |
| 2118 | 2280 | /* Spawn the kthread for this CPU. */ |
| 2119 | − t = kthread_run(rcu_nocb_kthread, rdp_spawn, | |
| 2281 | + t = kthread_run(rcu_nocb_cb_kthread, rdp, | |
| 2120 | 2282 | "rcuo%c/%d", rcu_state.abbr, cpu); |
| 2121 | − if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo kthread, OOM is now expected behavior\n", __func__)) | |
| 2283 | + if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo CB kthread, OOM is now expected behavior\n", __func__)) | |
| 2122 | 2284 | return; |
| 2123 | − WRITE_ONCE(rdp_spawn->nocb_kthread, t); | |
| 2285 | + WRITE_ONCE(rdp->nocb_cb_kthread, t); | |
| 2286 | + WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread); | |
| 2124 | 2287 | } |
| 2125 | 2288 | |
| 2126 | 2289 | /* |
| @@ -2147,27 +2310,28 @@ static void __init rcu_spawn_nocb_kthreads(void) · 20 unmodified lines | ||
| 2147 | 2310 | rcu_spawn_cpu_nocb_kthread(cpu); |
| 2148 | 2311 | } |
| 2149 | 2312 | |
| 2150 | −/* How many follower CPU IDs per leader? Default of -1 for sqrt(nr_cpu_ids). */ | |
| 2151 | −static int rcu_nocb_leader_stride = -1; | |
| 2152 | −module_param(rcu_nocb_leader_stride, int, 0444); | |
| 2313 | +/* How many CB CPU IDs per GP kthread? Default of -1 for sqrt(nr_cpu_ids). */ | |
| 2314 | +static int rcu_nocb_gp_stride = -1; | |
| 2315 | +module_param(rcu_nocb_gp_stride, int, 0444); | |
| 2153 | 2316 | |
| 2154 | 2317 | /* |
| 2155 | − * Initialize leader-follower relationships for all no-CBs CPU. | |
| 2318 | + * Initialize GP-CB relationships for all no-CBs CPU. | |
| 2156 | 2319 | */ |
| 2157 | 2320 | static void __init rcu_organize_nocb_kthreads(void) |
| 2158 | 2321 | { |
| 2159 | 2322 | int cpu; |
| 2160 | − int ls = rcu_nocb_leader_stride; | |
| 2161 | − int nl = 0; /* Next leader. */ | |
| 2323 | + bool firsttime = true; | |
| 2324 | + int ls = rcu_nocb_gp_stride; | |
| 2325 | + int nl = 0; /* Next GP kthread. */ | |
| 2162 | 2326 | struct rcu_data *rdp; |
| 2163 | − struct rcu_data *rdp_leader = NULL; /* Suppress misguided gcc warn. */ | |
| 2327 | + struct rcu_data *rdp_gp = NULL; /* Suppress misguided gcc warn. */ | |
| 2164 | 2328 | struct rcu_data *rdp_prev = NULL; |
| 2165 | 2329 | |
| 2166 | 2330 | if (!cpumask_available(rcu_nocb_mask)) |
| 2167 | 2331 | return; |
| 2168 | 2332 | if (ls == -1) { |
| 2169 | − ls = int_sqrt(nr_cpu_ids); | |
| 2170 | − rcu_nocb_leader_stride = ls; | |
| 2333 | + ls = nr_cpu_ids / int_sqrt(nr_cpu_ids); | |
| 2334 | + rcu_nocb_gp_stride = ls; | |
| 2171 | 2335 | } |
| 2172 | 2336 | |
| 2173 | 2337 | /* |
| @@ -2178,39 +2342,24 @@ static void __init rcu_organize_nocb_kthreads(void) · 4 unmodified lines | ||
| 2178 | 2342 | for_each_cpu(cpu, rcu_nocb_mask) { |
| 2179 | 2343 | rdp = per_cpu_ptr(&rcu_data, cpu); |
| 2180 | 2344 | if (rdp->cpu >= nl) { |
| 2181 | − /* New leader, set up for followers & next leader. */ | |
| 2345 | + /* New GP kthread, set up for CBs & next GP. */ | |
| 2182 | 2346 | nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls; |
| 2183 | − rdp->nocb_leader = rdp; | |
| 2184 | − rdp_leader = rdp; | |
| 2347 | + rdp->nocb_gp_rdp = rdp; | |
| 2348 | + rdp_gp = rdp; | |
| 2349 | + if (!firsttime && dump_tree) | |
| 2350 | + pr_cont("\n"); | |
| 2351 | + firsttime = false; | |
| 2352 | + pr_alert("%s: No-CB GP kthread CPU %d:", __func__, cpu); | |
| 2185 | 2353 | } else { |
| 2186 | − /* Another follower, link to previous leader. */ | |
| 2187 | − rdp->nocb_leader = rdp_leader; | |
| 2188 | − rdp_prev->nocb_next_follower = rdp; | |
| 2354 | + /* Another CB kthread, link to previous GP kthread. */ | |
| 2355 | + rdp->nocb_gp_rdp = rdp_gp; | |
| 2356 | + rdp_prev->nocb_next_cb_rdp = rdp; | |
| 2357 | + pr_alert(" %d", cpu); | |
| 2189 | 2358 | } |
| 2190 | 2359 | rdp_prev = rdp; |
| 2191 | 2360 | } |
| 2192 | 2361 | } |
| 2193 | 2362 | |
| 2194 | −/* Prevent __call_rcu() from enqueuing callbacks on no-CBs CPUs */ | |
| 2195 | −static bool init_nocb_callback_list(struct rcu_data *rdp) | |
| 2196 | −{ | |
| 2197 | − if (!rcu_is_nocb_cpu(rdp->cpu)) | |
| 2198 | − return false; | |
| 2199 | − | |
| 2200 | − /* If there are early-boot callbacks, move them to nocb lists. */ | |
| 2201 | − if (!rcu_segcblist_empty(&rdp->cblist)) { | |
| 2202 | − rdp->nocb_head = rcu_segcblist_head(&rdp->cblist); | |
| 2203 | − rdp->nocb_tail = rcu_segcblist_tail(&rdp->cblist); | |
| 2204 | − atomic_long_set(&rdp->nocb_q_count, | |
| 2205 | − rcu_segcblist_n_cbs(&rdp->cblist)); | |
| 2206 | − atomic_long_set(&rdp->nocb_q_count_lazy, | |
| 2207 | − rcu_segcblist_n_lazy_cbs(&rdp->cblist)); | |
| 2208 | − rcu_segcblist_init(&rdp->cblist); | |
| 2209 | − } | |
| 2210 | − rcu_segcblist_disable(&rdp->cblist); | |
| 2211 | − return true; | |
| 2212 | −} | |
| 2213 | − | |
| 2214 | 2363 | /* |
| 2215 | 2364 | * Bind the current task to the offloaded CPUs. If there are no offloaded |
| 2216 | 2365 | * CPUs, leave the task unbound. Splat if the bind attempt fails. |
| @@ -2223,20 +2372,101 @@ void rcu_bind_current_to_nocb(void) · 6 unmodified lines | ||
| 2223 | 2372 | EXPORT_SYMBOL_GPL(rcu_bind_current_to_nocb); |
| 2224 | 2373 | |
| 2225 | 2374 | /* |
| 2226 | − * Return the number of RCU callbacks still queued from the specified | |
| 2227 | − * CPU, which must be a nocbs CPU. | |
| 2375 | + * Dump out nocb grace-period kthread state for the specified rcu_data | |
| 2376 | + * structure. | |
| 2228 | 2377 | */ |
| 2229 | −static unsigned long rcu_get_n_cbs_nocb_cpu(struct rcu_data *rdp) | |
| 2378 | +static void show_rcu_nocb_gp_state(struct rcu_data *rdp) | |
| 2230 | 2379 | { |
| 2231 | − return atomic_long_read(&rdp->nocb_q_count); | |
| 2380 | + struct rcu_node *rnp = rdp->mynode; | |
| 2381 | + | |
| 2382 | + pr_info("nocb GP %d %c%c%c%c%c%c %c[%c%c] %c%c:%ld rnp %d:%d %lu\n", | |
| 2383 | + rdp->cpu, | |
| 2384 | + "kK"[!!rdp->nocb_gp_kthread], | |
| 2385 | + "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)], | |
| 2386 | + "dD"[!!rdp->nocb_defer_wakeup], | |
| 2387 | + "tT"[timer_pending(&rdp->nocb_timer)], | |
| 2388 | + "bB"[timer_pending(&rdp->nocb_bypass_timer)], | |
| 2389 | + "sS"[!!rdp->nocb_gp_sleep], | |
| 2390 | + ".W"[swait_active(&rdp->nocb_gp_wq)], | |
| 2391 | + ".W"[swait_active(&rnp->nocb_gp_wq[0])], | |
| 2392 | + ".W"[swait_active(&rnp->nocb_gp_wq[1])], | |
| 2393 | + ".B"[!!rdp->nocb_gp_bypass], | |
| 2394 | + ".G"[!!rdp->nocb_gp_gp], | |
| 2395 | + (long)rdp->nocb_gp_seq, | |
| 2396 | + rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops)); | |
| 2397 | +} | |
| 2398 | + | |
| 2399 | +/* Dump out nocb kthread state for the specified rcu_data structure. */ | |
| 2400 | +static void show_rcu_nocb_state(struct rcu_data *rdp) | |
| 2401 | +{ | |
| 2402 | + struct rcu_segcblist *rsclp = &rdp->cblist; | |
| 2403 | + bool waslocked; | |
| 2404 | + bool wastimer; | |
| 2405 | + bool wassleep; | |
| 2406 | + | |
| 2407 | + if (rdp->nocb_gp_rdp == rdp) | |
| 2408 | + show_rcu_nocb_gp_state(rdp); | |
| 2409 | + | |
| 2410 | + pr_info(" CB %d->%d %c%c%c%c%c%c F%ld L%ld C%d %c%c%c%c%c q%ld\n", | |
| 2411 | + rdp->cpu, rdp->nocb_gp_rdp->cpu, | |
| 2412 | + "kK"[!!rdp->nocb_cb_kthread], | |
| 2413 | + "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)], | |
| 2414 | + "cC"[!!atomic_read(&rdp->nocb_lock_contended)], | |
| 2415 | + "lL"[raw_spin_is_locked(&rdp->nocb_lock)], | |
| 2416 | + "sS"[!!rdp->nocb_cb_sleep], | |
| 2417 | + ".W"[swait_active(&rdp->nocb_cb_wq)], | |
| 2418 | + jiffies - rdp->nocb_bypass_first, | |
| 2419 | + jiffies - rdp->nocb_nobypass_last, | |
| 2420 | + rdp->nocb_nobypass_count, | |
| 2421 | + ".D"[rcu_segcblist_ready_cbs(rsclp)], | |
| 2422 | + ".W"[!rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)], | |
| 2423 | + ".R"[!rcu_segcblist_restempty(rsclp, RCU_WAIT_TAIL)], | |
| 2424 | + ".N"[!rcu_segcblist_restempty(rsclp, RCU_NEXT_READY_TAIL)], | |
| 2425 | + ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)], | |
| 2426 | + rcu_segcblist_n_cbs(&rdp->cblist)); | |
| 2427 | + | |
| 2428 | + /* It is OK for GP kthreads to have GP state. */ | |
| 2429 | + if (rdp->nocb_gp_rdp == rdp) | |
| 2430 | + return; | |
| 2431 | + | |
| 2432 | + waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock); | |
| 2433 | + wastimer = timer_pending(&rdp->nocb_timer); | |
| 2434 | + wassleep = swait_active(&rdp->nocb_gp_wq); | |
| 2435 | + if (!rdp->nocb_defer_wakeup && !rdp->nocb_gp_sleep && | |
| 2436 | + !waslocked && !wastimer && !wassleep) | |
| 2437 | + return; /* Nothing untowards. */ | |
| 2438 | + | |
| 2439 | + pr_info(" !!! %c%c%c%c %c\n", | |
| 2440 | + "lL"[waslocked], | |
| 2441 | + "dD"[!!rdp->nocb_defer_wakeup], | |
| 2442 | + "tT"[wastimer], | |
| 2443 | + "sS"[!!rdp->nocb_gp_sleep], | |
| 2444 | + ".W"[wassleep]); | |
| 2232 | 2445 | } |
| 2233 | 2446 | |
| 2234 | 2447 | #else /* #ifdef CONFIG_RCU_NOCB_CPU */ |
| 2235 | 2448 | |
| 2236 | −static bool rcu_nocb_cpu_needs_barrier(int cpu) | |
| 2449 | +/* No ->nocb_lock to acquire. */ | |
| 2450 | +static void rcu_nocb_lock(struct rcu_data *rdp) | |
| 2451 | +{ | |
| 2452 | +} | |
| 2453 | + | |
| 2454 | +/* No ->nocb_lock to release. */ | |
| 2455 | +static void rcu_nocb_unlock(struct rcu_data *rdp) | |
| 2237 | 2456 | { |
| 2238 | − WARN_ON_ONCE(1); /* Should be dead code. */ | |
| 2239 | − return false; | |
| 2457 | +} | |
| 2458 | + | |
| 2459 | +/* No ->nocb_lock to release. */ | |
| 2460 | +static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, | |
| 2461 | + unsigned long flags) | |
| 2462 | +{ | |
| 2463 | + local_irq_restore(flags); | |
| 2464 | +} | |
| 2465 | + | |
| 2466 | +/* Lockdep check that ->cblist may be safely accessed. */ | |
| 2467 | +static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) | |
| 2468 | +{ | |
| 2469 | + lockdep_assert_irqs_disabled(); | |
| 2240 | 2470 | } |
| 2241 | 2471 | |
| 2242 | 2472 | static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq) |
| @@ -2252,19 +2482,24 @@ static void rcu_init_one_nocb(struct rcu_node *rnp) · 9 unmodified lines | ||
| 2252 | 2482 | { |
| 2253 | 2483 | } |
| 2254 | 2484 | |
| 2255 | −static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp, | |
| 2256 | − bool lazy, unsigned long flags) | |
| 2485 | +static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, | |
| 2486 | + unsigned long j) | |
| 2257 | 2487 | { |
| 2258 | − return false; | |
| 2488 | + return true; | |
| 2259 | 2489 | } |
| 2260 | 2490 | |
| 2261 | −static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp, | |
| 2262 | − struct rcu_data *rdp, | |
| 2263 | − unsigned long flags) | |
| 2491 | +static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, | |
| 2492 | + bool *was_alldone, unsigned long flags) | |
| 2264 | 2493 | { |
| 2265 | 2494 | return false; |
| 2266 | 2495 | } |
| 2267 | 2496 | |
| 2497 | +static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, | |
| 2498 | + unsigned long flags) | |
| 2499 | +{ | |
| 2500 | + WARN_ON_ONCE(1); /* Should be dead code! */ | |
| 2501 | +} | |
| 2502 | + | |
| 2268 | 2503 | static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) |
| 2269 | 2504 | { |
| 2270 | 2505 | } |
| @@ -2286,14 +2521,8 @@ static void __init rcu_spawn_nocb_kthreads(void) · 15 unmodified lines | ||
| 2286 | 2521 | { |
| 2287 | 2522 | } |
| 2288 | 2523 | |
| 2289 | −static bool init_nocb_callback_list(struct rcu_data *rdp) | |
| 2290 | −{ | |
| 2291 | − return false; | |
| 2292 | −} | |
| 2293 | − | |
| 2294 | −static unsigned long rcu_get_n_cbs_nocb_cpu(struct rcu_data *rdp) | |
| 2524 | +static void show_rcu_nocb_state(struct rcu_data *rdp) | |
| 2295 | 2525 | { |
| 2296 | − return 0; | |
| 2297 | 2526 | } |
| 2298 | 2527 | |
| 2299 | 2528 | #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ |
| @@ -527,6 +527,8 @@ static void check_cpu_stall(struct rcu_data *rdp) · 526 unmodified lines | ||
| 527 | 527 | |
| 528 | 528 | /* We haven't checked in, so go dump stack. */ |
| 529 | 529 | print_cpu_stall(); |
| 530 | + if (rcu_cpu_stall_ftrace_dump) | |
| 531 | + rcu_ftrace_dump(DUMP_ALL); | |
| 530 | 532 | |
| 531 | 533 | } else if (rcu_gp_in_progress() && |
| 532 | 534 | ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY) && |
| @@ -534,6 +536,8 @@ static void check_cpu_stall(struct rcu_data *rdp) · 1 unmodified line | ||
| 534 | 536 | |
| 535 | 537 | /* They had a few time units to dump stack, so complain. */ |
| 536 | 538 | print_other_cpu_stall(gs2); |
| 539 | + if (rcu_cpu_stall_ftrace_dump) | |
| 540 | + rcu_ftrace_dump(DUMP_ALL); | |
| 537 | 541 | } |
| 538 | 542 | } |
| 539 | 543 | |
| @@ -585,6 +589,11 @@ void show_rcu_gp_kthreads(void) · 45 unmodified lines | ||
| 585 | 589 | cpu, (long)rdp->gp_seq_needed); |
| 586 | 590 | } |
| 587 | 591 | } |
| 592 | + for_each_possible_cpu(cpu) { | |
| 593 | + rdp = per_cpu_ptr(&rcu_data, cpu); | |
| 594 | + if (rcu_segcblist_is_offloaded(&rdp->cblist)) | |
| 595 | + show_rcu_nocb_state(rdp); | |
| 596 | + } | |
| 588 | 597 | /* sched_show_task(rcu_state.gp_kthread); */ |
| 589 | 598 | } |
| 590 | 599 | EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads); |
| @@ -61,9 +61,15 @@ module_param(rcu_normal_after_boot, int, 0); · 60 unmodified lines | ||
| 61 | 61 | |
| 62 | 62 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
| 63 | 63 | /** |
| 64 | − * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section? | |
| 64 | + * rcu_read_lock_held_common() - might we be in RCU-sched read-side critical section? | |
| 65 | + * @ret: Best guess answer if lockdep cannot be relied on | |
| 65 | 66 | * |
| 66 | − * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an | |
| 67 | + * Returns true if lockdep must be ignored, in which case *ret contains | |
| 68 | + * the best guess described below. Otherwise returns false, in which | |
| 69 | + * case *ret tells the caller nothing and the caller should instead | |
| 70 | + * consult lockdep. | |
| 71 | + * | |
| 72 | + * If CONFIG_DEBUG_LOCK_ALLOC is selected, set *ret to nonzero iff in an | |
| 67 | 73 | * RCU-sched read-side critical section. In absence of |
| 68 | 74 | * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side |
| 69 | 75 | * critical section unless it can prove otherwise. Note that disabling |
| @@ -75,35 +81,45 @@ module_param(rcu_normal_after_boot, int, 0); · 5 unmodified lines | ||
| 75 | 81 | * Check debug_lockdep_rcu_enabled() to prevent false positives during boot |
| 76 | 82 | * and while lockdep is disabled. |
| 77 | 83 | * |
| 78 | − * Note that if the CPU is in the idle loop from an RCU point of | |
| 79 | − * view (ie: that we are in the section between rcu_idle_enter() and | |
| 80 | − * rcu_idle_exit()) then rcu_read_lock_held() returns false even if the CPU | |
| 81 | − * did an rcu_read_lock(). The reason for this is that RCU ignores CPUs | |
| 82 | − * that are in such a section, considering these as in extended quiescent | |
| 83 | − * state, so such a CPU is effectively never in an RCU read-side critical | |
| 84 | − * section regardless of what RCU primitives it invokes. This state of | |
| 85 | − * affairs is required --- we need to keep an RCU-free window in idle | |
| 86 | − * where the CPU may possibly enter into low power mode. This way we can | |
| 87 | − * notice an extended quiescent state to other CPUs that started a grace | |
| 88 | − * period. Otherwise we would delay any grace period as long as we run in | |
| 89 | − * the idle task. | |
| 84 | + * Note that if the CPU is in the idle loop from an RCU point of view (ie: | |
| 85 | + * that we are in the section between rcu_idle_enter() and rcu_idle_exit()) | |
| 86 | + * then rcu_read_lock_held() sets *ret to false even if the CPU did an | |
| 87 | + * rcu_read_lock(). The reason for this is that RCU ignores CPUs that are | |
| 88 | + * in such a section, considering these as in extended quiescent state, | |
| 89 | + * so such a CPU is effectively never in an RCU read-side critical section | |
| 90 | + * regardless of what RCU primitives it invokes. This state of affairs is | |
| 91 | + * required --- we need to keep an RCU-free window in idle where the CPU may | |
| 92 | + * possibly enter into low power mode. This way we can notice an extended | |
| 93 | + * quiescent state to other CPUs that started a grace period. Otherwise | |
| 94 | + * we would delay any grace period as long as we run in the idle task. | |
| 90 | 95 | * |
| 91 | − * Similarly, we avoid claiming an SRCU read lock held if the current | |
| 96 | + * Similarly, we avoid claiming an RCU read lock held if the current | |
| 92 | 97 | * CPU is offline. |
| 93 | 98 | */ |
| 99 | +static bool rcu_read_lock_held_common(bool *ret) | |
| 100 | +{ | |
| 101 | + if (!debug_lockdep_rcu_enabled()) { | |
| 102 | + *ret = 1; | |
| 103 | + return true; | |
| 104 | + } | |
| 105 | + if (!rcu_is_watching()) { | |
| 106 | + *ret = 0; | |
| 107 | + return true; | |
| 108 | + } | |
| 109 | + if (!rcu_lockdep_current_cpu_online()) { | |
| 110 | + *ret = 0; | |
| 111 | + return true; | |
| 112 | + } | |
| 113 | + return false; | |
| 114 | +} | |
| 115 | + | |
| 94 | 116 | int rcu_read_lock_sched_held(void) |
| 95 | 117 | { |
| 96 | − int lockdep_opinion = 0; | |
| 118 | + bool ret; | |
| 97 | 119 | |
| 98 | − if (!debug_lockdep_rcu_enabled()) | |
| 99 | − return 1; | |
| 100 | − if (!rcu_is_watching()) | |
| 101 | − return 0; | |
| 102 | − if (!rcu_lockdep_current_cpu_online()) | |
| 103 | − return 0; | |
| 104 | − if (debug_locks) | |
| 105 | − lockdep_opinion = lock_is_held(&rcu_sched_lock_map); | |
| 106 | − return lockdep_opinion || !preemptible(); | |
| 120 | + if (rcu_read_lock_held_common(&ret)) | |
| 121 | + return ret; | |
| 122 | + return lock_is_held(&rcu_sched_lock_map) || !preemptible(); | |
| 107 | 123 | } |
| 108 | 124 | EXPORT_SYMBOL(rcu_read_lock_sched_held); |
| 109 | 125 | #endif |
| @@ -136,8 +152,7 @@ static atomic_t rcu_expedited_nesting = ATOMIC_INIT(1); · 26 unmodified lines | ||
| 136 | 152 | */ |
| 137 | 153 | bool rcu_gp_is_expedited(void) |
| 138 | 154 | { |
| 139 | − return rcu_expedited || atomic_read(&rcu_expedited_nesting) || | |
| 140 | − rcu_scheduler_active == RCU_SCHEDULER_INIT; | |
| 155 | + return rcu_expedited || atomic_read(&rcu_expedited_nesting); | |
| 141 | 156 | } |
| 142 | 157 | EXPORT_SYMBOL_GPL(rcu_gp_is_expedited); |
| 143 | 158 | |
| @@ -261,12 +276,10 @@ NOKPROBE_SYMBOL(debug_lockdep_rcu_enabled); · 117 unmodified lines | ||
| 261 | 276 | */ |
| 262 | 277 | int rcu_read_lock_held(void) |
| 263 | 278 | { |
| 264 | − if (!debug_lockdep_rcu_enabled()) | |
| 265 | − return 1; | |
| 266 | − if (!rcu_is_watching()) | |
| 267 | − return 0; | |
| 268 | − if (!rcu_lockdep_current_cpu_online()) | |
| 269 | − return 0; | |
| 279 | + bool ret; | |
| 280 | + | |
| 281 | + if (rcu_read_lock_held_common(&ret)) | |
| 282 | + return ret; | |
| 270 | 283 | return lock_is_held(&rcu_lock_map); |
| 271 | 284 | } |
| 272 | 285 | EXPORT_SYMBOL_GPL(rcu_read_lock_held); |
| @@ -288,16 +301,28 @@ EXPORT_SYMBOL_GPL(rcu_read_lock_held); · 15 unmodified lines | ||
| 288 | 301 | */ |
| 289 | 302 | int rcu_read_lock_bh_held(void) |
| 290 | 303 | { |
| 291 | − if (!debug_lockdep_rcu_enabled()) | |
| 292 | − return 1; | |
| 293 | − if (!rcu_is_watching()) | |
| 294 | − return 0; | |
| 295 | − if (!rcu_lockdep_current_cpu_online()) | |
| 296 | − return 0; | |
| 304 | + bool ret; | |
| 305 | + | |
| 306 | + if (rcu_read_lock_held_common(&ret)) | |
| 307 | + return ret; | |
| 297 | 308 | return in_softirq() || irqs_disabled(); |
| 298 | 309 | } |
| 299 | 310 | EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held); |
| 300 | 311 | |
| 312 | +int rcu_read_lock_any_held(void) | |
| 313 | +{ | |
| 314 | + bool ret; | |
| 315 | + | |
| 316 | + if (rcu_read_lock_held_common(&ret)) | |
| 317 | + return ret; | |
| 318 | + if (lock_is_held(&rcu_lock_map) || | |
| 319 | + lock_is_held(&rcu_bh_lock_map) || | |
| 320 | + lock_is_held(&rcu_sched_lock_map)) | |
| 321 | + return 1; | |
| 322 | + return !preemptible(); | |
| 323 | +} | |
| 324 | +EXPORT_SYMBOL_GPL(rcu_read_lock_any_held); | |
| 325 | + | |
| 301 | 326 | #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
| 302 | 327 | |
| 303 | 328 | /** |
| @@ -437,6 +462,8 @@ EXPORT_SYMBOL_GPL(rcutorture_sched_setaffinity); · 133 unmodified lines | ||
| 437 | 462 | #endif |
| 438 | 463 | |
| 439 | 464 | #ifdef CONFIG_RCU_STALL_COMMON |
| 465 | +int rcu_cpu_stall_ftrace_dump __read_mostly; | |
| 466 | +module_param(rcu_cpu_stall_ftrace_dump, int, 0644); | |
| 440 | 467 | int rcu_cpu_stall_suppress __read_mostly; /* 1 = suppress stall warnings. */ |
| 441 | 468 | EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress); |
| 442 | 469 | module_param(rcu_cpu_stall_suppress, int, 0644); |
| @@ -3486,8 +3486,36 @@ void scheduler_tick(void) · 3485 unmodified lines | ||
| 3486 | 3486 | |
| 3487 | 3487 | struct tick_work { |
| 3488 | 3488 | int cpu; |
| 3489 | + atomic_t state; | |
| 3489 | 3490 | struct delayed_work work; |
| 3490 | 3491 | }; |
| 3492 | +/* Values for ->state, see diagram below. */ | |
| 3493 | +#define TICK_SCHED_REMOTE_OFFLINE 0 | |
| 3494 | +#define TICK_SCHED_REMOTE_OFFLINING 1 | |
| 3495 | +#define TICK_SCHED_REMOTE_RUNNING 2 | |
| 3496 | + | |
| 3497 | +/* | |
| 3498 | + * State diagram for ->state: | |
| 3499 | + * | |
| 3500 | + * | |
| 3501 | + * TICK_SCHED_REMOTE_OFFLINE | |
| 3502 | + * | ^ | |
| 3503 | + * | | | |
| 3504 | + * | | sched_tick_remote() | |
| 3505 | + * | | | |
| 3506 | + * | | | |
| 3507 | + * +--TICK_SCHED_REMOTE_OFFLINING | |
| 3508 | + * | ^ | |
| 3509 | + * | | | |
| 3510 | + * sched_tick_start() | | sched_tick_stop() | |
| 3511 | + * | | | |
| 3512 | + * V | | |
| 3513 | + * TICK_SCHED_REMOTE_RUNNING | |
| 3514 | + * | |
| 3515 | + * | |
| 3516 | + * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote() | |
| 3517 | + * and sched_tick_start() are happy to leave the state in RUNNING. | |
| 3518 | + */ | |
| 3491 | 3519 | |
| 3492 | 3520 | static struct tick_work __percpu *tick_work_cpu; |
| 3493 | 3521 | |
| @@ -3500,6 +3528,7 @@ static void sched_tick_remote(struct work_struct *work) · 6 unmodified lines | ||
| 3500 | 3528 | struct task_struct *curr; |
| 3501 | 3529 | struct rq_flags rf; |
| 3502 | 3530 | u64 delta; |
| 3531 | + int os; | |
| 3503 | 3532 | |
| 3504 | 3533 | /* |
| 3505 | 3534 | * Handle the tick only if it appears the remote CPU is running in full |
| @@ -3513,7 +3542,7 @@ static void sched_tick_remote(struct work_struct *work) · 7 unmodified lines | ||
| 3513 | 3542 | |
| 3514 | 3543 | rq_lock_irq(rq, &rf); |
| 3515 | 3544 | curr = rq->curr; |
| 3516 | − if (is_idle_task(curr)) | |
| 3545 | + if (is_idle_task(curr) || cpu_is_offline(cpu)) | |
| 3517 | 3546 | goto out_unlock; |
| 3518 | 3547 | |
| 3519 | 3548 | update_rq_clock(rq); |
| @@ -3533,13 +3562,18 @@ out_requeue: · 13 unmodified lines | ||
| 3533 | 3562 | /* |
| 3534 | 3563 | * Run the remote tick once per second (1Hz). This arbitrary |
| 3535 | 3564 | * frequency is large enough to avoid overload but short enough |
| 3536 | − * to keep scheduler internal stats reasonably up to date. | |
| 3565 | + * to keep scheduler internal stats reasonably up to date. But | |
| 3566 | + * first update state to reflect hotplug activity if required. | |
| 3537 | 3567 | */ |
| 3538 | − queue_delayed_work(system_unbound_wq, dwork, HZ); | |
| 3568 | + os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING); | |
| 3569 | + WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE); | |
| 3570 | + if (os == TICK_SCHED_REMOTE_RUNNING) | |
| 3571 | + queue_delayed_work(system_unbound_wq, dwork, HZ); | |
| 3539 | 3572 | } |
| 3540 | 3573 | |
| 3541 | 3574 | static void sched_tick_start(int cpu) |
| 3542 | 3575 | { |
| 3576 | + int os; | |
| 3543 | 3577 | struct tick_work *twork; |
| 3544 | 3578 | |
| 3545 | 3579 | if (housekeeping_cpu(cpu, HK_FLAG_TICK)) |
| @@ -3548,15 +3582,20 @@ static void sched_tick_start(int cpu) · 2 unmodified lines | ||
| 3548 | 3582 | WARN_ON_ONCE(!tick_work_cpu); |
| 3549 | 3583 | |
| 3550 | 3584 | twork = per_cpu_ptr(tick_work_cpu, cpu); |
| 3551 | − twork->cpu = cpu; | |
| 3552 | − INIT_DELAYED_WORK(&twork->work, sched_tick_remote); | |
| 3553 | − queue_delayed_work(system_unbound_wq, &twork->work, HZ); | |
| 3585 | + os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING); | |
| 3586 | + WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING); | |
| 3587 | + if (os == TICK_SCHED_REMOTE_OFFLINE) { | |
| 3588 | + twork->cpu = cpu; | |
| 3589 | + INIT_DELAYED_WORK(&twork->work, sched_tick_remote); | |
| 3590 | + queue_delayed_work(system_unbound_wq, &twork->work, HZ); | |
| 3591 | + } | |
| 3554 | 3592 | } |
| 3555 | 3593 | |
| 3556 | 3594 | #ifdef CONFIG_HOTPLUG_CPU |
| 3557 | 3595 | static void sched_tick_stop(int cpu) |
| 3558 | 3596 | { |
| 3559 | 3597 | struct tick_work *twork; |
| 3598 | + int os; | |
| 3560 | 3599 | |
| 3561 | 3600 | if (housekeeping_cpu(cpu, HK_FLAG_TICK)) |
| 3562 | 3601 | return; |
| @@ -3564,7 +3603,10 @@ static void sched_tick_stop(int cpu) · 1 unmodified line | ||
| 3564 | 3603 | WARN_ON_ONCE(!tick_work_cpu); |
| 3565 | 3604 | |
| 3566 | 3605 | twork = per_cpu_ptr(tick_work_cpu, cpu); |
| 3567 | − cancel_delayed_work_sync(&twork->work); | |
| 3606 | + /* There cannot be competing actions, but don't rely on stop-machine. */ | |
| 3607 | + os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING); | |
| 3608 | + WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING); | |
| 3609 | + /* Don't cancel, as this would mess up the state machine. */ | |
| 3568 | 3610 | } |
| 3569 | 3611 | #endif /* CONFIG_HOTPLUG_CPU */ |
| 3570 | 3612 | |
| @@ -3572,7 +3614,6 @@ int __init sched_tick_offload_init(void) · 1 unmodified line | ||
| 3572 | 3614 | { |
| 3573 | 3615 | tick_work_cpu = alloc_percpu(struct tick_work); |
| 3574 | 3616 | BUG_ON(!tick_work_cpu); |
| 3575 | − | |
| 3576 | 3617 | return 0; |
| 3577 | 3618 | } |
| 3578 | 3619 | |
| @@ -241,13 +241,14 @@ static void do_idle(void) · 240 unmodified lines | ||
| 241 | 241 | check_pgt_cache(); |
| 242 | 242 | rmb(); |
| 243 | 243 | |
| 244 | + local_irq_disable(); | |
| 245 | + | |
| 244 | 246 | if (cpu_is_offline(cpu)) { |
| 245 | − tick_nohz_idle_stop_tick_protected(); | |
| 247 | + tick_nohz_idle_stop_tick(); | |
| 246 | 248 | cpuhp_report_idle_dead(); |
| 247 | 249 | arch_cpu_idle_dead(); |
| 248 | 250 | } |
| 249 | 251 | |
| 250 | − local_irq_disable(); | |
| 251 | 252 | arch_cpu_idle_enter(); |
| 252 | 253 | |
| 253 | 254 | /* |
| @@ -263,7 +263,6 @@ static void torture_onoff_cleanup(void) · 262 unmodified lines | ||
| 263 | 263 | onoff_task = NULL; |
| 264 | 264 | #endif /* #ifdef CONFIG_HOTPLUG_CPU */ |
| 265 | 265 | } |
| 266 | −EXPORT_SYMBOL_GPL(torture_onoff_cleanup); | |
| 267 | 266 | |
| 268 | 267 | /* |
| 269 | 268 | * Print online/offline testing statistics. |
| @@ -449,7 +448,6 @@ static void torture_shuffle_cleanup(void) · 179 unmodified lines | ||
| 449 | 448 | } |
| 450 | 449 | shuffler_task = NULL; |
| 451 | 450 | } |
| 452 | −EXPORT_SYMBOL_GPL(torture_shuffle_cleanup); | |
| 453 | 451 | |
| 454 | 452 | /* |
| 455 | 453 | * Variables for auto-shutdown. This allows "lights out" torture runs |
| @@ -6,22 +6,22 @@ · 5 unmodified lines | ||
| 6 | 6 | |
| 7 | 7 | /* |
| 8 | 8 | * Traverse the ftrace_global_list, invoking all entries. The reason that we |
| 9 | − * can use rcu_dereference_raw_notrace() is that elements removed from this list | |
| 9 | + * can use rcu_dereference_raw_check() is that elements removed from this list | |
| 10 | 10 | * are simply leaked, so there is no need to interact with a grace-period |
| 11 | − * mechanism. The rcu_dereference_raw_notrace() calls are needed to handle | |
| 11 | + * mechanism. The rcu_dereference_raw_check() calls are needed to handle | |
| 12 | 12 | * concurrent insertions into the ftrace_global_list. |
| 13 | 13 | * |
| 14 | 14 | * Silly Alpha and silly pointer-speculation compiler optimizations! |
| 15 | 15 | */ |
| 16 | 16 | #define do_for_each_ftrace_op(op, list) \ |
| 17 | − op = rcu_dereference_raw_notrace(list); \ | |
| 17 | + op = rcu_dereference_raw_check(list); \ | |
| 18 | 18 | do |
| 19 | 19 | |
| 20 | 20 | /* |
| 21 | 21 | * Optimized for just a single item in the list (as that is the normal case). |
| 22 | 22 | */ |
| 23 | 23 | #define while_for_each_ftrace_op(op) \ |
| 24 | − while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \ | |
| 24 | + while (likely(op = rcu_dereference_raw_check((op)->next)) && \ | |
| 25 | 25 | unlikely((op) != &ftrace_list_end)) |
| 26 | 26 | |
| 27 | 27 | extern struct ftrace_ops __rcu *ftrace_ops_list; |
| @@ -2642,10 +2642,10 @@ static void ftrace_exports(struct ring_buffer_event *event) · 2641 unmodified lines | ||
| 2642 | 2642 | |
| 2643 | 2643 | preempt_disable_notrace(); |
| 2644 | 2644 | |
| 2645 | − export = rcu_dereference_raw_notrace(ftrace_exports_list); | |
| 2645 | + export = rcu_dereference_raw_check(ftrace_exports_list); | |
| 2646 | 2646 | while (export) { |
| 2647 | 2647 | trace_process_export(export, event); |
| 2648 | − export = rcu_dereference_raw_notrace(export->next); | |
| 2648 | + export = rcu_dereference_raw_check(export->next); | |
| 2649 | 2649 | } |
| 2650 | 2650 | |
| 2651 | 2651 | preempt_enable_notrace(); |
| @@ -124,7 +124,8 @@ struct fib_table *fib_get_table(struct net *net, u32 id) · 123 unmodified lines | ||
| 124 | 124 | h = id & (FIB_TABLE_HASHSZ - 1); |
| 125 | 125 | |
| 126 | 126 | head = &net->ipv4.fib_table_hash[h]; |
| 127 | − hlist_for_each_entry_rcu(tb, head, tb_hlist) { | |
| 127 | + hlist_for_each_entry_rcu(tb, head, tb_hlist, | |
| 128 | + lockdep_rtnl_is_held()) { | |
| 128 | 129 | if (tb->tb_id == id) |
| 129 | 130 | return tb; |
| 130 | 131 | } |
| @@ -227,7 +227,7 @@ then · 226 unmodified lines | ||
| 227 | 227 | must_continue=yes |
| 228 | 228 | fi |
| 229 | 229 | last_ts="`tail $resdir/console.log | grep '^\[ *[0-9]\+\.[0-9]\+]' | tail -1 | sed -e 's/^\[ *//' -e 's/\..*$//'`" |
| 230 | − if test -z "last_ts" | |
| 230 | + if test -z "$last_ts" | |
| 231 | 231 | then |
| 232 | 232 | last_ts=0 |
| 233 | 233 | fi |
| @@ -3,3 +3,4 @@ rcutree.gp_preinit_delay=12 · 2 unmodified lines | ||
| 3 | 3 | rcutree.gp_init_delay=3 |
| 4 | 4 | rcutree.gp_cleanup_delay=3 |
| 5 | 5 | rcutree.kthread_prio=2 |
| 6 | +threadirqs | |