The current preemptible RCU implementation goes through three phases during bootup. In the first phase, there is only one CPU that is running with preemption disabled, so that a no-op is a synchronous grace period. In the second mid-boot phase, the scheduler is running, but RCU has not yet gotten its kthreads spawned (and, for expedited grace periods, workqueues are not yet running. During this time, any attempt to do a synchronous grace period will hang the system (or complain bitterly, depending). In the third and final phase, RCU is fully operational and everything works normally. This has been OK for some time, but there has recently been some synchronous grace periods showing up during the second mid-boot phase. This code worked "by accident" for awhile, but started failing as soon as expedited RCU grace periods switched over to workqueues in commit 8b355e3bc140 ("rcu: Drive expedited grace periods from workqueue"). Note that the code was buggy even before this commit, as it was subject to failure on real-time systems that forced all expedited grace periods to run as normal grace periods (for example, using the rcu_normal ksysfs parameter). The callchain from the failure case is as follows: early_amd_iommu_init() |-> acpi_put_table(ivrs_base); |-> acpi_tb_put_table(table_desc); |-> acpi_tb_invalidate_table(table_desc); |-> acpi_tb_release_table(...) |-> acpi_os_unmap_memory |-> acpi_os_unmap_iomem |-> acpi_os_map_cleanup |-> synchronize_rcu_expedited The kernel showing this callchain was built with CONFIG_PREEMPT_RCU=y, which caused the code to try using workqueues before they were initialized, which did not go well. This commit therefore reworks RCU to permit synchronous grace periods to proceed during this mid-boot phase. This commit is therefore a fix to a regression introduced in v4.9, and is therefore being put forward post-merge-window in v4.10. This commit sets a flag from the existing rcu_scheduler_starting() function which causes all synchronous grace periods to take the expedited path. The expedited path now checks this flag, using the requesting task to drive the expedited grace period forward during the mid-boot phase. Finally, this flag is updated by a core_initcall() function named rcu_exp_runtime_mode(), which causes the runtime codepaths to be used. Note that this arrangement assumes that tasks are not sent POSIX signals (or anything similar) from the time that the first task is spawned through core_initcall() time.
Unified · Split
| @@ -444,6 +444,10 @@ bool __rcu_is_watching(void); · 443 unmodified lines | ||
| 444 | 444 | #error "Unknown RCU implementation specified to kernel configuration" |
| 445 | 445 | #endif |
| 446 | 446 | |
| 447 | +#define RCU_SCHEDULER_INACTIVE 0 | |
| 448 | +#define RCU_SCHEDULER_INIT 1 | |
| 449 | +#define RCU_SCHEDULER_RUNNING 2 | |
| 450 | + | |
| 447 | 451 | /* |
| 448 | 452 | * init_rcu_head_on_stack()/destroy_rcu_head_on_stack() are needed for dynamic |
| 449 | 453 | * initialization and destruction of rcu_head on the stack. rcu_head structures |
| @@ -136,6 +136,7 @@ int rcu_jiffies_till_stall_check(void); · 135 unmodified lines | ||
| 136 | 136 | #define TPS(x) tracepoint_string(x) |
| 137 | 137 | |
| 138 | 138 | void rcu_early_boot_tests(void); |
| 139 | +void rcu_test_sync_prims(void); | |
| 139 | 140 | |
| 140 | 141 | /* |
| 141 | 142 | * This function really isn't for public consumption, but RCU is special in |
| @@ -60,12 +60,17 @@ EXPORT_SYMBOL_GPL(rcu_scheduler_active); · 59 unmodified lines | ||
| 60 | 60 | |
| 61 | 61 | /* |
| 62 | 62 | * During boot, we forgive RCU lockdep issues. After this function is |
| 63 | − * invoked, we start taking RCU lockdep issues seriously. | |
| 63 | + * invoked, we start taking RCU lockdep issues seriously. Note that unlike | |
| 64 | + * Tree RCU, Tiny RCU transitions directly from RCU_SCHEDULER_INACTIVE | |
| 65 | + * to RCU_SCHEDULER_RUNNING, skipping the RCU_SCHEDULER_INIT stage. | |
| 66 | + * The reason for this is that Tiny RCU does not need kthreads, so does | |
| 67 | + * not have to care about the fact that the scheduler is half-initialized | |
| 68 | + * at a certain phase of the boot process. | |
| 64 | 69 | */ |
| 65 | 70 | void __init rcu_scheduler_starting(void) |
| 66 | 71 | { |
| 67 | 72 | WARN_ON(nr_context_switches() > 0); |
| 68 | − rcu_scheduler_active = 1; | |
| 73 | + rcu_scheduler_active = RCU_SCHEDULER_RUNNING; | |
| 69 | 74 | } |
| 70 | 75 | |
| 71 | 76 | #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
| @@ -127,13 +127,16 @@ int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */ · 126 unmodified lines | ||
| 127 | 127 | int sysctl_panic_on_rcu_stall __read_mostly; |
| 128 | 128 | |
| 129 | 129 | /* |
| 130 | − * The rcu_scheduler_active variable transitions from zero to one just | |
| 131 | − * before the first task is spawned. So when this variable is zero, RCU | |
| 132 | − * can assume that there is but one task, allowing RCU to (for example) | |
| 130 | + * The rcu_scheduler_active variable is initialized to the value | |
| 131 | + * RCU_SCHEDULER_INACTIVE and transitions RCU_SCHEDULER_INIT just before the | |
| 132 | + * first task is spawned. So when this variable is RCU_SCHEDULER_INACTIVE, | |
| 133 | + * RCU can assume that there is but one task, allowing RCU to (for example) | |
| 133 | 134 | * optimize synchronize_rcu() to a simple barrier(). When this variable |
| 134 | − * is one, RCU must actually do all the hard work required to detect real | |
| 135 | − * grace periods. This variable is also used to suppress boot-time false | |
| 136 | − * positives from lockdep-RCU error checking. | |
| 135 | + * is RCU_SCHEDULER_INIT, RCU must actually do all the hard work required | |
| 136 | + * to detect real grace periods. This variable is also used to suppress | |
| 137 | + * boot-time false positives from lockdep-RCU error checking. Finally, it | |
| 138 | + * transitions from RCU_SCHEDULER_INIT to RCU_SCHEDULER_RUNNING after RCU | |
| 139 | + * is fully initialized, including all of its kthreads having been spawned. | |
| 137 | 140 | */ |
| 138 | 141 | int rcu_scheduler_active __read_mostly; |
| 139 | 142 | EXPORT_SYMBOL_GPL(rcu_scheduler_active); |
| @@ -3980,18 +3983,22 @@ static int __init rcu_spawn_gp_kthread(void) · 3840 unmodified lines | ||
| 3980 | 3983 | early_initcall(rcu_spawn_gp_kthread); |
| 3981 | 3984 | |
| 3982 | 3985 | /* |
| 3983 | − * This function is invoked towards the end of the scheduler's initialization | |
| 3984 | − * process. Before this is called, the idle task might contain | |
| 3985 | − * RCU read-side critical sections (during which time, this idle | |
| 3986 | − * task is booting the system). After this function is called, the | |
| 3987 | − * idle tasks are prohibited from containing RCU read-side critical | |
| 3988 | − * sections. This function also enables RCU lockdep checking. | |
| 3986 | + * This function is invoked towards the end of the scheduler's | |
| 3987 | + * initialization process. Before this is called, the idle task might | |
| 3988 | + * contain synchronous grace-period primitives (during which time, this idle | |
| 3989 | + * task is booting the system, and such primitives are no-ops). After this | |
| 3990 | + * function is called, any synchronous grace-period primitives are run as | |
| 3991 | + * expedited, with the requesting task driving the grace period forward. | |
| 3992 | + * A later core_initcall() rcu_exp_runtime_mode() will switch to full | |
| 3993 | + * runtime RCU functionality. | |
| 3989 | 3994 | */ |
| 3990 | 3995 | void rcu_scheduler_starting(void) |
| 3991 | 3996 | { |
| 3992 | 3997 | WARN_ON(num_online_cpus() != 1); |
| 3993 | 3998 | WARN_ON(nr_context_switches() > 0); |
| 3994 | − rcu_scheduler_active = 1; | |
| 3999 | + rcu_test_sync_prims(); | |
| 4000 | + rcu_scheduler_active = RCU_SCHEDULER_INIT; | |
| 4001 | + rcu_test_sync_prims(); | |
| 3995 | 4002 | } |
| 3996 | 4003 | |
| 3997 | 4004 | /* |
| @@ -531,6 +531,20 @@ struct rcu_exp_work { · 530 unmodified lines | ||
| 531 | 531 | struct work_struct rew_work; |
| 532 | 532 | }; |
| 533 | 533 | |
| 534 | +/* | |
| 535 | + * Common code to drive an expedited grace period forward, used by | |
| 536 | + * workqueues and mid-boot-time tasks. | |
| 537 | + */ | |
| 538 | +static void rcu_exp_sel_wait_wake(struct rcu_state *rsp, | |
| 539 | + smp_call_func_t func, unsigned long s) | |
| 540 | +{ | |
| 541 | + /* Initialize the rcu_node tree in preparation for the wait. */ | |
| 542 | + sync_rcu_exp_select_cpus(rsp, func); | |
| 543 | + | |
| 544 | + /* Wait and clean up, including waking everyone. */ | |
| 545 | + rcu_exp_wait_wake(rsp, s); | |
| 546 | +} | |
| 547 | + | |
| 534 | 548 | /* |
| 535 | 549 | * Work-queue handler to drive an expedited grace period forward. |
| 536 | 550 | */ |
| @@ -538,12 +552,8 @@ static void wait_rcu_exp_gp(struct work_struct *wp) · 1 unmodified line | ||
| 538 | 552 | { |
| 539 | 553 | struct rcu_exp_work *rewp; |
| 540 | 554 | |
| 541 | − /* Initialize the rcu_node tree in preparation for the wait. */ | |
| 542 | 555 | rewp = container_of(wp, struct rcu_exp_work, rew_work); |
| 543 | − sync_rcu_exp_select_cpus(rewp->rew_rsp, rewp->rew_func); | |
| 544 | − | |
| 545 | − /* Wait and clean up, including waking everyone. */ | |
| 546 | − rcu_exp_wait_wake(rewp->rew_rsp, rewp->rew_s); | |
| 556 | + rcu_exp_sel_wait_wake(rewp->rew_rsp, rewp->rew_func, rewp->rew_s); | |
| 547 | 557 | } |
| 548 | 558 | |
| 549 | 559 | /* |
| @@ -569,12 +579,18 @@ static void _synchronize_rcu_expedited(struct rcu_state *rsp, · 19 unmodified lines | ||
| 569 | 579 | if (exp_funnel_lock(rsp, s)) |
| 570 | 580 | return; /* Someone else did our work for us. */ |
| 571 | 581 | |
| 572 | − /* Marshall arguments and schedule the expedited grace period. */ | |
| 573 | − rew.rew_func = func; | |
| 574 | − rew.rew_rsp = rsp; | |
| 575 | − rew.rew_s = s; | |
| 576 | − INIT_WORK_ONSTACK(&rew.rew_work, wait_rcu_exp_gp); | |
| 577 | − schedule_work(&rew.rew_work); | |
| 582 | + /* Ensure that load happens before action based on it. */ | |
| 583 | + if (unlikely(rcu_scheduler_active == RCU_SCHEDULER_INIT)) { | |
| 584 | + /* Direct call during scheduler init and early_initcalls(). */ | |
| 585 | + rcu_exp_sel_wait_wake(rsp, func, s); | |
| 586 | + } else { | |
| 587 | + /* Marshall arguments & schedule the expedited grace period. */ | |
| 588 | + rew.rew_func = func; | |
| 589 | + rew.rew_rsp = rsp; | |
| 590 | + rew.rew_s = s; | |
| 591 | + INIT_WORK_ONSTACK(&rew.rew_work, wait_rcu_exp_gp); | |
| 592 | + schedule_work(&rew.rew_work); | |
| 593 | + } | |
| 578 | 594 | |
| 579 | 595 | /* Wait for expedited grace period to complete. */ |
| 580 | 596 | rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id()); |
| @@ -676,6 +692,8 @@ void synchronize_rcu_expedited(void) · 95 unmodified lines | ||
| 676 | 692 | { |
| 677 | 693 | struct rcu_state *rsp = rcu_state_p; |
| 678 | 694 | |
| 695 | + if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) | |
| 696 | + return; | |
| 679 | 697 | _synchronize_rcu_expedited(rsp, sync_rcu_exp_handler); |
| 680 | 698 | } |
| 681 | 699 | EXPORT_SYMBOL_GPL(synchronize_rcu_expedited); |
| @@ -693,3 +711,15 @@ void synchronize_rcu_expedited(void) · 11 unmodified lines | ||
| 693 | 711 | EXPORT_SYMBOL_GPL(synchronize_rcu_expedited); |
| 694 | 712 | |
| 695 | 713 | #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ |
| 714 | + | |
| 715 | +/* | |
| 716 | + * Switch to run-time mode once Tree RCU has fully initialized. | |
| 717 | + */ | |
| 718 | +static int __init rcu_exp_runtime_mode(void) | |
| 719 | +{ | |
| 720 | + rcu_test_sync_prims(); | |
| 721 | + rcu_scheduler_active = RCU_SCHEDULER_RUNNING; | |
| 722 | + rcu_test_sync_prims(); | |
| 723 | + return 0; | |
| 724 | +} | |
| 725 | +core_initcall(rcu_exp_runtime_mode); | |
| @@ -670,7 +670,7 @@ void synchronize_rcu(void) · 669 unmodified lines | ||
| 670 | 670 | lock_is_held(&rcu_lock_map) || |
| 671 | 671 | lock_is_held(&rcu_sched_lock_map), |
| 672 | 672 | "Illegal synchronize_rcu() in RCU read-side critical section"); |
| 673 | − if (!rcu_scheduler_active) | |
| 673 | + if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) | |
| 674 | 674 | return; |
| 675 | 675 | if (rcu_gp_is_expedited()) |
| 676 | 676 | synchronize_rcu_expedited(); |
| @@ -121,11 +121,14 @@ EXPORT_SYMBOL(rcu_read_lock_sched_held); · 120 unmodified lines | ||
| 121 | 121 | * Should expedited grace-period primitives always fall back to their |
| 122 | 122 | * non-expedited counterparts? Intended for use within RCU. Note |
| 123 | 123 | * that if the user specifies both rcu_expedited and rcu_normal, then |
| 124 | − * rcu_normal wins. | |
| 124 | + * rcu_normal wins. (Except during the time period during boot from | |
| 125 | + * when the first task is spawned until the rcu_exp_runtime_mode() | |
| 126 | + * core_initcall() is invoked, at which point everything is expedited.) | |
| 125 | 127 | */ |
| 126 | 128 | bool rcu_gp_is_normal(void) |
| 127 | 129 | { |
| 128 | − return READ_ONCE(rcu_normal); | |
| 130 | + return READ_ONCE(rcu_normal) && | |
| 131 | + rcu_scheduler_active != RCU_SCHEDULER_INIT; | |
| 129 | 132 | } |
| 130 | 133 | EXPORT_SYMBOL_GPL(rcu_gp_is_normal); |
| 131 | 134 | |
| @@ -135,13 +138,14 @@ static atomic_t rcu_expedited_nesting = · 3 unmodified lines | ||
| 135 | 138 | /* |
| 136 | 139 | * Should normal grace-period primitives be expedited? Intended for |
| 137 | 140 | * use within RCU. Note that this function takes the rcu_expedited |
| 138 | − * sysfs/boot variable into account as well as the rcu_expedite_gp() | |
| 139 | − * nesting. So looping on rcu_unexpedite_gp() until rcu_gp_is_expedited() | |
| 140 | − * returns false is a -really- bad idea. | |
| 141 | + * sysfs/boot variable and rcu_scheduler_active into account as well | |
| 142 | + * as the rcu_expedite_gp() nesting. So looping on rcu_unexpedite_gp() | |
| 143 | + * until rcu_gp_is_expedited() returns false is a -really- bad idea. | |
| 141 | 144 | */ |
| 142 | 145 | bool rcu_gp_is_expedited(void) |
| 143 | 146 | { |
| 144 | − return rcu_expedited || atomic_read(&rcu_expedited_nesting); | |
| 147 | + return rcu_expedited || atomic_read(&rcu_expedited_nesting) || | |
| 148 | + rcu_scheduler_active == RCU_SCHEDULER_INIT; | |
| 145 | 149 | } |
| 146 | 150 | EXPORT_SYMBOL_GPL(rcu_gp_is_expedited); |
| 147 | 151 | |
| @@ -257,7 +261,7 @@ EXPORT_SYMBOL_GPL(rcu_callback_map); · 109 unmodified lines | ||
| 257 | 261 | |
| 258 | 262 | int notrace debug_lockdep_rcu_enabled(void) |
| 259 | 263 | { |
| 260 | − return rcu_scheduler_active && debug_locks && | |
| 264 | + return rcu_scheduler_active != RCU_SCHEDULER_INACTIVE && debug_locks && | |
| 261 | 265 | current->lockdep_recursion == 0; |
| 262 | 266 | } |
| 263 | 267 | EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled); |
| @@ -591,7 +595,7 @@ EXPORT_SYMBOL_GPL(call_rcu_tasks); · 327 unmodified lines | ||
| 591 | 595 | void synchronize_rcu_tasks(void) |
| 592 | 596 | { |
| 593 | 597 | /* Complain if the scheduler has not started. */ |
| 594 | − RCU_LOCKDEP_WARN(!rcu_scheduler_active, | |
| 598 | + RCU_LOCKDEP_WARN(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE, | |
| 595 | 599 | "synchronize_rcu_tasks called too soon"); |
| 596 | 600 | |
| 597 | 601 | /* Wait for the grace period. */ |
| @@ -813,6 +817,23 @@ static void rcu_spawn_tasks_kthread(void) · 215 unmodified lines | ||
| 813 | 817 | |
| 814 | 818 | #endif /* #ifdef CONFIG_TASKS_RCU */ |
| 815 | 819 | |
| 820 | +/* | |
| 821 | + * Test each non-SRCU synchronous grace-period wait API. This is | |
| 822 | + * useful just after a change in mode for these primitives, and | |
| 823 | + * during early boot. | |
| 824 | + */ | |
| 825 | +void rcu_test_sync_prims(void) | |
| 826 | +{ | |
| 827 | + if (!IS_ENABLED(CONFIG_PROVE_RCU)) | |
| 828 | + return; | |
| 829 | + synchronize_rcu(); | |
| 830 | + synchronize_rcu_bh(); | |
| 831 | + synchronize_sched(); | |
| 832 | + synchronize_rcu_expedited(); | |
| 833 | + synchronize_rcu_bh_expedited(); | |
| 834 | + synchronize_sched_expedited(); | |
| 835 | +} | |
| 836 | + | |
| 816 | 837 | #ifdef CONFIG_PROVE_RCU |
| 817 | 838 | |
| 818 | 839 | /* |
| @@ -865,6 +886,7 @@ void rcu_early_boot_tests(void) · 46 unmodified lines | ||
| 865 | 886 | early_boot_test_call_rcu_bh(); |
| 866 | 887 | if (rcu_self_test_sched) |
| 867 | 888 | early_boot_test_call_rcu_sched(); |
| 889 | + rcu_test_sync_prims(); | |
| 868 | 890 | } |
| 869 | 891 | |
| 870 | 892 | static int rcu_verify_early_boot_tests(void) |