History

srcu: Add boot-time control over srcu_node array allocation

This commit adds an srcu_tree.convert_to_big kernel parameter that either refuses to convert at all (0), converts immediately at init_srcu_struct() time (1), or lets rcutorture convert it (2). An addition contention-based dynamic conversion choice will be added, along with documentation. [ paulmck: Apply callback-scanning feedback from Neeraj Upadhyay. ]

Paul E. McKenney committed 4 years ago

Commit c69a00a1 parent 0b56f952 files · +45 −16Browse files

Unified · Split

2 files
  1. Documentation/admin-guide/kernel-parameters.txt +13 −0
  2. kernel/rcu/srcutree.c +32 −16
Documentation/admin-guide/kernel-parameters.txt+13−0
@@ -5608,6 +5608,19 @@ · 5607 unmodified lines
5608 off: Disable mitigation and remove5608 off: Disable mitigation and remove
5609 performance impact to RDRAND and RDSEED5609 performance impact to RDRAND and RDSEED
5610 5610
5611 srcutree.convert_to_big [KNL]
5612 Specifies under what conditions an SRCU tree
5613 srcu_struct structure will be converted to big
5614 form, that is, with an rcu_node tree:
5615
5616 0: Never.
5617 1: At init_srcu_struct() time.
5618 2: When rcutorture decides to.
5619
5620 Either way, the srcu_node tree will be sized based
5621 on the actual runtime number of CPUs (nr_cpu_ids)
5622 instead of the compile-time CONFIG_NR_CPUS.
5623
5611 srcutree.counter_wrap_check [KNL]5624 srcutree.counter_wrap_check [KNL]
5612 Specifies how frequently to check for5625 Specifies how frequently to check for
5613 grace-period sequence counter wrap for the5626 grace-period sequence counter wrap for the
kernel/rcu/srcutree.c+32−16
@@ -39,6 +39,15 @@ module_param(exp_holdoff, ulong, 0444); · 38 unmodified lines
39static ulong counter_wrap_check = (ULONG_MAX >> 2);39static ulong counter_wrap_check = (ULONG_MAX >> 2);
40module_param(counter_wrap_check, ulong, 0444);40module_param(counter_wrap_check, ulong, 0444);
41 41
42/*
43 * Control conversion to SRCU_SIZE_BIG:
44 * 0: Don't convert at all (default).
45 * 1: Convert at init_srcu_struct() time.
46 * 2: Convert when rcutorture invokes srcu_torture_stats_print().
47 */
48static int convert_to_big;
49module_param(convert_to_big, int, 0444);
50
42/* Early-boot callback-management, so early that no lock is required! */51/* Early-boot callback-management, so early that no lock is required! */
43static LIST_HEAD(srcu_boot_list);52static LIST_HEAD(srcu_boot_list);
44static bool __read_mostly srcu_init_done;53static bool __read_mostly srcu_init_done;
@@ -123,7 +132,7 @@ static inline bool srcu_invl_snp_seq(unsigned long s) · 78 unmodified lines
123 * Allocated and initialize SRCU combining tree. Returns @true if132 * Allocated and initialize SRCU combining tree. Returns @true if
124 * allocation succeeded and @false otherwise.133 * allocation succeeded and @false otherwise.
125 */134 */
126static bool init_srcu_struct_nodes(struct srcu_struct *ssp)135static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)
127{136{
128 int cpu;137 int cpu;
129 int i;138 int i;
@@ -135,7 +144,7 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp) · 5 unmodified lines
135 144
136 /* Initialize geometry if it has not already been initialized. */145 /* Initialize geometry if it has not already been initialized. */
137 rcu_init_geometry();146 rcu_init_geometry();
138 ssp->node = kcalloc(rcu_num_nodes, sizeof(*ssp->node), GFP_ATOMIC);147 ssp->node = kcalloc(rcu_num_nodes, sizeof(*ssp->node), gfp_flags);
139 if (!ssp->node)148 if (!ssp->node)
140 return false;149 return false;
141 150
@@ -213,17 +222,19 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) · 71 unmodified lines
213 if (!ssp->sda)222 if (!ssp->sda)
214 return -ENOMEM;223 return -ENOMEM;
215 init_srcu_struct_data(ssp);224 init_srcu_struct_data(ssp);
216 if (!init_srcu_struct_nodes(ssp)) {
217 if (!is_static) {
218 free_percpu(ssp->sda);
219 ssp->sda = NULL;
220 return -ENOMEM;
221 }
222 } else {
223 ssp->srcu_size_state = SRCU_SIZE_BIG;
224 }
225 ssp->srcu_gp_seq_needed_exp = 0;225 ssp->srcu_gp_seq_needed_exp = 0;
226 ssp->srcu_last_gp_end = ktime_get_mono_fast_ns();226 ssp->srcu_last_gp_end = ktime_get_mono_fast_ns();
227 if (READ_ONCE(ssp->srcu_size_state) == SRCU_SIZE_SMALL && convert_to_big == 1) {
228 if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) {
229 if (!is_static) {
230 free_percpu(ssp->sda);
231 ssp->sda = NULL;
232 return -ENOMEM;
233 }
234 } else {
235 WRITE_ONCE(ssp->srcu_size_state, SRCU_SIZE_BIG);
236 }
237 }
227 smp_store_release(&ssp->srcu_gp_seq_needed, 0); /* Init done. */238 smp_store_release(&ssp->srcu_gp_seq_needed, 0); /* Init done. */
228 return 0;239 return 0;
229}240}
@@ -594,7 +605,8 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 364 unmodified lines
594 /* A new grace period can start at this point. But only one. */605 /* A new grace period can start at this point. But only one. */
595 606
596 /* Initiate callback invocation as needed. */607 /* Initiate callback invocation as needed. */
597 if (smp_load_acquire(&ssp->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER) {608 ss_state = smp_load_acquire(&ssp->srcu_size_state);
609 if (ss_state < SRCU_SIZE_WAIT_BARRIER) {
598 srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, 0), cbdelay);610 srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, 0), cbdelay);
599 } else {611 } else {
600 idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs);612 idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs);
@@ -603,13 +615,16 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 2 unmodified lines
603 cbs = false;615 cbs = false;
604 last_lvl = snp >= ssp->level[rcu_num_lvls - 1];616 last_lvl = snp >= ssp->level[rcu_num_lvls - 1];
605 if (last_lvl)617 if (last_lvl)
606 cbs = snp->srcu_have_cbs[idx] == gpseq;618 cbs = ss_state < SRCU_SIZE_BIG || snp->srcu_have_cbs[idx] == gpseq;
607 snp->srcu_have_cbs[idx] = gpseq;619 snp->srcu_have_cbs[idx] = gpseq;
608 rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);620 rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);
609 sgsne = snp->srcu_gp_seq_needed_exp;621 sgsne = snp->srcu_gp_seq_needed_exp;
610 if (srcu_invl_snp_seq(sgsne) || ULONG_CMP_LT(sgsne, gpseq))622 if (srcu_invl_snp_seq(sgsne) || ULONG_CMP_LT(sgsne, gpseq))
611 WRITE_ONCE(snp->srcu_gp_seq_needed_exp, gpseq);623 WRITE_ONCE(snp->srcu_gp_seq_needed_exp, gpseq);
612 mask = snp->srcu_data_have_cbs[idx];624 if (ss_state < SRCU_SIZE_BIG)
625 mask = ~0;
626 else
627 mask = snp->srcu_data_have_cbs[idx];
613 snp->srcu_data_have_cbs[idx] = 0;628 snp->srcu_data_have_cbs[idx] = 0;
614 spin_unlock_irq_rcu_node(snp);629 spin_unlock_irq_rcu_node(snp);
615 if (cbs)630 if (cbs)
@@ -645,10 +660,9 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 29 unmodified lines
645 }660 }
646 661
647 /* Transition to big if needed. */662 /* Transition to big if needed. */
648 ss_state = smp_load_acquire(&ssp->srcu_size_state);
649 if (ss_state != SRCU_SIZE_SMALL && ss_state != SRCU_SIZE_BIG) {663 if (ss_state != SRCU_SIZE_SMALL && ss_state != SRCU_SIZE_BIG) {
650 if (ss_state == SRCU_SIZE_ALLOC)664 if (ss_state == SRCU_SIZE_ALLOC)
651 init_srcu_struct_nodes(ssp);665 init_srcu_struct_nodes(ssp, GFP_KERNEL);
652 else666 else
653 smp_store_release(&ssp->srcu_size_state, ss_state + 1);667 smp_store_release(&ssp->srcu_size_state, ss_state + 1);
654 }668 }
@@ -1494,6 +1508,8 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf) · 839 unmodified lines
1494 s1 += c1;1508 s1 += c1;
1495 }1509 }
1496 pr_cont(" T(%ld,%ld)\n", s0, s1);1510 pr_cont(" T(%ld,%ld)\n", s0, s1);
1511 if (READ_ONCE(ssp->srcu_size_state) == SRCU_SIZE_SMALL && convert_to_big == 2)
1512 WRITE_ONCE(ssp->srcu_size_state, SRCU_SIZE_ALLOC);
1497}1513}
1498EXPORT_SYMBOL_GPL(srcu_torture_stats_print);1514EXPORT_SYMBOL_GPL(srcu_torture_stats_print);
1499 1515