History

srcu: Use invalid initial value for srcu_node GP sequence numbers

Currently, tree SRCU relies on the srcu_node structures being initialized at the same time that the srcu_struct itself is initialized, and thus use the initial grace-period sequence number as the initial value for the srcu_node structure's ->srcu_have_cbs[] and ->srcu_gp_seq_needed_exp fields. Although this has a high probability of also working when the srcu_node array is allocated and initialized at some random later time, it would be better to avoid leaving such things to chance. This commit therefore initializes these fields with 0x2, which is a recognizable invalid value. It then adds the required checks for this invalid value in order to avoid confusion on long-running kernels (especially those on 32-bit systems) that allocate and initialize srcu_node arrays late in life.

Paul E. McKenney committed 4 years ago

Commit cbdc98e1 parent aeb9b391 file · +27 −9Browse files

Unified · Split

kernel/rcu/srcutree.c+27−9
@@ -107,6 +107,18 @@ static void init_srcu_struct_data(struct srcu_struct *ssp) · 106 unmodified lines
107107 }
108108 }
109109
110+/* Invalid seq state, used during snp node initialization */
111+#define SRCU_SNP_INIT_SEQ 0x2
112+
113+/*
114+ * Check whether sequence number corresponding to snp node,
115+ * is invalid.
116+ */
117+static inline bool srcu_invl_snp_seq(unsigned long s)
118+{
119+ return rcu_seq_state(s) == SRCU_SNP_INIT_SEQ;
120+}
121+
110122 /*
111123 * Allocated and initialize SRCU combining tree. Returns @true if
112124 * allocation succeeded and @false otherwise.
@@ -139,10 +151,10 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp) · 26 unmodified lines
139151 WARN_ON_ONCE(ARRAY_SIZE(snp->srcu_have_cbs) !=
140152 ARRAY_SIZE(snp->srcu_data_have_cbs));
141153 for (i = 0; i < ARRAY_SIZE(snp->srcu_have_cbs); i++) {
142− snp->srcu_have_cbs[i] = 0;
154+ snp->srcu_have_cbs[i] = SRCU_SNP_INIT_SEQ;
143155 snp->srcu_data_have_cbs[i] = 0;
144156 }
145− snp->srcu_gp_seq_needed_exp = 0;
157+ snp->srcu_gp_seq_needed_exp = SRCU_SNP_INIT_SEQ;
146158 snp->grplo = -1;
147159 snp->grphi = -1;
148160 if (snp == &ssp->node[0]) {
@@ -386,8 +398,7 @@ static bool srcu_readers_active(struct srcu_struct *ssp) · 237 unmodified lines
386398 */
387399 static unsigned long srcu_get_delay(struct srcu_struct *ssp)
388400 {
389− if (ULONG_CMP_LT(READ_ONCE(ssp->srcu_gp_seq),
390− READ_ONCE(ssp->srcu_gp_seq_needed_exp)))
401+ if (ULONG_CMP_LT(READ_ONCE(ssp->srcu_gp_seq), READ_ONCE(ssp->srcu_gp_seq_needed_exp)))
391402 return 0;
392403 return SRCU_INTERVAL;
393404 }
@@ -561,6 +572,7 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 167 unmodified lines
561572 int idx;
562573 unsigned long mask;
563574 struct srcu_data *sdp;
575+ unsigned long sgsne;
564576 struct srcu_node *snp;
565577 int ss_state;
566578
@@ -594,7 +606,8 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 27 unmodified lines
594606 cbs = snp->srcu_have_cbs[idx] == gpseq;
595607 snp->srcu_have_cbs[idx] = gpseq;
596608 rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);
597− if (ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, gpseq))
609+ sgsne = snp->srcu_gp_seq_needed_exp;
610+ if (srcu_invl_snp_seq(sgsne) || ULONG_CMP_LT(sgsne, gpseq))
598611 WRITE_ONCE(snp->srcu_gp_seq_needed_exp, gpseq);
599612 mask = snp->srcu_data_have_cbs[idx];
600613 snp->srcu_data_have_cbs[idx] = 0;
@@ -652,14 +665,17 @@ static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp · 51 unmodified lines
652665 unsigned long s)
653666 {
654667 unsigned long flags;
668+ unsigned long sgsne;
655669
656670 if (snp)
657671 for (; snp != NULL; snp = snp->srcu_parent) {
672+ sgsne = READ_ONCE(snp->srcu_gp_seq_needed_exp);
658673 if (rcu_seq_done(&ssp->srcu_gp_seq, s) ||
659− ULONG_CMP_GE(READ_ONCE(snp->srcu_gp_seq_needed_exp), s))
674+ (!srcu_invl_snp_seq(sgsne) && ULONG_CMP_GE(sgsne, s)))
660675 return;
661676 spin_lock_irqsave_rcu_node(snp, flags);
662− if (ULONG_CMP_GE(snp->srcu_gp_seq_needed_exp, s)) {
677+ sgsne = snp->srcu_gp_seq_needed_exp;
678+ if (!srcu_invl_snp_seq(sgsne) && ULONG_CMP_GE(sgsne, s)) {
663679 spin_unlock_irqrestore_rcu_node(snp, flags);
664680 return;
665681 }
@@ -687,6 +703,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, · 21 unmodified lines
687703 {
688704 unsigned long flags;
689705 int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs);
706+ unsigned long sgsne;
690707 struct srcu_node *snp;
691708 struct srcu_node *snp_leaf = smp_load_acquire(&sdp->mynode);
692709 unsigned long snp_seq;
@@ -698,7 +715,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, · 5 unmodified lines
698715 return; /* GP already done and CBs recorded. */
699716 spin_lock_irqsave_rcu_node(snp, flags);
700717 snp_seq = snp->srcu_have_cbs[idx];
701− if (ULONG_CMP_GE(snp_seq, s)) {
718+ if (!srcu_invl_snp_seq(snp_seq) && ULONG_CMP_GE(snp_seq, s)) {
702719 if (snp == snp_leaf && snp_seq == s)
703720 snp->srcu_data_have_cbs[idx] |= sdp->grpmask;
704721 spin_unlock_irqrestore_rcu_node(snp, flags);
@@ -713,7 +730,8 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, · 8 unmodified lines
713730 snp->srcu_have_cbs[idx] = s;
714731 if (snp == snp_leaf)
715732 snp->srcu_data_have_cbs[idx] |= sdp->grpmask;
716− if (!do_norm && ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, s))
733+ sgsne = snp->srcu_gp_seq_needed_exp;
734+ if (!do_norm && (srcu_invl_snp_seq(sgsne) || ULONG_CMP_LT(sgsne, s)))
717735 WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);
718736 spin_unlock_irqrestore_rcu_node(snp, flags);
719737 }