The RCU_FAST_NO_HZ code relies on a number of per-CPU variables. This works, but is hidden from someone scanning the data structures in rcutree.h. This commit therefore converts these per-CPU variables to fields in the per-CPU rcu_dynticks structures.
Unified · Split
| @@ -84,6 +84,20 @@ struct rcu_dynticks { · 83 unmodified lines | ||
| 84 | 84 | /* Process level is worth LLONG_MAX/2. */ |
| 85 | 85 | int dynticks_nmi_nesting; /* Track NMI nesting level. */ |
| 86 | 86 | atomic_t dynticks; /* Even value for idle, else odd. */ |
| 87 | +#ifdef CONFIG_RCU_FAST_NO_HZ | |
| 88 | + int dyntick_drain; /* Prepare-for-idle state variable. */ | |
| 89 | + unsigned long dyntick_holdoff; | |
| 90 | + /* No retries for the jiffy of failure. */ | |
| 91 | + struct timer_list idle_gp_timer; | |
| 92 | + /* Wake up CPU sleeping with callbacks. */ | |
| 93 | + unsigned long idle_gp_timer_expires; | |
| 94 | + /* When to wake up CPU (for repost). */ | |
| 95 | + bool idle_first_pass; /* First pass of attempt to go idle? */ | |
| 96 | + unsigned long nonlazy_posted; | |
| 97 | + /* # times non-lazy CBs posted to CPU. */ | |
| 98 | + unsigned long nonlazy_posted_snap; | |
| 99 | + /* idle-period nonlazy_posted snapshot. */ | |
| 100 | +#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ | |
| 87 | 101 | }; |
| 88 | 102 | |
| 89 | 103 | /* RCU's kthread states for tracing. */ |
| @@ -1962,21 +1962,6 @@ static void rcu_idle_count_callbacks_posted(void) · 1961 unmodified lines | ||
| 1962 | 1962 | #define RCU_IDLE_GP_DELAY 6 /* Roughly one grace period. */ |
| 1963 | 1963 | #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */ |
| 1964 | 1964 | |
| 1965 | −/* Loop counter for rcu_prepare_for_idle(). */ | |
| 1966 | −static DEFINE_PER_CPU(int, rcu_dyntick_drain); | |
| 1967 | −/* If rcu_dyntick_holdoff==jiffies, don't try to enter dyntick-idle mode. */ | |
| 1968 | −static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff); | |
| 1969 | −/* Timer to awaken the CPU if it enters dyntick-idle mode with callbacks. */ | |
| 1970 | −static DEFINE_PER_CPU(struct timer_list, rcu_idle_gp_timer); | |
| 1971 | −/* Scheduled expiry time for rcu_idle_gp_timer to allow reposting. */ | |
| 1972 | −static DEFINE_PER_CPU(unsigned long, rcu_idle_gp_timer_expires); | |
| 1973 | −/* Enable special processing on first attempt to enter dyntick-idle mode. */ | |
| 1974 | −static DEFINE_PER_CPU(bool, rcu_idle_first_pass); | |
| 1975 | −/* Running count of non-lazy callbacks posted, never decremented. */ | |
| 1976 | −static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted); | |
| 1977 | −/* Snapshot of rcu_nonlazy_posted to detect meaningful exits from idle. */ | |
| 1978 | −static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted_snap); | |
| 1979 | − | |
| 1980 | 1965 | /* |
| 1981 | 1966 | * Allow the CPU to enter dyntick-idle mode if either: (1) There are no |
| 1982 | 1967 | * callbacks on this CPU, (2) this CPU has not yet attempted to enter |
| @@ -1988,13 +1973,15 @@ static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted_snap); · 5 unmodified lines | ||
| 1988 | 1973 | */ |
| 1989 | 1974 | int rcu_needs_cpu(int cpu) |
| 1990 | 1975 | { |
| 1976 | + struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); | |
| 1977 | + | |
| 1991 | 1978 | /* Flag a new idle sojourn to the idle-entry state machine. */ |
| 1992 | − per_cpu(rcu_idle_first_pass, cpu) = 1; | |
| 1979 | + rdtp->idle_first_pass = 1; | |
| 1993 | 1980 | /* If no callbacks, RCU doesn't need the CPU. */ |
| 1994 | 1981 | if (!rcu_cpu_has_callbacks(cpu)) |
| 1995 | 1982 | return 0; |
| 1996 | 1983 | /* Otherwise, RCU needs the CPU only if it recently tried and failed. */ |
| 1997 | − return per_cpu(rcu_dyntick_holdoff, cpu) == jiffies; | |
| 1984 | + return rdtp->dyntick_holdoff == jiffies; | |
| 1998 | 1985 | } |
| 1999 | 1986 | |
| 2000 | 1987 | /* |
| @@ -2075,21 +2062,24 @@ static void rcu_idle_gp_timer_func(unsigned long cpu_in) · 74 unmodified lines | ||
| 2075 | 2062 | */ |
| 2076 | 2063 | static void rcu_prepare_for_idle_init(int cpu) |
| 2077 | 2064 | { |
| 2078 | − per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1; | |
| 2079 | − setup_timer(&per_cpu(rcu_idle_gp_timer, cpu), | |
| 2080 | − rcu_idle_gp_timer_func, cpu); | |
| 2081 | − per_cpu(rcu_idle_gp_timer_expires, cpu) = jiffies - 1; | |
| 2082 | − per_cpu(rcu_idle_first_pass, cpu) = 1; | |
| 2065 | + struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); | |
| 2066 | + | |
| 2067 | + rdtp->dyntick_holdoff = jiffies - 1; | |
| 2068 | + setup_timer(&rdtp->idle_gp_timer, rcu_idle_gp_timer_func, cpu); | |
| 2069 | + rdtp->idle_gp_timer_expires = jiffies - 1; | |
| 2070 | + rdtp->idle_first_pass = 1; | |
| 2083 | 2071 | } |
| 2084 | 2072 | |
| 2085 | 2073 | /* |
| 2086 | 2074 | * Clean up for exit from idle. Because we are exiting from idle, there |
| 2087 | − * is no longer any point to rcu_idle_gp_timer, so cancel it. This will | |
| 2075 | + * is no longer any point to ->idle_gp_timer, so cancel it. This will | |
| 2088 | 2076 | * do nothing if this timer is not active, so just cancel it unconditionally. |
| 2089 | 2077 | */ |
| 2090 | 2078 | static void rcu_cleanup_after_idle(int cpu) |
| 2091 | 2079 | { |
| 2092 | − del_timer(&per_cpu(rcu_idle_gp_timer, cpu)); | |
| 2080 | + struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); | |
| 2081 | + | |
| 2082 | + del_timer(&rdtp->idle_gp_timer); | |
| 2093 | 2083 | trace_rcu_prep_idle("Cleanup after idle"); |
| 2094 | 2084 | } |
| 2095 | 2085 | |
| @@ -2108,42 +2098,41 @@ static void rcu_cleanup_after_idle(int cpu) · 12 unmodified lines | ||
| 2108 | 2098 | * Because it is not legal to invoke rcu_process_callbacks() with irqs |
| 2109 | 2099 | * disabled, we do one pass of force_quiescent_state(), then do a |
| 2110 | 2100 | * invoke_rcu_core() to cause rcu_process_callbacks() to be invoked |
| 2111 | − * later. The per-cpu rcu_dyntick_drain variable controls the sequencing. | |
| 2101 | + * later. The ->dyntick_drain field controls the sequencing. | |
| 2112 | 2102 | * |
| 2113 | 2103 | * The caller must have disabled interrupts. |
| 2114 | 2104 | */ |
| 2115 | 2105 | static void rcu_prepare_for_idle(int cpu) |
| 2116 | 2106 | { |
| 2117 | 2107 | struct timer_list *tp; |
| 2108 | + struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); | |
| 2118 | 2109 | |
| 2119 | 2110 | /* |
| 2120 | 2111 | * If this is an idle re-entry, for example, due to use of |
| 2121 | 2112 | * RCU_NONIDLE() or the new idle-loop tracing API within the idle |
| 2122 | 2113 | * loop, then don't take any state-machine actions, unless the |
| 2123 | 2114 | * momentary exit from idle queued additional non-lazy callbacks. |
| 2124 | − * Instead, repost the rcu_idle_gp_timer if this CPU has callbacks | |
| 2115 | + * Instead, repost the ->idle_gp_timer if this CPU has callbacks | |
| 2125 | 2116 | * pending. |
| 2126 | 2117 | */ |
| 2127 | − if (!per_cpu(rcu_idle_first_pass, cpu) && | |
| 2128 | − (per_cpu(rcu_nonlazy_posted, cpu) == | |
| 2129 | − per_cpu(rcu_nonlazy_posted_snap, cpu))) { | |
| 2118 | + if (!rdtp->idle_first_pass && | |
| 2119 | + (rdtp->nonlazy_posted == rdtp->nonlazy_posted_snap)) { | |
| 2130 | 2120 | if (rcu_cpu_has_callbacks(cpu)) { |
| 2131 | − tp = &per_cpu(rcu_idle_gp_timer, cpu); | |
| 2132 | − mod_timer_pinned(tp, per_cpu(rcu_idle_gp_timer_expires, cpu)); | |
| 2121 | + tp = &rdtp->idle_gp_timer; | |
| 2122 | + mod_timer_pinned(tp, rdtp->idle_gp_timer_expires); | |
| 2133 | 2123 | } |
| 2134 | 2124 | return; |
| 2135 | 2125 | } |
| 2136 | − per_cpu(rcu_idle_first_pass, cpu) = 0; | |
| 2137 | − per_cpu(rcu_nonlazy_posted_snap, cpu) = | |
| 2138 | − per_cpu(rcu_nonlazy_posted, cpu) - 1; | |
| 2126 | + rdtp->idle_first_pass = 0; | |
| 2127 | + rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted - 1; | |
| 2139 | 2128 | |
| 2140 | 2129 | /* |
| 2141 | 2130 | * If there are no callbacks on this CPU, enter dyntick-idle mode. |
| 2142 | 2131 | * Also reset state to avoid prejudicing later attempts. |
| 2143 | 2132 | */ |
| 2144 | 2133 | if (!rcu_cpu_has_callbacks(cpu)) { |
| 2145 | − per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1; | |
| 2146 | − per_cpu(rcu_dyntick_drain, cpu) = 0; | |
| 2134 | + rdtp->dyntick_holdoff = jiffies - 1; | |
| 2135 | + rdtp->dyntick_drain = 0; | |
| 2147 | 2136 | trace_rcu_prep_idle("No callbacks"); |
| 2148 | 2137 | return; |
| 2149 | 2138 | } |
| @@ -2152,38 +2141,37 @@ static void rcu_prepare_for_idle(int cpu) · 2 unmodified lines | ||
| 2152 | 2141 | * If in holdoff mode, just return. We will presumably have |
| 2153 | 2142 | * refrained from disabling the scheduling-clock tick. |
| 2154 | 2143 | */ |
| 2155 | − if (per_cpu(rcu_dyntick_holdoff, cpu) == jiffies) { | |
| 2144 | + if (rdtp->dyntick_holdoff == jiffies) { | |
| 2156 | 2145 | trace_rcu_prep_idle("In holdoff"); |
| 2157 | 2146 | return; |
| 2158 | 2147 | } |
| 2159 | 2148 | |
| 2160 | − /* Check and update the rcu_dyntick_drain sequencing. */ | |
| 2161 | − if (per_cpu(rcu_dyntick_drain, cpu) <= 0) { | |
| 2149 | + /* Check and update the ->dyntick_drain sequencing. */ | |
| 2150 | + if (rdtp->dyntick_drain <= 0) { | |
| 2162 | 2151 | /* First time through, initialize the counter. */ |
| 2163 | − per_cpu(rcu_dyntick_drain, cpu) = RCU_IDLE_FLUSHES; | |
| 2164 | − } else if (per_cpu(rcu_dyntick_drain, cpu) <= RCU_IDLE_OPT_FLUSHES && | |
| 2152 | + rdtp->dyntick_drain = RCU_IDLE_FLUSHES; | |
| 2153 | + } else if (rdtp->dyntick_drain <= RCU_IDLE_OPT_FLUSHES && | |
| 2165 | 2154 | !rcu_pending(cpu) && |
| 2166 | 2155 | !local_softirq_pending()) { |
| 2167 | 2156 | /* Can we go dyntick-idle despite still having callbacks? */ |
| 2168 | − per_cpu(rcu_dyntick_drain, cpu) = 0; | |
| 2169 | − per_cpu(rcu_dyntick_holdoff, cpu) = jiffies; | |
| 2157 | + rdtp->dyntick_drain = 0; | |
| 2158 | + rdtp->dyntick_holdoff = jiffies; | |
| 2170 | 2159 | if (rcu_cpu_has_nonlazy_callbacks(cpu)) { |
| 2171 | 2160 | trace_rcu_prep_idle("Dyntick with callbacks"); |
| 2172 | − per_cpu(rcu_idle_gp_timer_expires, cpu) = | |
| 2161 | + rdtp->idle_gp_timer_expires = | |
| 2173 | 2162 | jiffies + RCU_IDLE_GP_DELAY; |
| 2174 | 2163 | } else { |
| 2175 | − per_cpu(rcu_idle_gp_timer_expires, cpu) = | |
| 2164 | + rdtp->idle_gp_timer_expires = | |
| 2176 | 2165 | jiffies + RCU_IDLE_LAZY_GP_DELAY; |
| 2177 | 2166 | trace_rcu_prep_idle("Dyntick with lazy callbacks"); |
| 2178 | 2167 | } |
| 2179 | − tp = &per_cpu(rcu_idle_gp_timer, cpu); | |
| 2180 | − mod_timer_pinned(tp, per_cpu(rcu_idle_gp_timer_expires, cpu)); | |
| 2181 | − per_cpu(rcu_nonlazy_posted_snap, cpu) = | |
| 2182 | − per_cpu(rcu_nonlazy_posted, cpu); | |
| 2168 | + tp = &rdtp->idle_gp_timer; | |
| 2169 | + mod_timer_pinned(tp, rdtp->idle_gp_timer_expires); | |
| 2170 | + rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted; | |
| 2183 | 2171 | return; /* Nothing more to do immediately. */ |
| 2184 | − } else if (--per_cpu(rcu_dyntick_drain, cpu) <= 0) { | |
| 2172 | + } else if (--(rdtp->dyntick_drain) <= 0) { | |
| 2185 | 2173 | /* We have hit the limit, so time to give up. */ |
| 2186 | − per_cpu(rcu_dyntick_holdoff, cpu) = jiffies; | |
| 2174 | + rdtp->dyntick_holdoff = jiffies; | |
| 2187 | 2175 | trace_rcu_prep_idle("Begin holdoff"); |
| 2188 | 2176 | invoke_rcu_core(); /* Force the CPU out of dyntick-idle. */ |
| 2189 | 2177 | return; |
| @@ -2229,7 +2217,7 @@ static void rcu_prepare_for_idle(int cpu) · 39 unmodified lines | ||
| 2229 | 2217 | */ |
| 2230 | 2218 | static void rcu_idle_count_callbacks_posted(void) |
| 2231 | 2219 | { |
| 2232 | − __this_cpu_add(rcu_nonlazy_posted, 1); | |
| 2220 | + __this_cpu_add(rcu_dynticks.nonlazy_posted, 1); | |
| 2233 | 2221 | } |
| 2234 | 2222 | |
| 2235 | 2223 | #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */ |
| @@ -2240,11 +2228,12 @@ static void rcu_idle_count_callbacks_posted(void) · 4 unmodified lines | ||
| 2240 | 2228 | |
| 2241 | 2229 | static void print_cpu_stall_fast_no_hz(char *cp, int cpu) |
| 2242 | 2230 | { |
| 2243 | − struct timer_list *tltp = &per_cpu(rcu_idle_gp_timer, cpu); | |
| 2231 | + struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); | |
| 2232 | + struct timer_list *tltp = &rdtp->idle_gp_timer; | |
| 2244 | 2233 | |
| 2245 | 2234 | sprintf(cp, "drain=%d %c timer=%lu", |
| 2246 | − per_cpu(rcu_dyntick_drain, cpu), | |
| 2247 | − per_cpu(rcu_dyntick_holdoff, cpu) == jiffies ? 'H' : '.', | |
| 2235 | + rdtp->dyntick_drain, | |
| 2236 | + rdtp->dyntick_holdoff == jiffies ? 'H' : '.', | |
| 2248 | 2237 | timer_pending(tltp) ? tltp->expires - jiffies : -1); |
| 2249 | 2238 | } |
| 2250 | 2239 | |