Currently, lookup_chain_cache() provides both 'lookup' and 'add' functionalities in a function. However, each is useful. So this patch makes lookup_chain_cache() only do 'lookup' functionality and makes add_chain_cahce() only do 'add' functionality. And it's more readable than before.
Showing the changes to kernel/locking (1 of 1 file). Show all
Unified · Split
| @@ -2151,20 +2151,26 @@ static int check_no_collision(struct task_struct *curr, · 2150 unmodified lines | ||
| 2151 | 2151 | } |
| 2152 | 2152 | |
| 2153 | 2153 | /* |
| 2154 | − * Look up a dependency chain. If the key is not present yet then | |
| 2155 | − * add it and return 1 - in this case the new dependency chain is | |
| 2156 | − * validated. If the key is already hashed, return 0. | |
| 2157 | − * (On return with 1 graph_lock is held.) | |
| 2154 | + * Adds a dependency chain into chain hashtable. And must be called with | |
| 2155 | + * graph_lock held. | |
| 2156 | + * | |
| 2157 | + * Return 0 if fail, and graph_lock is released. | |
| 2158 | + * Return 1 if succeed, with graph_lock held. | |
| 2158 | 2159 | */ |
| 2159 | −static inline int lookup_chain_cache(struct task_struct *curr, | |
| 2160 | − struct held_lock *hlock, | |
| 2161 | − u64 chain_key) | |
| 2160 | +static inline int add_chain_cache(struct task_struct *curr, | |
| 2161 | + struct held_lock *hlock, | |
| 2162 | + u64 chain_key) | |
| 2162 | 2163 | { |
| 2163 | 2164 | struct lock_class *class = hlock_class(hlock); |
| 2164 | 2165 | struct hlist_head *hash_head = chainhashentry(chain_key); |
| 2165 | 2166 | struct lock_chain *chain; |
| 2166 | 2167 | int i, j; |
| 2167 | 2168 | |
| 2169 | + /* | |
| 2170 | + * Allocate a new chain entry from the static array, and add | |
| 2171 | + * it to the hash: | |
| 2172 | + */ | |
| 2173 | + | |
| 2168 | 2174 | /* |
| 2169 | 2175 | * We might need to take the graph lock, ensure we've got IRQs |
| 2170 | 2176 | * disabled to make this an IRQ-safe lock.. for recursion reasons |
| @@ -2172,43 +2178,7 @@ static inline int lookup_chain_cache(struct task_struct *curr, · 1 unmodified line | ||
| 2172 | 2178 | */ |
| 2173 | 2179 | if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) |
| 2174 | 2180 | return 0; |
| 2175 | − /* | |
| 2176 | − * We can walk it lock-free, because entries only get added | |
| 2177 | − * to the hash: | |
| 2178 | − */ | |
| 2179 | − hlist_for_each_entry_rcu(chain, hash_head, entry) { | |
| 2180 | − if (chain->chain_key == chain_key) { | |
| 2181 | −cache_hit: | |
| 2182 | − debug_atomic_inc(chain_lookup_hits); | |
| 2183 | − if (!check_no_collision(curr, hlock, chain)) | |
| 2184 | − return 0; | |
| 2185 | 2181 | |
| 2186 | − if (very_verbose(class)) | |
| 2187 | − printk("\nhash chain already cached, key: " | |
| 2188 | − "%016Lx tail class: [%p] %s\n", | |
| 2189 | − (unsigned long long)chain_key, | |
| 2190 | − class->key, class->name); | |
| 2191 | − return 0; | |
| 2192 | − } | |
| 2193 | − } | |
| 2194 | − if (very_verbose(class)) | |
| 2195 | − printk("\nnew hash chain, key: %016Lx tail class: [%p] %s\n", | |
| 2196 | − (unsigned long long)chain_key, class->key, class->name); | |
| 2197 | − /* | |
| 2198 | − * Allocate a new chain entry from the static array, and add | |
| 2199 | − * it to the hash: | |
| 2200 | − */ | |
| 2201 | − if (!graph_lock()) | |
| 2202 | − return 0; | |
| 2203 | − /* | |
| 2204 | − * We have to walk the chain again locked - to avoid duplicates: | |
| 2205 | − */ | |
| 2206 | − hlist_for_each_entry(chain, hash_head, entry) { | |
| 2207 | − if (chain->chain_key == chain_key) { | |
| 2208 | − graph_unlock(); | |
| 2209 | − goto cache_hit; | |
| 2210 | − } | |
| 2211 | − } | |
| 2212 | 2182 | if (unlikely(nr_lock_chains >= MAX_LOCKDEP_CHAINS)) { |
| 2213 | 2183 | if (!debug_locks_off_graph_unlock()) |
| 2214 | 2184 | return 0; |
| @@ -2260,6 +2230,78 @@ cache_hit: · 45 unmodified lines | ||
| 2260 | 2230 | return 1; |
| 2261 | 2231 | } |
| 2262 | 2232 | |
| 2233 | +/* | |
| 2234 | + * Look up a dependency chain. | |
| 2235 | + */ | |
| 2236 | +static inline struct lock_chain *lookup_chain_cache(u64 chain_key) | |
| 2237 | +{ | |
| 2238 | + struct hlist_head *hash_head = chainhashentry(chain_key); | |
| 2239 | + struct lock_chain *chain; | |
| 2240 | + | |
| 2241 | + /* | |
| 2242 | + * We can walk it lock-free, because entries only get added | |
| 2243 | + * to the hash: | |
| 2244 | + */ | |
| 2245 | + hlist_for_each_entry_rcu(chain, hash_head, entry) { | |
| 2246 | + if (chain->chain_key == chain_key) { | |
| 2247 | + debug_atomic_inc(chain_lookup_hits); | |
| 2248 | + return chain; | |
| 2249 | + } | |
| 2250 | + } | |
| 2251 | + return NULL; | |
| 2252 | +} | |
| 2253 | + | |
| 2254 | +/* | |
| 2255 | + * If the key is not present yet in dependency chain cache then | |
| 2256 | + * add it and return 1 - in this case the new dependency chain is | |
| 2257 | + * validated. If the key is already hashed, return 0. | |
| 2258 | + * (On return with 1 graph_lock is held.) | |
| 2259 | + */ | |
| 2260 | +static inline int lookup_chain_cache_add(struct task_struct *curr, | |
| 2261 | + struct held_lock *hlock, | |
| 2262 | + u64 chain_key) | |
| 2263 | +{ | |
| 2264 | + struct lock_class *class = hlock_class(hlock); | |
| 2265 | + struct lock_chain *chain = lookup_chain_cache(chain_key); | |
| 2266 | + | |
| 2267 | + if (chain) { | |
| 2268 | +cache_hit: | |
| 2269 | + if (!check_no_collision(curr, hlock, chain)) | |
| 2270 | + return 0; | |
| 2271 | + | |
| 2272 | + if (very_verbose(class)) { | |
| 2273 | + printk("\nhash chain already cached, key: " | |
| 2274 | + "%016Lx tail class: [%p] %s\n", | |
| 2275 | + (unsigned long long)chain_key, | |
| 2276 | + class->key, class->name); | |
| 2277 | + } | |
| 2278 | + | |
| 2279 | + return 0; | |
| 2280 | + } | |
| 2281 | + | |
| 2282 | + if (very_verbose(class)) { | |
| 2283 | + printk("\nnew hash chain, key: %016Lx tail class: [%p] %s\n", | |
| 2284 | + (unsigned long long)chain_key, class->key, class->name); | |
| 2285 | + } | |
| 2286 | + | |
| 2287 | + if (!graph_lock()) | |
| 2288 | + return 0; | |
| 2289 | + | |
| 2290 | + /* | |
| 2291 | + * We have to walk the chain again locked - to avoid duplicates: | |
| 2292 | + */ | |
| 2293 | + chain = lookup_chain_cache(chain_key); | |
| 2294 | + if (chain) { | |
| 2295 | + graph_unlock(); | |
| 2296 | + goto cache_hit; | |
| 2297 | + } | |
| 2298 | + | |
| 2299 | + if (!add_chain_cache(curr, hlock, chain_key)) | |
| 2300 | + return 0; | |
| 2301 | + | |
| 2302 | + return 1; | |
| 2303 | +} | |
| 2304 | + | |
| 2263 | 2305 | static int validate_chain(struct task_struct *curr, struct lockdep_map *lock, |
| 2264 | 2306 | struct held_lock *hlock, int chain_head, u64 chain_key) |
| 2265 | 2307 | { |
| @@ -2270,11 +2312,11 @@ static int validate_chain(struct task_struct *curr, struct lockdep_map *lock, · 4 unmodified lines | ||
| 2270 | 2312 | * |
| 2271 | 2313 | * We look up the chain_key and do the O(N^2) check and update of |
| 2272 | 2314 | * the dependencies only if this is a new dependency chain. |
| 2273 | − * (If lookup_chain_cache() returns with 1 it acquires | |
| 2315 | + * (If lookup_chain_cache_add() return with 1 it acquires | |
| 2274 | 2316 | * graph_lock for us) |
| 2275 | 2317 | */ |
| 2276 | 2318 | if (!hlock->trylock && hlock->check && |
| 2277 | − lookup_chain_cache(curr, hlock, chain_key)) { | |
| 2319 | + lookup_chain_cache_add(curr, hlock, chain_key)) { | |
| 2278 | 2320 | /* |
| 2279 | 2321 | * Check whether last held lock: |
| 2280 | 2322 | * |
| @@ -2302,14 +2344,17 @@ static int validate_chain(struct task_struct *curr, struct lockdep_map *lock, · 21 unmodified lines | ||
| 2302 | 2344 | * Add dependency only if this lock is not the head |
| 2303 | 2345 | * of the chain, and if it's not a secondary read-lock: |
| 2304 | 2346 | */ |
| 2305 | − if (!chain_head && ret != 2) | |
| 2347 | + if (!chain_head && ret != 2) { | |
| 2306 | 2348 | if (!check_prevs_add(curr, hlock)) |
| 2307 | 2349 | return 0; |
| 2350 | + } | |
| 2351 | + | |
| 2308 | 2352 | graph_unlock(); |
| 2309 | − } else | |
| 2310 | − /* after lookup_chain_cache(): */ | |
| 2353 | + } else { | |
| 2354 | + /* after lookup_chain_cache_add(): */ | |
| 2311 | 2355 | if (unlikely(!debug_locks)) |
| 2312 | 2356 | return 0; |
| 2357 | + } | |
| 2313 | 2358 | |
| 2314 | 2359 | return 1; |
| 2315 | 2360 | } |