Pull locking updates from Ingo Molnar: "The main changes in this cycle are: - rwsem scalability improvements, phase #2, by Waiman Long, which are rather impressive: "On a 2-socket 40-core 80-thread Skylake system with 40 reader and writer locking threads, the min/mean/max locking operations done in a 5-second testing window before the patchset were: 40 readers, Iterations Min/Mean/Max = 1,807/1,808/1,810 40 writers, Iterations Min/Mean/Max = 1,807/50,344/151,255 After the patchset, they became: 40 readers, Iterations Min/Mean/Max = 30,057/31,359/32,741 40 writers, Iterations Min/Mean/Max = 94,466/95,845/97,098" There's a lot of changes to the locking implementation that makes it similar to qrwlock, including owner handoff for more fair locking. Another microbenchmark shows how across the spectrum the improvements are: "With a locking microbenchmark running on 5.1 based kernel, the total locking rates (in kops/s) on a 2-socket Skylake system with equal numbers of readers and writers (mixed) before and after this patchset were: # of Threads Before Patch After Patch ------------ ------------ ----------- 2 2,618 4,193 4 1,202 3,726 8 802 3,622 16 729 3,359 32 319 2,826 64 102 2,744" The changes are extensive and the patch-set has been through several iterations addressing various locking workloads. There might be more regressions, but unless they are pathological I believe we want to use this new implementation as the baseline going forward. - jump-label optimizations by Daniel Bristot de Oliveira: the primary motivation was to remove IPI disturbance of isolated RT-workload CPUs, which resulted in the implementation of batched jump-label updates. Beyond the improvement of the real-time characteristics kernel, in one test this patchset improved static key update overhead from 57 msecs to just 1.4 msecs - which is a nice speedup as well. - atomic64_t cross-arch type cleanups by Mark Rutland: over the last ~10 years of atomic64_t existence the various types used by the APIs only had to be self-consistent within each architecture - which means they became wildly inconsistent across architectures. Mark puts and end to this by reworking all the atomic64 implementations to use 's64' as the base type for atomic64_t, and to ensure that this type is consistently used for parameters and return values in the API, avoiding further problems in this area. - A large set of small improvements to lockdep by Yuyang Du: type cleanups, output cleanups, function return type and othr cleanups all around the place. - A set of percpu ops cleanups and fixes by Peter Zijlstra. - Misc other changes - please see the Git log for more details" * 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (82 commits) locking/lockdep: increase size of counters for lockdep statistics locking/atomics: Use sed(1) instead of non-standard head(1) option locking/lockdep: Move mark_lock() inside CONFIG_TRACE_IRQFLAGS && CONFIG_PROVE_LOCKING x86/jump_label: Make tp_vec_nr static x86/percpu: Optimize raw_cpu_xchg() x86/percpu, sched/fair: Avoid local_clock() x86/percpu, x86/irq: Relax {set,get}_irq_regs() x86/percpu: Relax smp_processor_id() x86/percpu: Differentiate this_cpu_{}() and __this_cpu_{}() locking/rwsem: Guard against making count negative locking/rwsem: Adaptive disabling of reader optimistic spinning locking/rwsem: Enable time-based spinning on reader-owned rwsem locking/rwsem: Make rwsem->owner an atomic_long_t locking/rwsem: Enable readers spinning on writer locking/rwsem: Clarify usage of owner's nonspinaable bit locking/rwsem: Wake up almost all readers in wait queue locking/rwsem: More optimal RT task handling of null owner locking/rwsem: Always release wait_lock before waking up tasks locking/rwsem: Implement lock handoff to prevent lock starvation locking/rwsem: Make rwsem_spin_on_owner() return owner state ...
Showing the changes to include/linux (7 of 55 files). Show all
Unified · Split
| @@ -215,6 +215,9 @@ extern void arch_jump_label_transform(struct jump_entry *entry, · 214 unmodified lines | ||
| 215 | 215 | enum jump_label_type type); |
| 216 | 216 | extern void arch_jump_label_transform_static(struct jump_entry *entry, |
| 217 | 217 | enum jump_label_type type); |
| 218 | +extern bool arch_jump_label_transform_queue(struct jump_entry *entry, | |
| 219 | + enum jump_label_type type); | |
| 220 | +extern void arch_jump_label_transform_apply(void); | |
| 218 | 221 | extern int jump_label_text_reserved(void *start, void *end); |
| 219 | 222 | extern void static_key_slow_inc(struct static_key *key); |
| 220 | 223 | extern void static_key_slow_dec(struct static_key *key); |
| @@ -203,11 +203,17 @@ struct lock_list { · 202 unmodified lines | ||
| 203 | 203 | struct lock_list *parent; |
| 204 | 204 | }; |
| 205 | 205 | |
| 206 | −/* | |
| 207 | − * We record lock dependency chains, so that we can cache them: | |
| 206 | +/** | |
| 207 | + * struct lock_chain - lock dependency chain record | |
| 208 | + * | |
| 209 | + * @irq_context: the same as irq_context in held_lock below | |
| 210 | + * @depth: the number of held locks in this chain | |
| 211 | + * @base: the index in chain_hlocks for this chain | |
| 212 | + * @entry: the collided lock chains in lock_chain hash list | |
| 213 | + * @chain_key: the hash key of this lock_chain | |
| 208 | 214 | */ |
| 209 | 215 | struct lock_chain { |
| 210 | − /* see BUILD_BUG_ON()s in lookup_chain_cache() */ | |
| 216 | + /* see BUILD_BUG_ON()s in add_chain_cache() */ | |
| 211 | 217 | unsigned int irq_context : 2, |
| 212 | 218 | depth : 6, |
| 213 | 219 | base : 24; |
| @@ -217,12 +223,8 @@ struct lock_chain { · 3 unmodified lines | ||
| 217 | 223 | }; |
| 218 | 224 | |
| 219 | 225 | #define MAX_LOCKDEP_KEYS_BITS 13 |
| 220 | −/* | |
| 221 | − * Subtract one because we offset hlock->class_idx by 1 in order | |
| 222 | − * to make 0 mean no class. This avoids overflowing the class_idx | |
| 223 | − * bitfield and hitting the BUG in hlock_class(). | |
| 224 | − */ | |
| 225 | −#define MAX_LOCKDEP_KEYS ((1UL << MAX_LOCKDEP_KEYS_BITS) - 1) | |
| 226 | +#define MAX_LOCKDEP_KEYS (1UL << MAX_LOCKDEP_KEYS_BITS) | |
| 227 | +#define INITIAL_CHAIN_KEY -1 | |
| 226 | 228 | |
| 227 | 229 | struct held_lock { |
| 228 | 230 | /* |
| @@ -247,6 +249,11 @@ struct held_lock { · 18 unmodified lines | ||
| 247 | 249 | u64 waittime_stamp; |
| 248 | 250 | u64 holdtime_stamp; |
| 249 | 251 | #endif |
| 252 | + /* | |
| 253 | + * class_idx is zero-indexed; it points to the element in | |
| 254 | + * lock_classes this held lock instance belongs to. class_idx is in | |
| 255 | + * the range from 0 to (MAX_LOCKDEP_KEYS-1) inclusive. | |
| 256 | + */ | |
| 250 | 257 | unsigned int class_idx:MAX_LOCKDEP_KEYS_BITS; |
| 251 | 258 | /* |
| 252 | 259 | * The lock-stack is unified in that the lock chains of interrupt |
| @@ -281,6 +288,8 @@ extern void lockdep_free_key_range(void *start, unsigned long size); · 28 unmodified lines | ||
| 281 | 288 | extern asmlinkage void lockdep_sys_exit(void); |
| 282 | 289 | extern void lockdep_set_selftest_task(struct task_struct *task); |
| 283 | 290 | |
| 291 | +extern void lockdep_init_task(struct task_struct *task); | |
| 292 | + | |
| 284 | 293 | extern void lockdep_off(void); |
| 285 | 294 | extern void lockdep_on(void); |
| 286 | 295 | |
| @@ -385,7 +394,7 @@ extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie); · 98 unmodified lines | ||
| 385 | 394 | WARN_ON(debug_locks && !lockdep_is_held(l)); \ |
| 386 | 395 | } while (0) |
| 387 | 396 | |
| 388 | −#define lockdep_assert_held_exclusive(l) do { \ | |
| 397 | +#define lockdep_assert_held_write(l) do { \ | |
| 389 | 398 | WARN_ON(debug_locks && !lockdep_is_held_type(l, 0)); \ |
| 390 | 399 | } while (0) |
| 391 | 400 | |
| @@ -405,6 +414,10 @@ extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie); · 13 unmodified lines | ||
| 405 | 414 | |
| 406 | 415 | #else /* !CONFIG_LOCKDEP */ |
| 407 | 416 | |
| 417 | +static inline void lockdep_init_task(struct task_struct *task) | |
| 418 | +{ | |
| 419 | +} | |
| 420 | + | |
| 408 | 421 | static inline void lockdep_off(void) |
| 409 | 422 | { |
| 410 | 423 | } |
| @@ -466,7 +479,7 @@ struct lockdep_map { }; · 55 unmodified lines | ||
| 466 | 479 | #define lockdep_is_held_type(l, r) (1) |
| 467 | 480 | |
| 468 | 481 | #define lockdep_assert_held(l) do { (void)(l); } while (0) |
| 469 | −#define lockdep_assert_held_exclusive(l) do { (void)(l); } while (0) | |
| 482 | +#define lockdep_assert_held_write(l) do { (void)(l); } while (0) | |
| 470 | 483 | #define lockdep_assert_held_read(l) do { (void)(l); } while (0) |
| 471 | 484 | #define lockdep_assert_held_once(l) do { (void)(l); } while (0) |
| 472 | 485 | |
| @@ -497,7 +510,6 @@ enum xhlock_context_t { · 24 unmodified lines | ||
| 497 | 510 | { .name = (_name), .key = (void *)(_key), } |
| 498 | 511 | |
| 499 | 512 | static inline void lockdep_invariant_state(bool force) {} |
| 500 | −static inline void lockdep_init_task(struct task_struct *task) {} | |
| 501 | 513 | static inline void lockdep_free_task(struct task_struct *task) {} |
| 502 | 514 | |
| 503 | 515 | #ifdef CONFIG_LOCK_STAT |
| @@ -121,7 +121,7 @@ static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem, · 120 unmodified lines | ||
| 121 | 121 | lock_release(&sem->rw_sem.dep_map, 1, ip); |
| 122 | 122 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER |
| 123 | 123 | if (!read) |
| 124 | − sem->rw_sem.owner = RWSEM_OWNER_UNKNOWN; | |
| 124 | + atomic_long_set(&sem->rw_sem.owner, RWSEM_OWNER_UNKNOWN); | |
| 125 | 125 | #endif |
| 126 | 126 | } |
| 127 | 127 | |
| @@ -131,7 +131,7 @@ static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem, · 3 unmodified lines | ||
| 131 | 131 | lock_acquire(&sem->rw_sem.dep_map, 0, 1, read, 1, NULL, ip); |
| 132 | 132 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER |
| 133 | 133 | if (!read) |
| 134 | − sem->rw_sem.owner = current; | |
| 134 | + atomic_long_set(&sem->rw_sem.owner, (long)current); | |
| 135 | 135 | #endif |
| 136 | 136 | } |
| 137 | 137 | |
| @@ -34,12 +34,13 @@ · 33 unmodified lines | ||
| 34 | 34 | */ |
| 35 | 35 | struct rw_semaphore { |
| 36 | 36 | atomic_long_t count; |
| 37 | −#ifdef CONFIG_RWSEM_SPIN_ON_OWNER | |
| 38 | 37 | /* |
| 39 | − * Write owner. Used as a speculative check to see | |
| 40 | − * if the owner is running on the cpu. | |
| 38 | + * Write owner or one of the read owners as well flags regarding | |
| 39 | + * the current state of the rwsem. Can be used as a speculative | |
| 40 | + * check to see if the write owner is running on the cpu. | |
| 41 | 41 | */ |
| 42 | − struct task_struct *owner; | |
| 42 | + atomic_long_t owner; | |
| 43 | +#ifdef CONFIG_RWSEM_SPIN_ON_OWNER | |
| 43 | 44 | struct optimistic_spin_queue osq; /* spinner MCS lock */ |
| 44 | 45 | #endif |
| 45 | 46 | raw_spinlock_t wait_lock; |
| @@ -50,10 +51,10 @@ struct rw_semaphore { · 4 unmodified lines | ||
| 50 | 51 | }; |
| 51 | 52 | |
| 52 | 53 | /* |
| 53 | − * Setting bit 1 of the owner field but not bit 0 will indicate | |
| 54 | + * Setting all bits of the owner field except bit 0 will indicate | |
| 54 | 55 | * that the rwsem is writer-owned with an unknown owner. |
| 55 | 56 | */ |
| 56 | −#define RWSEM_OWNER_UNKNOWN ((struct task_struct *)-2L) | |
| 57 | +#define RWSEM_OWNER_UNKNOWN (-2L) | |
| 57 | 58 | |
| 58 | 59 | /* In all implementations count != 0 means locked */ |
| 59 | 60 | static inline int rwsem_is_locked(struct rw_semaphore *sem) |
| @@ -73,13 +74,14 @@ static inline int rwsem_is_locked(struct rw_semaphore *sem) · 13 unmodified lines | ||
| 73 | 74 | #endif |
| 74 | 75 | |
| 75 | 76 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER |
| 76 | −#define __RWSEM_OPT_INIT(lockname) , .osq = OSQ_LOCK_UNLOCKED, .owner = NULL | |
| 77 | +#define __RWSEM_OPT_INIT(lockname) , .osq = OSQ_LOCK_UNLOCKED | |
| 77 | 78 | #else |
| 78 | 79 | #define __RWSEM_OPT_INIT(lockname) |
| 79 | 80 | #endif |
| 80 | 81 | |
| 81 | 82 | #define __RWSEM_INITIALIZER(name) \ |
| 82 | 83 | { __RWSEM_INIT_COUNT(name), \ |
| 84 | + .owner = ATOMIC_LONG_INIT(0), \ | |
| 83 | 85 | .wait_list = LIST_HEAD_INIT((name).wait_list), \ |
| 84 | 86 | .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock) \ |
| 85 | 87 | __RWSEM_OPT_INIT(name) \ |
| @@ -51,6 +51,11 @@ static inline void wake_q_init(struct wake_q_head *head) · 50 unmodified lines | ||
| 51 | 51 | head->lastp = &head->first; |
| 52 | 52 | } |
| 53 | 53 | |
| 54 | +static inline bool wake_q_empty(struct wake_q_head *head) | |
| 55 | +{ | |
| 56 | + return head->first == WAKE_Q_TAIL; | |
| 57 | +} | |
| 58 | + | |
| 54 | 59 | extern void wake_q_add(struct wake_q_head *head, struct task_struct *task); |
| 55 | 60 | extern void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task); |
| 56 | 61 | extern void wake_up_q(struct wake_q_head *head); |
| @@ -180,29 +180,46 @@ static inline int get_boot_cpu_id(void) · 179 unmodified lines | ||
| 180 | 180 | |
| 181 | 181 | #endif /* !SMP */ |
| 182 | 182 | |
| 183 | −/* | |
| 184 | − * smp_processor_id(): get the current CPU ID. | |
| 183 | +/** | |
| 184 | + * raw_processor_id() - get the current (unstable) CPU id | |
| 185 | + * | |
| 186 | + * For then you know what you are doing and need an unstable | |
| 187 | + * CPU id. | |
| 188 | + */ | |
| 189 | + | |
| 190 | +/** | |
| 191 | + * smp_processor_id() - get the current (stable) CPU id | |
| 192 | + * | |
| 193 | + * This is the normal accessor to the CPU id and should be used | |
| 194 | + * whenever possible. | |
| 195 | + * | |
| 196 | + * The CPU id is stable when: | |
| 185 | 197 | * |
| 186 | − * if DEBUG_PREEMPT is enabled then we check whether it is | |
| 187 | − * used in a preemption-safe way. (smp_processor_id() is safe | |
| 188 | − * if it's used in a preemption-off critical section, or in | |
| 189 | − * a thread that is bound to the current CPU.) | |
| 198 | + * - IRQs are disabled; | |
| 199 | + * - preemption is disabled; | |
| 200 | + * - the task is CPU affine. | |
| 190 | 201 | * |
| 191 | − * NOTE: raw_smp_processor_id() is for internal use only | |
| 192 | − * (smp_processor_id() is the preferred variant), but in rare | |
| 193 | − * instances it might also be used to turn off false positives | |
| 194 | − * (i.e. smp_processor_id() use that the debugging code reports but | |
| 195 | − * which use for some reason is legal). Don't use this to hack around | |
| 196 | − * the warning message, as your code might not work under PREEMPT. | |
| 202 | + * When CONFIG_DEBUG_PREEMPT; we verify these assumption and WARN | |
| 203 | + * when smp_processor_id() is used when the CPU id is not stable. | |
| 197 | 204 | */ |
| 205 | + | |
| 206 | +/* | |
| 207 | + * Allow the architecture to differentiate between a stable and unstable read. | |
| 208 | + * For example, x86 uses an IRQ-safe asm-volatile read for the unstable but a | |
| 209 | + * regular asm read for the stable. | |
| 210 | + */ | |
| 211 | +#ifndef __smp_processor_id | |
| 212 | +#define __smp_processor_id(x) raw_smp_processor_id(x) | |
| 213 | +#endif | |
| 214 | + | |
| 198 | 215 | #ifdef CONFIG_DEBUG_PREEMPT |
| 199 | 216 | extern unsigned int debug_smp_processor_id(void); |
| 200 | 217 | # define smp_processor_id() debug_smp_processor_id() |
| 201 | 218 | #else |
| 202 | −# define smp_processor_id() raw_smp_processor_id() | |
| 219 | +# define smp_processor_id() __smp_processor_id() | |
| 203 | 220 | #endif |
| 204 | 221 | |
| 205 | −#define get_cpu() ({ preempt_disable(); smp_processor_id(); }) | |
| 222 | +#define get_cpu() ({ preempt_disable(); __smp_processor_id(); }) | |
| 206 | 223 | #define put_cpu() preempt_enable() |
| 207 | 224 | |
| 208 | 225 | /* |
| @@ -174,7 +174,7 @@ typedef struct { · 173 unmodified lines | ||
| 174 | 174 | |
| 175 | 175 | #ifdef CONFIG_64BIT |
| 176 | 176 | typedef struct { |
| 177 | − long counter; | |
| 177 | + s64 counter; | |
| 178 | 178 | } atomic64_t; |
| 179 | 179 | #endif |
| 180 | 180 | |