The owner field in the rw_semaphore structure is used primarily for optimistic spinning. However, identifying the rwsem owner can also be helpful in debugging as well as tracing locking related issues when analyzing crash dump. The owner field may also store state information that can be important to the operation of the rwsem. So the owner field is now made a permanent member of the rw_semaphore structure irrespective of CONFIG_RWSEM_SPIN_ON_OWNER.
Unified · Split
| @@ -34,12 +34,12 @@ · 33 unmodified lines | |||
| 34 | */ | 34 | */ |
| 35 | struct rw_semaphore { | 35 | struct rw_semaphore { |
| 36 | atomic_long_t count; | 36 | atomic_long_t count; |
| 37 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER | ||
| 38 | /* | 37 | /* |
| 39 | * Write owner. Used as a speculative check to see | 38 | * Write owner or one of the read owners. Can be used as a |
| 40 | * if the owner is running on the cpu. | 39 | * speculative check to see if the owner is running on the cpu. |
| 41 | */ | 40 | */ |
| 42 | struct task_struct *owner; | 41 | struct task_struct *owner; |
| 42 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER | ||
| 43 | struct optimistic_spin_queue osq; /* spinner MCS lock */ | 43 | struct optimistic_spin_queue osq; /* spinner MCS lock */ |
| 44 | #endif | 44 | #endif |
| 45 | raw_spinlock_t wait_lock; | 45 | raw_spinlock_t wait_lock; |
| @@ -73,13 +73,14 @@ static inline int rwsem_is_locked(struct rw_semaphore *sem) · 27 unmodified lines | |||
| 73 | #endif | 73 | #endif |
| 74 | 74 | ||
| 75 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER | 75 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER |
| 76 | #define __RWSEM_OPT_INIT(lockname) , .osq = OSQ_LOCK_UNLOCKED, .owner = NULL | 76 | #define __RWSEM_OPT_INIT(lockname) , .osq = OSQ_LOCK_UNLOCKED |
| 77 | #else | 77 | #else |
| 78 | #define __RWSEM_OPT_INIT(lockname) | 78 | #define __RWSEM_OPT_INIT(lockname) |
| 79 | #endif | 79 | #endif |
| 80 | 80 | ||
| 81 | #define __RWSEM_INITIALIZER(name) \ | 81 | #define __RWSEM_INITIALIZER(name) \ |
| 82 | { __RWSEM_INIT_COUNT(name), \ | 82 | { __RWSEM_INIT_COUNT(name), \ |
| 83 | .owner = NULL, \ | ||
| 83 | .wait_list = LIST_HEAD_INIT((name).wait_list), \ | 84 | .wait_list = LIST_HEAD_INIT((name).wait_list), \ |
| 84 | .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock) \ | 85 | .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock) \ |
| 85 | __RWSEM_OPT_INIT(name) \ | 86 | __RWSEM_OPT_INIT(name) \ |
| @@ -86,8 +86,8 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name, · 85 unmodified lines | |||
| 86 | atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE); | 86 | atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE); |
| 87 | raw_spin_lock_init(&sem->wait_lock); | 87 | raw_spin_lock_init(&sem->wait_lock); |
| 88 | INIT_LIST_HEAD(&sem->wait_list); | 88 | INIT_LIST_HEAD(&sem->wait_list); |
| 89 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER | ||
| 90 | sem->owner = NULL; | 89 | sem->owner = NULL; |
| 90 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER | ||
| 91 | osq_lock_init(&sem->osq); | 91 | osq_lock_init(&sem->osq); |
| 92 | #endif | 92 | #endif |
| 93 | } | 93 | } |
| @@ -61,7 +61,6 @@ · 60 unmodified lines | |||
| 61 | #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS | 61 | #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS |
| 62 | #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) | 62 | #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) |
| 63 | 63 | ||
| 64 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER | ||
| 65 | /* | 64 | /* |
| 66 | * All writes to owner are protected by WRITE_ONCE() to make sure that | 65 | * All writes to owner are protected by WRITE_ONCE() to make sure that |
| 67 | * store tearing can't happen as optimistic spinners may read and use | 66 | * store tearing can't happen as optimistic spinners may read and use |
| @@ -126,7 +125,6 @@ static inline bool rwsem_has_anonymous_owner(struct task_struct *owner) · 58 unmodified lines | |||
| 126 | * real owner or one of the real owners. The only exception is when the | 125 | * real owner or one of the real owners. The only exception is when the |
| 127 | * unlock is done by up_read_non_owner(). | 126 | * unlock is done by up_read_non_owner(). |
| 128 | */ | 127 | */ |
| 129 | #define rwsem_clear_reader_owned rwsem_clear_reader_owned | ||
| 130 | static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) | 128 | static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) |
| 131 | { | 129 | { |
| 132 | unsigned long val = (unsigned long)current | RWSEM_READER_OWNED | 130 | unsigned long val = (unsigned long)current | RWSEM_READER_OWNED |
| @@ -135,28 +133,7 @@ static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) · 2 unmodified lines | |||
| 135 | cmpxchg_relaxed((unsigned long *)&sem->owner, val, | 133 | cmpxchg_relaxed((unsigned long *)&sem->owner, val, |
| 136 | RWSEM_READER_OWNED | RWSEM_ANONYMOUSLY_OWNED); | 134 | RWSEM_READER_OWNED | RWSEM_ANONYMOUSLY_OWNED); |
| 137 | } | 135 | } |
| 138 | #endif | ||
| 139 | |||
| 140 | #else | 136 | #else |
| 141 | static inline void rwsem_set_owner(struct rw_semaphore *sem) | ||
| 142 | { | ||
| 143 | } | ||
| 144 | |||
| 145 | static inline void rwsem_clear_owner(struct rw_semaphore *sem) | ||
| 146 | { | ||
| 147 | } | ||
| 148 | |||
| 149 | static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, | ||
| 150 | struct task_struct *owner) | ||
| 151 | { | ||
| 152 | } | ||
| 153 | |||
| 154 | static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) | ||
| 155 | { | ||
| 156 | } | ||
| 157 | #endif | ||
| 158 | |||
| 159 | #ifndef rwsem_clear_reader_owned | ||
| 160 | static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) | 137 | static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) |
| 161 | { | 138 | { |
| 162 | } | 139 | } |
| @@ -1095,7 +1095,7 @@ config PROVE_LOCKING · 1094 unmodified lines | |||
| 1095 | select DEBUG_SPINLOCK | 1095 | select DEBUG_SPINLOCK |
| 1096 | select DEBUG_MUTEXES | 1096 | select DEBUG_MUTEXES |
| 1097 | select DEBUG_RT_MUTEXES if RT_MUTEXES | 1097 | select DEBUG_RT_MUTEXES if RT_MUTEXES |
| 1098 | select DEBUG_RWSEMS if RWSEM_SPIN_ON_OWNER | 1098 | select DEBUG_RWSEMS |
| 1099 | select DEBUG_WW_MUTEX_SLOWPATH | 1099 | select DEBUG_WW_MUTEX_SLOWPATH |
| 1100 | select DEBUG_LOCK_ALLOC | 1100 | select DEBUG_LOCK_ALLOC |
| 1101 | select TRACE_IRQFLAGS | 1101 | select TRACE_IRQFLAGS |
| @@ -1199,10 +1199,10 @@ config DEBUG_WW_MUTEX_SLOWPATH · 97 unmodified lines | |||
| 1199 | 1199 | ||
| 1200 | config DEBUG_RWSEMS | 1200 | config DEBUG_RWSEMS |
| 1201 | bool "RW Semaphore debugging: basic checks" | 1201 | bool "RW Semaphore debugging: basic checks" |
| 1202 | depends on DEBUG_KERNEL && RWSEM_SPIN_ON_OWNER | 1202 | depends on DEBUG_KERNEL |
| 1203 | help | 1203 | help |
| 1204 | This debugging feature allows mismatched rw semaphore locks and unlocks | 1204 | This debugging feature allows mismatched rw semaphore locks |
| 1205 | to be detected and reported. | 1205 | and unlocks to be detected and reported. |
| 1206 | 1206 | ||
| 1207 | config DEBUG_LOCK_ALLOC | 1207 | config DEBUG_LOCK_ALLOC |
| 1208 | bool "Lock debugging: detect incorrect freeing of live locks" | 1208 | bool "Lock debugging: detect incorrect freeing of live locks" |