As preparation for replacing the embedded rwsem, give percpu-rwsem its own lockdep_map.
Showing the changes to include/linux (1 of 5 files). Show all
Unified · Split
| @@ -15,8 +15,17 @@ struct percpu_rw_semaphore { · 14 unmodified lines | |||
| 15 | struct rw_semaphore rw_sem; /* slowpath */ | 15 | struct rw_semaphore rw_sem; /* slowpath */ |
| 16 | struct rcuwait writer; /* blocked writer */ | 16 | struct rcuwait writer; /* blocked writer */ |
| 17 | int readers_block; | 17 | int readers_block; |
| 18 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
| 19 | struct lockdep_map dep_map; | ||
| 20 | #endif | ||
| 18 | }; | 21 | }; |
| 19 | 22 | ||
| 23 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
| 24 | #define __PERCPU_RWSEM_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname }, | ||
| 25 | #else | ||
| 26 | #define __PERCPU_RWSEM_DEP_MAP_INIT(lockname) | ||
| 27 | #endif | ||
| 28 | |||
| 20 | #define __DEFINE_PERCPU_RWSEM(name, is_static) \ | 29 | #define __DEFINE_PERCPU_RWSEM(name, is_static) \ |
| 21 | static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name); \ | 30 | static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name); \ |
| 22 | is_static struct percpu_rw_semaphore name = { \ | 31 | is_static struct percpu_rw_semaphore name = { \ |
| @@ -24,7 +33,9 @@ is_static struct percpu_rw_semaphore name = { \ · 1 unmodified line | |||
| 24 | .read_count = &__percpu_rwsem_rc_##name, \ | 33 | .read_count = &__percpu_rwsem_rc_##name, \ |
| 25 | .rw_sem = __RWSEM_INITIALIZER(name.rw_sem), \ | 34 | .rw_sem = __RWSEM_INITIALIZER(name.rw_sem), \ |
| 26 | .writer = __RCUWAIT_INITIALIZER(name.writer), \ | 35 | .writer = __RCUWAIT_INITIALIZER(name.writer), \ |
| 36 | __PERCPU_RWSEM_DEP_MAP_INIT(name) \ | ||
| 27 | } | 37 | } |
| 38 | |||
| 28 | #define DEFINE_PERCPU_RWSEM(name) \ | 39 | #define DEFINE_PERCPU_RWSEM(name) \ |
| 29 | __DEFINE_PERCPU_RWSEM(name, /* not static */) | 40 | __DEFINE_PERCPU_RWSEM(name, /* not static */) |
| 30 | #define DEFINE_STATIC_PERCPU_RWSEM(name) \ | 41 | #define DEFINE_STATIC_PERCPU_RWSEM(name) \ |
| @@ -37,7 +48,7 @@ static inline void percpu_down_read(struct percpu_rw_semaphore *sem) · 6 unmodified lines | |||
| 37 | { | 48 | { |
| 38 | might_sleep(); | 49 | might_sleep(); |
| 39 | 50 | ||
| 40 | rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_); | 51 | rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); |
| 41 | 52 | ||
| 42 | preempt_disable(); | 53 | preempt_disable(); |
| 43 | /* | 54 | /* |
| @@ -76,13 +87,15 @@ static inline int percpu_down_read_trylock(struct percpu_rw_semaphore *sem) · 32 unmodified lines | |||
| 76 | */ | 87 | */ |
| 77 | 88 | ||
| 78 | if (ret) | 89 | if (ret) |
| 79 | rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 1, _RET_IP_); | 90 | rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_); |
| 80 | 91 | ||
| 81 | return ret; | 92 | return ret; |
| 82 | } | 93 | } |
| 83 | 94 | ||
| 84 | static inline void percpu_up_read(struct percpu_rw_semaphore *sem) | 95 | static inline void percpu_up_read(struct percpu_rw_semaphore *sem) |
| 85 | { | 96 | { |
| 97 | rwsem_release(&sem->dep_map, _RET_IP_); | ||
| 98 | |||
| 86 | preempt_disable(); | 99 | preempt_disable(); |
| 87 | /* | 100 | /* |
| 88 | * Same as in percpu_down_read(). | 101 | * Same as in percpu_down_read(). |
| @@ -92,8 +105,6 @@ static inline void percpu_up_read(struct percpu_rw_semaphore *sem) · 3 unmodified lines | |||
| 92 | else | 105 | else |
| 93 | __percpu_up_read(sem); /* Unconditional memory barrier */ | 106 | __percpu_up_read(sem); /* Unconditional memory barrier */ |
| 94 | preempt_enable(); | 107 | preempt_enable(); |
| 95 | |||
| 96 | rwsem_release(&sem->rw_sem.dep_map, _RET_IP_); | ||
| 97 | } | 108 | } |
| 98 | 109 | ||
| 99 | extern void percpu_down_write(struct percpu_rw_semaphore *); | 110 | extern void percpu_down_write(struct percpu_rw_semaphore *); |
| @@ -110,15 +121,13 @@ extern void percpu_free_rwsem(struct percpu_rw_semaphore *); · 10 unmodified lines | |||
| 110 | __percpu_init_rwsem(sem, #sem, &rwsem_key); \ | 121 | __percpu_init_rwsem(sem, #sem, &rwsem_key); \ |
| 111 | }) | 122 | }) |
| 112 | 123 | ||
| 113 | #define percpu_rwsem_is_held(sem) lockdep_is_held(&(sem)->rw_sem) | 124 | #define percpu_rwsem_is_held(sem) lockdep_is_held(sem) |
| 114 | 125 | #define percpu_rwsem_assert_held(sem) lockdep_assert_held(sem) | |
| 115 | #define percpu_rwsem_assert_held(sem) \ | ||
| 116 | lockdep_assert_held(&(sem)->rw_sem) | ||
| 117 | 126 | ||
| 118 | static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem, | 127 | static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem, |
| 119 | bool read, unsigned long ip) | 128 | bool read, unsigned long ip) |
| 120 | { | 129 | { |
| 121 | lock_release(&sem->rw_sem.dep_map, ip); | 130 | lock_release(&sem->dep_map, ip); |
| 122 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER | 131 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER |
| 123 | if (!read) | 132 | if (!read) |
| 124 | atomic_long_set(&sem->rw_sem.owner, RWSEM_OWNER_UNKNOWN); | 133 | atomic_long_set(&sem->rw_sem.owner, RWSEM_OWNER_UNKNOWN); |
| @@ -128,7 +137,7 @@ static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem, · 3 unmodified lines | |||
| 128 | static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem, | 137 | static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem, |
| 129 | bool read, unsigned long ip) | 138 | bool read, unsigned long ip) |
| 130 | { | 139 | { |
| 131 | lock_acquire(&sem->rw_sem.dep_map, 0, 1, read, 1, NULL, ip); | 140 | lock_acquire(&sem->dep_map, 0, 1, read, 1, NULL, ip); |
| 132 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER | 141 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER |
| 133 | if (!read) | 142 | if (!read) |
| 134 | atomic_long_set(&sem->rw_sem.owner, (long)current); | 143 | atomic_long_set(&sem->rw_sem.owner, (long)current); |