Removing explicit calls to mmiowb() from driver code means that we must now call into the generic mmiowb_spin_{lock,unlock}() functions from the core spinlock code. In order to elide barriers following critical sections without any I/O writes, we also hook into the asm-generic I/O routines.
Unified · Split
| @@ -19,6 +19,7 @@ · 18 unmodified lines | ||
| 19 | 19 | #include <asm-generic/iomap.h> |
| 20 | 20 | #endif |
| 21 | 21 | |
| 22 | +#include <asm/mmiowb.h> | |
| 22 | 23 | #include <asm-generic/pci_iomap.h> |
| 23 | 24 | |
| 24 | 25 | #ifndef mmiowb |
| @@ -49,7 +50,7 @@ · 24 unmodified lines | ||
| 49 | 50 | |
| 50 | 51 | /* serialize device access against a spin_unlock, usually handled there. */ |
| 51 | 52 | #ifndef __io_aw |
| 52 | −#define __io_aw() barrier() | |
| 53 | +#define __io_aw() mmiowb_set_pending() | |
| 53 | 54 | #endif |
| 54 | 55 | |
| 55 | 56 | #ifndef __io_pbw |
| @@ -57,6 +57,7 @@ · 56 unmodified lines | ||
| 57 | 57 | #include <linux/stringify.h> |
| 58 | 58 | #include <linux/bottom_half.h> |
| 59 | 59 | #include <asm/barrier.h> |
| 60 | +#include <asm/mmiowb.h> | |
| 60 | 61 | |
| 61 | 62 | |
| 62 | 63 | /* |
| @@ -178,6 +179,7 @@ static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock) · 115 unmodified lines | ||
| 178 | 179 | { |
| 179 | 180 | __acquire(lock); |
| 180 | 181 | arch_spin_lock(&lock->raw_lock); |
| 182 | + mmiowb_spin_lock(); | |
| 181 | 183 | } |
| 182 | 184 | |
| 183 | 185 | #ifndef arch_spin_lock_flags |
| @@ -189,15 +191,22 @@ do_raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long *flags) __acquires(lo · 5 unmodified lines | ||
| 189 | 191 | { |
| 190 | 192 | __acquire(lock); |
| 191 | 193 | arch_spin_lock_flags(&lock->raw_lock, *flags); |
| 194 | + mmiowb_spin_lock(); | |
| 192 | 195 | } |
| 193 | 196 | |
| 194 | 197 | static inline int do_raw_spin_trylock(raw_spinlock_t *lock) |
| 195 | 198 | { |
| 196 | − return arch_spin_trylock(&(lock)->raw_lock); | |
| 199 | + int ret = arch_spin_trylock(&(lock)->raw_lock); | |
| 200 | + | |
| 201 | + if (ret) | |
| 202 | + mmiowb_spin_lock(); | |
| 203 | + | |
| 204 | + return ret; | |
| 197 | 205 | } |
| 198 | 206 | |
| 199 | 207 | static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) |
| 200 | 208 | { |
| 209 | + mmiowb_spin_unlock(); | |
| 201 | 210 | arch_spin_unlock(&lock->raw_lock); |
| 202 | 211 | __release(lock); |
| 203 | 212 | } |
| @@ -111,6 +111,7 @@ void do_raw_spin_lock(raw_spinlock_t *lock) · 110 unmodified lines | ||
| 111 | 111 | { |
| 112 | 112 | debug_spin_lock_before(lock); |
| 113 | 113 | arch_spin_lock(&lock->raw_lock); |
| 114 | + mmiowb_spin_lock(); | |
| 114 | 115 | debug_spin_lock_after(lock); |
| 115 | 116 | } |
| 116 | 117 | |
| @@ -118,8 +119,10 @@ int do_raw_spin_trylock(raw_spinlock_t *lock) · 1 unmodified line | ||
| 118 | 119 | { |
| 119 | 120 | int ret = arch_spin_trylock(&lock->raw_lock); |
| 120 | 121 | |
| 121 | − if (ret) | |
| 122 | + if (ret) { | |
| 123 | + mmiowb_spin_lock(); | |
| 122 | 124 | debug_spin_lock_after(lock); |
| 125 | + } | |
| 123 | 126 | #ifndef CONFIG_SMP |
| 124 | 127 | /* |
| 125 | 128 | * Must not happen on UP: |
| @@ -131,6 +134,7 @@ int do_raw_spin_trylock(raw_spinlock_t *lock) · 5 unmodified lines | ||
| 131 | 134 | |
| 132 | 135 | void do_raw_spin_unlock(raw_spinlock_t *lock) |
| 133 | 136 | { |
| 137 | + mmiowb_spin_unlock(); | |
| 134 | 138 | debug_spin_unlock(lock); |
| 135 | 139 | arch_spin_unlock(&lock->raw_lock); |
| 136 | 140 | } |