Without branch hints, the very unlikely chance of the loop repeating due to cmpxchg failure is unrolled with gcc-4 that I have tested. Improve this for architectures with a native cas/cmpxchg. llsc archs should try to implement this natively.
Showing the changes to include (6 of 6 files). Show all
Unified · Split
| @@ -225,8 +225,14 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v) · 224 unmodified lines | |||
| 225 | ({ \ | 225 | ({ \ |
| 226 | int c, old; \ | 226 | int c, old; \ |
| 227 | c = atomic_read(v); \ | 227 | c = atomic_read(v); \ |
| 228 | while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ | 228 | for (;;) { \ |
| 229 | if (unlikely(c == (u))) \ | ||
| 230 | break; \ | ||
| 231 | old = atomic_cmpxchg((v), c, c + (a)); \ | ||
| 232 | if (likely(old == c)) \ | ||
| 233 | break; \ | ||
| 229 | c = old; \ | 234 | c = old; \ |
| 235 | } \ | ||
| 230 | c != (u); \ | 236 | c != (u); \ |
| 231 | }) | 237 | }) |
| 232 | #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) | 238 | #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) |
| @@ -95,8 +95,14 @@ ia64_atomic64_sub (__s64 i, atomic64_t *v) · 94 unmodified lines | |||
| 95 | ({ \ | 95 | ({ \ |
| 96 | int c, old; \ | 96 | int c, old; \ |
| 97 | c = atomic_read(v); \ | 97 | c = atomic_read(v); \ |
| 98 | while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ | 98 | for (;;) { \ |
| 99 | if (unlikely(c == (u))) \ | ||
| 100 | break; \ | ||
| 101 | old = atomic_cmpxchg((v), c, c + (a)); \ | ||
| 102 | if (likely(old == c)) \ | ||
| 103 | break; \ | ||
| 99 | c = old; \ | 104 | c = old; \ |
| 105 | } \ | ||
| 100 | c != (u); \ | 106 | c != (u); \ |
| 101 | }) | 107 | }) |
| 102 | #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) | 108 | #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) |
| @@ -175,8 +175,14 @@ static inline void atomic_set_mask(unsigned long mask, unsigned long *v) · 174 unmodified lines | |||
| 175 | ({ \ | 175 | ({ \ |
| 176 | int c, old; \ | 176 | int c, old; \ |
| 177 | c = atomic_read(v); \ | 177 | c = atomic_read(v); \ |
| 178 | while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ | 178 | for (;;) { \ |
| 179 | if (unlikely(c == (u))) \ | ||
| 180 | break; \ | ||
| 181 | old = atomic_cmpxchg((v), c, c + (a)); \ | ||
| 182 | if (likely(old == c)) \ | ||
| 183 | break; \ | ||
| 179 | c = old; \ | 184 | c = old; \ |
| 185 | } \ | ||
| 180 | c != (u); \ | 186 | c != (u); \ |
| 181 | }) | 187 | }) |
| 182 | #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) | 188 | #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) |
| @@ -89,10 +89,15 @@ static __inline__ int atomic_cmpxchg(atomic_t *v, int old, int new) · 88 unmodified lines | |||
| 89 | static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) | 89 | static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) |
| 90 | { | 90 | { |
| 91 | int c, old; | 91 | int c, old; |
| 92 | |||
| 93 | c = atomic_read(v); | 92 | c = atomic_read(v); |
| 94 | while (c != u && (old = atomic_cmpxchg(v, c, c + a)) != c) | 93 | for (;;) { |
| 94 | if (unlikely(c == u)) | ||
| 95 | break; | ||
| 96 | old = atomic_cmpxchg(v, c, c + a); | ||
| 97 | if (likely(old == c)) | ||
| 98 | break; | ||
| 95 | c = old; | 99 | c = old; |
| 100 | } | ||
| 96 | return c != u; | 101 | return c != u; |
| 97 | } | 102 | } |
| 98 | 103 | ||
| @@ -167,10 +172,15 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, · 68 unmodified lines | |||
| 167 | long long a, long long u) | 172 | long long a, long long u) |
| 168 | { | 173 | { |
| 169 | long long c, old; | 174 | long long c, old; |
| 170 | |||
| 171 | c = atomic64_read(v); | 175 | c = atomic64_read(v); |
| 172 | while (c != u && (old = atomic64_cmpxchg(v, c, c + a)) != c) | 176 | for (;;) { |
| 177 | if (unlikely(c == u)) | ||
| 178 | break; | ||
| 179 | old = atomic64_cmpxchg(v, c, c + a); | ||
| 180 | if (likely(old == c)) | ||
| 181 | break; | ||
| 173 | c = old; | 182 | c = old; |
| 183 | } | ||
| 174 | return c != u; | 184 | return c != u; |
| 175 | } | 185 | } |
| 176 | 186 | ||
| @@ -78,9 +78,15 @@ extern int atomic64_sub_ret(int, atomic64_t *); · 77 unmodified lines | |||
| 78 | ({ \ | 78 | ({ \ |
| 79 | int c, old; \ | 79 | int c, old; \ |
| 80 | c = atomic_read(v); \ | 80 | c = atomic_read(v); \ |
| 81 | while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ | 81 | for (;;) { \ |
| 82 | if (unlikely(c == (u))) \ | ||
| 83 | break; \ | ||
| 84 | old = atomic_cmpxchg((v), c, c + (a)); \ | ||
| 85 | if (likely(old == c)) \ | ||
| 86 | break; \ | ||
| 82 | c = old; \ | 87 | c = old; \ |
| 83 | c != (u); \ | 88 | } \ |
| 89 | likely(c != (u)); \ | ||
| 84 | }) | 90 | }) |
| 85 | #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) | 91 | #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) |
| 86 | 92 | ||
| @@ -405,8 +405,14 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) · 404 unmodified lines | |||
| 405 | ({ \ | 405 | ({ \ |
| 406 | int c, old; \ | 406 | int c, old; \ |
| 407 | c = atomic_read(v); \ | 407 | c = atomic_read(v); \ |
| 408 | while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ | 408 | for (;;) { \ |
| 409 | if (unlikely(c == (u))) \ | ||
| 410 | break; \ | ||
| 411 | old = atomic_cmpxchg((v), c, c + (a)); \ | ||
| 412 | if (likely(old == c)) \ | ||
| 413 | break; \ | ||
| 409 | c = old; \ | 414 | c = old; \ |
| 415 | } \ | ||
| 410 | c != (u); \ | 416 | c != (u); \ |
| 411 | }) | 417 | }) |
| 412 | #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) | 418 | #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) |