Adds the required KCSAN instrumentation for barriers if CONFIG_SMP. KCSAN supports modeling the effects of: smp_mb() smp_rmb() smp_wmb() smp_store_release()
Showing the changes to include (2 of 2 files). Show all
Unified · Split
| @@ -14,6 +14,7 @@ · 13 unmodified lines | ||
| 14 | 14 | #ifndef __ASSEMBLY__ |
| 15 | 15 | |
| 16 | 16 | #include <linux/compiler.h> |
| 17 | +#include <linux/kcsan-checks.h> | |
| 17 | 18 | #include <asm/rwonce.h> |
| 18 | 19 | |
| 19 | 20 | #ifndef nop |
| @@ -62,15 +63,15 @@ · 42 unmodified lines | ||
| 62 | 63 | #ifdef CONFIG_SMP |
| 63 | 64 | |
| 64 | 65 | #ifndef smp_mb |
| 65 | −#define smp_mb() __smp_mb() | |
| 66 | +#define smp_mb() do { kcsan_mb(); __smp_mb(); } while (0) | |
| 66 | 67 | #endif |
| 67 | 68 | |
| 68 | 69 | #ifndef smp_rmb |
| 69 | −#define smp_rmb() __smp_rmb() | |
| 70 | +#define smp_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0) | |
| 70 | 71 | #endif |
| 71 | 72 | |
| 72 | 73 | #ifndef smp_wmb |
| 73 | −#define smp_wmb() __smp_wmb() | |
| 74 | +#define smp_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0) | |
| 74 | 75 | #endif |
| 75 | 76 | |
| 76 | 77 | #else /* !CONFIG_SMP */ |
| @@ -123,19 +124,19 @@ do { \ · 46 unmodified lines | ||
| 123 | 124 | #ifdef CONFIG_SMP |
| 124 | 125 | |
| 125 | 126 | #ifndef smp_store_mb |
| 126 | −#define smp_store_mb(var, value) __smp_store_mb(var, value) | |
| 127 | +#define smp_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0) | |
| 127 | 128 | #endif |
| 128 | 129 | |
| 129 | 130 | #ifndef smp_mb__before_atomic |
| 130 | −#define smp_mb__before_atomic() __smp_mb__before_atomic() | |
| 131 | +#define smp_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0) | |
| 131 | 132 | #endif |
| 132 | 133 | |
| 133 | 134 | #ifndef smp_mb__after_atomic |
| 134 | −#define smp_mb__after_atomic() __smp_mb__after_atomic() | |
| 135 | +#define smp_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0) | |
| 135 | 136 | #endif |
| 136 | 137 | |
| 137 | 138 | #ifndef smp_store_release |
| 138 | −#define smp_store_release(p, v) __smp_store_release(p, v) | |
| 139 | +#define smp_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0) | |
| 139 | 140 | #endif |
| 140 | 141 | |
| 141 | 142 | #ifndef smp_load_acquire |
| @@ -178,13 +179,13 @@ do { \ · 36 unmodified lines | ||
| 178 | 179 | #endif /* CONFIG_SMP */ |
| 179 | 180 | |
| 180 | 181 | /* Barriers for virtual machine guests when talking to an SMP host */ |
| 181 | −#define virt_mb() __smp_mb() | |
| 182 | −#define virt_rmb() __smp_rmb() | |
| 183 | −#define virt_wmb() __smp_wmb() | |
| 184 | −#define virt_store_mb(var, value) __smp_store_mb(var, value) | |
| 185 | −#define virt_mb__before_atomic() __smp_mb__before_atomic() | |
| 186 | −#define virt_mb__after_atomic() __smp_mb__after_atomic() | |
| 187 | −#define virt_store_release(p, v) __smp_store_release(p, v) | |
| 182 | +#define virt_mb() do { kcsan_mb(); __smp_mb(); } while (0) | |
| 183 | +#define virt_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0) | |
| 184 | +#define virt_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0) | |
| 185 | +#define virt_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0) | |
| 186 | +#define virt_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0) | |
| 187 | +#define virt_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0) | |
| 188 | +#define virt_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0) | |
| 188 | 189 | #define virt_load_acquire(p) __smp_load_acquire(p) |
| 189 | 190 | |
| 190 | 191 | /** |
| @@ -171,7 +171,7 @@ do { \ · 170 unmodified lines | ||
| 171 | 171 | * Architectures that can implement ACQUIRE better need to take care. |
| 172 | 172 | */ |
| 173 | 173 | #ifndef smp_mb__after_spinlock |
| 174 | −#define smp_mb__after_spinlock() do { } while (0) | |
| 174 | +#define smp_mb__after_spinlock() kcsan_mb() | |
| 175 | 175 | #endif |
| 176 | 176 | |
| 177 | 177 | #ifdef CONFIG_DEBUG_SPINLOCK |