Adds the required KCSAN instrumentation for barriers if CONFIG_SMP. KCSAN supports modeling the effects of: smp_mb() smp_rmb() smp_wmb() smp_store_release()
Showing the changes to include (2 of 2 files). Show all
Unified · Split
| @@ -14,6 +14,7 @@ · 13 unmodified lines | |||
| 14 | #ifndef __ASSEMBLY__ | 14 | #ifndef __ASSEMBLY__ |
| 15 | 15 | ||
| 16 | #include <linux/compiler.h> | 16 | #include <linux/compiler.h> |
| 17 | #include <linux/kcsan-checks.h> | ||
| 17 | #include <asm/rwonce.h> | 18 | #include <asm/rwonce.h> |
| 18 | 19 | ||
| 19 | #ifndef nop | 20 | #ifndef nop |
| @@ -62,15 +63,15 @@ · 42 unmodified lines | |||
| 62 | #ifdef CONFIG_SMP | 63 | #ifdef CONFIG_SMP |
| 63 | 64 | ||
| 64 | #ifndef smp_mb | 65 | #ifndef smp_mb |
| 65 | #define smp_mb() __smp_mb() | 66 | #define smp_mb() do { kcsan_mb(); __smp_mb(); } while (0) |
| 66 | #endif | 67 | #endif |
| 67 | 68 | ||
| 68 | #ifndef smp_rmb | 69 | #ifndef smp_rmb |
| 69 | #define smp_rmb() __smp_rmb() | 70 | #define smp_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0) |
| 70 | #endif | 71 | #endif |
| 71 | 72 | ||
| 72 | #ifndef smp_wmb | 73 | #ifndef smp_wmb |
| 73 | #define smp_wmb() __smp_wmb() | 74 | #define smp_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0) |
| 74 | #endif | 75 | #endif |
| 75 | 76 | ||
| 76 | #else /* !CONFIG_SMP */ | 77 | #else /* !CONFIG_SMP */ |
| @@ -123,19 +124,19 @@ do { \ · 46 unmodified lines | |||
| 123 | #ifdef CONFIG_SMP | 124 | #ifdef CONFIG_SMP |
| 124 | 125 | ||
| 125 | #ifndef smp_store_mb | 126 | #ifndef smp_store_mb |
| 126 | #define smp_store_mb(var, value) __smp_store_mb(var, value) | 127 | #define smp_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0) |
| 127 | #endif | 128 | #endif |
| 128 | 129 | ||
| 129 | #ifndef smp_mb__before_atomic | 130 | #ifndef smp_mb__before_atomic |
| 130 | #define smp_mb__before_atomic() __smp_mb__before_atomic() | 131 | #define smp_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0) |
| 131 | #endif | 132 | #endif |
| 132 | 133 | ||
| 133 | #ifndef smp_mb__after_atomic | 134 | #ifndef smp_mb__after_atomic |
| 134 | #define smp_mb__after_atomic() __smp_mb__after_atomic() | 135 | #define smp_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0) |
| 135 | #endif | 136 | #endif |
| 136 | 137 | ||
| 137 | #ifndef smp_store_release | 138 | #ifndef smp_store_release |
| 138 | #define smp_store_release(p, v) __smp_store_release(p, v) | 139 | #define smp_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0) |
| 139 | #endif | 140 | #endif |
| 140 | 141 | ||
| 141 | #ifndef smp_load_acquire | 142 | #ifndef smp_load_acquire |
| @@ -178,13 +179,13 @@ do { \ · 36 unmodified lines | |||
| 178 | #endif /* CONFIG_SMP */ | 179 | #endif /* CONFIG_SMP */ |
| 179 | 180 | ||
| 180 | /* Barriers for virtual machine guests when talking to an SMP host */ | 181 | /* Barriers for virtual machine guests when talking to an SMP host */ |
| 181 | #define virt_mb() __smp_mb() | 182 | #define virt_mb() do { kcsan_mb(); __smp_mb(); } while (0) |
| 182 | #define virt_rmb() __smp_rmb() | 183 | #define virt_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0) |
| 183 | #define virt_wmb() __smp_wmb() | 184 | #define virt_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0) |
| 184 | #define virt_store_mb(var, value) __smp_store_mb(var, value) | 185 | #define virt_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0) |
| 185 | #define virt_mb__before_atomic() __smp_mb__before_atomic() | 186 | #define virt_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0) |
| 186 | #define virt_mb__after_atomic() __smp_mb__after_atomic() | 187 | #define virt_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0) |
| 187 | #define virt_store_release(p, v) __smp_store_release(p, v) | 188 | #define virt_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0) |
| 188 | #define virt_load_acquire(p) __smp_load_acquire(p) | 189 | #define virt_load_acquire(p) __smp_load_acquire(p) |
| 189 | 190 | ||
| 190 | /** | 191 | /** |
| @@ -171,7 +171,7 @@ do { \ · 170 unmodified lines | |||
| 171 | * Architectures that can implement ACQUIRE better need to take care. | 171 | * Architectures that can implement ACQUIRE better need to take care. |
| 172 | */ | 172 | */ |
| 173 | #ifndef smp_mb__after_spinlock | 173 | #ifndef smp_mb__after_spinlock |
| 174 | #define smp_mb__after_spinlock() do { } while (0) | 174 | #define smp_mb__after_spinlock() kcsan_mb() |
| 175 | #endif | 175 | #endif |
| 176 | 176 | ||
| 177 | #ifdef CONFIG_DEBUG_SPINLOCK | 177 | #ifdef CONFIG_DEBUG_SPINLOCK |