Fengguang Wu, Oleg Nesterov and Peter Zijlstra tracked down a kernel crash to a GCC bug: GCC miscompiles certain 'asm goto' constructs, as outlined here: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670 Implement a workaround suggested by Jakub Jelinek.
Unified · Split
| @@ -16,7 +16,7 @@ · 15 unmodified lines | ||
| 16 | 16 | |
| 17 | 17 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 18 | 18 | { |
| 19 | − asm goto("1:\n\t" | |
| 19 | + asm_volatile_goto("1:\n\t" | |
| 20 | 20 | JUMP_LABEL_NOP "\n\t" |
| 21 | 21 | ".pushsection __jump_table, \"aw\"\n\t" |
| 22 | 22 | ".word 1b, %l[l_yes], %c0\n\t" |
| @@ -22,7 +22,7 @@ · 21 unmodified lines | ||
| 22 | 22 | |
| 23 | 23 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 24 | 24 | { |
| 25 | − asm goto("1:\tnop\n\t" | |
| 25 | + asm_volatile_goto("1:\tnop\n\t" | |
| 26 | 26 | "nop\n\t" |
| 27 | 27 | ".pushsection __jump_table, \"aw\"\n\t" |
| 28 | 28 | WORD_INSN " 1b, %l[l_yes], %0\n\t" |
| @@ -19,7 +19,7 @@ · 18 unmodified lines | ||
| 19 | 19 | |
| 20 | 20 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 21 | 21 | { |
| 22 | − asm goto("1:\n\t" | |
| 22 | + asm_volatile_goto("1:\n\t" | |
| 23 | 23 | "nop\n\t" |
| 24 | 24 | ".pushsection __jump_table, \"aw\"\n\t" |
| 25 | 25 | JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" |
| @@ -15,7 +15,7 @@ · 14 unmodified lines | ||
| 15 | 15 | |
| 16 | 16 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 17 | 17 | { |
| 18 | − asm goto("0: brcl 0,0\n" | |
| 18 | + asm_volatile_goto("0: brcl 0,0\n" | |
| 19 | 19 | ".pushsection __jump_table, \"aw\"\n" |
| 20 | 20 | ASM_ALIGN "\n" |
| 21 | 21 | ASM_PTR " 0b, %l[label], %0\n" |
| @@ -9,7 +9,7 @@ · 8 unmodified lines | ||
| 9 | 9 | |
| 10 | 10 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 11 | 11 | { |
| 12 | − asm goto("1:\n\t" | |
| 12 | + asm_volatile_goto("1:\n\t" | |
| 13 | 13 | "nop\n\t" |
| 14 | 14 | "nop\n\t" |
| 15 | 15 | ".pushsection __jump_table, \"aw\"\n\t" |
| @@ -374,7 +374,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit) · 373 unmodified lines | ||
| 374 | 374 | * Catch too early usage of this before alternatives |
| 375 | 375 | * have run. |
| 376 | 376 | */ |
| 377 | − asm goto("1: jmp %l[t_warn]\n" | |
| 377 | + asm_volatile_goto("1: jmp %l[t_warn]\n" | |
| 378 | 378 | "2:\n" |
| 379 | 379 | ".section .altinstructions,\"a\"\n" |
| 380 | 380 | " .long 1b - .\n" |
| @@ -388,7 +388,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit) · 7 unmodified lines | ||
| 388 | 388 | |
| 389 | 389 | #endif |
| 390 | 390 | |
| 391 | − asm goto("1: jmp %l[t_no]\n" | |
| 391 | + asm_volatile_goto("1: jmp %l[t_no]\n" | |
| 392 | 392 | "2:\n" |
| 393 | 393 | ".section .altinstructions,\"a\"\n" |
| 394 | 394 | " .long 1b - .\n" |
| @@ -453,7 +453,7 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit) · 58 unmodified lines | ||
| 453 | 453 | * have. Thus, we force the jump to the widest, 4-byte, signed relative |
| 454 | 454 | * offset even though the last would often fit in less bytes. |
| 455 | 455 | */ |
| 456 | − asm goto("1: .byte 0xe9\n .long %l[t_dynamic] - 2f\n" | |
| 456 | + asm_volatile_goto("1: .byte 0xe9\n .long %l[t_dynamic] - 2f\n" | |
| 457 | 457 | "2:\n" |
| 458 | 458 | ".section .altinstructions,\"a\"\n" |
| 459 | 459 | " .long 1b - .\n" /* src offset */ |
| @@ -18,7 +18,7 @@ · 17 unmodified lines | ||
| 18 | 18 | |
| 19 | 19 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 20 | 20 | { |
| 21 | − asm goto("1:" | |
| 21 | + asm_volatile_goto("1:" | |
| 22 | 22 | ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" |
| 23 | 23 | ".pushsection __jump_table, \"aw\" \n\t" |
| 24 | 24 | _ASM_ALIGN "\n\t" |
| @@ -20,7 +20,7 @@ · 19 unmodified lines | ||
| 20 | 20 | static inline void __mutex_fastpath_lock(atomic_t *v, |
| 21 | 21 | void (*fail_fn)(atomic_t *)) |
| 22 | 22 | { |
| 23 | − asm volatile goto(LOCK_PREFIX " decl %0\n" | |
| 23 | + asm_volatile_goto(LOCK_PREFIX " decl %0\n" | |
| 24 | 24 | " jns %l[exit]\n" |
| 25 | 25 | : : "m" (v->counter) |
| 26 | 26 | : "memory", "cc" |
| @@ -75,7 +75,7 @@ static inline int __mutex_fastpath_lock_retval(atomic_t *count) · 48 unmodified lines | ||
| 75 | 75 | static inline void __mutex_fastpath_unlock(atomic_t *v, |
| 76 | 76 | void (*fail_fn)(atomic_t *)) |
| 77 | 77 | { |
| 78 | − asm volatile goto(LOCK_PREFIX " incl %0\n" | |
| 78 | + asm_volatile_goto(LOCK_PREFIX " incl %0\n" | |
| 79 | 79 | " jg %l[exit]\n" |
| 80 | 80 | : : "m" (v->counter) |
| 81 | 81 | : "memory", "cc" |
| @@ -65,6 +65,21 @@ · 64 unmodified lines | ||
| 65 | 65 | #define __visible __attribute__((externally_visible)) |
| 66 | 66 | #endif |
| 67 | 67 | |
| 68 | +/* | |
| 69 | + * GCC 'asm goto' miscompiles certain code sequences: | |
| 70 | + * | |
| 71 | + * http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670 | |
| 72 | + * | |
| 73 | + * Work it around via a compiler barrier quirk suggested by Jakub Jelinek. | |
| 74 | + * Fixed in GCC 4.8.2 and later versions. | |
| 75 | + * | |
| 76 | + * (asm goto is automatically volatile - the naming reflects this.) | |
| 77 | + */ | |
| 78 | +#if GCC_VERSION <= 40801 | |
| 79 | +# define asm_volatile_goto(x...) do { asm goto(x); asm (""); } while (0) | |
| 80 | +#else | |
| 81 | +# define asm_volatile_goto(x...) do { asm goto(x); } while (0) | |
| 82 | +#endif | |
| 68 | 83 | |
| 69 | 84 | #ifdef CONFIG_ARCH_USE_BUILTIN_BSWAP |
| 70 | 85 | #if GCC_VERSION >= 40400 |