Fengguang Wu, Oleg Nesterov and Peter Zijlstra tracked down a kernel crash to a GCC bug: GCC miscompiles certain 'asm goto' constructs, as outlined here: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670 Implement a workaround suggested by Jakub Jelinek.
Unified · Split
| @@ -16,7 +16,7 @@ · 15 unmodified lines | |||
| 16 | 16 | ||
| 17 | static __always_inline bool arch_static_branch(struct static_key *key) | 17 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 18 | { | 18 | { |
| 19 | asm goto("1:\n\t" | 19 | asm_volatile_goto("1:\n\t" |
| 20 | JUMP_LABEL_NOP "\n\t" | 20 | JUMP_LABEL_NOP "\n\t" |
| 21 | ".pushsection __jump_table, \"aw\"\n\t" | 21 | ".pushsection __jump_table, \"aw\"\n\t" |
| 22 | ".word 1b, %l[l_yes], %c0\n\t" | 22 | ".word 1b, %l[l_yes], %c0\n\t" |
| @@ -22,7 +22,7 @@ · 21 unmodified lines | |||
| 22 | 22 | ||
| 23 | static __always_inline bool arch_static_branch(struct static_key *key) | 23 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 24 | { | 24 | { |
| 25 | asm goto("1:\tnop\n\t" | 25 | asm_volatile_goto("1:\tnop\n\t" |
| 26 | "nop\n\t" | 26 | "nop\n\t" |
| 27 | ".pushsection __jump_table, \"aw\"\n\t" | 27 | ".pushsection __jump_table, \"aw\"\n\t" |
| 28 | WORD_INSN " 1b, %l[l_yes], %0\n\t" | 28 | WORD_INSN " 1b, %l[l_yes], %0\n\t" |
| @@ -19,7 +19,7 @@ · 18 unmodified lines | |||
| 19 | 19 | ||
| 20 | static __always_inline bool arch_static_branch(struct static_key *key) | 20 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 21 | { | 21 | { |
| 22 | asm goto("1:\n\t" | 22 | asm_volatile_goto("1:\n\t" |
| 23 | "nop\n\t" | 23 | "nop\n\t" |
| 24 | ".pushsection __jump_table, \"aw\"\n\t" | 24 | ".pushsection __jump_table, \"aw\"\n\t" |
| 25 | JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" | 25 | JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" |
| @@ -15,7 +15,7 @@ · 14 unmodified lines | |||
| 15 | 15 | ||
| 16 | static __always_inline bool arch_static_branch(struct static_key *key) | 16 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 17 | { | 17 | { |
| 18 | asm goto("0: brcl 0,0\n" | 18 | asm_volatile_goto("0: brcl 0,0\n" |
| 19 | ".pushsection __jump_table, \"aw\"\n" | 19 | ".pushsection __jump_table, \"aw\"\n" |
| 20 | ASM_ALIGN "\n" | 20 | ASM_ALIGN "\n" |
| 21 | ASM_PTR " 0b, %l[label], %0\n" | 21 | ASM_PTR " 0b, %l[label], %0\n" |
| @@ -9,7 +9,7 @@ · 8 unmodified lines | |||
| 9 | 9 | ||
| 10 | static __always_inline bool arch_static_branch(struct static_key *key) | 10 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 11 | { | 11 | { |
| 12 | asm goto("1:\n\t" | 12 | asm_volatile_goto("1:\n\t" |
| 13 | "nop\n\t" | 13 | "nop\n\t" |
| 14 | "nop\n\t" | 14 | "nop\n\t" |
| 15 | ".pushsection __jump_table, \"aw\"\n\t" | 15 | ".pushsection __jump_table, \"aw\"\n\t" |
| @@ -374,7 +374,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit) · 373 unmodified lines | |||
| 374 | * Catch too early usage of this before alternatives | 374 | * Catch too early usage of this before alternatives |
| 375 | * have run. | 375 | * have run. |
| 376 | */ | 376 | */ |
| 377 | asm goto("1: jmp %l[t_warn]\n" | 377 | asm_volatile_goto("1: jmp %l[t_warn]\n" |
| 378 | "2:\n" | 378 | "2:\n" |
| 379 | ".section .altinstructions,\"a\"\n" | 379 | ".section .altinstructions,\"a\"\n" |
| 380 | " .long 1b - .\n" | 380 | " .long 1b - .\n" |
| @@ -388,7 +388,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit) · 7 unmodified lines | |||
| 388 | 388 | ||
| 389 | #endif | 389 | #endif |
| 390 | 390 | ||
| 391 | asm goto("1: jmp %l[t_no]\n" | 391 | asm_volatile_goto("1: jmp %l[t_no]\n" |
| 392 | "2:\n" | 392 | "2:\n" |
| 393 | ".section .altinstructions,\"a\"\n" | 393 | ".section .altinstructions,\"a\"\n" |
| 394 | " .long 1b - .\n" | 394 | " .long 1b - .\n" |
| @@ -453,7 +453,7 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit) · 58 unmodified lines | |||
| 453 | * have. Thus, we force the jump to the widest, 4-byte, signed relative | 453 | * have. Thus, we force the jump to the widest, 4-byte, signed relative |
| 454 | * offset even though the last would often fit in less bytes. | 454 | * offset even though the last would often fit in less bytes. |
| 455 | */ | 455 | */ |
| 456 | asm goto("1: .byte 0xe9\n .long %l[t_dynamic] - 2f\n" | 456 | asm_volatile_goto("1: .byte 0xe9\n .long %l[t_dynamic] - 2f\n" |
| 457 | "2:\n" | 457 | "2:\n" |
| 458 | ".section .altinstructions,\"a\"\n" | 458 | ".section .altinstructions,\"a\"\n" |
| 459 | " .long 1b - .\n" /* src offset */ | 459 | " .long 1b - .\n" /* src offset */ |
| @@ -18,7 +18,7 @@ · 17 unmodified lines | |||
| 18 | 18 | ||
| 19 | static __always_inline bool arch_static_branch(struct static_key *key) | 19 | static __always_inline bool arch_static_branch(struct static_key *key) |
| 20 | { | 20 | { |
| 21 | asm goto("1:" | 21 | asm_volatile_goto("1:" |
| 22 | ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" | 22 | ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" |
| 23 | ".pushsection __jump_table, \"aw\" \n\t" | 23 | ".pushsection __jump_table, \"aw\" \n\t" |
| 24 | _ASM_ALIGN "\n\t" | 24 | _ASM_ALIGN "\n\t" |
| @@ -20,7 +20,7 @@ · 19 unmodified lines | |||
| 20 | static inline void __mutex_fastpath_lock(atomic_t *v, | 20 | static inline void __mutex_fastpath_lock(atomic_t *v, |
| 21 | void (*fail_fn)(atomic_t *)) | 21 | void (*fail_fn)(atomic_t *)) |
| 22 | { | 22 | { |
| 23 | asm volatile goto(LOCK_PREFIX " decl %0\n" | 23 | asm_volatile_goto(LOCK_PREFIX " decl %0\n" |
| 24 | " jns %l[exit]\n" | 24 | " jns %l[exit]\n" |
| 25 | : : "m" (v->counter) | 25 | : : "m" (v->counter) |
| 26 | : "memory", "cc" | 26 | : "memory", "cc" |
| @@ -75,7 +75,7 @@ static inline int __mutex_fastpath_lock_retval(atomic_t *count) · 48 unmodified lines | |||
| 75 | static inline void __mutex_fastpath_unlock(atomic_t *v, | 75 | static inline void __mutex_fastpath_unlock(atomic_t *v, |
| 76 | void (*fail_fn)(atomic_t *)) | 76 | void (*fail_fn)(atomic_t *)) |
| 77 | { | 77 | { |
| 78 | asm volatile goto(LOCK_PREFIX " incl %0\n" | 78 | asm_volatile_goto(LOCK_PREFIX " incl %0\n" |
| 79 | " jg %l[exit]\n" | 79 | " jg %l[exit]\n" |
| 80 | : : "m" (v->counter) | 80 | : : "m" (v->counter) |
| 81 | : "memory", "cc" | 81 | : "memory", "cc" |
| @@ -65,6 +65,21 @@ · 64 unmodified lines | |||
| 65 | #define __visible __attribute__((externally_visible)) | 65 | #define __visible __attribute__((externally_visible)) |
| 66 | #endif | 66 | #endif |
| 67 | 67 | ||
| 68 | /* | ||
| 69 | * GCC 'asm goto' miscompiles certain code sequences: | ||
| 70 | * | ||
| 71 | * http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670 | ||
| 72 | * | ||
| 73 | * Work it around via a compiler barrier quirk suggested by Jakub Jelinek. | ||
| 74 | * Fixed in GCC 4.8.2 and later versions. | ||
| 75 | * | ||
| 76 | * (asm goto is automatically volatile - the naming reflects this.) | ||
| 77 | */ | ||
| 78 | #if GCC_VERSION <= 40801 | ||
| 79 | # define asm_volatile_goto(x...) do { asm goto(x); asm (""); } while (0) | ||
| 80 | #else | ||
| 81 | # define asm_volatile_goto(x...) do { asm goto(x); } while (0) | ||
| 82 | #endif | ||
| 68 | 83 | ||
| 69 | #ifdef CONFIG_ARCH_USE_BUILTIN_BSWAP | 84 | #ifdef CONFIG_ARCH_USE_BUILTIN_BSWAP |
| 70 | #if GCC_VERSION >= 40400 | 85 | #if GCC_VERSION >= 40400 |