History

LoongArch: Add ORC stack unwinder support

The kernel CONFIG_UNWINDER_ORC option enables the ORC unwinder, which is similar in concept to a DWARF unwinder. The difference is that the format of the ORC data is much simpler than DWARF, which in turn allows the ORC unwinder to be much simpler and faster. The ORC data consists of unwind tables which are generated by objtool. After analyzing all the code paths of a .o file, it determines information about the stack state at each instruction address in the file and outputs that information to the .orc_unwind and .orc_unwind_ip sections. The per-object ORC sections are combined at link time and are sorted and post-processed at boot time. The unwinder uses the resulting data to correlate instruction addresses with their stack states at run time. Most of the logic are similar with x86, in order to get ra info before ra is saved into stack, add ra_reg and ra_offset into orc_entry. At the same time, modify some arch-specific code to silence the objtool warnings.

Tiezhu Yang committed 2 years ago

Commit cb8a2ef1 parent e91c5e437 files · +875 −42Browse files

Unified · Split

37 files
  1. arch/loongarch/Kconfig +2 −0
  2. arch/loongarch/Kconfig.debug +11 −0
  3. arch/loongarch/Makefile +21 −2
  4. arch/loongarch/include/asm/Kbuild +2 −0
  5. arch/loongarch/include/asm/bug.h +1 −0
  6. arch/loongarch/include/asm/exception.h +2 −0
  7. arch/loongarch/include/asm/module.h +7 −0
  8. arch/loongarch/include/asm/orc_header.h +18 −0
  9. arch/loongarch/include/asm/orc_lookup.h +31 −0
  10. arch/loongarch/include/asm/orc_types.h +58 −0
  11. arch/loongarch/include/asm/stackframe.h +3 −0
  12. arch/loongarch/include/asm/unwind.h +18 −2
  13. arch/loongarch/include/asm/unwind_hints.h +28 −0
  14. arch/loongarch/kernel/Makefile +4 −0
  15. arch/loongarch/kernel/entry.S +5 −0
  16. arch/loongarch/kernel/fpu.S +7 −0
  17. arch/loongarch/kernel/genex.S +6 −0
  18. arch/loongarch/kernel/lbt.S +3 −0
  19. arch/loongarch/kernel/mcount_dyn.S +6 −0
  20. arch/loongarch/kernel/module.c +18 −4
  21. arch/loongarch/kernel/relocate_kernel.S +4 −3
  22. arch/loongarch/kernel/rethook_trampoline.S +1 −0
  23. arch/loongarch/kernel/setup.c +2 −0
  24. arch/loongarch/kernel/stacktrace.c +1 −0
  25. arch/loongarch/kernel/traps.c +29 −13
  26. arch/loongarch/kernel/unwind_orc.c +528 −0
  27. arch/loongarch/kernel/vmlinux.lds.S +3 −0
  28. arch/loongarch/kvm/switch.S +8 −1
  29. arch/loongarch/lib/clear_user.S +3 −0
  30. arch/loongarch/lib/copy_user.S +3 −0
  31. arch/loongarch/lib/memcpy.S +3 −0
  32. arch/loongarch/lib/memset.S +3 −0
  33. arch/loongarch/mm/tlb.c +12 −15
  34. arch/loongarch/mm/tlbex.S +9 −0
  35. arch/loongarch/vdso/Makefile +1 −0
  36. include/linux/compiler.h +9 −0
  37. scripts/Makefile +5 −2
arch/loongarch/Kconfig+2−0
@@ -136,6 +136,7 @@ config LOONGARCH · 135 unmodified lines
136 select HAVE_KVM136 select HAVE_KVM
137 select HAVE_MOD_ARCH_SPECIFIC137 select HAVE_MOD_ARCH_SPECIFIC
138 select HAVE_NMI138 select HAVE_NMI
139 select HAVE_OBJTOOL if AS_HAS_EXPLICIT_RELOCS
139 select HAVE_PCI140 select HAVE_PCI
140 select HAVE_PERF_EVENTS141 select HAVE_PERF_EVENTS
141 select HAVE_PERF_REGS142 select HAVE_PERF_REGS
@@ -148,6 +149,7 @@ config LOONGARCH · 6 unmodified lines
148 select HAVE_SAMPLE_FTRACE_DIRECT149 select HAVE_SAMPLE_FTRACE_DIRECT
149 select HAVE_SAMPLE_FTRACE_DIRECT_MULTI150 select HAVE_SAMPLE_FTRACE_DIRECT_MULTI
150 select HAVE_SETUP_PER_CPU_AREA if NUMA151 select HAVE_SETUP_PER_CPU_AREA if NUMA
152 select HAVE_STACK_VALIDATION if HAVE_OBJTOOL
151 select HAVE_STACKPROTECTOR153 select HAVE_STACKPROTECTOR
152 select HAVE_SYSCALL_TRACEPOINTS154 select HAVE_SYSCALL_TRACEPOINTS
153 select HAVE_TIF_NOHZ155 select HAVE_TIF_NOHZ
arch/loongarch/Kconfig.debug+11−0
@@ -26,4 +26,15 @@ config UNWINDER_PROLOGUE · 25 unmodified lines
26 Some of the addresses it reports may be incorrect (but better than the26 Some of the addresses it reports may be incorrect (but better than the
27 Guess unwinder).27 Guess unwinder).
28 28
29config UNWINDER_ORC
30 bool "ORC unwinder"
31 select OBJTOOL
32 help
33 This option enables the ORC (Oops Rewind Capability) unwinder for
34 unwinding kernel stack traces. It uses a custom data format which is
35 a simplified version of the DWARF Call Frame Information standard.
36
37 Enabling this option will increase the kernel's runtime memory usage
38 by roughly 2-4MB, depending on your kernel config.
39
29endchoice40endchoice
arch/loongarch/Makefile+21−2
@@ -26,6 +26,18 @@ endif · 25 unmodified lines
2632bit-emul = elf32loongarch2632bit-emul = elf32loongarch
2764bit-emul = elf64loongarch2764bit-emul = elf64loongarch
28 28
29ifdef CONFIG_UNWINDER_ORC
30orc_hash_h := arch/$(SRCARCH)/include/generated/asm/orc_hash.h
31orc_hash_sh := $(srctree)/scripts/orc_hash.sh
32targets += $(orc_hash_h)
33quiet_cmd_orc_hash = GEN $@
34 cmd_orc_hash = mkdir -p $(dir $@); \
35 $(CONFIG_SHELL) $(orc_hash_sh) < $< > $@
36$(orc_hash_h): $(srctree)/arch/loongarch/include/asm/orc_types.h $(orc_hash_sh) FORCE
37 $(call if_changed,orc_hash)
38archprepare: $(orc_hash_h)
39endif
40
29ifdef CONFIG_DYNAMIC_FTRACE41ifdef CONFIG_DYNAMIC_FTRACE
30KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY42KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY
31CC_FLAGS_FTRACE := -fpatchable-function-entry=243CC_FLAGS_FTRACE := -fpatchable-function-entry=2
@@ -72,8 +84,6 @@ KBUILD_CFLAGS_KERNEL += $(call cc-option,-mdirect-extern-access) · 40 unmodified lines
72KBUILD_CFLAGS_KERNEL += $(call cc-option,-fdirect-access-external-data)84KBUILD_CFLAGS_KERNEL += $(call cc-option,-fdirect-access-external-data)
73KBUILD_AFLAGS_MODULE += $(call cc-option,-fno-direct-access-external-data)85KBUILD_AFLAGS_MODULE += $(call cc-option,-fno-direct-access-external-data)
74KBUILD_CFLAGS_MODULE += $(call cc-option,-fno-direct-access-external-data)86KBUILD_CFLAGS_MODULE += $(call cc-option,-fno-direct-access-external-data)
75KBUILD_AFLAGS_MODULE += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax)
76KBUILD_CFLAGS_MODULE += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax)
77else87else
78cflags-y += $(call cc-option,-mno-explicit-relocs)88cflags-y += $(call cc-option,-mno-explicit-relocs)
79KBUILD_AFLAGS_KERNEL += -Wa,-mla-global-with-pcrel89KBUILD_AFLAGS_KERNEL += -Wa,-mla-global-with-pcrel
@@ -82,6 +92,15 @@ KBUILD_AFLAGS_MODULE += -Wa,-mla-global-with-abs · 2 unmodified lines
82KBUILD_CFLAGS_MODULE += -fplt -Wa,-mla-global-with-abs,-mla-local-with-abs92KBUILD_CFLAGS_MODULE += -fplt -Wa,-mla-global-with-abs,-mla-local-with-abs
83endif93endif
84 94
95KBUILD_AFLAGS += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax)
96KBUILD_CFLAGS += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax)
97KBUILD_AFLAGS += $(call cc-option,-mthin-add-sub) $(call cc-option,-Wa$(comma)-mthin-add-sub)
98KBUILD_CFLAGS += $(call cc-option,-mthin-add-sub) $(call cc-option,-Wa$(comma)-mthin-add-sub)
99
100ifdef CONFIG_OBJTOOL
101KBUILD_CFLAGS += -fno-jump-tables
102endif
103
85KBUILD_RUSTFLAGS_MODULE += -Crelocation-model=pic104KBUILD_RUSTFLAGS_MODULE += -Crelocation-model=pic
86 105
87ifeq ($(CONFIG_RELOCATABLE),y)106ifeq ($(CONFIG_RELOCATABLE),y)
arch/loongarch/include/asm/Kbuild+2−0
@@ -1,4 +1,6 @@
1# SPDX-License-Identifier: GPL-2.01# SPDX-License-Identifier: GPL-2.0
2generated-y += orc_hash.h
3
2generic-y += dma-contiguous.h4generic-y += dma-contiguous.h
3generic-y += mcs_spinlock.h5generic-y += mcs_spinlock.h
4generic-y += parport.h6generic-y += parport.h
arch/loongarch/include/asm/bug.h+1−0
@@ -44,6 +44,7 @@ · 43 unmodified lines
44do { \44do { \
45 instrumentation_begin(); \45 instrumentation_begin(); \
46 __BUG_FLAGS(BUGFLAG_WARNING|(flags)); \46 __BUG_FLAGS(BUGFLAG_WARNING|(flags)); \
47 annotate_reachable(); \
47 instrumentation_end(); \48 instrumentation_end(); \
48} while (0)49} while (0)
49 50
arch/loongarch/include/asm/exception.h+2−0
@@ -6,6 +6,8 @@ · 5 unmodified lines
6#include <asm/ptrace.h>6#include <asm/ptrace.h>
7#include <linux/kprobes.h>7#include <linux/kprobes.h>
8 8
9extern void *exception_table[];
10
9void show_registers(struct pt_regs *regs);11void show_registers(struct pt_regs *regs);
10 12
11asmlinkage void cache_parity_error(void);13asmlinkage void cache_parity_error(void);
arch/loongarch/include/asm/module.h+7−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
6#define _ASM_MODULE_H6#define _ASM_MODULE_H
7 7
8#include <asm/inst.h>8#include <asm/inst.h>
9#include <asm/orc_types.h>
9#include <asm-generic/module.h>10#include <asm-generic/module.h>
10 11
11#define RELA_STACK_DEPTH 1612#define RELA_STACK_DEPTH 16
@@ -21,6 +22,12 @@ struct mod_arch_specific { · 9 unmodified lines
21 struct mod_section plt;22 struct mod_section plt;
22 struct mod_section plt_idx;23 struct mod_section plt_idx;
23 24
25#ifdef CONFIG_UNWINDER_ORC
26 unsigned int num_orcs;
27 int *orc_unwind_ip;
28 struct orc_entry *orc_unwind;
29#endif
30
24 /* For CONFIG_DYNAMIC_FTRACE */31 /* For CONFIG_DYNAMIC_FTRACE */
25 struct plt_entry *ftrace_trampolines;32 struct plt_entry *ftrace_trampolines;
26};33};
arch/loongarch/include/asm/orc_header.h+18−0
@@ -0,0 +1,18 @@
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2
3#ifndef _ORC_HEADER_H
4#define _ORC_HEADER_H
5
6#include <linux/types.h>
7#include <linux/compiler.h>
8#include <asm/orc_hash.h>
9
10/*
11 * The header is currently a 20-byte hash of the ORC entry definition; see
12 * scripts/orc_hash.sh.
13 */
14#define ORC_HEADER \
15 __used __section(".orc_header") __aligned(4) \
16 static const u8 orc_header[] = { ORC_HASH }
17
18#endif /* _ORC_HEADER_H */
arch/loongarch/include/asm/orc_lookup.h+31−0
@@ -0,0 +1,31 @@
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2#ifndef _ORC_LOOKUP_H
3#define _ORC_LOOKUP_H
4
5/*
6 * This is a lookup table for speeding up access to the .orc_unwind table.
7 * Given an input address offset, the corresponding lookup table entry
8 * specifies a subset of the .orc_unwind table to search.
9 *
10 * Each block represents the end of the previous range and the start of the
11 * next range. An extra block is added to give the last range an end.
12 *
13 * The block size should be a power of 2 to avoid a costly 'div' instruction.
14 *
15 * A block size of 256 was chosen because it roughly doubles unwinder
16 * performance while only adding ~5% to the ORC data footprint.
17 */
18#define LOOKUP_BLOCK_ORDER 8
19#define LOOKUP_BLOCK_SIZE (1 << LOOKUP_BLOCK_ORDER)
20
21#ifndef LINKER_SCRIPT
22
23extern unsigned int orc_lookup[];
24extern unsigned int orc_lookup_end[];
25
26#define LOOKUP_START_IP (unsigned long)_stext
27#define LOOKUP_STOP_IP (unsigned long)_etext
28
29#endif /* LINKER_SCRIPT */
30
31#endif /* _ORC_LOOKUP_H */
arch/loongarch/include/asm/orc_types.h+58−0
@@ -0,0 +1,58 @@
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2#ifndef _ORC_TYPES_H
3#define _ORC_TYPES_H
4
5#include <linux/types.h>
6
7/*
8 * The ORC_REG_* registers are base registers which are used to find other
9 * registers on the stack.
10 *
11 * ORC_REG_PREV_SP, also known as DWARF Call Frame Address (CFA), is the
12 * address of the previous frame: the caller's SP before it called the current
13 * function.
14 *
15 * ORC_REG_UNDEFINED means the corresponding register's value didn't change in
16 * the current frame.
17 *
18 * The most commonly used base registers are SP and FP -- which the previous SP
19 * is usually based on -- and PREV_SP and UNDEFINED -- which the previous FP is
20 * usually based on.
21 *
22 * The rest of the base registers are needed for special cases like entry code
23 * and GCC realigned stacks.
24 */
25#define ORC_REG_UNDEFINED 0
26#define ORC_REG_PREV_SP 1
27#define ORC_REG_SP 2
28#define ORC_REG_FP 3
29#define ORC_REG_MAX 4
30
31#define ORC_TYPE_UNDEFINED 0
32#define ORC_TYPE_END_OF_STACK 1
33#define ORC_TYPE_CALL 2
34#define ORC_TYPE_REGS 3
35#define ORC_TYPE_REGS_PARTIAL 4
36
37#ifndef __ASSEMBLY__
38/*
39 * This struct is more or less a vastly simplified version of the DWARF Call
40 * Frame Information standard. It contains only the necessary parts of DWARF
41 * CFI, simplified for ease of access by the in-kernel unwinder. It tells the
42 * unwinder how to find the previous SP and FP (and sometimes entry regs) on
43 * the stack for a given code address. Each instance of the struct corresponds
44 * to one or more code locations.
45 */
46struct orc_entry {
47 s16 sp_offset;
48 s16 fp_offset;
49 s16 ra_offset;
50 unsigned int sp_reg:4;
51 unsigned int fp_reg:4;
52 unsigned int ra_reg:4;
53 unsigned int type:3;
54 unsigned int signal:1;
55};
56#endif /* __ASSEMBLY__ */
57
58#endif /* _ORC_TYPES_H */
arch/loongarch/include/asm/stackframe.h+3−0
@@ -13,6 +13,7 @@ · 12 unmodified lines
13#include <asm/asm-offsets.h>13#include <asm/asm-offsets.h>
14#include <asm/loongarch.h>14#include <asm/loongarch.h>
15#include <asm/thread_info.h>15#include <asm/thread_info.h>
16#include <asm/unwind_hints.h>
16 17
17/* Make the addition of cfi info a little easier. */18/* Make the addition of cfi info a little easier. */
18 .macro cfi_rel_offset reg offset=0 docfi=019 .macro cfi_rel_offset reg offset=0 docfi=0
@@ -162,6 +163,7 @@ · 143 unmodified lines
162 li.w t0, CSR_CRMD_WE163 li.w t0, CSR_CRMD_WE
163 csrxchg t0, t0, LOONGARCH_CSR_CRMD164 csrxchg t0, t0, LOONGARCH_CSR_CRMD
164#endif165#endif
166 UNWIND_HINT_REGS
165 .endm167 .endm
166 168
167 .macro SAVE_ALL docfi=0169 .macro SAVE_ALL docfi=0
@@ -219,6 +221,7 @@ · 51 unmodified lines
219 221
220 .macro RESTORE_SP_AND_RET docfi=0222 .macro RESTORE_SP_AND_RET docfi=0
221 cfi_ld sp, PT_R3, \docfi223 cfi_ld sp, PT_R3, \docfi
224 UNWIND_HINT_FUNC
222 ertn225 ertn
223 .endm226 .endm
224 227
arch/loongarch/include/asm/unwind.h+18−2
@@ -16,6 +16,7 @@ · 15 unmodified lines
16enum unwinder_type {16enum unwinder_type {
17 UNWINDER_GUESS,17 UNWINDER_GUESS,
18 UNWINDER_PROLOGUE,18 UNWINDER_PROLOGUE,
19 UNWINDER_ORC,
19};20};
20 21
21struct unwind_state {22struct unwind_state {
@@ -24,7 +25,7 @@ struct unwind_state { · 2 unmodified lines
24 struct task_struct *task;25 struct task_struct *task;
25 bool first, error, reset;26 bool first, error, reset;
26 int graph_idx;27 int graph_idx;
27 unsigned long sp, pc, ra;28 unsigned long sp, fp, pc, ra;
28};29};
29 30
30bool default_next_frame(struct unwind_state *state);31bool default_next_frame(struct unwind_state *state);
@@ -61,14 +62,17 @@ static __always_inline void __unwind_start(struct unwind_state *state, · 30 unmodified lines
61 state->sp = regs->regs[3];62 state->sp = regs->regs[3];
62 state->pc = regs->csr_era;63 state->pc = regs->csr_era;
63 state->ra = regs->regs[1];64 state->ra = regs->regs[1];
65 state->fp = regs->regs[22];
64 } else if (task && task != current) {66 } else if (task && task != current) {
65 state->sp = thread_saved_fp(task);67 state->sp = thread_saved_fp(task);
66 state->pc = thread_saved_ra(task);68 state->pc = thread_saved_ra(task);
67 state->ra = 0;69 state->ra = 0;
70 state->fp = 0;
68 } else {71 } else {
69 state->sp = (unsigned long)__builtin_frame_address(0);72 state->sp = (unsigned long)__builtin_frame_address(0);
70 state->pc = (unsigned long)__builtin_return_address(0);73 state->pc = (unsigned long)__builtin_return_address(0);
71 state->ra = 0;74 state->ra = 0;
75 state->fp = 0;
72 }76 }
73 state->task = task;77 state->task = task;
74 get_stack_info(state->sp, state->task, &state->stack_info);78 get_stack_info(state->sp, state->task, &state->stack_info);
@@ -77,6 +81,18 @@ static __always_inline void __unwind_start(struct unwind_state *state, · 2 unmodified lines
77 81
78static __always_inline unsigned long __unwind_get_return_address(struct unwind_state *state)82static __always_inline unsigned long __unwind_get_return_address(struct unwind_state *state)
79{83{
80 return unwind_done(state) ? 0 : state->pc;84 if (unwind_done(state))
85 return 0;
86
87 return __kernel_text_address(state->pc) ? state->pc : 0;
81}88}
89
90#ifdef CONFIG_UNWINDER_ORC
91void unwind_init(void);
92void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size);
93#else
94static inline void unwind_init(void) {}
95static inline void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size) {}
96#endif
97
82#endif /* _ASM_UNWIND_H */98#endif /* _ASM_UNWIND_H */
arch/loongarch/include/asm/unwind_hints.h+28−0
@@ -0,0 +1,28 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_LOONGARCH_UNWIND_HINTS_H
3#define _ASM_LOONGARCH_UNWIND_HINTS_H
4
5#include <linux/objtool.h>
6#include <asm/orc_types.h>
7
8#ifdef __ASSEMBLY__
9
10.macro UNWIND_HINT_UNDEFINED
11 UNWIND_HINT type=UNWIND_HINT_TYPE_UNDEFINED
12.endm
13
14.macro UNWIND_HINT_END_OF_STACK
15 UNWIND_HINT type=UNWIND_HINT_TYPE_END_OF_STACK
16.endm
17
18.macro UNWIND_HINT_REGS
19 UNWIND_HINT sp_reg=ORC_REG_SP type=UNWIND_HINT_TYPE_REGS
20.endm
21
22.macro UNWIND_HINT_FUNC
23 UNWIND_HINT sp_reg=ORC_REG_SP type=UNWIND_HINT_TYPE_CALL
24.endm
25
26#endif /* __ASSEMBLY__ */
27
28#endif /* _ASM_LOONGARCH_UNWIND_HINTS_H */
arch/loongarch/kernel/Makefile+4−0
@@ -3,6 +3,8 @@ · 2 unmodified lines
3# Makefile for the Linux/LoongArch kernel.3# Makefile for the Linux/LoongArch kernel.
4#4#
5 5
6OBJECT_FILES_NON_STANDARD_head.o := y
7
6extra-y := vmlinux.lds8extra-y := vmlinux.lds
7 9
8obj-y += head.o cpu-probe.o cacheinfo.o env.o setup.o entry.o genex.o \10obj-y += head.o cpu-probe.o cacheinfo.o env.o setup.o entry.o genex.o \
@@ -21,6 +23,7 @@ obj-$(CONFIG_ARCH_STRICT_ALIGN) += unaligned.o · 12 unmodified lines
21 23
22CFLAGS_module.o += $(call cc-option,-Wno-override-init,)24CFLAGS_module.o += $(call cc-option,-Wno-override-init,)
23CFLAGS_syscall.o += $(call cc-option,-Wno-override-init,)25CFLAGS_syscall.o += $(call cc-option,-Wno-override-init,)
26CFLAGS_traps.o += $(call cc-option,-Wno-override-init,)
24CFLAGS_perf_event.o += $(call cc-option,-Wno-override-init,)27CFLAGS_perf_event.o += $(call cc-option,-Wno-override-init,)
25 28
26ifdef CONFIG_FUNCTION_TRACER29ifdef CONFIG_FUNCTION_TRACER
@@ -62,6 +65,7 @@ obj-$(CONFIG_CRASH_DUMP) += crash_dump.o · 35 unmodified lines
62 65
63obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o66obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o
64obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o67obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o
68obj-$(CONFIG_UNWINDER_ORC) += unwind_orc.o
65 69
66obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o70obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o
67obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o71obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
arch/loongarch/kernel/entry.S+5−0
@@ -14,11 +14,13 @@ · 13 unmodified lines
14#include <asm/regdef.h>14#include <asm/regdef.h>
15#include <asm/stackframe.h>15#include <asm/stackframe.h>
16#include <asm/thread_info.h>16#include <asm/thread_info.h>
17#include <asm/unwind_hints.h>
17 18
18 .text19 .text
19 .cfi_sections .debug_frame20 .cfi_sections .debug_frame
20 .align 521 .align 5
21SYM_CODE_START(handle_syscall)22SYM_CODE_START(handle_syscall)
23 UNWIND_HINT_UNDEFINED
22 csrrd t0, PERCPU_BASE_KS24 csrrd t0, PERCPU_BASE_KS
23 la.pcrel t1, kernelsp25 la.pcrel t1, kernelsp
24 add.d t1, t1, t026 add.d t1, t1, t0
@@ -57,6 +59,7 @@ SYM_CODE_START(handle_syscall) · 32 unmodified lines
57 cfi_st fp, PT_R2259 cfi_st fp, PT_R22
58 60
59 SAVE_STATIC61 SAVE_STATIC
62 UNWIND_HINT_REGS
60 63
61#ifdef CONFIG_KGDB64#ifdef CONFIG_KGDB
62 li.w t1, CSR_CRMD_WE65 li.w t1, CSR_CRMD_WE
@@ -75,6 +78,7 @@ SYM_CODE_END(handle_syscall) · 12 unmodified lines
75_ASM_NOKPROBE(handle_syscall)78_ASM_NOKPROBE(handle_syscall)
76 79
77SYM_CODE_START(ret_from_fork)80SYM_CODE_START(ret_from_fork)
81 UNWIND_HINT_REGS
78 bl schedule_tail # a0 = struct task_struct *prev82 bl schedule_tail # a0 = struct task_struct *prev
79 move a0, sp83 move a0, sp
80 bl syscall_exit_to_user_mode84 bl syscall_exit_to_user_mode
@@ -84,6 +88,7 @@ SYM_CODE_START(ret_from_fork) · 3 unmodified lines
84SYM_CODE_END(ret_from_fork)88SYM_CODE_END(ret_from_fork)
85 89
86SYM_CODE_START(ret_from_kernel_thread)90SYM_CODE_START(ret_from_kernel_thread)
91 UNWIND_HINT_REGS
87 bl schedule_tail # a0 = struct task_struct *prev92 bl schedule_tail # a0 = struct task_struct *prev
88 move a0, s193 move a0, s1
89 jirl ra, s0, 094 jirl ra, s0, 0
arch/loongarch/kernel/fpu.S+7−0
@@ -15,6 +15,7 @@ · 14 unmodified lines
15#include <asm/fpregdef.h>15#include <asm/fpregdef.h>
16#include <asm/loongarch.h>16#include <asm/loongarch.h>
17#include <asm/regdef.h>17#include <asm/regdef.h>
18#include <asm/unwind_hints.h>
18 19
19#define FPU_REG_WIDTH 820#define FPU_REG_WIDTH 8
20#define LSX_REG_WIDTH 1621#define LSX_REG_WIDTH 16
@@ -526,3 +527,9 @@ SYM_FUNC_END(_restore_lasx_context) · 505 unmodified lines
526.L_fpu_fault:527.L_fpu_fault:
527 li.w a0, -EFAULT # failure528 li.w a0, -EFAULT # failure
528 jr ra529 jr ra
530
531#ifdef CONFIG_CPU_HAS_LBT
532STACK_FRAME_NON_STANDARD _restore_fp
533STACK_FRAME_NON_STANDARD _restore_lsx
534STACK_FRAME_NON_STANDARD _restore_lasx
535#endif
arch/loongarch/kernel/genex.S+6−0
@@ -32,6 +32,7 @@ SYM_FUNC_START(__arch_cpu_idle) · 31 unmodified lines
32SYM_FUNC_END(__arch_cpu_idle)32SYM_FUNC_END(__arch_cpu_idle)
33 33
34SYM_CODE_START(handle_vint)34SYM_CODE_START(handle_vint)
35 UNWIND_HINT_UNDEFINED
35 BACKUP_T0T136 BACKUP_T0T1
36 SAVE_ALL37 SAVE_ALL
37 la_abs t1, __arch_cpu_idle38 la_abs t1, __arch_cpu_idle
@@ -49,6 +50,7 @@ SYM_CODE_START(handle_vint) · 11 unmodified lines
49SYM_CODE_END(handle_vint)50SYM_CODE_END(handle_vint)
50 51
51SYM_CODE_START(except_vec_cex)52SYM_CODE_START(except_vec_cex)
53 UNWIND_HINT_UNDEFINED
52 b cache_parity_error54 b cache_parity_error
53SYM_CODE_END(except_vec_cex)55SYM_CODE_END(except_vec_cex)
54 56
@@ -67,6 +69,7 @@ SYM_CODE_END(except_vec_cex) · 12 unmodified lines
67 .macro BUILD_HANDLER exception handler prep69 .macro BUILD_HANDLER exception handler prep
68 .align 570 .align 5
69 SYM_CODE_START(handle_\exception)71 SYM_CODE_START(handle_\exception)
72 UNWIND_HINT_UNDEFINED
70 666:73 666:
71 BACKUP_T0T174 BACKUP_T0T1
72 SAVE_ALL75 SAVE_ALL
@@ -77,7 +80,9 @@ SYM_CODE_END(except_vec_cex) · 4 unmodified lines
77 668:80 668:
78 RESTORE_ALL_AND_RET81 RESTORE_ALL_AND_RET
79 SYM_CODE_END(handle_\exception)82 SYM_CODE_END(handle_\exception)
83 .pushsection ".data", "aw", %progbits
80 SYM_DATA(unwind_hint_\exception, .word 668b - 666b)84 SYM_DATA(unwind_hint_\exception, .word 668b - 666b)
85 .popsection
81 .endm86 .endm
82 87
83 BUILD_HANDLER ade ade badv88 BUILD_HANDLER ade ade badv
@@ -94,6 +99,7 @@ SYM_CODE_END(except_vec_cex) · 10 unmodified lines
94 BUILD_HANDLER reserved reserved none /* others */99 BUILD_HANDLER reserved reserved none /* others */
95 100
96SYM_CODE_START(handle_sys)101SYM_CODE_START(handle_sys)
102 UNWIND_HINT_UNDEFINED
97 la_abs t0, handle_syscall103 la_abs t0, handle_syscall
98 jr t0104 jr t0
99SYM_CODE_END(handle_sys)105SYM_CODE_END(handle_sys)
arch/loongarch/kernel/lbt.S+3−0
@@ -11,6 +11,7 @@ · 10 unmodified lines
11#include <asm/asm-offsets.h>11#include <asm/asm-offsets.h>
12#include <asm/errno.h>12#include <asm/errno.h>
13#include <asm/regdef.h>13#include <asm/regdef.h>
14#include <asm/unwind_hints.h>
14 15
15#define SCR_REG_WIDTH 816#define SCR_REG_WIDTH 8
16 17
@@ -153,3 +154,5 @@ SYM_FUNC_END(_restore_ftop_context) · 136 unmodified lines
153.L_lbt_fault:154.L_lbt_fault:
154 li.w a0, -EFAULT # failure155 li.w a0, -EFAULT # failure
155 jr ra156 jr ra
157
158STACK_FRAME_NON_STANDARD _restore_ftop_context
arch/loongarch/kernel/mcount_dyn.S+6−0
@@ -73,6 +73,7 @@ SYM_FUNC_START(ftrace_stub) · 72 unmodified lines
73SYM_FUNC_END(ftrace_stub)73SYM_FUNC_END(ftrace_stub)
74 74
75SYM_CODE_START(ftrace_common)75SYM_CODE_START(ftrace_common)
76 UNWIND_HINT_UNDEFINED
76 PTR_ADDI a0, ra, -8 /* arg0: ip */77 PTR_ADDI a0, ra, -8 /* arg0: ip */
77 move a1, t0 /* arg1: parent_ip */78 move a1, t0 /* arg1: parent_ip */
78 la.pcrel t1, function_trace_op79 la.pcrel t1, function_trace_op
@@ -113,12 +114,14 @@ ftrace_common_return: · 34 unmodified lines
113SYM_CODE_END(ftrace_common)114SYM_CODE_END(ftrace_common)
114 115
115SYM_CODE_START(ftrace_caller)116SYM_CODE_START(ftrace_caller)
117 UNWIND_HINT_UNDEFINED
116 ftrace_regs_entry allregs=0118 ftrace_regs_entry allregs=0
117 b ftrace_common119 b ftrace_common
118SYM_CODE_END(ftrace_caller)120SYM_CODE_END(ftrace_caller)
119 121
120#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS122#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
121SYM_CODE_START(ftrace_regs_caller)123SYM_CODE_START(ftrace_regs_caller)
124 UNWIND_HINT_UNDEFINED
122 ftrace_regs_entry allregs=1125 ftrace_regs_entry allregs=1
123 b ftrace_common126 b ftrace_common
124SYM_CODE_END(ftrace_regs_caller)127SYM_CODE_END(ftrace_regs_caller)
@@ -126,6 +129,7 @@ SYM_CODE_END(ftrace_regs_caller) · 1 unmodified line
126 129
127#ifdef CONFIG_FUNCTION_GRAPH_TRACER130#ifdef CONFIG_FUNCTION_GRAPH_TRACER
128SYM_CODE_START(ftrace_graph_caller)131SYM_CODE_START(ftrace_graph_caller)
132 UNWIND_HINT_UNDEFINED
129 PTR_L a0, sp, PT_ERA133 PTR_L a0, sp, PT_ERA
130 PTR_ADDI a0, a0, -8 /* arg0: self_addr */134 PTR_ADDI a0, a0, -8 /* arg0: self_addr */
131 PTR_ADDI a1, sp, PT_R1 /* arg1: parent */135 PTR_ADDI a1, sp, PT_R1 /* arg1: parent */
@@ -134,6 +138,7 @@ SYM_CODE_START(ftrace_graph_caller) · 2 unmodified lines
134SYM_CODE_END(ftrace_graph_caller)138SYM_CODE_END(ftrace_graph_caller)
135 139
136SYM_CODE_START(return_to_handler)140SYM_CODE_START(return_to_handler)
141 UNWIND_HINT_UNDEFINED
137 /* Save return value regs */142 /* Save return value regs */
138 PTR_ADDI sp, sp, -FGRET_REGS_SIZE143 PTR_ADDI sp, sp, -FGRET_REGS_SIZE
139 PTR_S a0, sp, FGRET_REGS_A0144 PTR_S a0, sp, FGRET_REGS_A0
@@ -155,6 +160,7 @@ SYM_CODE_END(return_to_handler) · 15 unmodified lines
155 160
156#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS161#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
157SYM_CODE_START(ftrace_stub_direct_tramp)162SYM_CODE_START(ftrace_stub_direct_tramp)
163 UNWIND_HINT_UNDEFINED
158 jr t0164 jr t0
159SYM_CODE_END(ftrace_stub_direct_tramp)165SYM_CODE_END(ftrace_stub_direct_tramp)
160#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */166#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
arch/loongarch/kernel/module.c+18−4
@@ -20,6 +20,7 @@ · 19 unmodified lines
20#include <linux/kernel.h>20#include <linux/kernel.h>
21#include <asm/alternative.h>21#include <asm/alternative.h>
22#include <asm/inst.h>22#include <asm/inst.h>
23#include <asm/unwind.h>
23 24
24static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top)25static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top)
25{26{
@@ -515,15 +516,28 @@ static void module_init_ftrace_plt(const Elf_Ehdr *hdr, · 489 unmodified lines
515int module_finalize(const Elf_Ehdr *hdr,516int module_finalize(const Elf_Ehdr *hdr,
516 const Elf_Shdr *sechdrs, struct module *mod)517 const Elf_Shdr *sechdrs, struct module *mod)
517{518{
518 const Elf_Shdr *s, *se;
519 const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;519 const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
520 const Elf_Shdr *s, *alt = NULL, *orc = NULL, *orc_ip = NULL, *ftrace = NULL;
520 521
521 for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {522 for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
522 if (!strcmp(".altinstructions", secstrs + s->sh_name))523 if (!strcmp(".altinstructions", secstrs + s->sh_name))
523 apply_alternatives((void *)s->sh_addr, (void *)s->sh_addr + s->sh_size);524 alt = s;
525 if (!strcmp(".orc_unwind", secstrs + s->sh_name))
526 orc = s;
527 if (!strcmp(".orc_unwind_ip", secstrs + s->sh_name))
528 orc_ip = s;
524 if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name))529 if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name))
525 module_init_ftrace_plt(hdr, s, mod);530 ftrace = s;
526 }531 }
527 532
533 if (alt)
534 apply_alternatives((void *)alt->sh_addr, (void *)alt->sh_addr + alt->sh_size);
535
536 if (orc && orc_ip)
537 unwind_module_init(mod, (void *)orc_ip->sh_addr, orc_ip->sh_size, (void *)orc->sh_addr, orc->sh_size);
538
539 if (ftrace)
540 module_init_ftrace_plt(hdr, ftrace, mod);
541
528 return 0;542 return 0;
529}543}
arch/loongarch/kernel/relocate_kernel.S+4−3
@@ -15,6 +15,7 @@ · 14 unmodified lines
15#include <asm/addrspace.h>15#include <asm/addrspace.h>
16 16
17SYM_CODE_START(relocate_new_kernel)17SYM_CODE_START(relocate_new_kernel)
18 UNWIND_HINT_UNDEFINED
18 /*19 /*
19 * a0: EFI boot flag for the new kernel20 * a0: EFI boot flag for the new kernel
20 * a1: Command line pointer for the new kernel21 * a1: Command line pointer for the new kernel
@@ -90,6 +91,7 @@ SYM_CODE_END(relocate_new_kernel) · 69 unmodified lines
90 * then start at the entry point from LOONGARCH_IOCSR_MBUF0.91 * then start at the entry point from LOONGARCH_IOCSR_MBUF0.
91 */92 */
92SYM_CODE_START(kexec_smp_wait)93SYM_CODE_START(kexec_smp_wait)
94 UNWIND_HINT_UNDEFINED
931: li.w t0, 0x100 /* wait for init loop */951: li.w t0, 0x100 /* wait for init loop */
942: addi.w t0, t0, -1 /* limit mailbox access */962: addi.w t0, t0, -1 /* limit mailbox access */
95 bnez t0, 2b97 bnez t0, 2b
@@ -106,6 +108,5 @@ SYM_CODE_END(kexec_smp_wait) · 10 unmodified lines
106 108
107relocate_new_kernel_end:109relocate_new_kernel_end:
108 110
109SYM_DATA_START(relocate_new_kernel_size)111 .section ".data"
110 PTR relocate_new_kernel_end - relocate_new_kernel112SYM_DATA(relocate_new_kernel_size, .long relocate_new_kernel_end - relocate_new_kernel)
111SYM_DATA_END(relocate_new_kernel_size)
arch/loongarch/kernel/rethook_trampoline.S+1−0
@@ -76,6 +76,7 @@ · 75 unmodified lines
76 .endm76 .endm
77 77
78SYM_CODE_START(arch_rethook_trampoline)78SYM_CODE_START(arch_rethook_trampoline)
79 UNWIND_HINT_UNDEFINED
79 addi.d sp, sp, -PT_SIZE80 addi.d sp, sp, -PT_SIZE
80 save_all_base_regs81 save_all_base_regs
81 82
arch/loongarch/kernel/setup.c+2−0
@@ -47,6 +47,7 @@ · 46 unmodified lines
47#include <asm/sections.h>47#include <asm/sections.h>
48#include <asm/setup.h>48#include <asm/setup.h>
49#include <asm/time.h>49#include <asm/time.h>
50#include <asm/unwind.h>
50 51
51#define SMBIOS_BIOSSIZE_OFFSET 0x0952#define SMBIOS_BIOSSIZE_OFFSET 0x09
52#define SMBIOS_BIOSEXTERN_OFFSET 0x1353#define SMBIOS_BIOSEXTERN_OFFSET 0x13
@@ -587,6 +588,7 @@ static void __init prefill_possible_map(void) · 534 unmodified lines
587void __init setup_arch(char **cmdline_p)588void __init setup_arch(char **cmdline_p)
588{589{
589 cpu_probe();590 cpu_probe();
591 unwind_init();
590 592
591 init_environ();593 init_environ();
592 efi_init();594 efi_init();
arch/loongarch/kernel/stacktrace.c+1−0
@@ -29,6 +29,7 @@ void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie, · 28 unmodified lines
29 regs->csr_era = thread_saved_ra(task);29 regs->csr_era = thread_saved_ra(task);
30 }30 }
31 regs->regs[1] = 0;31 regs->regs[1] = 0;
32 regs->regs[22] = 0;
32 }33 }
33 34
34 for (unwind_start(&state, task, regs);35 for (unwind_start(&state, task, regs);
arch/loongarch/kernel/traps.c+29−13
@@ -53,6 +53,32 @@ · 52 unmodified lines
53 53
54#include "access-helper.h"54#include "access-helper.h"
55 55
56void *exception_table[EXCCODE_INT_START] = {
57 [0 ... EXCCODE_INT_START - 1] = handle_reserved,
58
59 [EXCCODE_TLBI] = handle_tlb_load,
60 [EXCCODE_TLBL] = handle_tlb_load,
61 [EXCCODE_TLBS] = handle_tlb_store,
62 [EXCCODE_TLBM] = handle_tlb_modify,
63 [EXCCODE_TLBNR] = handle_tlb_protect,
64 [EXCCODE_TLBNX] = handle_tlb_protect,
65 [EXCCODE_TLBPE] = handle_tlb_protect,
66 [EXCCODE_ADE] = handle_ade,
67 [EXCCODE_ALE] = handle_ale,
68 [EXCCODE_BCE] = handle_bce,
69 [EXCCODE_SYS] = handle_sys,
70 [EXCCODE_BP] = handle_bp,
71 [EXCCODE_INE] = handle_ri,
72 [EXCCODE_IPE] = handle_ri,
73 [EXCCODE_FPDIS] = handle_fpu,
74 [EXCCODE_LSXDIS] = handle_lsx,
75 [EXCCODE_LASXDIS] = handle_lasx,
76 [EXCCODE_FPE] = handle_fpe,
77 [EXCCODE_WATCH] = handle_watch,
78 [EXCCODE_BTDIS] = handle_lbt,
79};
80EXPORT_SYMBOL_GPL(exception_table);
81
56static void show_backtrace(struct task_struct *task, const struct pt_regs *regs,82static void show_backtrace(struct task_struct *task, const struct pt_regs *regs,
57 const char *loglvl, bool user)83 const char *loglvl, bool user)
58{84{
@@ -1150,19 +1176,9 @@ void __init trap_init(void) · 1091 unmodified lines
1150 for (i = EXCCODE_INT_START; i <= EXCCODE_INT_END; i++)1176 for (i = EXCCODE_INT_START; i <= EXCCODE_INT_END; i++)
1151 set_handler(i * VECSIZE, handle_vint, VECSIZE);1177 set_handler(i * VECSIZE, handle_vint, VECSIZE);
1152 1178
1153 set_handler(EXCCODE_ADE * VECSIZE, handle_ade, VECSIZE);1179 /* Set exception vector handler */
1154 set_handler(EXCCODE_ALE * VECSIZE, handle_ale, VECSIZE);1180 for (i = EXCCODE_ADE; i <= EXCCODE_BTDIS; i++)
1155 set_handler(EXCCODE_BCE * VECSIZE, handle_bce, VECSIZE);1181 set_handler(i * VECSIZE, exception_table[i], VECSIZE);
1156 set_handler(EXCCODE_SYS * VECSIZE, handle_sys, VECSIZE);
1157 set_handler(EXCCODE_BP * VECSIZE, handle_bp, VECSIZE);
1158 set_handler(EXCCODE_INE * VECSIZE, handle_ri, VECSIZE);
1159 set_handler(EXCCODE_IPE * VECSIZE, handle_ri, VECSIZE);
1160 set_handler(EXCCODE_FPDIS * VECSIZE, handle_fpu, VECSIZE);
1161 set_handler(EXCCODE_LSXDIS * VECSIZE, handle_lsx, VECSIZE);
1162 set_handler(EXCCODE_LASXDIS * VECSIZE, handle_lasx, VECSIZE);
1163 set_handler(EXCCODE_FPE * VECSIZE, handle_fpe, VECSIZE);
1164 set_handler(EXCCODE_BTDIS * VECSIZE, handle_lbt, VECSIZE);
1165 set_handler(EXCCODE_WATCH * VECSIZE, handle_watch, VECSIZE);
1166 1182
1167 cache_error_setup();1183 cache_error_setup();
1168 1184
arch/loongarch/kernel/unwind_orc.c+528−0
@@ -0,0 +1,528 @@
1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/objtool.h>
3#include <linux/module.h>
4#include <linux/sort.h>
5#include <asm/exception.h>
6#include <asm/orc_header.h>
7#include <asm/orc_lookup.h>
8#include <asm/orc_types.h>
9#include <asm/ptrace.h>
10#include <asm/setup.h>
11#include <asm/stacktrace.h>
12#include <asm/tlb.h>
13#include <asm/unwind.h>
14
15ORC_HEADER;
16
17#define orc_warn(fmt, ...) \
18 printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__)
19
20extern int __start_orc_unwind_ip[];
21extern int __stop_orc_unwind_ip[];
22extern struct orc_entry __start_orc_unwind[];
23extern struct orc_entry __stop_orc_unwind[];
24
25static bool orc_init __ro_after_init;
26static unsigned int lookup_num_blocks __ro_after_init;
27
28/* Fake frame pointer entry -- used as a fallback for generated code */
29static struct orc_entry orc_fp_entry = {
30 .sp_reg = ORC_REG_FP,
31 .sp_offset = 16,
32 .fp_reg = ORC_REG_PREV_SP,
33 .fp_offset = -16,
34 .ra_reg = ORC_REG_PREV_SP,
35 .ra_offset = -8,
36 .type = ORC_TYPE_CALL
37};
38
39/*
40 * If we crash with IP==0, the last successfully executed instruction
41 * was probably an indirect function call with a NULL function pointer,
42 * and we don't have unwind information for NULL.
43 * This hardcoded ORC entry for IP==0 allows us to unwind from a NULL function
44 * pointer into its parent and then continue normally from there.
45 */
46static struct orc_entry orc_null_entry = {
47 .sp_reg = ORC_REG_SP,
48 .sp_offset = sizeof(long),
49 .fp_reg = ORC_REG_UNDEFINED,
50 .type = ORC_TYPE_CALL
51};
52
53static inline unsigned long orc_ip(const int *ip)
54{
55 return (unsigned long)ip + *ip;
56}
57
58static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table,
59 unsigned int num_entries, unsigned long ip)
60{
61 int *first = ip_table;
62 int *mid = first, *found = first;
63 int *last = ip_table + num_entries - 1;
64
65 if (!num_entries)
66 return NULL;
67
68 /*
69 * Do a binary range search to find the rightmost duplicate of a given
70 * starting address. Some entries are section terminators which are
71 * "weak" entries for ensuring there are no gaps. They should be
72 * ignored when they conflict with a real entry.
73 */
74 while (first <= last) {
75 mid = first + ((last - first) / 2);
76
77 if (orc_ip(mid) <= ip) {
78 found = mid;
79 first = mid + 1;
80 } else
81 last = mid - 1;
82 }
83
84 return u_table + (found - ip_table);
85}
86
87#ifdef CONFIG_MODULES
88static struct orc_entry *orc_module_find(unsigned long ip)
89{
90 struct module *mod;
91
92 mod = __module_address(ip);
93 if (!mod || !mod->arch.orc_unwind || !mod->arch.orc_unwind_ip)
94 return NULL;
95
96 return __orc_find(mod->arch.orc_unwind_ip, mod->arch.orc_unwind, mod->arch.num_orcs, ip);
97}
98#else
99static struct orc_entry *orc_module_find(unsigned long ip)
100{
101 return NULL;
102}
103#endif
104
105#ifdef CONFIG_DYNAMIC_FTRACE
106static struct orc_entry *orc_find(unsigned long ip);
107
108/*
109 * Ftrace dynamic trampolines do not have orc entries of their own.
110 * But they are copies of the ftrace entries that are static and
111 * defined in ftrace_*.S, which do have orc entries.
112 *
113 * If the unwinder comes across a ftrace trampoline, then find the
114 * ftrace function that was used to create it, and use that ftrace
115 * function's orc entry, as the placement of the return code in
116 * the stack will be identical.
117 */
118static struct orc_entry *orc_ftrace_find(unsigned long ip)
119{
120 struct ftrace_ops *ops;
121 unsigned long tramp_addr, offset;
122
123 ops = ftrace_ops_trampoline(ip);
124 if (!ops)
125 return NULL;
126
127 /* Set tramp_addr to the start of the code copied by the trampoline */
128 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
129 tramp_addr = (unsigned long)ftrace_regs_caller;
130 else
131 tramp_addr = (unsigned long)ftrace_caller;
132
133 /* Now place tramp_addr to the location within the trampoline ip is at */
134 offset = ip - ops->trampoline;
135 tramp_addr += offset;
136
137 /* Prevent unlikely recursion */
138 if (ip == tramp_addr)
139 return NULL;
140
141 return orc_find(tramp_addr);
142}
143#else
144static struct orc_entry *orc_ftrace_find(unsigned long ip)
145{
146 return NULL;
147}
148#endif
149
150static struct orc_entry *orc_find(unsigned long ip)
151{
152 static struct orc_entry *orc;
153
154 if (ip == 0)
155 return &orc_null_entry;
156
157 /* For non-init vmlinux addresses, use the fast lookup table: */
158 if (ip >= LOOKUP_START_IP && ip < LOOKUP_STOP_IP) {
159 unsigned int idx, start, stop;
160
161 idx = (ip - LOOKUP_START_IP) / LOOKUP_BLOCK_SIZE;
162
163 if (unlikely((idx >= lookup_num_blocks-1))) {
164 orc_warn("WARNING: bad lookup idx: idx=%u num=%u ip=%pB\n",
165 idx, lookup_num_blocks, (void *)ip);
166 return NULL;
167 }
168
169 start = orc_lookup[idx];
170 stop = orc_lookup[idx + 1] + 1;
171
172 if (unlikely((__start_orc_unwind + start >= __stop_orc_unwind) ||
173 (__start_orc_unwind + stop > __stop_orc_unwind))) {
174 orc_warn("WARNING: bad lookup value: idx=%u num=%u start=%u stop=%u ip=%pB\n",
175 idx, lookup_num_blocks, start, stop, (void *)ip);
176 return NULL;
177 }
178
179 return __orc_find(__start_orc_unwind_ip + start,
180 __start_orc_unwind + start, stop - start, ip);
181 }
182
183 /* vmlinux .init slow lookup: */
184 if (is_kernel_inittext(ip))
185 return __orc_find(__start_orc_unwind_ip, __start_orc_unwind,
186 __stop_orc_unwind_ip - __start_orc_unwind_ip, ip);
187
188 /* Module lookup: */
189 orc = orc_module_find(ip);
190 if (orc)
191 return orc;
192
193 return orc_ftrace_find(ip);
194}
195
196#ifdef CONFIG_MODULES
197
198static DEFINE_MUTEX(sort_mutex);
199static int *cur_orc_ip_table = __start_orc_unwind_ip;
200static struct orc_entry *cur_orc_table = __start_orc_unwind;
201
202static void orc_sort_swap(void *_a, void *_b, int size)
203{
204 int delta = _b - _a;
205 int *a = _a, *b = _b, tmp;
206 struct orc_entry *orc_a, *orc_b;
207
208 /* Swap the .orc_unwind_ip entries: */
209 tmp = *a;
210 *a = *b + delta;
211 *b = tmp - delta;
212
213 /* Swap the corresponding .orc_unwind entries: */
214 orc_a = cur_orc_table + (a - cur_orc_ip_table);
215 orc_b = cur_orc_table + (b - cur_orc_ip_table);
216 swap(*orc_a, *orc_b);
217}
218
219static int orc_sort_cmp(const void *_a, const void *_b)
220{
221 const int *a = _a, *b = _b;
222 unsigned long a_val = orc_ip(a);
223 unsigned long b_val = orc_ip(b);
224 struct orc_entry *orc_a;
225
226 if (a_val > b_val)
227 return 1;
228 if (a_val < b_val)
229 return -1;
230
231 /*
232 * The "weak" section terminator entries need to always be first
233 * to ensure the lookup code skips them in favor of real entries.
234 * These terminator entries exist to handle any gaps created by
235 * whitelisted .o files which didn't get objtool generation.
236 */
237 orc_a = cur_orc_table + (a - cur_orc_ip_table);
238
239 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
240}
241
242void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size,
243 void *_orc, size_t orc_size)
244{
245 int *orc_ip = _orc_ip;
246 struct orc_entry *orc = _orc;
247 unsigned int num_entries = orc_ip_size / sizeof(int);
248
249 WARN_ON_ONCE(orc_ip_size % sizeof(int) != 0 ||
250 orc_size % sizeof(*orc) != 0 ||
251 num_entries != orc_size / sizeof(*orc));
252
253 /*
254 * The 'cur_orc_*' globals allow the orc_sort_swap() callback to
255 * associate an .orc_unwind_ip table entry with its corresponding
256 * .orc_unwind entry so they can both be swapped.
257 */
258 mutex_lock(&sort_mutex);
259 cur_orc_ip_table = orc_ip;
260 cur_orc_table = orc;
261 sort(orc_ip, num_entries, sizeof(int), orc_sort_cmp, orc_sort_swap);
262 mutex_unlock(&sort_mutex);
263
264 mod->arch.orc_unwind_ip = orc_ip;
265 mod->arch.orc_unwind = orc;
266 mod->arch.num_orcs = num_entries;
267}
268#endif
269
270void __init unwind_init(void)
271{
272 int i;
273 size_t orc_size = (void *)__stop_orc_unwind - (void *)__start_orc_unwind;
274 size_t orc_ip_size = (void *)__stop_orc_unwind_ip - (void *)__start_orc_unwind_ip;
275 size_t num_entries = orc_ip_size / sizeof(int);
276 struct orc_entry *orc;
277
278 if (!num_entries || orc_ip_size % sizeof(int) != 0 ||
279 orc_size % sizeof(struct orc_entry) != 0 ||
280 num_entries != orc_size / sizeof(struct orc_entry)) {
281 orc_warn("WARNING: Bad or missing .orc_unwind table. Disabling unwinder.\n");
282 return;
283 }
284
285 /*
286 * Note, the orc_unwind and orc_unwind_ip tables were already
287 * sorted at build time via the 'sorttable' tool.
288 * It's ready for binary search straight away, no need to sort it.
289 */
290
291 /* Initialize the fast lookup table: */
292 lookup_num_blocks = orc_lookup_end - orc_lookup;
293 for (i = 0; i < lookup_num_blocks-1; i++) {
294 orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind,
295 num_entries, LOOKUP_START_IP + (LOOKUP_BLOCK_SIZE * i));
296 if (!orc) {
297 orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n");
298 return;
299 }
300
301 orc_lookup[i] = orc - __start_orc_unwind;
302 }
303
304 /* Initialize the ending block: */
305 orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, num_entries, LOOKUP_STOP_IP);
306 if (!orc) {
307 orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n");
308 return;
309 }
310 orc_lookup[lookup_num_blocks-1] = orc - __start_orc_unwind;
311
312 orc_init = true;
313}
314
315static inline bool on_stack(struct stack_info *info, unsigned long addr, size_t len)
316{
317 unsigned long begin = info->begin;
318 unsigned long end = info->end;
319
320 return (info->type != STACK_TYPE_UNKNOWN &&
321 addr >= begin && addr < end && addr + len > begin && addr + len <= end);
322}
323
324static bool stack_access_ok(struct unwind_state *state, unsigned long addr, size_t len)
325{
326 struct stack_info *info = &state->stack_info;
327
328 if (on_stack(info, addr, len))
329 return true;
330
331 return !get_stack_info(addr, state->task, info) && on_stack(info, addr, len);
332}
333
334unsigned long unwind_get_return_address(struct unwind_state *state)
335{
336 return __unwind_get_return_address(state);
337}
338EXPORT_SYMBOL_GPL(unwind_get_return_address);
339
340void unwind_start(struct unwind_state *state, struct task_struct *task,
341 struct pt_regs *regs)
342{
343 __unwind_start(state, task, regs);
344 state->type = UNWINDER_ORC;
345 if (!unwind_done(state) && !__kernel_text_address(state->pc))
346 unwind_next_frame(state);
347}
348EXPORT_SYMBOL_GPL(unwind_start);
349
350static bool is_entry_func(unsigned long addr)
351{
352 extern u32 kernel_entry;
353 extern u32 kernel_entry_end;
354
355 return addr >= (unsigned long)&kernel_entry && addr < (unsigned long)&kernel_entry_end;
356}
357
358static inline unsigned long bt_address(unsigned long ra)
359{
360 extern unsigned long eentry;
361
362 if (__kernel_text_address(ra))
363 return ra;
364
365 if (__module_text_address(ra))
366 return ra;
367
368 if (ra >= eentry && ra < eentry + EXCCODE_INT_END * VECSIZE) {
369 unsigned long func;
370 unsigned long type = (ra - eentry) / VECSIZE;
371 unsigned long offset = (ra - eentry) % VECSIZE;
372
373 switch (type) {
374 case 0 ... EXCCODE_INT_START - 1:
375 func = (unsigned long)exception_table[type];
376 break;
377 case EXCCODE_INT_START ... EXCCODE_INT_END:
378 func = (unsigned long)handle_vint;
379 break;
380 default:
381 func = (unsigned long)handle_reserved;
382 break;
383 }
384
385 return func + offset;
386 }
387
388 return ra;
389}
390
391bool unwind_next_frame(struct unwind_state *state)
392{
393 unsigned long *p, pc;
394 struct pt_regs *regs;
395 struct orc_entry *orc;
396 struct stack_info *info = &state->stack_info;
397
398 if (unwind_done(state))
399 return false;
400
401 /* Don't let modules unload while we're reading their ORC data. */
402 preempt_disable();
403
404 if (is_entry_func(state->pc))
405 goto end;
406
407 orc = orc_find(state->pc);
408 if (!orc) {
409 /*
410 * As a fallback, try to assume this code uses a frame pointer.
411 * This is useful for generated code, like BPF, which ORC
412 * doesn't know about. This is just a guess, so the rest of
413 * the unwind is no longer considered reliable.
414 */
415 orc = &orc_fp_entry;
416 state->error = true;
417 } else {
418 if (orc->type == ORC_TYPE_UNDEFINED)
419 goto err;
420
421 if (orc->type == ORC_TYPE_END_OF_STACK)
422 goto end;
423 }
424
425 switch (orc->sp_reg) {
426 case ORC_REG_SP:
427 if (info->type == STACK_TYPE_IRQ && state->sp == info->end)
428 orc->type = ORC_TYPE_REGS;
429 else
430 state->sp = state->sp + orc->sp_offset;
431 break;
432 case ORC_REG_FP:
433 state->sp = state->fp;
434 break;
435 default:
436 orc_warn("unknown SP base reg %d at %pB\n", orc->sp_reg, (void *)state->pc);
437 goto err;
438 }
439
440 switch (orc->fp_reg) {
441 case ORC_REG_PREV_SP:
442 p = (unsigned long *)(state->sp + orc->fp_offset);
443 if (!stack_access_ok(state, (unsigned long)p, sizeof(unsigned long)))
444 goto err;
445
446 state->fp = *p;
447 break;
448 case ORC_REG_UNDEFINED:
449 /* Nothing. */
450 break;
451 default:
452 orc_warn("unknown FP base reg %d at %pB\n", orc->fp_reg, (void *)state->pc);
453 goto err;
454 }
455
456 switch (orc->type) {
457 case ORC_TYPE_CALL:
458 if (orc->ra_reg == ORC_REG_PREV_SP) {
459 p = (unsigned long *)(state->sp + orc->ra_offset);
460 if (!stack_access_ok(state, (unsigned long)p, sizeof(unsigned long)))
461 goto err;
462
463 pc = unwind_graph_addr(state, *p, state->sp);
464 pc -= LOONGARCH_INSN_SIZE;
465 } else if (orc->ra_reg == ORC_REG_UNDEFINED) {
466 if (!state->ra || state->ra == state->pc)
467 goto err;
468
469 pc = unwind_graph_addr(state, state->ra, state->sp);
470 pc -= LOONGARCH_INSN_SIZE;
471 state->ra = 0;
472 } else {
473 orc_warn("unknown ra base reg %d at %pB\n", orc->ra_reg, (void *)state->pc);
474 goto err;
475 }
476 break;
477 case ORC_TYPE_REGS:
478 if (info->type == STACK_TYPE_IRQ && state->sp == info->end)
479 regs = (struct pt_regs *)info->next_sp;
480 else
481 regs = (struct pt_regs *)state->sp;
482
483 if (!stack_access_ok(state, (unsigned long)regs, sizeof(*regs)))
484 goto err;
485
486 if ((info->end == (unsigned long)regs + sizeof(*regs)) &&
487 !regs->regs[3] && !regs->regs[1])
488 goto end;
489
490 if (user_mode(regs))
491 goto end;
492
493 pc = regs->csr_era;
494 if (!__kernel_text_address(pc))
495 goto err;
496
497 state->sp = regs->regs[3];
498 state->ra = regs->regs[1];
499 state->fp = regs->regs[22];
500 get_stack_info(state->sp, state->task, info);
501
502 break;
503 default:
504 orc_warn("unknown .orc_unwind entry type %d at %pB\n", orc->type, (void *)state->pc);
505 goto err;
506 }
507
508 state->pc = bt_address(pc);
509 if (!state->pc) {
510 pr_err("cannot find unwind pc at %pK\n", (void *)pc);
511 goto err;
512 }
513
514 if (!__kernel_text_address(state->pc))
515 goto err;
516
517 preempt_enable();
518 return true;
519
520err:
521 state->error = true;
522
523end:
524 preempt_enable();
525 state->stack_info.type = STACK_TYPE_UNKNOWN;
526 return false;
527}
528EXPORT_SYMBOL_GPL(unwind_next_frame);
arch/loongarch/kernel/vmlinux.lds.S+3−0
@@ -2,6 +2,7 @@ · 1 unmodified line
2#include <linux/sizes.h>2#include <linux/sizes.h>
3#include <asm/asm-offsets.h>3#include <asm/asm-offsets.h>
4#include <asm/thread_info.h>4#include <asm/thread_info.h>
5#include <asm/orc_lookup.h>
5 6
6#define PAGE_SIZE _PAGE_SIZE7#define PAGE_SIZE _PAGE_SIZE
7#define RO_EXCEPTION_TABLE_ALIGN 48#define RO_EXCEPTION_TABLE_ALIGN 4
@@ -122,6 +123,8 @@ SECTIONS · 114 unmodified lines
122 }123 }
123#endif124#endif
124 125
126 ORC_UNWIND_TABLE
127
125 .sdata : {128 .sdata : {
126 *(.sdata)129 *(.sdata)
127 }130 }
arch/loongarch/kvm/switch.S+8−1
@@ -8,7 +8,7 @@ · 7 unmodified lines
8#include <asm/asmmacro.h>8#include <asm/asmmacro.h>
9#include <asm/loongarch.h>9#include <asm/loongarch.h>
10#include <asm/regdef.h>10#include <asm/regdef.h>
11#include <asm/stackframe.h>11#include <asm/unwind_hints.h>
12 12
13#define HGPR_OFFSET(x) (PT_R0 + 8*x)13#define HGPR_OFFSET(x) (PT_R0 + 8*x)
14#define GGPR_OFFSET(x) (KVM_ARCH_GGPR + 8*x)14#define GGPR_OFFSET(x) (KVM_ARCH_GGPR + 8*x)
@@ -112,6 +112,7 @@ · 97 unmodified lines
112 .text112 .text
113 .cfi_sections .debug_frame113 .cfi_sections .debug_frame
114SYM_CODE_START(kvm_exc_entry)114SYM_CODE_START(kvm_exc_entry)
115 UNWIND_HINT_UNDEFINED
115 csrwr a2, KVM_TEMP_KS116 csrwr a2, KVM_TEMP_KS
116 csrrd a2, KVM_VCPU_KS117 csrrd a2, KVM_VCPU_KS
117 addi.d a2, a2, KVM_VCPU_ARCH118 addi.d a2, a2, KVM_VCPU_ARCH
@@ -279,3 +280,9 @@ SYM_FUNC_END(kvm_restore_lasx) · 161 unmodified lines
279 .section ".rodata"280 .section ".rodata"
280SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry)281SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry)
281SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest)282SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest)
283
284#ifdef CONFIG_CPU_HAS_LBT
285STACK_FRAME_NON_STANDARD kvm_restore_fpu
286STACK_FRAME_NON_STANDARD kvm_restore_lsx
287STACK_FRAME_NON_STANDARD kvm_restore_lasx
288#endif
arch/loongarch/lib/clear_user.S+3−0
@@ -10,6 +10,7 @@ · 9 unmodified lines
10#include <asm/asm-extable.h>10#include <asm/asm-extable.h>
11#include <asm/cpu.h>11#include <asm/cpu.h>
12#include <asm/regdef.h>12#include <asm/regdef.h>
13#include <asm/unwind_hints.h>
13 14
14SYM_FUNC_START(__clear_user)15SYM_FUNC_START(__clear_user)
15 /*16 /*
@@ -204,3 +205,5 @@ SYM_FUNC_START(__clear_user_fast) · 188 unmodified lines
204 _asm_extable 28b, .Lsmall_fixup205 _asm_extable 28b, .Lsmall_fixup
205 _asm_extable 29b, .Lexit206 _asm_extable 29b, .Lexit
206SYM_FUNC_END(__clear_user_fast)207SYM_FUNC_END(__clear_user_fast)
208
209STACK_FRAME_NON_STANDARD __clear_user_fast
arch/loongarch/lib/copy_user.S+3−0
@@ -10,6 +10,7 @@ · 9 unmodified lines
10#include <asm/asm-extable.h>10#include <asm/asm-extable.h>
11#include <asm/cpu.h>11#include <asm/cpu.h>
12#include <asm/regdef.h>12#include <asm/regdef.h>
13#include <asm/unwind_hints.h>
13 14
14SYM_FUNC_START(__copy_user)15SYM_FUNC_START(__copy_user)
15 /*16 /*
@@ -278,3 +279,5 @@ SYM_FUNC_START(__copy_user_fast) · 262 unmodified lines
278 _asm_extable 58b, .Lexit279 _asm_extable 58b, .Lexit
279 _asm_extable 59b, .Lexit280 _asm_extable 59b, .Lexit
280SYM_FUNC_END(__copy_user_fast)281SYM_FUNC_END(__copy_user_fast)
282
283STACK_FRAME_NON_STANDARD __copy_user_fast
arch/loongarch/lib/memcpy.S+3−0
@@ -9,6 +9,7 @@ · 8 unmodified lines
9#include <asm/asmmacro.h>9#include <asm/asmmacro.h>
10#include <asm/cpu.h>10#include <asm/cpu.h>
11#include <asm/regdef.h>11#include <asm/regdef.h>
12#include <asm/unwind_hints.h>
12 13
13.section .noinstr.text, "ax"14.section .noinstr.text, "ax"
14 15
@@ -197,3 +198,5 @@ SYM_FUNC_START(__memcpy_fast) · 182 unmodified lines
197 jr ra198 jr ra
198SYM_FUNC_END(__memcpy_fast)199SYM_FUNC_END(__memcpy_fast)
199_ASM_NOKPROBE(__memcpy_fast)200_ASM_NOKPROBE(__memcpy_fast)
201
202STACK_FRAME_NON_STANDARD __memcpy_small
arch/loongarch/lib/memset.S+3−0
@@ -9,6 +9,7 @@ · 8 unmodified lines
9#include <asm/asmmacro.h>9#include <asm/asmmacro.h>
10#include <asm/cpu.h>10#include <asm/cpu.h>
11#include <asm/regdef.h>11#include <asm/regdef.h>
12#include <asm/unwind_hints.h>
12 13
13.macro fill_to_64 r014.macro fill_to_64 r0
14 bstrins.d \r0, \r0, 15, 815 bstrins.d \r0, \r0, 15, 8
@@ -166,3 +167,5 @@ SYM_FUNC_START(__memset_fast) · 151 unmodified lines
166 jr ra167 jr ra
167SYM_FUNC_END(__memset_fast)168SYM_FUNC_END(__memset_fast)
168_ASM_NOKPROBE(__memset_fast)169_ASM_NOKPROBE(__memset_fast)
170
171STACK_FRAME_NON_STANDARD __memset_fast
arch/loongarch/mm/tlb.c+12−15
@@ -9,8 +9,9 @@ · 8 unmodified lines
9#include <linux/hugetlb.h>9#include <linux/hugetlb.h>
10#include <linux/export.h>10#include <linux/export.h>
11 11
12#include <asm/cpu.h>
13#include <asm/bootinfo.h>12#include <asm/bootinfo.h>
13#include <asm/cpu.h>
14#include <asm/exception.h>
14#include <asm/mmu_context.h>15#include <asm/mmu_context.h>
15#include <asm/pgtable.h>16#include <asm/pgtable.h>
16#include <asm/tlb.h>17#include <asm/tlb.h>
@@ -266,24 +267,20 @@ static void setup_tlb_handler(int cpu) · 249 unmodified lines
266 setup_ptwalker();267 setup_ptwalker();
267 local_flush_tlb_all();268 local_flush_tlb_all();
268 269
270 if (cpu_has_ptw) {
271 exception_table[EXCCODE_TLBI] = handle_tlb_load_ptw;
272 exception_table[EXCCODE_TLBL] = handle_tlb_load_ptw;
273 exception_table[EXCCODE_TLBS] = handle_tlb_store_ptw;
274 exception_table[EXCCODE_TLBM] = handle_tlb_modify_ptw;
275 }
276
269 /* The tlb handlers are generated only once */277 /* The tlb handlers are generated only once */
270 if (cpu == 0) {278 if (cpu == 0) {
271 memcpy((void *)tlbrentry, handle_tlb_refill, 0x80);279 memcpy((void *)tlbrentry, handle_tlb_refill, 0x80);
272 local_flush_icache_range(tlbrentry, tlbrentry + 0x80);280 local_flush_icache_range(tlbrentry, tlbrentry + 0x80);
273 if (!cpu_has_ptw) {281
274 set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);282 for (int i = EXCCODE_TLBL; i <= EXCCODE_TLBPE; i++)
275 set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);283 set_handler(i * VECSIZE, exception_table[i], VECSIZE);
276 set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
277 set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
278 } else {
279 set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load_ptw, VECSIZE);
280 set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load_ptw, VECSIZE);
281 set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store_ptw, VECSIZE);
282 set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify_ptw, VECSIZE);
283 }
284 set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE);
285 set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE);
286 set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE);
287 } else {284 } else {
288 int vec_sz __maybe_unused;285 int vec_sz __maybe_unused;
289 void *addr __maybe_unused;286 void *addr __maybe_unused;
arch/loongarch/mm/tlbex.S+9−0
@@ -18,6 +18,7 @@ · 17 unmodified lines
18 18
19 .macro tlb_do_page_fault, write19 .macro tlb_do_page_fault, write
20 SYM_CODE_START(tlb_do_page_fault_\write)20 SYM_CODE_START(tlb_do_page_fault_\write)
21 UNWIND_HINT_UNDEFINED
21 SAVE_ALL22 SAVE_ALL
22 csrrd a2, LOONGARCH_CSR_BADV23 csrrd a2, LOONGARCH_CSR_BADV
23 move a0, sp24 move a0, sp
@@ -32,6 +33,7 @@ · 8 unmodified lines
32 tlb_do_page_fault 133 tlb_do_page_fault 1
33 34
34SYM_CODE_START(handle_tlb_protect)35SYM_CODE_START(handle_tlb_protect)
36 UNWIND_HINT_UNDEFINED
35 BACKUP_T0T137 BACKUP_T0T1
36 SAVE_ALL38 SAVE_ALL
37 move a0, sp39 move a0, sp
@@ -44,6 +46,7 @@ SYM_CODE_START(handle_tlb_protect) · 6 unmodified lines
44SYM_CODE_END(handle_tlb_protect)46SYM_CODE_END(handle_tlb_protect)
45 47
46SYM_CODE_START(handle_tlb_load)48SYM_CODE_START(handle_tlb_load)
49 UNWIND_HINT_UNDEFINED
47 csrwr t0, EXCEPTION_KS050 csrwr t0, EXCEPTION_KS0
48 csrwr t1, EXCEPTION_KS151 csrwr t1, EXCEPTION_KS1
49 csrwr ra, EXCEPTION_KS252 csrwr ra, EXCEPTION_KS2
@@ -190,6 +193,7 @@ nopage_tlb_load: · 140 unmodified lines
190SYM_CODE_END(handle_tlb_load)193SYM_CODE_END(handle_tlb_load)
191 194
192SYM_CODE_START(handle_tlb_load_ptw)195SYM_CODE_START(handle_tlb_load_ptw)
196 UNWIND_HINT_UNDEFINED
193 csrwr t0, LOONGARCH_CSR_KS0197 csrwr t0, LOONGARCH_CSR_KS0
194 csrwr t1, LOONGARCH_CSR_KS1198 csrwr t1, LOONGARCH_CSR_KS1
195 la_abs t0, tlb_do_page_fault_0199 la_abs t0, tlb_do_page_fault_0
@@ -197,6 +201,7 @@ SYM_CODE_START(handle_tlb_load_ptw) · 1 unmodified line
197SYM_CODE_END(handle_tlb_load_ptw)201SYM_CODE_END(handle_tlb_load_ptw)
198 202
199SYM_CODE_START(handle_tlb_store)203SYM_CODE_START(handle_tlb_store)
204 UNWIND_HINT_UNDEFINED
200 csrwr t0, EXCEPTION_KS0205 csrwr t0, EXCEPTION_KS0
201 csrwr t1, EXCEPTION_KS1206 csrwr t1, EXCEPTION_KS1
202 csrwr ra, EXCEPTION_KS2207 csrwr ra, EXCEPTION_KS2
@@ -346,6 +351,7 @@ nopage_tlb_store: · 143 unmodified lines
346SYM_CODE_END(handle_tlb_store)351SYM_CODE_END(handle_tlb_store)
347 352
348SYM_CODE_START(handle_tlb_store_ptw)353SYM_CODE_START(handle_tlb_store_ptw)
354 UNWIND_HINT_UNDEFINED
349 csrwr t0, LOONGARCH_CSR_KS0355 csrwr t0, LOONGARCH_CSR_KS0
350 csrwr t1, LOONGARCH_CSR_KS1356 csrwr t1, LOONGARCH_CSR_KS1
351 la_abs t0, tlb_do_page_fault_1357 la_abs t0, tlb_do_page_fault_1
@@ -353,6 +359,7 @@ SYM_CODE_START(handle_tlb_store_ptw) · 1 unmodified line
353SYM_CODE_END(handle_tlb_store_ptw)359SYM_CODE_END(handle_tlb_store_ptw)
354 360
355SYM_CODE_START(handle_tlb_modify)361SYM_CODE_START(handle_tlb_modify)
362 UNWIND_HINT_UNDEFINED
356 csrwr t0, EXCEPTION_KS0363 csrwr t0, EXCEPTION_KS0
357 csrwr t1, EXCEPTION_KS1364 csrwr t1, EXCEPTION_KS1
358 csrwr ra, EXCEPTION_KS2365 csrwr ra, EXCEPTION_KS2
@@ -500,6 +507,7 @@ nopage_tlb_modify: · 141 unmodified lines
500SYM_CODE_END(handle_tlb_modify)507SYM_CODE_END(handle_tlb_modify)
501 508
502SYM_CODE_START(handle_tlb_modify_ptw)509SYM_CODE_START(handle_tlb_modify_ptw)
510 UNWIND_HINT_UNDEFINED
503 csrwr t0, LOONGARCH_CSR_KS0511 csrwr t0, LOONGARCH_CSR_KS0
504 csrwr t1, LOONGARCH_CSR_KS1512 csrwr t1, LOONGARCH_CSR_KS1
505 la_abs t0, tlb_do_page_fault_1513 la_abs t0, tlb_do_page_fault_1
@@ -507,6 +515,7 @@ SYM_CODE_START(handle_tlb_modify_ptw) · 1 unmodified line
507SYM_CODE_END(handle_tlb_modify_ptw)515SYM_CODE_END(handle_tlb_modify_ptw)
508 516
509SYM_CODE_START(handle_tlb_refill)517SYM_CODE_START(handle_tlb_refill)
518 UNWIND_HINT_UNDEFINED
510 csrwr t0, LOONGARCH_CSR_TLBRSAVE519 csrwr t0, LOONGARCH_CSR_TLBRSAVE
511 csrrd t0, LOONGARCH_CSR_PGD520 csrrd t0, LOONGARCH_CSR_PGD
512 lddir t0, t0, 3521 lddir t0, t0, 3
arch/loongarch/vdso/Makefile+1−0
@@ -4,6 +4,7 @@ · 3 unmodified lines
4KASAN_SANITIZE := n4KASAN_SANITIZE := n
5UBSAN_SANITIZE := n5UBSAN_SANITIZE := n
6KCOV_INSTRUMENT := n6KCOV_INSTRUMENT := n
7OBJECT_FILES_NON_STANDARD := y
7 8
8# Include the generic Makefile to check the built vdso.9# Include the generic Makefile to check the built vdso.
9include $(srctree)/lib/vdso/Makefile10include $(srctree)/lib/vdso/Makefile
include/linux/compiler.h+9−0
@@ -116,6 +116,14 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 115 unmodified lines
116 */116 */
117#define __stringify_label(n) #n117#define __stringify_label(n) #n
118 118
119#define __annotate_reachable(c) ({ \
120 asm volatile(__stringify_label(c) ":\n\t" \
121 ".pushsection .discard.reachable\n\t" \
122 ".long " __stringify_label(c) "b - .\n\t" \
123 ".popsection\n\t"); \
124})
125#define annotate_reachable() __annotate_reachable(__COUNTER__)
126
119#define __annotate_unreachable(c) ({ \127#define __annotate_unreachable(c) ({ \
120 asm volatile(__stringify_label(c) ":\n\t" \128 asm volatile(__stringify_label(c) ":\n\t" \
121 ".pushsection .discard.unreachable\n\t" \129 ".pushsection .discard.unreachable\n\t" \
@@ -128,6 +136,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 6 unmodified lines
128#define __annotate_jump_table __section(".rodata..c_jump_table")136#define __annotate_jump_table __section(".rodata..c_jump_table")
129 137
130#else /* !CONFIG_OBJTOOL */138#else /* !CONFIG_OBJTOOL */
139#define annotate_reachable()
131#define annotate_unreachable()140#define annotate_unreachable()
132#define __annotate_jump_table141#define __annotate_jump_table
133#endif /* CONFIG_OBJTOOL */142#endif /* CONFIG_OBJTOOL */
scripts/Makefile+5−2
@@ -31,9 +31,12 @@ HOSTLDLIBS_sign-file = $(shell $(HOSTPKG_CONFIG) --libs libcrypto 2> /dev/null | · 30 unmodified lines
31 31
32ifdef CONFIG_UNWINDER_ORC32ifdef CONFIG_UNWINDER_ORC
33ifeq ($(ARCH),x86_64)33ifeq ($(ARCH),x86_64)
34ARCH := x8634SRCARCH := x86
35endif35endif
36HOSTCFLAGS_sorttable.o += -I$(srctree)/tools/arch/x86/include36ifeq ($(ARCH),loongarch)
37SRCARCH := loongarch
38endif
39HOSTCFLAGS_sorttable.o += -I$(srctree)/tools/arch/$(SRCARCH)/include
37HOSTCFLAGS_sorttable.o += -DUNWINDER_ORC_ENABLED40HOSTCFLAGS_sorttable.o += -DUNWINDER_ORC_ENABLED
38endif41endif
39 42