/* SPDX-License-Identifier: GPL-2.0 *//* * This file contains the 64-bit "server" PowerPC variant * of the low level exception handling including exception * vectors, exception return, part of the slb and stab * handling and other fixed offset specific things. * * This file is meant to be #included from head_64.S due to * position dependent assembly. * * Most of this originates from head_64.S and thus has the same * copyright history. * */ #include <linux/linkage.h>#include <asm/hw_irq.h>#include <asm/exception-64s.h>#include <asm/ptrace.h>#include <asm/cpuidle.h>#include <asm/head-64.h>#include <asm/feature-fixups.h>#include <asm/kup.h> /* * Following are fixed section helper macros. * * EXC_REAL_BEGIN/END - real, unrelocated exception vectors * EXC_VIRT_BEGIN/END - virt (AIL), unrelocated exception vectors * TRAMP_REAL_BEGIN - real, unrelocated helpers (virt may call these) * TRAMP_VIRT_BEGIN - virt, unreloc helpers (in practice, real can use) * EXC_COMMON - After switching to virtual, relocated mode. */ #define EXC_REAL_BEGIN(name, start, size) \ FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size) #define EXC_REAL_END(name, start, size) \ FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size) #define EXC_VIRT_BEGIN(name, start, size) \ FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) #define EXC_VIRT_END(name, start, size) \ FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) #define EXC_COMMON_BEGIN(name) \ USE_TEXT_SECTION(); \ .balign IFETCH_ALIGN_BYTES; \ .global name; \ _ASM_NOKPROBE_SYMBOL(name); \ DEFINE_FIXED_SYMBOL(name, text); \name: #define TRAMP_REAL_BEGIN(name) \ FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name) #define TRAMP_VIRT_BEGIN(name) \ FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name) #define EXC_REAL_NONE(start, size) \ FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \ FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size) #define EXC_VIRT_NONE(start, size) \ FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \ FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size) /* * We're short on space and time in the exception prolog, so we can't * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. * Instead we get the base of the kernel from paca->kernelbase and or in the low * part of label. This requires that the label be within 64KB of kernelbase, and * that kernelbase be 64K aligned. */#define LOAD_HANDLER(reg, label) \ ld reg,PACAKBASE(r13); /* get high part of &label */ \ ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) #define __LOAD_HANDLER(reg, label, section) \ ld reg,PACAKBASE(r13); \ ori reg,reg,(ABS_ADDR(label, section))@l /* * Branches from unrelocated code (e.g., interrupts) to labels outside * head-y require >64K offsets. */#define __LOAD_FAR_HANDLER(reg, label, section) \ ld reg,PACAKBASE(r13); \ ori reg,reg,(ABS_ADDR(label, section))@l; \ addis reg,reg,(ABS_ADDR(label, section))@h /* * Interrupt code generation macros */#define IVEC .L_IVEC_\name\() /* Interrupt vector address */#define IHSRR .L_IHSRR_\name\() /* Sets SRR or HSRR registers */#define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */#define IAREA .L_IAREA_\name\() /* PACA save area */#define IVIRT .L_IVIRT_\name\() /* Has virt mode entry point */#define IISIDE .L_IISIDE_\name\() /* Uses SRR0/1 not DAR/DSISR */#define ICFAR .L_ICFAR_\name\() /* Uses CFAR */#define ICFAR_IF_HVMODE .L_ICFAR_IF_HVMODE_\name\() /* Uses CFAR if HV */#define IDAR .L_IDAR_\name\() /* Uses DAR (or SRR0) */#define IDSISR .L_IDSISR_\name\() /* Uses DSISR (or SRR1) */#define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */#define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */#define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */#define IKVM_REAL .L_IKVM_REAL_\name\() /* Real entry tests KVM */#define __IKVM_REAL(name) .L_IKVM_REAL_ ## name#define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */#define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */#define __ISTACK(name) .L_ISTACK_ ## name#define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */#define IMSR_R12 .L_IMSR_R12_\name\() /* Assumes MSR saved to r12 */ #define INT_DEFINE_BEGIN(n) \.macro int_define_ ## n name #define INT_DEFINE_END(n) \.endm ; \int_define_ ## n n ; \do_define_int n .macro do_define_int name .ifndef IVEC .error "IVEC not defined" .endif .ifndef IHSRR IHSRR=0 .endif .ifndef IHSRR_IF_HVMODE IHSRR_IF_HVMODE=0 .endif .ifndef IAREA IAREA=PACA_EXGEN .endif .ifndef IVIRT IVIRT=1 .endif .ifndef IISIDE IISIDE=0 .endif .ifndef ICFAR ICFAR=1 .endif .ifndef ICFAR_IF_HVMODE ICFAR_IF_HVMODE=0 .endif .ifndef IDAR IDAR=0 .endif .ifndef IDSISR IDSISR=0 .endif .ifndef IBRANCH_TO_COMMON IBRANCH_TO_COMMON=1 .endif .ifndef IREALMODE_COMMON IREALMODE_COMMON=0 .else .if ! IBRANCH_TO_COMMON .error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0" .endif .endif .ifndef IMASK IMASK=0 .endif .ifndef IKVM_REAL IKVM_REAL=0 .endif .ifndef IKVM_VIRT IKVM_VIRT=0 .endif .ifndef ISTACK ISTACK=1 .endif .ifndef IKUAP IKUAP=1 .endif .ifndef IMSR_R12 IMSR_R12=0 .endif.endm /* * All interrupts which set HSRR registers, as well as SRESET and MCE and * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken, * so they all generally need to test whether they were taken in guest context. * * Note: SRESET and MCE may also be sent to the guest by the hypervisor, and be * taken with MSR[HV]=0. * * Interrupts which set SRR registers (with the above exceptions) do not * elevate to MSR[HV]=1 mode, though most can be taken when running with * MSR[HV]=1 (e.g., bare metal kernel and userspace). So these interrupts do * not need to test whether a guest is running because they get delivered to * the guest directly, including nested HV KVM guests. * * The exception is PR KVM, where the guest runs with MSR[PR]=1 and the host * runs with MSR[HV]=0, so the host takes all interrupts on behalf of the * guest. PR KVM runs with LPCR[AIL]=0 which causes interrupts to always be * delivered to the real-mode entry point, therefore such interrupts only test * KVM in their real mode handlers, and only when PR KVM is possible. * * Interrupts that are taken in MSR[HV]=0 and escalate to MSR[HV]=1 are always * delivered in real-mode when the MMU is in hash mode because the MMU * registers are not set appropriately to translate host addresses. In nested * radix mode these can be delivered in virt-mode as the host translations are * used implicitly (see: effective LPID, effective PID). */ /* * If an interrupt is taken while a guest is running, it is immediately routed * to KVM to handle. */ .macro KVMTEST name handler#ifdef CONFIG_KVM_BOOK3S_64_HANDLER lbz r10,HSTATE_IN_GUEST(r13) cmpwi r10,0 /* HSRR variants have the 0x2 bit added to their trap number */ .if IHSRR_IF_HVMODE BEGIN_FTR_SECTION li r10,(IVEC + 0x2) FTR_SECTION_ELSE li r10,(IVEC) ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) .elseif IHSRR li r10,(IVEC + 0x2) .else li r10,(IVEC) .endif bne \handler#endif.endm /* * This is the BOOK3S interrupt entry code macro. * * This can result in one of several things happening: * - Branch to the _common handler, relocated, in virtual mode. * These are normal interrupts (synchronous and asynchronous) handled by * the kernel. * - Branch to KVM, relocated but real mode interrupts remain in real mode. * These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by * / intended for host or guest kernel, but KVM must always be involved * because the machine state is set for guest execution. * - Branch to the masked handler, unrelocated. * These occur when maskable asynchronous interrupts are taken with the * irq_soft_mask set. * - Branch to an "early" handler in real mode but relocated. * This is done if early=1. MCE and HMI use these to handle errors in real * mode. * - Fall through and continue executing in real, unrelocated mode. * This is done if early=2. */ .macro GEN_BRANCH_TO_COMMON name, virt .if IREALMODE_COMMON LOAD_HANDLER(r10, \name\()_common) mtctr r10 bctr .else .if \virt#ifndef CONFIG_RELOCATABLE b \name\()_common_virt#else LOAD_HANDLER(r10, \name\()_common_virt) mtctr r10 bctr#endif .else LOAD_HANDLER(r10, \name\()_common_real) mtctr r10 bctr .endif .endif.endm .macro GEN_INT_ENTRY name, virt, ool=0 SET_SCRATCH0(r13) /* save r13 */ GET_PACA(r13) std r9,IAREA+EX_R9(r13) /* save r9 */BEGIN_FTR_SECTION mfspr r9,SPRN_PPREND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) HMT_MEDIUM std r10,IAREA+EX_R10(r13) /* save r10 */ .if ICFARBEGIN_FTR_SECTION mfspr r10,SPRN_CFAREND_FTR_SECTION_IFSET(CPU_FTR_CFAR) .elseif ICFAR_IF_HVMODEBEGIN_FTR_SECTION BEGIN_FTR_SECTION_NESTED(69) mfspr r10,SPRN_CFAR END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 69)FTR_SECTION_ELSE BEGIN_FTR_SECTION_NESTED(69) li r10,0 END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 69)ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) .endif .if \ool .if !\virt b tramp_real_\name .pushsection .text TRAMP_REAL_BEGIN(tramp_real_\name) .else b tramp_virt_\name .pushsection .text TRAMP_VIRT_BEGIN(tramp_virt_\name) .endif .endif BEGIN_FTR_SECTION std r9,IAREA+EX_PPR(r13)END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) .if ICFAR || ICFAR_IF_HVMODEBEGIN_FTR_SECTION std r10,IAREA+EX_CFAR(r13)END_FTR_SECTION_IFSET(CPU_FTR_CFAR) .endif INTERRUPT_TO_KERNEL mfctr r10 std r10,IAREA+EX_CTR(r13) mfcr r9 std r11,IAREA+EX_R11(r13) /* save r11 - r12 */ std r12,IAREA+EX_R12(r13) /* * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI], * because a d-side MCE will clobber those registers so is * not recoverable if they are live. */ GET_SCRATCH0(r10) std r10,IAREA+EX_R13(r13) .if IDAR && !IISIDE .if IHSRR mfspr r10,SPRN_HDAR .else mfspr r10,SPRN_DAR .endif std r10,IAREA+EX_DAR(r13) .endif .if IDSISR && !IISIDE .if IHSRR mfspr r10,SPRN_HDSISR .else mfspr r10,SPRN_DSISR .endif stw r10,IAREA+EX_DSISR(r13) .endif .if IHSRR_IF_HVMODE BEGIN_FTR_SECTION mfspr r11,SPRN_HSRR0 /* save HSRR0 */ mfspr r12,SPRN_HSRR1 /* and HSRR1 */ FTR_SECTION_ELSE mfspr r11,SPRN_SRR0 /* save SRR0 */ mfspr r12,SPRN_SRR1 /* and SRR1 */ ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) .elseif IHSRR mfspr r11,SPRN_HSRR0 /* save HSRR0 */ mfspr r12,SPRN_HSRR1 /* and HSRR1 */ .else mfspr r11,SPRN_SRR0 /* save SRR0 */ mfspr r12,SPRN_SRR1 /* and SRR1 */ .endif .if IBRANCH_TO_COMMON GEN_BRANCH_TO_COMMON \name \virt .endif .if \ool .popsection .endif.endm /* * __GEN_COMMON_ENTRY is required to receive the branch from interrupt * entry, except in the case of the real-mode handlers which require * __GEN_REALMODE_COMMON_ENTRY. * * This switches to virtual mode and sets MSR[RI]. */.macro __GEN_COMMON_ENTRY nameDEFINE_FIXED_SYMBOL(\name\()_common_real, text)\name\()_common_real: .if IKVM_REAL KVMTEST \name kvm_interrupt .endif ld r10,PACAKMSR(r13) /* get MSR value for kernel */ /* MSR[RI] is clear iff using SRR regs */ .if IHSRR_IF_HVMODE BEGIN_FTR_SECTION xori r10,r10,MSR_RI END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) .elseif ! IHSRR xori r10,r10,MSR_RI .endif mtmsrd r10 .if IVIRT .if IKVM_VIRT b 1f /* skip the virt test coming from real */ .endif .balign IFETCH_ALIGN_BYTESDEFINE_FIXED_SYMBOL(\name\()_common_virt, text)\name\()_common_virt: .if IKVM_VIRT KVMTEST \name kvm_interrupt1: .endif .endif /* IVIRT */.endm /* * Don't switch to virt mode. Used for early MCE and HMI handlers that * want to run in real mode. */.macro __GEN_REALMODE_COMMON_ENTRY nameDEFINE_FIXED_SYMBOL(\name\()_common_real, text)\name\()_common_real: .if IKVM_REAL KVMTEST \name kvm_interrupt .endif.endm .macro __GEN_COMMON_BODY name .if IMASK .if ! ISTACK .error "No support for masked interrupt to use custom stack" .endif /* If coming from user, skip soft-mask tests. */ andi. r10,r12,MSR_PR bne 3f /* * Kernel code running below __end_soft_masked may be * implicitly soft-masked if it is within the regions * in the soft mask table. */ LOAD_HANDLER(r10, __end_soft_masked) cmpld r11,r10 bge+ 1f /* SEARCH_SOFT_MASK_TABLE clobbers r9,r10,r12 */ mtctr r12 stw r9,PACA_EXGEN+EX_CCR(r13) SEARCH_SOFT_MASK_TABLE cmpdi r12,0 mfctr r12 /* Restore r12 to SRR1 */ lwz r9,PACA_EXGEN+EX_CCR(r13) beq 1f /* Not in soft-mask table */ li r10,IMASK b 2f /* In soft-mask table, always mask */ /* Test the soft mask state against our interrupt's bit */1: lbz r10,PACAIRQSOFTMASK(r13)2: andi. r10,r10,IMASK /* Associate vector numbers with bits in paca->irq_happened */ .if IVEC == 0x500 || IVEC == 0xea0 li r10,PACA_IRQ_EE .elseif IVEC == 0x900 li r10,PACA_IRQ_DEC .elseif IVEC == 0xa00 || IVEC == 0xe80 li r10,PACA_IRQ_DBELL .elseif IVEC == 0xe60 li r10,PACA_IRQ_HMI .elseif IVEC == 0xf00 li r10,PACA_IRQ_PMI .else .abort "Bad maskable vector" .endif .if IHSRR_IF_HVMODE BEGIN_FTR_SECTION bne masked_Hinterrupt FTR_SECTION_ELSE bne masked_interrupt ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) .elseif IHSRR bne masked_Hinterrupt .else bne masked_interrupt .endif .endif .if ISTACK andi. r10,r12,MSR_PR /* See if coming from user */3: mr r10,r1 /* Save r1 */ subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */ beq- 100f ld r1,PACAKSAVE(r13) /* kernel stack to use */100: tdgei r1,-INT_FRAME_SIZE /* trap if r1 is in userspace */Lines 1–500 of 3085. Load more
/* SPDX-License-Identifier: GPL-2.0 */ /* * This file contains the 64-bit "server" PowerPC variant * of the low level exception handling including exception * vectors, exception return, part of the slb and stab * handling and other fixed offset specific things. * * This file is meant to be #included from head_64.S due to * position dependent assembly. * * Most of this originates from head_64.S and thus has the same * copyright history. * */ #include <linux/linkage.h> #include <asm/hw_irq.h> #include <asm/exception-64s.h> #include <asm/ptrace.h> #include <asm/cpuidle.h> #include <asm/head-64.h> #include <asm/feature-fixups.h> #include <asm/kup.h> /* * Following are fixed section helper macros. * * EXC_REAL_BEGIN/END - real, unrelocated exception vectors * EXC_VIRT_BEGIN/END - virt (AIL), unrelocated exception vectors * TRAMP_REAL_BEGIN - real, unrelocated helpers (virt may call these) * TRAMP_VIRT_BEGIN - virt, unreloc helpers (in practice, real can use) * EXC_COMMON - After switching to virtual, relocated mode. */ #define EXC_REAL_BEGIN(name, start, size) \ FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size) #define EXC_REAL_END(name, start, size) \ FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size) #define EXC_VIRT_BEGIN(name, start, size) \ FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) #define EXC_VIRT_END(name, start, size) \ FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) #define EXC_COMMON_BEGIN(name) \ USE_TEXT_SECTION(); \ .balign IFETCH_ALIGN_BYTES; \ .global name; \ _ASM_NOKPROBE_SYMBOL(name); \ DEFINE_FIXED_SYMBOL(name, text); \ name: #define TRAMP_REAL_BEGIN(name) \ FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name) #define TRAMP_VIRT_BEGIN(name) \ FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name) #define EXC_REAL_NONE(start, size) \ FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \ FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size) #define EXC_VIRT_NONE(start, size) \ FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \ FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size) /* * We're short on space and time in the exception prolog, so we can't * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. * Instead we get the base of the kernel from paca->kernelbase and or in the low * part of label. This requires that the label be within 64KB of kernelbase, and * that kernelbase be 64K aligned. */ #define LOAD_HANDLER(reg, label) \ ld reg,PACAKBASE(r13); /* get high part of &label */ \ ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) #define __LOAD_HANDLER(reg, label, section) \ ld reg,PACAKBASE(r13); \ ori reg,reg,(ABS_ADDR(label, section))@l /* * Branches from unrelocated code (e.g., interrupts) to labels outside * head-y require >64K offsets. */ #define __LOAD_FAR_HANDLER(reg, label, section) \ ld reg,PACAKBASE(r13); \ ori reg,reg,(ABS_ADDR(label, section))@l; \ addis reg,reg,(ABS_ADDR(label, section))@h /* * Interrupt code generation macros */ #define IVEC .L_IVEC_\name\() /* Interrupt vector address */ #define IHSRR .L_IHSRR_\name\() /* Sets SRR or HSRR registers */ #define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */ #define IAREA .L_IAREA_\name\() /* PACA save area */ #define IVIRT .L_IVIRT_\name\() /* Has virt mode entry point */ #define IISIDE .L_IISIDE_\name\() /* Uses SRR0/1 not DAR/DSISR */ #define ICFAR .L_ICFAR_\name\() /* Uses CFAR */ #define ICFAR_IF_HVMODE .L_ICFAR_IF_HVMODE_\name\() /* Uses CFAR if HV */ #define IDAR .L_IDAR_\name\() /* Uses DAR (or SRR0) */ #define IDSISR .L_IDSISR_\name\() /* Uses DSISR (or SRR1) */ #define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */ #define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */ #define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */ #define IKVM_REAL .L_IKVM_REAL_\name\() /* Real entry tests KVM */ #define __IKVM_REAL(name) .L_IKVM_REAL_ ## name #define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */ #define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */ #define __ISTACK(name) .L_ISTACK_ ## name #define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */ #define IMSR_R12 .L_IMSR_R12_\name\() /* Assumes MSR saved to r12 */ #define INT_DEFINE_BEGIN(n) \ .macro int_define_ ## n name #define INT_DEFINE_END(n) \ .endm ; \ int_define_ ## n n ; \ do_define_int n .macro do_define_int name .ifndef IVEC .error "IVEC not defined" .endif .ifndef IHSRR IHSRR=0 .endif .ifndef IHSRR_IF_HVMODE IHSRR_IF_HVMODE=0 .endif .ifndef IAREA IAREA=PACA_EXGEN .endif .ifndef IVIRT IVIRT=1 .endif .ifndef IISIDE IISIDE=0 .endif .ifndef ICFAR ICFAR=1 .endif .ifndef ICFAR_IF_HVMODE ICFAR_IF_HVMODE=0 .endif .ifndef IDAR IDAR=0 .endif .ifndef IDSISR IDSISR=0 .endif .ifndef IBRANCH_TO_COMMON IBRANCH_TO_COMMON=1 .endif .ifndef IREALMODE_COMMON IREALMODE_COMMON=0 .else .if ! IBRANCH_TO_COMMON .error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0" .endif .endif .ifndef IMASK IMASK=0 .endif .ifndef IKVM_REAL IKVM_REAL=0 .endif .ifndef IKVM_VIRT IKVM_VIRT=0 .endif .ifndef ISTACK ISTACK=1 .endif .ifndef IKUAP IKUAP=1 .endif .ifndef IMSR_R12 IMSR_R12=0 .endif .endm /* * All interrupts which set HSRR registers, as well as SRESET and MCE and * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken, * so they all generally need to test whether they were taken in guest context. * * Note: SRESET and MCE may also be sent to the guest by the hypervisor, and be * taken with MSR[HV]=0. * * Interrupts which set SRR registers (with the above exceptions) do not * elevate to MSR[HV]=1 mode, though most can be taken when running with * MSR[HV]=1 (e.g., bare metal kernel and userspace). So these interrupts do * not need to test whether a guest is running because they get delivered to * the guest directly, including nested HV KVM guests. * * The exception is PR KVM, where the guest runs with MSR[PR]=1 and the host * runs with MSR[HV]=0, so the host takes all interrupts on behalf of the * guest. PR KVM runs with LPCR[AIL]=0 which causes interrupts to always be * delivered to the real-mode entry point, therefore such interrupts only test * KVM in their real mode handlers, and only when PR KVM is possible. * * Interrupts that are taken in MSR[HV]=0 and escalate to MSR[HV]=1 are always * delivered in real-mode when the MMU is in hash mode because the MMU * registers are not set appropriately to translate host addresses. In nested * radix mode these can be delivered in virt-mode as the host translations are * used implicitly (see: effective LPID, effective PID). */ /* * If an interrupt is taken while a guest is running, it is immediately routed * to KVM to handle. */ .macro KVMTEST name handler #ifdef CONFIG_KVM_BOOK3S_64_HANDLER lbz r10,HSTATE_IN_GUEST(r13) cmpwi r10,0 /* HSRR variants have the 0x2 bit added to their trap number */ .if IHSRR_IF_HVMODE BEGIN_FTR_SECTION li r10,(IVEC + 0x2) FTR_SECTION_ELSE li r10,(IVEC) ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) .elseif IHSRR li r10,(IVEC + 0x2) .else li r10,(IVEC) .endif bne \handler #endif .endm /* * This is the BOOK3S interrupt entry code macro. * * This can result in one of several things happening: * - Branch to the _common handler, relocated, in virtual mode. * These are normal interrupts (synchronous and asynchronous) handled by * the kernel. * - Branch to KVM, relocated but real mode interrupts remain in real mode. * These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by * / intended for host or guest kernel, but KVM must always be involved * because the machine state is set for guest execution. * - Branch to the masked handler, unrelocated. * These occur when maskable asynchronous interrupts are taken with the * irq_soft_mask set. * - Branch to an "early" handler in real mode but relocated. * This is done if early=1. MCE and HMI use these to handle errors in real * mode. * - Fall through and continue executing in real, unrelocated mode. * This is done if early=2. */ .macro GEN_BRANCH_TO_COMMON name, virt .if IREALMODE_COMMON LOAD_HANDLER(r10, \name\()_common) mtctr r10 bctr .else .if \virt #ifndef CONFIG_RELOCATABLE b \name\()_common_virt #else LOAD_HANDLER(r10, \name\()_common_virt) mtctr r10 bctr #endif .else LOAD_HANDLER(r10, \name\()_common_real) mtctr r10 bctr .endif .endif .endm .macro GEN_INT_ENTRY name, virt, ool=0 SET_SCRATCH0(r13) /* save r13 */ GET_PACA(r13) std r9,IAREA+EX_R9(r13) /* save r9 */ BEGIN_FTR_SECTION mfspr r9,SPRN_PPR END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) HMT_MEDIUM std r10,IAREA+EX_R10(r13) /* save r10 */ .if ICFAR BEGIN_FTR_SECTION mfspr r10,SPRN_CFAR END_FTR_SECTION_IFSET(CPU_FTR_CFAR) .elseif ICFAR_IF_HVMODE BEGIN_FTR_SECTION BEGIN_FTR_SECTION_NESTED(69) mfspr r10,SPRN_CFAR END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 69) FTR_SECTION_ELSE BEGIN_FTR_SECTION_NESTED(69) li r10,0 END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 69) ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) .endif .if \ool .if !\virt b tramp_real_\name .pushsection .text TRAMP_REAL_BEGIN(tramp_real_\name) .else b tramp_virt_\name .pushsection .text TRAMP_VIRT_BEGIN(tramp_virt_\name) .endif .endif BEGIN_FTR_SECTION std r9,IAREA+EX_PPR(r13) END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) .if ICFAR || ICFAR_IF_HVMODE BEGIN_FTR_SECTION std r10,IAREA+EX_CFAR(r13) END_FTR_SECTION_IFSET(CPU_FTR_CFAR) .endif INTERRUPT_TO_KERNEL mfctr r10 std r10,IAREA+EX_CTR(r13) mfcr r9 std r11,IAREA+EX_R11(r13) /* save r11 - r12 */ std r12,IAREA+EX_R12(r13) /* * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI], * because a d-side MCE will clobber those registers so is * not recoverable if they are live. */ GET_SCRATCH0(r10) std r10,IAREA+EX_R13(r13) .if IDAR && !IISIDE .if IHSRR mfspr r10,SPRN_HDAR .else mfspr r10,SPRN_DAR .endif std r10,IAREA+EX_DAR(r13) .endif .if IDSISR && !IISIDE .if IHSRR mfspr r10,SPRN_HDSISR .else mfspr r10,SPRN_DSISR .endif stw r10,IAREA+EX_DSISR(r13) .endif .if IHSRR_IF_HVMODE BEGIN_FTR_SECTION mfspr r11,SPRN_HSRR0 /* save HSRR0 */ mfspr r12,SPRN_HSRR1 /* and HSRR1 */ FTR_SECTION_ELSE mfspr r11,SPRN_SRR0 /* save SRR0 */ mfspr r12,SPRN_SRR1 /* and SRR1 */ ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) .elseif IHSRR mfspr r11,SPRN_HSRR0 /* save HSRR0 */ mfspr r12,SPRN_HSRR1 /* and HSRR1 */ .else mfspr r11,SPRN_SRR0 /* save SRR0 */ mfspr r12,SPRN_SRR1 /* and SRR1 */ .endif .if IBRANCH_TO_COMMON GEN_BRANCH_TO_COMMON \name \virt .endif .if \ool .popsection .endif .endm /* * __GEN_COMMON_ENTRY is required to receive the branch from interrupt * entry, except in the case of the real-mode handlers which require * __GEN_REALMODE_COMMON_ENTRY. * * This switches to virtual mode and sets MSR[RI]. */ .macro __GEN_COMMON_ENTRY name DEFINE_FIXED_SYMBOL(\name\()_common_real, text) \name\()_common_real: .if IKVM_REAL KVMTEST \name kvm_interrupt .endif ld r10,PACAKMSR(r13) /* get MSR value for kernel */ /* MSR[RI] is clear iff using SRR regs */ .if IHSRR_IF_HVMODE BEGIN_FTR_SECTION xori r10,r10,MSR_RI END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) .elseif ! IHSRR xori r10,r10,MSR_RI .endif mtmsrd r10 .if IVIRT .if IKVM_VIRT b 1f /* skip the virt test coming from real */ .endif .balign IFETCH_ALIGN_BYTES DEFINE_FIXED_SYMBOL(\name\()_common_virt, text) \name\()_common_virt: .if IKVM_VIRT KVMTEST \name kvm_interrupt 1: .endif .endif /* IVIRT */ .endm /* * Don't switch to virt mode. Used for early MCE and HMI handlers that * want to run in real mode. */ .macro __GEN_REALMODE_COMMON_ENTRY name DEFINE_FIXED_SYMBOL(\name\()_common_real, text) \name\()_common_real: .if IKVM_REAL KVMTEST \name kvm_interrupt .endif .endm .macro __GEN_COMMON_BODY name .if IMASK .if ! ISTACK .error "No support for masked interrupt to use custom stack" .endif /* If coming from user, skip soft-mask tests. */ andi. r10,r12,MSR_PR bne 3f /* * Kernel code running below __end_soft_masked may be * implicitly soft-masked if it is within the regions * in the soft mask table. */ LOAD_HANDLER(r10, __end_soft_masked) cmpld r11,r10 bge+ 1f /* SEARCH_SOFT_MASK_TABLE clobbers r9,r10,r12 */ mtctr r12 stw r9,PACA_EXGEN+EX_CCR(r13) SEARCH_SOFT_MASK_TABLE cmpdi r12,0 mfctr r12 /* Restore r12 to SRR1 */ lwz r9,PACA_EXGEN+EX_CCR(r13) beq 1f /* Not in soft-mask table */ li r10,IMASK b 2f /* In soft-mask table, always mask */ /* Test the soft mask state against our interrupt's bit */ 1: lbz r10,PACAIRQSOFTMASK(r13) 2: andi. r10,r10,IMASK /* Associate vector numbers with bits in paca->irq_happened */ .if IVEC == 0x500 || IVEC == 0xea0 li r10,PACA_IRQ_EE .elseif IVEC == 0x900 li r10,PACA_IRQ_DEC .elseif IVEC == 0xa00 || IVEC == 0xe80 li r10,PACA_IRQ_DBELL .elseif IVEC == 0xe60 li r10,PACA_IRQ_HMI .elseif IVEC == 0xf00 li r10,PACA_IRQ_PMI .else .abort "Bad maskable vector" .endif .if IHSRR_IF_HVMODE BEGIN_FTR_SECTION bne masked_Hinterrupt FTR_SECTION_ELSE bne masked_interrupt ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) .elseif IHSRR bne masked_Hinterrupt .else bne masked_interrupt .endif .endif .if ISTACK andi. r10,r12,MSR_PR /* See if coming from user */ 3: mr r10,r1 /* Save r1 */ subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */ beq- 100f ld r1,PACAKSAVE(r13) /* kernel stack to use */ 100: tdgei r1,-INT_FRAME_SIZE /* trap if r1 is in userspace */