There's nothing special enough about the espfix64 double fault fixup to justify writing it in assembly. Move it to C. This also fixes a bug: if the double fault came from an IST stack, the old asm code would return to a partially uninitialized stack frame.
Unified · Split
| @@ -828,6 +828,7 @@ ENTRY(native_iret) · 827 unmodified lines | |||
| 828 | jnz native_irq_return_ldt | 828 | jnz native_irq_return_ldt |
| 829 | #endif | 829 | #endif |
| 830 | 830 | ||
| 831 | .global native_irq_return_iret | ||
| 831 | native_irq_return_iret: | 832 | native_irq_return_iret: |
| 832 | iretq | 833 | iretq |
| 833 | _ASM_EXTABLE(native_irq_return_iret, bad_iret) | 834 | _ASM_EXTABLE(native_irq_return_iret, bad_iret) |
| @@ -922,37 +923,6 @@ ENTRY(retint_kernel) · 88 unmodified lines | |||
| 922 | CFI_ENDPROC | 923 | CFI_ENDPROC |
| 923 | END(common_interrupt) | 924 | END(common_interrupt) |
| 924 | 925 | ||
| 925 | /* | ||
| 926 | * If IRET takes a fault on the espfix stack, then we | ||
| 927 | * end up promoting it to a doublefault. In that case, | ||
| 928 | * modify the stack to make it look like we just entered | ||
| 929 | * the #GP handler from user space, similar to bad_iret. | ||
| 930 | */ | ||
| 931 | #ifdef CONFIG_X86_ESPFIX64 | ||
| 932 | ALIGN | ||
| 933 | __do_double_fault: | ||
| 934 | XCPT_FRAME 1 RDI+8 | ||
| 935 | movq RSP(%rdi),%rax /* Trap on the espfix stack? */ | ||
| 936 | sarq $PGDIR_SHIFT,%rax | ||
| 937 | cmpl $ESPFIX_PGD_ENTRY,%eax | ||
| 938 | jne do_double_fault /* No, just deliver the fault */ | ||
| 939 | cmpl $__KERNEL_CS,CS(%rdi) | ||
| 940 | jne do_double_fault | ||
| 941 | movq RIP(%rdi),%rax | ||
| 942 | cmpq $native_irq_return_iret,%rax | ||
| 943 | jne do_double_fault /* This shouldn't happen... */ | ||
| 944 | movq PER_CPU_VAR(kernel_stack),%rax | ||
| 945 | subq $(6*8-KERNEL_STACK_OFFSET),%rax /* Reset to original stack */ | ||
| 946 | movq %rax,RSP(%rdi) | ||
| 947 | movq $0,(%rax) /* Missing (lost) #GP error code */ | ||
| 948 | movq $general_protection,RIP(%rdi) | ||
| 949 | retq | ||
| 950 | CFI_ENDPROC | ||
| 951 | END(__do_double_fault) | ||
| 952 | #else | ||
| 953 | # define __do_double_fault do_double_fault | ||
| 954 | #endif | ||
| 955 | |||
| 956 | /* | 926 | /* |
| 957 | * APIC interrupts. | 927 | * APIC interrupts. |
| 958 | */ | 928 | */ |
| @@ -1124,7 +1094,7 @@ idtentry overflow do_overflow has_error_code=0 · 165 unmodified lines | |||
| 1124 | idtentry bounds do_bounds has_error_code=0 | 1094 | idtentry bounds do_bounds has_error_code=0 |
| 1125 | idtentry invalid_op do_invalid_op has_error_code=0 | 1095 | idtentry invalid_op do_invalid_op has_error_code=0 |
| 1126 | idtentry device_not_available do_device_not_available has_error_code=0 | 1096 | idtentry device_not_available do_device_not_available has_error_code=0 |
| 1127 | idtentry double_fault __do_double_fault has_error_code=1 paranoid=1 | 1097 | idtentry double_fault do_double_fault has_error_code=1 paranoid=1 |
| 1128 | idtentry coprocessor_segment_overrun do_coprocessor_segment_overrun has_error_code=0 | 1098 | idtentry coprocessor_segment_overrun do_coprocessor_segment_overrun has_error_code=0 |
| 1129 | idtentry invalid_TSS do_invalid_TSS has_error_code=1 | 1099 | idtentry invalid_TSS do_invalid_TSS has_error_code=1 |
| 1130 | idtentry segment_not_present do_segment_not_present has_error_code=1 | 1100 | idtentry segment_not_present do_segment_not_present has_error_code=1 |
| @@ -259,6 +259,30 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code) · 258 unmodified lines | |||
| 259 | static const char str[] = "double fault"; | 259 | static const char str[] = "double fault"; |
| 260 | struct task_struct *tsk = current; | 260 | struct task_struct *tsk = current; |
| 261 | 261 | ||
| 262 | #ifdef CONFIG_X86_ESPFIX64 | ||
| 263 | extern unsigned char native_irq_return_iret[]; | ||
| 264 | |||
| 265 | /* | ||
| 266 | * If IRET takes a non-IST fault on the espfix64 stack, then we | ||
| 267 | * end up promoting it to a doublefault. In that case, modify | ||
| 268 | * the stack to make it look like we just entered the #GP | ||
| 269 | * handler from user space, similar to bad_iret. | ||
| 270 | */ | ||
| 271 | if (((long)regs->sp >> PGDIR_SHIFT) == ESPFIX_PGD_ENTRY && | ||
| 272 | regs->cs == __KERNEL_CS && | ||
| 273 | regs->ip == (unsigned long)native_irq_return_iret) | ||
| 274 | { | ||
| 275 | struct pt_regs *normal_regs = task_pt_regs(current); | ||
| 276 | |||
| 277 | /* Fake a #GP(0) from userspace. */ | ||
| 278 | memmove(&normal_regs->ip, (void *)regs->sp, 5*8); | ||
| 279 | normal_regs->orig_ax = 0; /* Missing (lost) #GP error code */ | ||
| 280 | regs->ip = (unsigned long)general_protection; | ||
| 281 | regs->sp = (unsigned long)&normal_regs->orig_ax; | ||
| 282 | return; | ||
| 283 | } | ||
| 284 | #endif | ||
| 285 | |||
| 262 | exception_enter(); | 286 | exception_enter(); |
| 263 | /* Return not checked because double check cannot be ignored */ | 287 | /* Return not checked because double check cannot be ignored */ |
| 264 | notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV); | 288 | notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV); |