History

KVM: PPC: Book3S HV: Close race with testing for signals on guest entry

At present, interrupts are hard-disabled fairly late in the guest entry path, in the assembly code. Since we check for pending signals for the vCPU(s) task(s) earlier in the guest entry path, it is possible for a signal to be delivered before we enter the guest but not be noticed until after we exit the guest for some other reason. Similarly, it is possible for the scheduler to request a reschedule while we are in the guest entry path, and we won't notice until after we have run the guest, potentially for a whole timeslice. Furthermore, with a radix guest on POWER9, we can take the interrupt with the MMU on. In this case we end up leaving interrupts hard-disabled after the guest exit, and they are likely to stay hard-disabled until we exit to userspace or context-switch to another process. This was masking the fact that we were also not setting the RI (recoverable interrupt) bit in the MSR, meaning that if we had taken an interrupt, it would have crashed the host kernel with an unrecoverable interrupt message. To close these races, we need to check for signals and reschedule requests after hard-disabling interrupts, and then keep interrupts hard-disabled until we enter the guest. If there is a signal or a reschedule request from another CPU, it will send an IPI, which will cause a guest exit. This puts the interrupt disabling before we call kvmppc_start_thread() for all the secondary threads of this core that are going to run vCPUs. The reason for that is that once we have started the secondary threads there is no easy way to back out without going through at least part of the guest entry path. However, kvmppc_start_thread() includes some code for radix guests which needs to call smp_call_function(), which must be called with interrupts enabled. To solve this problem, this patch moves that code into a separate function that is called earlier. When the guest exit is caused by an external interrupt, a hypervisor doorbell or a hypervisor maintenance interrupt, we now handle these using the replay facility. __kvmppc_vcore_entry() now returns the trap number that caused the exit on this thread, and instead of the assembly code jumping to the handler entry, we return to C code with interrupts still hard-disabled and set the irq_happened flag in the PACA, so that when we do local_irq_enable() the appropriate handler gets called. With all this, we now have the interrupt soft-enable flag clear while we are in the guest. This is useful because code in the real-mode hypercall handlers that checks whether interrupts are enabled will now see that they are disabled, which is correct, since interrupts are hard-disabled in the real-mode code.

Paul Mackerras committed 9 years ago

Commit 8b24e691 parent 898b25b3 files · +127 −79Browse files

Unified · Split

3 files
  1. arch/powerpc/kvm/book3s_hv.c +110 −30
  2. arch/powerpc/kvm/book3s_hv_interrupts.S +1 −7
  3. arch/powerpc/kvm/book3s_hv_rmhandlers.S +16 −42
arch/powerpc/kvm/book3s_hv.c+110−30
@@ -647,6 +647,7 @@ static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, · 646 unmodified lines
647647 unsigned long stolen;
648648 unsigned long core_stolen;
649649 u64 now;
650+ unsigned long flags;
650651
651652 dt = vcpu->arch.dtl_ptr;
652653 vpa = vcpu->arch.vpa.pinned_addr;
@@ -654,10 +655,10 @@ static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, · 1 unmodified line
654655 core_stolen = vcore_stolen_time(vc, now);
655656 stolen = core_stolen - vcpu->arch.stolen_logged;
656657 vcpu->arch.stolen_logged = core_stolen;
657− spin_lock_irq(&vcpu->arch.tbacct_lock);
658+ spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
658659 stolen += vcpu->arch.busy_stolen;
659660 vcpu->arch.busy_stolen = 0;
660− spin_unlock_irq(&vcpu->arch.tbacct_lock);
661+ spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
661662 if (!dt || !vpa)
662663 return;
663664 memset(dt, 0, sizeof(struct dtl_entry));
@@ -2085,7 +2086,7 @@ static void kvmppc_end_cede(struct kvm_vcpu *vcpu) · 1421 unmodified lines
20852086 }
20862087 }
20872088
2088−extern void __kvmppc_vcore_entry(void);
2089+extern int __kvmppc_vcore_entry(void);
20892090
20902091 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
20912092 struct kvm_vcpu *vcpu)
@@ -2167,6 +2168,31 @@ static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu) · 75 unmodified lines
21672168 smp_call_function_single(cpu + i, do_nothing, NULL, 1);
21682169 }
21692170
2171+static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
2172+{
2173+ struct kvm *kvm = vcpu->kvm;
2174+
2175+ /*
2176+ * With radix, the guest can do TLB invalidations itself,
2177+ * and it could choose to use the local form (tlbiel) if
2178+ * it is invalidating a translation that has only ever been
2179+ * used on one vcpu. However, that doesn't mean it has
2180+ * only ever been used on one physical cpu, since vcpus
2181+ * can move around between pcpus. To cope with this, when
2182+ * a vcpu moves from one pcpu to another, we need to tell
2183+ * any vcpus running on the same core as this vcpu previously
2184+ * ran to flush the TLB. The TLB is shared between threads,
2185+ * so we use a single bit in .need_tlb_flush for all 4 threads.
2186+ */
2187+ if (vcpu->arch.prev_cpu != pcpu) {
2188+ if (vcpu->arch.prev_cpu >= 0 &&
2189+ cpu_first_thread_sibling(vcpu->arch.prev_cpu) !=
2190+ cpu_first_thread_sibling(pcpu))
2191+ radix_flush_cpu(kvm, vcpu->arch.prev_cpu, vcpu);
2192+ vcpu->arch.prev_cpu = pcpu;
2193+ }
2194+}
2195+
21702196 static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc)
21712197 {
21722198 int cpu;
@@ -2182,26 +2208,6 @@ static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc) · 9 unmodified lines
21822208 cpu += vcpu->arch.ptid;
21832209 vcpu->cpu = vc->pcpu;
21842210 vcpu->arch.thread_cpu = cpu;
2185−
2186− /*
2187− * With radix, the guest can do TLB invalidations itself,
2188− * and it could choose to use the local form (tlbiel) if
2189− * it is invalidating a translation that has only ever been
2190− * used on one vcpu. However, that doesn't mean it has
2191− * only ever been used on one physical cpu, since vcpus
2192− * can move around between pcpus. To cope with this, when
2193− * a vcpu moves from one pcpu to another, we need to tell
2194− * any vcpus running on the same core as this vcpu previously
2195− * ran to flush the TLB. The TLB is shared between threads,
2196− * so we use a single bit in .need_tlb_flush for all 4 threads.
2197− */
2198− if (kvm_is_radix(kvm) && vcpu->arch.prev_cpu != cpu) {
2199− if (vcpu->arch.prev_cpu >= 0 &&
2200− cpu_first_thread_sibling(vcpu->arch.prev_cpu) !=
2201− cpu_first_thread_sibling(cpu))
2202− radix_flush_cpu(kvm, vcpu->arch.prev_cpu, vcpu);
2203− vcpu->arch.prev_cpu = cpu;
2204− }
22052211 cpumask_set_cpu(cpu, &kvm->arch.cpu_in_guest);
22062212 }
22072213 tpaca = &paca[cpu];
@@ -2470,6 +2476,18 @@ static void collect_piggybacks(struct core_info *cip, int target_threads) · 262 unmodified lines
24702476 spin_unlock(&lp->lock);
24712477 }
24722478
2479+static bool recheck_signals(struct core_info *cip)
2480+{
2481+ int sub, i;
2482+ struct kvm_vcpu *vcpu;
2483+
2484+ for (sub = 0; sub < cip->n_subcores; ++sub)
2485+ for_each_runnable_thread(i, vcpu, cip->vc[sub])
2486+ if (signal_pending(vcpu->arch.run_task))
2487+ return true;
2488+ return false;
2489+}
2490+
24732491 static void post_guest_process(struct kvmppc_vcore *vc, bool is_master)
24742492 {
24752493 int still_running = 0, i;
@@ -2568,6 +2586,21 @@ static inline int kvmppc_set_host_core(unsigned int cpu) · 92 unmodified lines
25682586 return 0;
25692587 }
25702588
2589+static void set_irq_happened(int trap)
2590+{
2591+ switch (trap) {
2592+ case BOOK3S_INTERRUPT_EXTERNAL:
2593+ local_paca->irq_happened |= PACA_IRQ_EE;
2594+ break;
2595+ case BOOK3S_INTERRUPT_H_DOORBELL:
2596+ local_paca->irq_happened |= PACA_IRQ_DBELL;
2597+ break;
2598+ case BOOK3S_INTERRUPT_HMI:
2599+ local_paca->irq_happened |= PACA_IRQ_HMI;
2600+ break;
2601+ }
2602+}
2603+
25712604 /*
25722605 * Run a set of guest threads on a physical core.
25732606 * Called with vc->lock held.
@@ -2587,6 +2620,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 13 unmodified lines
25872620 int pcpu, thr;
25882621 int target_threads;
25892622 int controlled_threads;
2623+ int trap;
25902624
25912625 /*
25922626 * Remove from the list any threads that have a signal pending
@@ -2638,6 +2672,43 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 45 unmodified lines
26382672 if (vc->num_threads < target_threads)
26392673 collect_piggybacks(&core_info, target_threads);
26402674
2675+ /*
2676+ * On radix, arrange for TLB flushing if necessary.
2677+ * This has to be done before disabling interrupts since
2678+ * it uses smp_call_function().
2679+ */
2680+ pcpu = smp_processor_id();
2681+ if (kvm_is_radix(vc->kvm)) {
2682+ for (sub = 0; sub < core_info.n_subcores; ++sub)
2683+ for_each_runnable_thread(i, vcpu, core_info.vc[sub])
2684+ kvmppc_prepare_radix_vcpu(vcpu, pcpu);
2685+ }
2686+
2687+ /*
2688+ * Hard-disable interrupts, and check resched flag and signals.
2689+ * If we need to reschedule or deliver a signal, clean up
2690+ * and return without going into the guest(s).
2691+ */
2692+ local_irq_disable();
2693+ hard_irq_disable();
2694+ if (lazy_irq_pending() || need_resched() ||
2695+ recheck_signals(&core_info)) {
2696+ local_irq_enable();
2697+ vc->vcore_state = VCORE_INACTIVE;
2698+ /* Unlock all except the primary vcore */
2699+ for (sub = 1; sub < core_info.n_subcores; ++sub) {
2700+ pvc = core_info.vc[sub];
2701+ /* Put back on to the preempted vcores list */
2702+ kvmppc_vcore_preempt(pvc);
2703+ spin_unlock(&pvc->lock);
2704+ }
2705+ for (i = 0; i < controlled_threads; ++i)
2706+ kvmppc_release_hwthread(pcpu + i);
2707+ return;
2708+ }
2709+
2710+ kvmppc_clear_host_core(pcpu);
2711+
26412712 /* Decide on micro-threading (split-core) mode */
26422713 subcore_size = threads_per_subcore;
26432714 cmd_bit = stat_bit = 0;
@@ -2665,7 +2736,6 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 21 unmodified lines
26652736 /* order writes to split_info before kvm_split_mode pointer */
26662737 smp_wmb();
26672738 }
2668− pcpu = smp_processor_id();
26692739 for (thr = 0; thr < controlled_threads; ++thr)
26702740 paca[pcpu + thr].kvm_hstate.kvm_split_mode = sip;
26712741
@@ -2685,8 +2755,6 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 13 unmodified lines
26852755 }
26862756 }
26872757
2688− kvmppc_clear_host_core(pcpu);
2689−
26902758 /* Start all the threads */
26912759 active = 0;
26922760 for (sub = 0; sub < core_info.n_subcores; ++sub) {
@@ -2738,14 +2806,25 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 45 unmodified lines
27382806 for (sub = 0; sub < core_info.n_subcores; ++sub)
27392807 spin_unlock(&core_info.vc[sub]->lock);
27402808
2809+ /*
2810+ * Interrupts will be enabled once we get into the guest,
2811+ * so tell lockdep that we're about to enable interrupts.
2812+ */
2813+ trace_hardirqs_on();
2814+
27412815 guest_enter();
27422816
27432817 srcu_idx = srcu_read_lock(&vc->kvm->srcu);
27442818
2745− __kvmppc_vcore_entry();
2819+ trap = __kvmppc_vcore_entry();
27462820
27472821 srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
27482822
2823+ guest_exit();
2824+
2825+ trace_hardirqs_off();
2826+ set_irq_happened(trap);
2827+
27492828 spin_lock(&vc->lock);
27502829 /* prevent other vcpu threads from doing kvmppc_start_thread() now */
27512830 vc->vcore_state = VCORE_EXITING;
@@ -2773,6 +2852,10 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 21 unmodified lines
27732852 split_info.do_nap = 0;
27742853 }
27752854
2855+ kvmppc_set_host_core(pcpu);
2856+
2857+ local_irq_enable();
2858+
27762859 /* Let secondaries go back to the offline loop */
27772860 for (i = 0; i < controlled_threads; ++i) {
27782861 kvmppc_release_hwthread(pcpu + i);
@@ -2781,13 +2864,10 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 2 unmodified lines
27812864 cpumask_clear_cpu(pcpu + i, &vc->kvm->arch.cpu_in_guest);
27822865 }
27832866
2784− kvmppc_set_host_core(pcpu);
2785−
27862867 spin_unlock(&vc->lock);
27872868
27882869 /* make sure updates to secondary vcpu structs are visible now */
27892870 smp_mb();
2790− guest_exit();
27912871
27922872 for (sub = 0; sub < core_info.n_subcores; ++sub) {
27932873 pvc = core_info.vc[sub];
arch/powerpc/kvm/book3s_hv_interrupts.S+1−7
@@ -61,13 +61,6 @@ BEGIN_FTR_SECTION · 60 unmodified lines
6161 std r3, HSTATE_DABR(r13)
6262 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
6363
64− /* Hard-disable interrupts */
65− mfmsr r10
66− std r10, HSTATE_HOST_MSR(r13)
67− rldicl r10,r10,48,1
68− rotldi r10,r10,16
69− mtmsrd r10,1
70−
7164 /* Save host PMU registers */
7265 BEGIN_FTR_SECTION
7366 /* Work around P8 PMAE bug */
@@ -153,6 +146,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) · 79 unmodified lines
153146 *
154147 * R1 = host R1
155148 * R2 = host R2
149+ * R3 = trap number on this thread
156150 * R12 = exit handler id
157151 * R13 = PACA
158152 */
arch/powerpc/kvm/book3s_hv_rmhandlers.S+16−42
@@ -69,6 +69,7 @@ _GLOBAL_TOC(kvmppc_hv_entry_trampoline) · 68 unmodified lines
6969 std r0, PPC_LR_STKOFF(r1)
7070 stdu r1, -112(r1)
7171 mfmsr r10
72+ std r10, HSTATE_HOST_MSR(r13)
7273 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
7374 li r0,MSR_RI
7475 andc r0,r10,r0
@@ -165,6 +166,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) · 90 unmodified lines
165166 addi r1, r1, 112
166167 ld r7, HSTATE_HOST_MSR(r13)
167168
169+ /* Return the trap number on this thread as the return value */
170+ mr r3, r12
171+
168172 /*
169173 * If we came back from the guest via a relocation-on interrupt,
170174 * we will be in virtual mode at this point, which makes it a
@@ -174,59 +178,20 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) · 3 unmodified lines
174178 andi. r0, r0, MSR_IR /* in real mode? */
175179 bne .Lvirt_return
176180
177− cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
178− beq 11f
179− cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
180− beq 15f /* Invoke the H_DOORBELL handler */
181− cmpwi cr2, r12, BOOK3S_INTERRUPT_HMI
182− beq cr2, 14f /* HMI check */
183−
184− /* RFI into the highmem handler, or branch to interrupt handler */
181+ /* RFI into the highmem handler */
185182 mfmsr r6
186183 li r0, MSR_RI
187184 andc r6, r6, r0
188185 mtmsrd r6, 1 /* Clear RI in MSR */
189186 mtsrr0 r8
190187 mtsrr1 r7
191− /*
192− * BOOK3S_INTERRUPT_MACHINE_CHECK is handled at the
193− * time of guest exit
194− */
195188 RFI
196189
197− /* On POWER7, we have external interrupts set to use HSRR0/1 */
198−11: mtspr SPRN_HSRR0, r8
199− mtspr SPRN_HSRR1, r7
200− ba 0x500
201−
202−14: mtspr SPRN_HSRR0, r8
203− mtspr SPRN_HSRR1, r7
204− b hmi_exception_after_realmode
205−
206−15: mtspr SPRN_HSRR0, r8
207− mtspr SPRN_HSRR1, r7
208− ba 0xe80
209−
210− /* Virtual-mode return - can't get here for HMI or machine check */
190+ /* Virtual-mode return */
211191 .Lvirt_return:
212− cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
213− beq 16f
214− cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
215− beq 17f
216− andi. r0, r7, MSR_EE /* were interrupts hard-enabled? */
217− beq 18f
218− mtmsrd r7, 1 /* if so then re-enable them */
219−18: mtlr r8
192+ mtlr r8
220193 blr
221194
222−16: mtspr SPRN_HSRR0, r8 /* jump to reloc-on external vector */
223− mtspr SPRN_HSRR1, r7
224− b exc_virt_0x4500_hardware_interrupt
225−
226−17: mtspr SPRN_HSRR0, r8
227− mtspr SPRN_HSRR1, r7
228− b exc_virt_0x4e80_h_doorbell
229−
230195 kvmppc_primary_no_guest:
231196 /* We handle this much like a ceded vcpu */
232197 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */
@@ -1258,6 +1223,15 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) · 1025 unmodified lines
12581223
12591224 stw r12,VCPU_TRAP(r9)
12601225
1226+ /*
1227+ * Now that we have saved away SRR0/1 and HSRR0/1,
1228+ * interrupts are recoverable in principle, so set MSR_RI.
1229+ * This becomes important for relocation-on interrupts from
1230+ * the guest, which we can get in radix mode on POWER9.
1231+ */
1232+ li r0, MSR_RI
1233+ mtmsrd r0, 1
1234+
12611235 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
12621236 addi r3, r9, VCPU_TB_RMINTR
12631237 mr r4, r9