entry/kvm: Explicitly flush pending rcuog wakeup before last rescheduling point

Following the idle loop model, cleanly check for pending rcuog wakeup before the last rescheduling point upon resuming to guest mode. This way we can avoid to do it from rcu_user_enter() with the last resort self-IPI hack that enforces rescheduling.

Frederic Weisbecker committed 5 years ago

Commit 4ae7dc91 parent 47b8ff14 files · +50 −10Browse files

Unified · Split

4 files

arch/x86/kvm/x86.c +1 −0
include/linux/entry-kvm.h +14 −0
kernel/rcu/tree.c +34 −10
kernel/rcu/tree_plugin.h +1 −0

arch/x86/kvm/x86.c+1−0

@@ -1782,6 +1782,7 @@ EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr); · 1781 unmodified lines
1782		1782
1783	bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)	1783	bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
1784	{	1784	{
		1785	xfer_to_guest_mode_prepare();
1785	return vcpu->mode == EXITING_GUEST_MODE \|\| kvm_request_pending(vcpu) \|\|	1786	return vcpu->mode == EXITING_GUEST_MODE \|\| kvm_request_pending(vcpu) \|\|
1786	xfer_to_guest_mode_work_pending();	1787	xfer_to_guest_mode_work_pending();
1787	}	1788	}

include/linux/entry-kvm.h+14−0

@@ -46,6 +46,20 @@ static inline int arch_xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu, · 45 unmodified lines
46	*/	46	*/
47	int xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu);	47	int xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu);
48		48
		49	/**
		50	* xfer_to_guest_mode_prepare - Perform last minute preparation work that
		51	* need to be handled while IRQs are disabled
		52	* upon entering to guest.
		53	*
		54	* Has to be invoked with interrupts disabled before the last call
		55	* to xfer_to_guest_mode_work_pending().
		56	*/
		57	static inline void xfer_to_guest_mode_prepare(void)
		58	{
		59	lockdep_assert_irqs_disabled();
		60	rcu_nocb_flush_deferred_wakeup();
		61	}
		62
49	/**	63	/**
50	* __xfer_to_guest_mode_work_pending - Check if work is pending	64	* __xfer_to_guest_mode_work_pending - Check if work is pending
51	*	65	*

kernel/rcu/tree.c+34−10

 #ifdef CONFIG_NO_HZ_FULL
+#if !defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK)
 /*
  * An empty function that will trigger a reschedule on
- * IRQ tail once IRQs get re-enabled on userspace resume.
+ * IRQ tail once IRQs get re-enabled on userspace/guest resume.
  */
 static void late_wakeup_func(struct irq_work *work)
 {
 static DEFINE_PER_CPU(struct irq_work, late_wakeup_work) =
 	IRQ_WORK_INIT(late_wakeup_func);
+/*
+ * If either:
+ *
+ * 1) the task is about to enter in guest mode and $ARCH doesn't support KVM generic work
+ * 2) the task is about to enter in user mode and $ARCH doesn't support generic entry.
+ *
+ * In these cases the late RCU wake ups aren't supported in the resched loops and our
+ * last resort is to fire a local irq_work that will trigger a reschedule once IRQs
+ * get re-enabled again.
+ */
+noinstr static void rcu_irq_work_resched(void)
+{
+	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
+	if (IS_ENABLED(CONFIG_GENERIC_ENTRY) && !(current->flags & PF_VCPU))
+		return;
+	if (IS_ENABLED(CONFIG_KVM_XFER_TO_GUEST_WORK) && (current->flags & PF_VCPU))
+		return;
+	instrumentation_begin();
+	if (do_nocb_deferred_wakeup(rdp) && need_resched()) {
+		irq_work_queue(this_cpu_ptr(&late_wakeup_work));
+	}
+	instrumentation_end();
+}
+#else
+static inline void rcu_irq_work_resched(void) { }
+#endif
 /**
  * rcu_user_enter - inform RCU that we are resuming userspace.
  *
  */
 noinstr void rcu_user_enter(void)
 {
-	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
 	lockdep_assert_irqs_disabled();
 	/*
 	 * rescheduling opportunity in the entry code. Trigger a self IPI
 	 * that will fire and reschedule once we resume in user/guest mode.
 	 */
-	instrumentation_begin();
+	rcu_irq_work_resched();
-	if (!IS_ENABLED(CONFIG_GENERIC_ENTRY) || (current->flags & PF_VCPU)) {
-		if (do_nocb_deferred_wakeup(rdp) && need_resched())
-			irq_work_queue(this_cpu_ptr(&late_wakeup_work));
-	}
-	instrumentation_end();
 	rcu_eqs_enter(true);
 }

kernel/rcu/tree_plugin.h+1−0

@@ -2197,6 +2197,7 @@ void rcu_nocb_flush_deferred_wakeup(void) · 2196 unmodified lines
2197	{	2197	{
2198	do_nocb_deferred_wakeup(this_cpu_ptr(&rcu_data));	2198	do_nocb_deferred_wakeup(this_cpu_ptr(&rcu_data));
2199	}	2199	}
		2200	EXPORT_SYMBOL_GPL(rcu_nocb_flush_deferred_wakeup);
2200		2201
2201	void __init rcu_init_nohz(void)	2202	void __init rcu_init_nohz(void)
2202	{	2203	{

entry/kvm: Explicitly flush pending rcuog wakeup before last rescheduling point

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code