When transitioning to/from guest mode, it is necessary to inform lockdep, tracing, and RCU in a specific order, similar to the requirements for transitions to/from user mode. Additionally, it is necessary to perform vtime accounting for a window around running the guest, with RCU enabled, such that timer interrupts taken from the guest can be accounted as guest time. Most architectures don't handle all the necessary pieces, and a have a number of common bugs, including unsafe usage of RCU during the window between guest_enter() and guest_exit(). On x86, this was dealt with across commits: 87fa7f3e98a1310e ("x86/kvm: Move context tracking where it belongs") 0642391e2139a2c1 ("x86/kvm/vmx: Add hardirq tracing to guest enter/exit") 9fc975e9efd03e57 ("x86/kvm/svm: Add hardirq tracing on guest enter/exit") 3ebccdf373c21d86 ("x86/kvm/vmx: Move guest enter/exit into .noinstr.text") 135961e0a7d555fc ("x86/kvm/svm: Move guest enter/exit into .noinstr.text") 160457140187c5fb ("KVM: x86: Defer vtime accounting 'til after IRQ handling") bc908e091b326467 ("KVM: x86: Consolidate guest enter/exit logic to common helpers") ... but those fixes are specific to x86, and as the resulting logic (while correct) is split across generic helper functions and x86-specific helper functions, it is difficult to see that the entry/exit accounting is balanced. This patch adds generic helpers which architectures can use to handle guest entry/exit consistently and correctly. The guest_{enter,exit}() helpers are split into guest_timing_{enter,exit}() to perform vtime accounting, and guest_context_{enter,exit}() to perform the necessary context tracking and RCU management. The existing guest_{enter,exit}() heleprs are left as wrappers of these. Atop this, new guest_state_enter_irqoff() and guest_state_exit_irqoff() helpers are added to handle the ordering of lockdep, tracing, and RCU manageent. These are inteneded to mirror exit_to_user_mode() and enter_from_user_mode(). Subsequent patches will migrate architectures over to the new helpers, following a sequence: guest_timing_enter_irqoff(); guest_state_enter_irqoff(); < run the vcpu > guest_state_exit_irqoff(); < take any pending IRQs > guest_timing_exit_irqoff(); This sequences handles all of the above correctly, and more clearly balances the entry and exit portions, making it easier to understand. The existing helpers are marked as deprecated, and will be removed once all architectures have been converted. There should be no functional change as a result of this patch.
Showing the changes to include (1 of 1 file). Show all
Unified · Split
| @@ -29,7 +29,9 @@ · 28 unmodified lines | |||
| 29 | #include <linux/refcount.h> | 29 | #include <linux/refcount.h> |
| 30 | #include <linux/nospec.h> | 30 | #include <linux/nospec.h> |
| 31 | #include <linux/notifier.h> | 31 | #include <linux/notifier.h> |
| 32 | #include <linux/ftrace.h> | ||
| 32 | #include <linux/hashtable.h> | 33 | #include <linux/hashtable.h> |
| 34 | #include <linux/instrumentation.h> | ||
| 33 | #include <linux/interval_tree.h> | 35 | #include <linux/interval_tree.h> |
| 34 | #include <linux/rbtree.h> | 36 | #include <linux/rbtree.h> |
| 35 | #include <linux/xarray.h> | 37 | #include <linux/xarray.h> |
| @@ -368,8 +370,11 @@ struct kvm_vcpu { · 332 unmodified lines | |||
| 368 | u64 last_used_slot_gen; | 370 | u64 last_used_slot_gen; |
| 369 | }; | 371 | }; |
| 370 | 372 | ||
| 371 | /* must be called with irqs disabled */ | 373 | /* |
| 372 | static __always_inline void guest_enter_irqoff(void) | 374 | * Start accounting time towards a guest. |
| 375 | * Must be called before entering guest context. | ||
| 376 | */ | ||
| 377 | static __always_inline void guest_timing_enter_irqoff(void) | ||
| 373 | { | 378 | { |
| 374 | /* | 379 | /* |
| 375 | * This is running in ioctl context so its safe to assume that it's the | 380 | * This is running in ioctl context so its safe to assume that it's the |
| @@ -378,7 +383,18 @@ static __always_inline void guest_enter_irqoff(void) · 2 unmodified lines | |||
| 378 | instrumentation_begin(); | 383 | instrumentation_begin(); |
| 379 | vtime_account_guest_enter(); | 384 | vtime_account_guest_enter(); |
| 380 | instrumentation_end(); | 385 | instrumentation_end(); |
| 386 | } | ||
| 381 | 387 | ||
| 388 | /* | ||
| 389 | * Enter guest context and enter an RCU extended quiescent state. | ||
| 390 | * | ||
| 391 | * Between guest_context_enter_irqoff() and guest_context_exit_irqoff() it is | ||
| 392 | * unsafe to use any code which may directly or indirectly use RCU, tracing | ||
| 393 | * (including IRQ flag tracing), or lockdep. All code in this period must be | ||
| 394 | * non-instrumentable. | ||
| 395 | */ | ||
| 396 | static __always_inline void guest_context_enter_irqoff(void) | ||
| 397 | { | ||
| 382 | /* | 398 | /* |
| 383 | * KVM does not hold any references to rcu protected data when it | 399 | * KVM does not hold any references to rcu protected data when it |
| 384 | * switches CPU into a guest mode. In fact switching to a guest mode | 400 | * switches CPU into a guest mode. In fact switching to a guest mode |
| @@ -394,16 +410,79 @@ static __always_inline void guest_enter_irqoff(void) · 9 unmodified lines | |||
| 394 | } | 410 | } |
| 395 | } | 411 | } |
| 396 | 412 | ||
| 397 | static __always_inline void guest_exit_irqoff(void) | 413 | /* |
| 414 | * Deprecated. Architectures should move to guest_timing_enter_irqoff() and | ||
| 415 | * guest_state_enter_irqoff(). | ||
| 416 | */ | ||
| 417 | static __always_inline void guest_enter_irqoff(void) | ||
| 418 | { | ||
| 419 | guest_timing_enter_irqoff(); | ||
| 420 | guest_context_enter_irqoff(); | ||
| 421 | } | ||
| 422 | |||
| 423 | /** | ||
| 424 | * guest_state_enter_irqoff - Fixup state when entering a guest | ||
| 425 | * | ||
| 426 | * Entry to a guest will enable interrupts, but the kernel state is interrupts | ||
| 427 | * disabled when this is invoked. Also tell RCU about it. | ||
| 428 | * | ||
| 429 | * 1) Trace interrupts on state | ||
| 430 | * 2) Invoke context tracking if enabled to adjust RCU state | ||
| 431 | * 3) Tell lockdep that interrupts are enabled | ||
| 432 | * | ||
| 433 | * Invoked from architecture specific code before entering a guest. | ||
| 434 | * Must be called with interrupts disabled and the caller must be | ||
| 435 | * non-instrumentable. | ||
| 436 | * The caller has to invoke guest_timing_enter_irqoff() before this. | ||
| 437 | * | ||
| 438 | * Note: this is analogous to exit_to_user_mode(). | ||
| 439 | */ | ||
| 440 | static __always_inline void guest_state_enter_irqoff(void) | ||
| 441 | { | ||
| 442 | instrumentation_begin(); | ||
| 443 | trace_hardirqs_on_prepare(); | ||
| 444 | lockdep_hardirqs_on_prepare(CALLER_ADDR0); | ||
| 445 | instrumentation_end(); | ||
| 446 | |||
| 447 | guest_context_enter_irqoff(); | ||
| 448 | lockdep_hardirqs_on(CALLER_ADDR0); | ||
| 449 | } | ||
| 450 | |||
| 451 | /* | ||
| 452 | * Exit guest context and exit an RCU extended quiescent state. | ||
| 453 | * | ||
| 454 | * Between guest_context_enter_irqoff() and guest_context_exit_irqoff() it is | ||
| 455 | * unsafe to use any code which may directly or indirectly use RCU, tracing | ||
| 456 | * (including IRQ flag tracing), or lockdep. All code in this period must be | ||
| 457 | * non-instrumentable. | ||
| 458 | */ | ||
| 459 | static __always_inline void guest_context_exit_irqoff(void) | ||
| 398 | { | 460 | { |
| 399 | context_tracking_guest_exit(); | 461 | context_tracking_guest_exit(); |
| 462 | } | ||
| 400 | 463 | ||
| 464 | /* | ||
| 465 | * Stop accounting time towards a guest. | ||
| 466 | * Must be called after exiting guest context. | ||
| 467 | */ | ||
| 468 | static __always_inline void guest_timing_exit_irqoff(void) | ||
| 469 | { | ||
| 401 | instrumentation_begin(); | 470 | instrumentation_begin(); |
| 402 | /* Flush the guest cputime we spent on the guest */ | 471 | /* Flush the guest cputime we spent on the guest */ |
| 403 | vtime_account_guest_exit(); | 472 | vtime_account_guest_exit(); |
| 404 | instrumentation_end(); | 473 | instrumentation_end(); |
| 405 | } | 474 | } |
| 406 | 475 | ||
| 476 | /* | ||
| 477 | * Deprecated. Architectures should move to guest_state_exit_irqoff() and | ||
| 478 | * guest_timing_exit_irqoff(). | ||
| 479 | */ | ||
| 480 | static __always_inline void guest_exit_irqoff(void) | ||
| 481 | { | ||
| 482 | guest_context_exit_irqoff(); | ||
| 483 | guest_timing_exit_irqoff(); | ||
| 484 | } | ||
| 485 | |||
| 407 | static inline void guest_exit(void) | 486 | static inline void guest_exit(void) |
| 408 | { | 487 | { |
| 409 | unsigned long flags; | 488 | unsigned long flags; |
| @@ -413,6 +492,33 @@ static inline void guest_exit(void) · 3 unmodified lines | |||
| 413 | local_irq_restore(flags); | 492 | local_irq_restore(flags); |
| 414 | } | 493 | } |
| 415 | 494 | ||
| 495 | /** | ||
| 496 | * guest_state_exit_irqoff - Establish state when returning from guest mode | ||
| 497 | * | ||
| 498 | * Entry from a guest disables interrupts, but guest mode is traced as | ||
| 499 | * interrupts enabled. Also with NO_HZ_FULL RCU might be idle. | ||
| 500 | * | ||
| 501 | * 1) Tell lockdep that interrupts are disabled | ||
| 502 | * 2) Invoke context tracking if enabled to reactivate RCU | ||
| 503 | * 3) Trace interrupts off state | ||
| 504 | * | ||
| 505 | * Invoked from architecture specific code after exiting a guest. | ||
| 506 | * Must be invoked with interrupts disabled and the caller must be | ||
| 507 | * non-instrumentable. | ||
| 508 | * The caller has to invoke guest_timing_exit_irqoff() after this. | ||
| 509 | * | ||
| 510 | * Note: this is analogous to enter_from_user_mode(). | ||
| 511 | */ | ||
| 512 | static __always_inline void guest_state_exit_irqoff(void) | ||
| 513 | { | ||
| 514 | lockdep_hardirqs_off(CALLER_ADDR0); | ||
| 515 | guest_context_exit_irqoff(); | ||
| 516 | |||
| 517 | instrumentation_begin(); | ||
| 518 | trace_hardirqs_off_finish(); | ||
| 519 | instrumentation_end(); | ||
| 520 | } | ||
| 521 | |||
| 416 | static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) | 522 | static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) |
| 417 | { | 523 | { |
| 418 | /* | 524 | /* |