Pull scheduler fix from Borislav Petkov: - Add a missing memory barrier in the concurrency ID mm switching * tag 'sched_urgent_for_v6.9_rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched: Add missing memory barrier in switch_mm_cid
Unified · Split
| @@ -79,6 +79,9 @@ do { \ · 78 unmodified lines | |||
| 79 | #define __smp_mb__before_atomic() do { } while (0) | 79 | #define __smp_mb__before_atomic() do { } while (0) |
| 80 | #define __smp_mb__after_atomic() do { } while (0) | 80 | #define __smp_mb__after_atomic() do { } while (0) |
| 81 | 81 | ||
| 82 | /* Writing to CR3 provides a full memory barrier in switch_mm(). */ | ||
| 83 | #define smp_mb__after_switch_mm() do { } while (0) | ||
| 84 | |||
| 82 | #include <asm-generic/barrier.h> | 85 | #include <asm-generic/barrier.h> |
| 83 | 86 | ||
| 84 | #endif /* _ASM_X86_BARRIER_H */ | 87 | #endif /* _ASM_X86_BARRIER_H */ |
| @@ -294,5 +294,13 @@ do { \ · 293 unmodified lines | |||
| 294 | #define io_stop_wc() do { } while (0) | 294 | #define io_stop_wc() do { } while (0) |
| 295 | #endif | 295 | #endif |
| 296 | 296 | ||
| 297 | /* | ||
| 298 | * Architectures that guarantee an implicit smp_mb() in switch_mm() | ||
| 299 | * can override smp_mb__after_switch_mm. | ||
| 300 | */ | ||
| 301 | #ifndef smp_mb__after_switch_mm | ||
| 302 | # define smp_mb__after_switch_mm() smp_mb() | ||
| 303 | #endif | ||
| 304 | |||
| 297 | #endif /* !__ASSEMBLY__ */ | 305 | #endif /* !__ASSEMBLY__ */ |
| 298 | #endif /* __ASM_GENERIC_BARRIER_H */ | 306 | #endif /* __ASM_GENERIC_BARRIER_H */ |
| @@ -79,6 +79,8 @@ · 78 unmodified lines | |||
| 79 | # include <asm/paravirt_api_clock.h> | 79 | # include <asm/paravirt_api_clock.h> |
| 80 | #endif | 80 | #endif |
| 81 | 81 | ||
| 82 | #include <asm/barrier.h> | ||
| 83 | |||
| 82 | #include "cpupri.h" | 84 | #include "cpupri.h" |
| 83 | #include "cpudeadline.h" | 85 | #include "cpudeadline.h" |
| 84 | 86 | ||
| @@ -3445,13 +3447,19 @@ static inline void switch_mm_cid(struct rq *rq, · 3360 unmodified lines | |||
| 3445 | * between rq->curr store and load of {prev,next}->mm->pcpu_cid[cpu]. | 3447 | * between rq->curr store and load of {prev,next}->mm->pcpu_cid[cpu]. |
| 3446 | * Provide it here. | 3448 | * Provide it here. |
| 3447 | */ | 3449 | */ |
| 3448 | if (!prev->mm) // from kernel | 3450 | if (!prev->mm) { // from kernel |
| 3449 | smp_mb(); | 3451 | smp_mb(); |
| 3450 | /* | 3452 | } else { // from user |
| 3451 | * user -> user transition guarantees a memory barrier through | 3453 | /* |
| 3452 | * switch_mm() when current->mm changes. If current->mm is | 3454 | * user->user transition relies on an implicit |
| 3453 | * unchanged, no barrier is needed. | 3455 | * memory barrier in switch_mm() when |
| 3454 | */ | 3456 | * current->mm changes. If the architecture |
| 3457 | * switch_mm() does not have an implicit memory | ||
| 3458 | * barrier, it is emitted here. If current->mm | ||
| 3459 | * is unchanged, no barrier is needed. | ||
| 3460 | */ | ||
| 3461 | smp_mb__after_switch_mm(); | ||
| 3462 | } | ||
| 3455 | } | 3463 | } |
| 3456 | if (prev->mm_cid_active) { | 3464 | if (prev->mm_cid_active) { |
| 3457 | mm_cid_snapshot_time(rq, prev->mm); | 3465 | mm_cid_snapshot_time(rq, prev->mm); |