Pull scheduler fix from Borislav Petkov: - Add a missing memory barrier in the concurrency ID mm switching * tag 'sched_urgent_for_v6.9_rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched: Add missing memory barrier in switch_mm_cid
Unified · Split
| @@ -79,6 +79,9 @@ do { \ · 78 unmodified lines | ||
| 79 | 79 | #define __smp_mb__before_atomic() do { } while (0) |
| 80 | 80 | #define __smp_mb__after_atomic() do { } while (0) |
| 81 | 81 | |
| 82 | +/* Writing to CR3 provides a full memory barrier in switch_mm(). */ | |
| 83 | +#define smp_mb__after_switch_mm() do { } while (0) | |
| 84 | + | |
| 82 | 85 | #include <asm-generic/barrier.h> |
| 83 | 86 | |
| 84 | 87 | #endif /* _ASM_X86_BARRIER_H */ |
| @@ -294,5 +294,13 @@ do { \ · 293 unmodified lines | ||
| 294 | 294 | #define io_stop_wc() do { } while (0) |
| 295 | 295 | #endif |
| 296 | 296 | |
| 297 | +/* | |
| 298 | + * Architectures that guarantee an implicit smp_mb() in switch_mm() | |
| 299 | + * can override smp_mb__after_switch_mm. | |
| 300 | + */ | |
| 301 | +#ifndef smp_mb__after_switch_mm | |
| 302 | +# define smp_mb__after_switch_mm() smp_mb() | |
| 303 | +#endif | |
| 304 | + | |
| 297 | 305 | #endif /* !__ASSEMBLY__ */ |
| 298 | 306 | #endif /* __ASM_GENERIC_BARRIER_H */ |
| @@ -79,6 +79,8 @@ · 78 unmodified lines | ||
| 79 | 79 | # include <asm/paravirt_api_clock.h> |
| 80 | 80 | #endif |
| 81 | 81 | |
| 82 | +#include <asm/barrier.h> | |
| 83 | + | |
| 82 | 84 | #include "cpupri.h" |
| 83 | 85 | #include "cpudeadline.h" |
| 84 | 86 | |
| @@ -3445,13 +3447,19 @@ static inline void switch_mm_cid(struct rq *rq, · 3360 unmodified lines | ||
| 3445 | 3447 | * between rq->curr store and load of {prev,next}->mm->pcpu_cid[cpu]. |
| 3446 | 3448 | * Provide it here. |
| 3447 | 3449 | */ |
| 3448 | − if (!prev->mm) // from kernel | |
| 3450 | + if (!prev->mm) { // from kernel | |
| 3449 | 3451 | smp_mb(); |
| 3450 | − /* | |
| 3451 | − * user -> user transition guarantees a memory barrier through | |
| 3452 | − * switch_mm() when current->mm changes. If current->mm is | |
| 3453 | − * unchanged, no barrier is needed. | |
| 3454 | − */ | |
| 3452 | + } else { // from user | |
| 3453 | + /* | |
| 3454 | + * user->user transition relies on an implicit | |
| 3455 | + * memory barrier in switch_mm() when | |
| 3456 | + * current->mm changes. If the architecture | |
| 3457 | + * switch_mm() does not have an implicit memory | |
| 3458 | + * barrier, it is emitted here. If current->mm | |
| 3459 | + * is unchanged, no barrier is needed. | |
| 3460 | + */ | |
| 3461 | + smp_mb__after_switch_mm(); | |
| 3462 | + } | |
| 3455 | 3463 | } |
| 3456 | 3464 | if (prev->mm_cid_active) { |
| 3457 | 3465 | mm_cid_snapshot_time(rq, prev->mm); |