History

sched/mmcid: Revert the complex CID management

The CID management is a complex beast, which affects both scheduling and task migration. The compaction mechanism forces random tasks of a process into task work on exit to user space causing latency spikes. Revert back to the initial simple bitmap allocating mechanics, which are known to have scalability issues as that allows to gradually build up a replacement functionality in a reviewable way.

Thomas Gleixner committed 11 months ago

Commit 77d7dc81 parent 80adacc4 files · +64 −800Browse files

Unified · Split

4 files
  1. include/linux/mm_types.h +2 −51
  2. kernel/fork.c +2 −3
  3. kernel/sched/core.c +17 −500
  4. kernel/sched/sched.h +43 −246
include/linux/mm_types.h+2−51
@@ -922,13 +922,9 @@ struct vm_area_struct { · 921 unmodified lines
922#define vma_policy(vma) NULL922#define vma_policy(vma) NULL
923#endif923#endif
924 924
925#ifdef CONFIG_SCHED_MM_CID
926struct mm_cid {925struct mm_cid {
927 u64 time;926 unsigned int cid;
928 int cid;
929 int recent_cid;
930};927};
931#endif
932 928
933/*929/*
934 * Opaque type representing current mm_struct flag state. Must be accessed via930 * Opaque type representing current mm_struct flag state. Must be accessed via
@@ -1000,12 +996,6 @@ struct mm_struct { · 65 unmodified lines
1000 * runqueue locks.996 * runqueue locks.
1001 */997 */
1002 struct mm_cid __percpu *pcpu_cid;998 struct mm_cid __percpu *pcpu_cid;
1003 /*
1004 * @mm_cid_next_scan: Next mm_cid scan (in jiffies).
1005 *
1006 * When the next mm_cid scan is due (in jiffies).
1007 */
1008 unsigned long mm_cid_next_scan;
1009 /**999 /**
1010 * @nr_cpus_allowed: Number of CPUs allowed for mm.1000 * @nr_cpus_allowed: Number of CPUs allowed for mm.
1011 *1001 *
@@ -1013,14 +1003,6 @@ struct mm_struct { · 1 unmodified line
1013 * threads allowed CPUs.1003 * threads allowed CPUs.
1014 */1004 */
1015 unsigned int nr_cpus_allowed;1005 unsigned int nr_cpus_allowed;
1016 /**
1017 * @max_nr_cid: Maximum number of allowed concurrency
1018 * IDs allocated.
1019 *
1020 * Track the highest number of allowed concurrency IDs
1021 * allocated for the mm.
1022 */
1023 atomic_t max_nr_cid;
1024 /**1006 /**
1025 * @cpus_allowed_lock: Lock protecting mm cpus_allowed.1007 * @cpus_allowed_lock: Lock protecting mm cpus_allowed.
1026 *1008 *
@@ -1371,35 +1353,7 @@ static inline void vma_iter_init(struct vma_iterator *vmi, · 344 unmodified lines
1371 1353
1372#ifdef CONFIG_SCHED_MM_CID1354#ifdef CONFIG_SCHED_MM_CID
1373 1355
1374enum mm_cid_state {1356#define MM_CID_UNSET (~0U)
1375 MM_CID_UNSET = -1U, /* Unset state has lazy_put flag set. */
1376 MM_CID_LAZY_PUT = (1U << 31),
1377};
1378
1379static inline bool mm_cid_is_unset(int cid)
1380{
1381 return cid == MM_CID_UNSET;
1382}
1383
1384static inline bool mm_cid_is_lazy_put(int cid)
1385{
1386 return !mm_cid_is_unset(cid) && (cid & MM_CID_LAZY_PUT);
1387}
1388
1389static inline bool mm_cid_is_valid(int cid)
1390{
1391 return !(cid & MM_CID_LAZY_PUT);
1392}
1393
1394static inline int mm_cid_set_lazy_put(int cid)
1395{
1396 return cid | MM_CID_LAZY_PUT;
1397}
1398
1399static inline int mm_cid_clear_lazy_put(int cid)
1400{
1401 return cid & ~MM_CID_LAZY_PUT;
1402}
1403 1357
1404/*1358/*
1405 * mm_cpus_allowed: Union of all mm's threads allowed CPUs.1359 * mm_cpus_allowed: Union of all mm's threads allowed CPUs.
@@ -1432,11 +1386,8 @@ static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p) · 26 unmodified lines
1432 struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);1386 struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);
1433 1387
1434 pcpu_cid->cid = MM_CID_UNSET;1388 pcpu_cid->cid = MM_CID_UNSET;
1435 pcpu_cid->recent_cid = MM_CID_UNSET;
1436 pcpu_cid->time = 0;
1437 }1389 }
1438 mm->nr_cpus_allowed = p->nr_cpus_allowed;1390 mm->nr_cpus_allowed = p->nr_cpus_allowed;
1439 atomic_set(&mm->max_nr_cid, 0);
1440 raw_spin_lock_init(&mm->cpus_allowed_lock);1391 raw_spin_lock_init(&mm->cpus_allowed_lock);
1441 cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);1392 cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
1442 cpumask_clear(mm_cidmask(mm));1393 cpumask_clear(mm_cidmask(mm));
kernel/fork.c+2−3
@@ -955,10 +955,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) · 954 unmodified lines
955#endif955#endif
956 956
957#ifdef CONFIG_SCHED_MM_CID957#ifdef CONFIG_SCHED_MM_CID
958 tsk->mm_cid = -1;958 tsk->mm_cid = MM_CID_UNSET;
959 tsk->last_mm_cid = -1;959 tsk->last_mm_cid = MM_CID_UNSET;
960 tsk->mm_cid_active = 0;960 tsk->mm_cid_active = 0;
961 tsk->migrate_from_cpu = -1;
962#endif961#endif
963 return tsk;962 return tsk;
964 963
kernel/sched/core.c+17−500
@@ -2128,8 +2128,6 @@ void activate_task(struct rq *rq, struct task_struct *p, int flags) · 2127 unmodified lines
2128{2128{
2129 if (task_on_rq_migrating(p))2129 if (task_on_rq_migrating(p))
2130 flags |= ENQUEUE_MIGRATED;2130 flags |= ENQUEUE_MIGRATED;
2131 if (flags & ENQUEUE_MIGRATED)
2132 sched_mm_cid_migrate_to(rq, p);
2133 2131
2134 enqueue_task(rq, p, flags);2132 enqueue_task(rq, p, flags);
2135 2133
@@ -3329,7 +3327,6 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) · 1193 unmodified lines
3329 if (p->sched_class->migrate_task_rq)3327 if (p->sched_class->migrate_task_rq)
3330 p->sched_class->migrate_task_rq(p, new_cpu);3328 p->sched_class->migrate_task_rq(p, new_cpu);
3331 p->se.nr_migrations++;3329 p->se.nr_migrations++;
3332 sched_mm_cid_migrate_from(p);
3333 perf_event_task_migrate(p);3330 perf_event_task_migrate(p);
3334 }3331 }
3335 3332
@@ -5280,9 +5277,6 @@ context_switch(struct rq *rq, struct task_struct *prev, · 1944 unmodified lines
5280 *5277 *
5281 * kernel -> user switch + mmdrop_lazy_tlb() active5278 * kernel -> user switch + mmdrop_lazy_tlb() active
5282 * user -> user switch5279 * user -> user switch
5283 *
5284 * switch_mm_cid() needs to be updated if the barriers provided
5285 * by context_switch() are modified.
5286 */5280 */
5287 if (!next->mm) { // to kernel5281 if (!next->mm) { // to kernel
5288 enter_lazy_tlb(prev->active_mm, next);5282 enter_lazy_tlb(prev->active_mm, next);
@@ -5312,8 +5306,7 @@ context_switch(struct rq *rq, struct task_struct *prev, · 23 unmodified lines
5312 }5306 }
5313 }5307 }
5314 5308
5315 /* switch_mm_cid() requires the memory barriers above. */5309 switch_mm_cid(prev, next);
5316 switch_mm_cid(rq, prev, next);
5317 5310
5318 /*5311 /*
5319 * Tell rseq that the task was scheduled in. Must be after5312 * Tell rseq that the task was scheduled in. Must be after
@@ -5604,7 +5597,6 @@ void sched_tick(void) · 284 unmodified lines
5604 resched_latency = cpu_resched_latency(rq);5597 resched_latency = cpu_resched_latency(rq);
5605 calc_global_load_tick(rq);5598 calc_global_load_tick(rq);
5606 sched_core_tick(rq);5599 sched_core_tick(rq);
5607 task_tick_mm_cid(rq, donor);
5608 scx_tick(rq);5600 scx_tick(rq);
5609 5601
5610 rq_unlock(rq, &rf);5602 rq_unlock(rq, &rf);
@@ -10376,522 +10368,47 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count) · 4765 unmodified lines
10376}10368}
10377 10369
10378#ifdef CONFIG_SCHED_MM_CID10370#ifdef CONFIG_SCHED_MM_CID
10379
10380/*10371/*
10381 * @cid_lock: Guarantee forward-progress of cid allocation.10372 * When a task exits, the MM CID held by the task is not longer required as
10382 *10373 * the task cannot return to user space.
10383 * Concurrency ID allocation within a bitmap is mostly lock-free. The cid_lock
10384 * is only used when contention is detected by the lock-free allocation so
10385 * forward progress can be guaranteed.
10386 */
10387DEFINE_RAW_SPINLOCK(cid_lock);
10388
10389/*
10390 * @use_cid_lock: Select cid allocation behavior: lock-free vs spinlock.
10391 *
10392 * When @use_cid_lock is 0, the cid allocation is lock-free. When contention is
10393 * detected, it is set to 1 to ensure that all newly coming allocations are
10394 * serialized by @cid_lock until the allocation which detected contention
10395 * completes and sets @use_cid_lock back to 0. This guarantees forward progress
10396 * of a cid allocation.
10397 */
10398int use_cid_lock;
10399
10400/*
10401 * mm_cid remote-clear implements a lock-free algorithm to clear per-mm/cpu cid
10402 * concurrently with respect to the execution of the source runqueue context
10403 * switch.
10404 *
10405 * There is one basic properties we want to guarantee here:
10406 *
10407 * (1) Remote-clear should _never_ mark a per-cpu cid UNSET when it is actively
10408 * used by a task. That would lead to concurrent allocation of the cid and
10409 * userspace corruption.
10410 *
10411 * Provide this guarantee by introducing a Dekker memory ordering to guarantee
10412 * that a pair of loads observe at least one of a pair of stores, which can be
10413 * shown as:
10414 *
10415 * X = Y = 0
10416 *
10417 * w[X]=1 w[Y]=1
10418 * MB MB
10419 * r[Y]=y r[X]=x
10420 *
10421 * Which guarantees that x==0 && y==0 is impossible. But rather than using
10422 * values 0 and 1, this algorithm cares about specific state transitions of the
10423 * runqueue current task (as updated by the scheduler context switch), and the
10424 * per-mm/cpu cid value.
10425 *
10426 * Let's introduce task (Y) which has task->mm == mm and task (N) which has
10427 * task->mm != mm for the rest of the discussion. There are two scheduler state
10428 * transitions on context switch we care about:
10429 *
10430 * (TSA) Store to rq->curr with transition from (N) to (Y)
10431 *
10432 * (TSB) Store to rq->curr with transition from (Y) to (N)
10433 *
10434 * On the remote-clear side, there is one transition we care about:
10435 *
10436 * (TMA) cmpxchg to *pcpu_cid to set the LAZY flag
10437 *
10438 * There is also a transition to UNSET state which can be performed from all
10439 * sides (scheduler, remote-clear). It is always performed with a cmpxchg which
10440 * guarantees that only a single thread will succeed:
10441 *
10442 * (TMB) cmpxchg to *pcpu_cid to mark UNSET
10443 *
10444 * Just to be clear, what we do _not_ want to happen is a transition to UNSET
10445 * when a thread is actively using the cid (property (1)).
10446 *
10447 * Let's looks at the relevant combinations of TSA/TSB, and TMA transitions.
10448 *
10449 * Scenario A) (TSA)+(TMA) (from next task perspective)
10450 *
10451 * CPU0 CPU1
10452 *
10453 * Context switch CS-1 Remote-clear
10454 * - store to rq->curr: (N)->(Y) (TSA) - cmpxchg to *pcpu_id to LAZY (TMA)
10455 * (implied barrier after cmpxchg)
10456 * - switch_mm_cid()
10457 * - memory barrier (see switch_mm_cid()
10458 * comment explaining how this barrier
10459 * is combined with other scheduler
10460 * barriers)
10461 * - mm_cid_get (next)
10462 * - READ_ONCE(*pcpu_cid) - rcu_dereference(src_rq->curr)
10463 *
10464 * This Dekker ensures that either task (Y) is observed by the
10465 * rcu_dereference() or the LAZY flag is observed by READ_ONCE(), or both are
10466 * observed.
10467 *
10468 * If task (Y) store is observed by rcu_dereference(), it means that there is
10469 * still an active task on the cpu. Remote-clear will therefore not transition
10470 * to UNSET, which fulfills property (1).
10471 *
10472 * If task (Y) is not observed, but the lazy flag is observed by READ_ONCE(),
10473 * it will move its state to UNSET, which clears the percpu cid perhaps
10474 * uselessly (which is not an issue for correctness). Because task (Y) is not
10475 * observed, CPU1 can move ahead to set the state to UNSET. Because moving
10476 * state to UNSET is done with a cmpxchg expecting that the old state has the
10477 * LAZY flag set, only one thread will successfully UNSET.
10478 *
10479 * If both states (LAZY flag and task (Y)) are observed, the thread on CPU0
10480 * will observe the LAZY flag and transition to UNSET (perhaps uselessly), and
10481 * CPU1 will observe task (Y) and do nothing more, which is fine.
10482 *
10483 * What we are effectively preventing with this Dekker is a scenario where
10484 * neither LAZY flag nor store (Y) are observed, which would fail property (1)
10485 * because this would UNSET a cid which is actively used.
10486 */10374 */
10487
10488void sched_mm_cid_migrate_from(struct task_struct *t)
10489{
10490 t->migrate_from_cpu = task_cpu(t);
10491}
10492
10493static
10494int __sched_mm_cid_migrate_from_fetch_cid(struct rq *src_rq,
10495 struct task_struct *t,
10496 struct mm_cid *src_pcpu_cid)
10497{
10498 struct mm_struct *mm = t->mm;
10499 struct task_struct *src_task;
10500 int src_cid, last_mm_cid;
10501
10502 if (!mm)
10503 return -1;
10504
10505 last_mm_cid = t->last_mm_cid;
10506 /*
10507 * If the migrated task has no last cid, or if the current
10508 * task on src rq uses the cid, it means the source cid does not need
10509 * to be moved to the destination cpu.
10510 */
10511 if (last_mm_cid == -1)
10512 return -1;
10513 src_cid = READ_ONCE(src_pcpu_cid->cid);
10514 if (!mm_cid_is_valid(src_cid) || last_mm_cid != src_cid)
10515 return -1;
10516
10517 /*
10518 * If we observe an active task using the mm on this rq, it means we
10519 * are not the last task to be migrated from this cpu for this mm, so
10520 * there is no need to move src_cid to the destination cpu.
10521 */
10522 guard(rcu)();
10523 src_task = rcu_dereference(src_rq->curr);
10524 if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) {
10525 t->last_mm_cid = -1;
10526 return -1;
10527 }
10528
10529 return src_cid;
10530}
10531
10532static
10533int __sched_mm_cid_migrate_from_try_steal_cid(struct rq *src_rq,
10534 struct task_struct *t,
10535 struct mm_cid *src_pcpu_cid,
10536 int src_cid)
10537{
10538 struct task_struct *src_task;
10539 struct mm_struct *mm = t->mm;
10540 int lazy_cid;
10541
10542 if (src_cid == -1)
10543 return -1;
10544
10545 /*
10546 * Attempt to clear the source cpu cid to move it to the destination
10547 * cpu.
10548 */
10549 lazy_cid = mm_cid_set_lazy_put(src_cid);
10550 if (!try_cmpxchg(&src_pcpu_cid->cid, &src_cid, lazy_cid))
10551 return -1;
10552
10553 /*
10554 * The implicit barrier after cmpxchg per-mm/cpu cid before loading
10555 * rq->curr->mm matches the scheduler barrier in context_switch()
10556 * between store to rq->curr and load of prev and next task's
10557 * per-mm/cpu cid.
10558 *
10559 * The implicit barrier after cmpxchg per-mm/cpu cid before loading
10560 * rq->curr->mm_cid_active matches the barrier in
10561 * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and
10562 * sched_mm_cid_after_execve() between store to t->mm_cid_active and
10563 * load of per-mm/cpu cid.
10564 */
10565
10566 /*
10567 * If we observe an active task using the mm on this rq after setting
10568 * the lazy-put flag, this task will be responsible for transitioning
10569 * from lazy-put flag set to MM_CID_UNSET.
10570 */
10571 scoped_guard (rcu) {
10572 src_task = rcu_dereference(src_rq->curr);
10573 if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) {
10574 /*
10575 * We observed an active task for this mm, there is therefore
10576 * no point in moving this cid to the destination cpu.
10577 */
10578 t->last_mm_cid = -1;
10579 return -1;
10580 }
10581 }
10582
10583 /*
10584 * The src_cid is unused, so it can be unset.
10585 */
10586 if (!try_cmpxchg(&src_pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
10587 return -1;
10588 WRITE_ONCE(src_pcpu_cid->recent_cid, MM_CID_UNSET);
10589 return src_cid;
10590}
10591
10592/*
10593 * Migration to dst cpu. Called with dst_rq lock held.
10594 * Interrupts are disabled, which keeps the window of cid ownership without the
10595 * source rq lock held small.
10596 */
10597void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t)
10598{
10599 struct mm_cid *src_pcpu_cid, *dst_pcpu_cid;
10600 struct mm_struct *mm = t->mm;
10601 int src_cid, src_cpu;
10602 bool dst_cid_is_set;
10603 struct rq *src_rq;
10604
10605 lockdep_assert_rq_held(dst_rq);
10606
10607 if (!mm)
10608 return;
10609 src_cpu = t->migrate_from_cpu;
10610 if (src_cpu == -1) {
10611 t->last_mm_cid = -1;
10612 return;
10613 }
10614 /*
10615 * Move the src cid if the dst cid is unset. This keeps id
10616 * allocation closest to 0 in cases where few threads migrate around
10617 * many CPUs.
10618 *
10619 * If destination cid or recent cid is already set, we may have
10620 * to just clear the src cid to ensure compactness in frequent
10621 * migrations scenarios.
10622 *
10623 * It is not useful to clear the src cid when the number of threads is
10624 * greater or equal to the number of allowed CPUs, because user-space
10625 * can expect that the number of allowed cids can reach the number of
10626 * allowed CPUs.
10627 */
10628 dst_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(dst_rq));
10629 dst_cid_is_set = !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->cid)) ||
10630 !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->recent_cid));
10631 if (dst_cid_is_set && atomic_read(&mm->mm_users) >= READ_ONCE(mm->nr_cpus_allowed))
10632 return;
10633 src_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, src_cpu);
10634 src_rq = cpu_rq(src_cpu);
10635 src_cid = __sched_mm_cid_migrate_from_fetch_cid(src_rq, t, src_pcpu_cid);
10636 if (src_cid == -1)
10637 return;
10638 src_cid = __sched_mm_cid_migrate_from_try_steal_cid(src_rq, t, src_pcpu_cid,
10639 src_cid);
10640 if (src_cid == -1)
10641 return;
10642 if (dst_cid_is_set) {
10643 __mm_cid_put(mm, src_cid);
10644 return;
10645 }
10646 /* Move src_cid to dst cpu. */
10647 mm_cid_snapshot_time(dst_rq, mm);
10648 WRITE_ONCE(dst_pcpu_cid->cid, src_cid);
10649 WRITE_ONCE(dst_pcpu_cid->recent_cid, src_cid);
10650}
10651
10652static void sched_mm_cid_remote_clear(struct mm_struct *mm, struct mm_cid *pcpu_cid,
10653 int cpu)
10654{
10655 struct rq *rq = cpu_rq(cpu);
10656 struct task_struct *t;
10657 int cid, lazy_cid;
10658
10659 cid = READ_ONCE(pcpu_cid->cid);
10660 if (!mm_cid_is_valid(cid))
10661 return;
10662
10663 /*
10664 * Clear the cpu cid if it is set to keep cid allocation compact. If
10665 * there happens to be other tasks left on the source cpu using this
10666 * mm, the next task using this mm will reallocate its cid on context
10667 * switch.
10668 */
10669 lazy_cid = mm_cid_set_lazy_put(cid);
10670 if (!try_cmpxchg(&pcpu_cid->cid, &cid, lazy_cid))
10671 return;
10672
10673 /*
10674 * The implicit barrier after cmpxchg per-mm/cpu cid before loading
10675 * rq->curr->mm matches the scheduler barrier in context_switch()
10676 * between store to rq->curr and load of prev and next task's
10677 * per-mm/cpu cid.
10678 *
10679 * The implicit barrier after cmpxchg per-mm/cpu cid before loading
10680 * rq->curr->mm_cid_active matches the barrier in
10681 * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and
10682 * sched_mm_cid_after_execve() between store to t->mm_cid_active and
10683 * load of per-mm/cpu cid.
10684 */
10685
10686 /*
10687 * If we observe an active task using the mm on this rq after setting
10688 * the lazy-put flag, that task will be responsible for transitioning
10689 * from lazy-put flag set to MM_CID_UNSET.
10690 */
10691 scoped_guard (rcu) {
10692 t = rcu_dereference(rq->curr);
10693 if (READ_ONCE(t->mm_cid_active) && t->mm == mm)
10694 return;
10695 }
10696
10697 /*
10698 * The cid is unused, so it can be unset.
10699 * Disable interrupts to keep the window of cid ownership without rq
10700 * lock small.
10701 */
10702 scoped_guard (irqsave) {
10703 if (try_cmpxchg(&pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
10704 __mm_cid_put(mm, cid);
10705 }
10706}
10707
10708static void sched_mm_cid_remote_clear_old(struct mm_struct *mm, int cpu)
10709{
10710 struct rq *rq = cpu_rq(cpu);
10711 struct mm_cid *pcpu_cid;
10712 struct task_struct *curr;
10713 u64 rq_clock;
10714
10715 /*
10716 * rq->clock load is racy on 32-bit but one spurious clear once in a
10717 * while is irrelevant.
10718 */
10719 rq_clock = READ_ONCE(rq->clock);
10720 pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu);
10721
10722 /*
10723 * In order to take care of infrequently scheduled tasks, bump the time
10724 * snapshot associated with this cid if an active task using the mm is
10725 * observed on this rq.
10726 */
10727 scoped_guard (rcu) {
10728 curr = rcu_dereference(rq->curr);
10729 if (READ_ONCE(curr->mm_cid_active) && curr->mm == mm) {
10730 WRITE_ONCE(pcpu_cid->time, rq_clock);
10731 return;
10732 }
10733 }
10734
10735 if (rq_clock < pcpu_cid->time + SCHED_MM_CID_PERIOD_NS)
10736 return;
10737 sched_mm_cid_remote_clear(mm, pcpu_cid, cpu);
10738}
10739
10740static void sched_mm_cid_remote_clear_weight(struct mm_struct *mm, int cpu,
10741 int weight)
10742{
10743 struct mm_cid *pcpu_cid;
10744 int cid;
10745
10746 pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu);
10747 cid = READ_ONCE(pcpu_cid->cid);
10748 if (!mm_cid_is_valid(cid) || cid < weight)
10749 return;
10750 sched_mm_cid_remote_clear(mm, pcpu_cid, cpu);
10751}
10752
10753static void task_mm_cid_work(struct callback_head *work)
10754{
10755 unsigned long now = jiffies, old_scan, next_scan;
10756 struct task_struct *t = current;
10757 struct cpumask *cidmask;
10758 struct mm_struct *mm;
10759 int weight, cpu;
10760
10761 WARN_ON_ONCE(t != container_of(work, struct task_struct, cid_work));
10762
10763 work->next = work; /* Prevent double-add */
10764 if (t->flags & PF_EXITING)
10765 return;
10766 mm = t->mm;
10767 if (!mm)
10768 return;
10769 old_scan = READ_ONCE(mm->mm_cid_next_scan);
10770 next_scan = now + msecs_to_jiffies(MM_CID_SCAN_DELAY);
10771 if (!old_scan) {
10772 unsigned long res;
10773
10774 res = cmpxchg(&mm->mm_cid_next_scan, old_scan, next_scan);
10775 if (res != old_scan)
10776 old_scan = res;
10777 else
10778 old_scan = next_scan;
10779 }
10780 if (time_before(now, old_scan))
10781 return;
10782 if (!try_cmpxchg(&mm->mm_cid_next_scan, &old_scan, next_scan))
10783 return;
10784 cidmask = mm_cidmask(mm);
10785 /* Clear cids that were not recently used. */
10786 for_each_possible_cpu(cpu)
10787 sched_mm_cid_remote_clear_old(mm, cpu);
10788 weight = cpumask_weight(cidmask);
10789 /*
10790 * Clear cids that are greater or equal to the cidmask weight to
10791 * recompact it.
10792 */
10793 for_each_possible_cpu(cpu)
10794 sched_mm_cid_remote_clear_weight(mm, cpu, weight);
10795}
10796
10797void init_sched_mm_cid(struct task_struct *t)
10798{
10799 struct mm_struct *mm = t->mm;
10800 int mm_users = 0;
10801
10802 if (mm) {
10803 mm_users = atomic_read(&mm->mm_users);
10804 if (mm_users == 1)
10805 mm->mm_cid_next_scan = jiffies + msecs_to_jiffies(MM_CID_SCAN_DELAY);
10806 }
10807 t->cid_work.next = &t->cid_work; /* Protect against double add */
10808 init_task_work(&t->cid_work, task_mm_cid_work);
10809}
10810
10811void task_tick_mm_cid(struct rq *rq, struct task_struct *curr)
10812{
10813 struct callback_head *work = &curr->cid_work;
10814 unsigned long now = jiffies;
10815
10816 if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) ||
10817 work->next != work)
10818 return;
10819 if (time_before(now, READ_ONCE(curr->mm->mm_cid_next_scan)))
10820 return;
10821
10822 /* No page allocation under rq lock */
10823 task_work_add(curr, work, TWA_RESUME);
10824}
10825
10826void sched_mm_cid_exit_signals(struct task_struct *t)10375void sched_mm_cid_exit_signals(struct task_struct *t)
10827{10376{
10828 struct mm_struct *mm = t->mm;10377 struct mm_struct *mm = t->mm;
10829 struct rq *rq;
10830 10378
10831 if (!mm)10379 if (!mm || !t->mm_cid_active)
10832 return;10380 return;
10833 10381
10834 preempt_disable();10382 guard(preempt)();
10835 rq = this_rq();10383 t->mm_cid_active = 0;
10836 guard(rq_lock_irqsave)(rq);10384 if (t->mm_cid != MM_CID_UNSET) {
10837 preempt_enable_no_resched(); /* holding spinlock */10385 cpumask_clear_cpu(t->mm_cid, mm_cidmask(mm));
10838 WRITE_ONCE(t->mm_cid_active, 0);10386 t->mm_cid = MM_CID_UNSET;
10839 /*10387 }
10840 * Store t->mm_cid_active before loading per-mm/cpu cid.
10841 * Matches barrier in sched_mm_cid_remote_clear_old().
10842 */
10843 smp_mb();
10844 mm_cid_put(mm);
10845 t->last_mm_cid = t->mm_cid = -1;
10846}10388}
10847 10389
10390/* Deactivate MM CID allocation across execve() */
10848void sched_mm_cid_before_execve(struct task_struct *t)10391void sched_mm_cid_before_execve(struct task_struct *t)
10849{10392{
10850 struct mm_struct *mm = t->mm;10393 sched_mm_cid_exit_signals(t);
10851 struct rq *rq;
10852
10853 if (!mm)
10854 return;
10855
10856 preempt_disable();
10857 rq = this_rq();
10858 guard(rq_lock_irqsave)(rq);
10859 preempt_enable_no_resched(); /* holding spinlock */
10860 WRITE_ONCE(t->mm_cid_active, 0);
10861 /*
10862 * Store t->mm_cid_active before loading per-mm/cpu cid.
10863 * Matches barrier in sched_mm_cid_remote_clear_old().
10864 */
10865 smp_mb();
10866 mm_cid_put(mm);
10867 t->last_mm_cid = t->mm_cid = -1;
10868}10394}
10869 10395
10396/* Reactivate MM CID after successful execve() */
10870void sched_mm_cid_after_execve(struct task_struct *t)10397void sched_mm_cid_after_execve(struct task_struct *t)
10871{10398{
10872 struct mm_struct *mm = t->mm;10399 struct mm_struct *mm = t->mm;
10873 struct rq *rq;
10874 10400
10875 if (!mm)10401 if (!mm)
10876 return;10402 return;
10877 10403
10878 preempt_disable();10404 guard(preempt)();
10879 rq = this_rq();10405 t->mm_cid_active = 1;
10880 scoped_guard (rq_lock_irqsave, rq) {10406 mm_cid_select(t);
10881 preempt_enable_no_resched(); /* holding spinlock */
10882 WRITE_ONCE(t->mm_cid_active, 1);
10883 /*
10884 * Store t->mm_cid_active before loading per-mm/cpu cid.
10885 * Matches barrier in sched_mm_cid_remote_clear_old().
10886 */
10887 smp_mb();
10888 t->last_mm_cid = t->mm_cid = mm_cid_get(rq, t, mm);
10889 }
10890}10407}
10891 10408
10892void sched_mm_cid_fork(struct task_struct *t)10409void sched_mm_cid_fork(struct task_struct *t)
10893{10410{
10894 WARN_ON_ONCE(!t->mm || t->mm_cid != -1);10411 WARN_ON_ONCE(!t->mm || t->mm_cid != MM_CID_UNSET);
10895 t->mm_cid_active = 1;10412 t->mm_cid_active = 1;
10896}10413}
10897#endif /* CONFIG_SCHED_MM_CID */10414#endif /* CONFIG_SCHED_MM_CID */
kernel/sched/sched.h+43−246
@@ -3540,286 +3540,83 @@ extern void sched_dynamic_update(int mode); · 3539 unmodified lines
3540extern const char *preempt_modes[];3540extern const char *preempt_modes[];
3541 3541
3542#ifdef CONFIG_SCHED_MM_CID3542#ifdef CONFIG_SCHED_MM_CID
3543 3543static inline void init_sched_mm_cid(struct task_struct *t)
3544#define SCHED_MM_CID_PERIOD_NS (100ULL * 1000000) /* 100ms */
3545#define MM_CID_SCAN_DELAY 100 /* 100ms */
3546
3547extern raw_spinlock_t cid_lock;
3548extern int use_cid_lock;
3549
3550extern void sched_mm_cid_migrate_from(struct task_struct *t);
3551extern void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t);
3552extern void task_tick_mm_cid(struct rq *rq, struct task_struct *curr);
3553extern void init_sched_mm_cid(struct task_struct *t);
3554
3555static inline void __mm_cid_put(struct mm_struct *mm, int cid)
3556{
3557 if (cid < 0)
3558 return;
3559 cpumask_clear_cpu(cid, mm_cidmask(mm));
3560}
3561
3562/*
3563 * The per-mm/cpu cid can have the MM_CID_LAZY_PUT flag set or transition to
3564 * the MM_CID_UNSET state without holding the rq lock, but the rq lock needs to
3565 * be held to transition to other states.
3566 *
3567 * State transitions synchronized with cmpxchg or try_cmpxchg need to be
3568 * consistent across CPUs, which prevents use of this_cpu_cmpxchg.
3569 */
3570static inline void mm_cid_put_lazy(struct task_struct *t)
3571{3544{
3572 struct mm_struct *mm = t->mm;3545 struct mm_struct *mm = t->mm;
3573 struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;3546 unsigned int max_cid;
3574 int cid;
3575 3547
3576 lockdep_assert_irqs_disabled();3548 if (!mm)
3577 cid = __this_cpu_read(pcpu_cid->cid);
3578 if (!mm_cid_is_lazy_put(cid) ||
3579 !try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
3580 return;3549 return;
3581 __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
3582}
3583 3550
3584static inline int mm_cid_pcpu_unset(struct mm_struct *mm)3551 /* Preset last_mm_cid */
3585{3552 max_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
3586 struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;3553 t->last_mm_cid = max_cid - 1;
3587 int cid, res;
3588
3589 lockdep_assert_irqs_disabled();
3590 cid = __this_cpu_read(pcpu_cid->cid);
3591 for (;;) {
3592 if (mm_cid_is_unset(cid))
3593 return MM_CID_UNSET;
3594 /*
3595 * Attempt transition from valid or lazy-put to unset.
3596 */
3597 res = cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, cid, MM_CID_UNSET);
3598 if (res == cid)
3599 break;
3600 cid = res;
3601 }
3602 return cid;
3603}3554}
3604 3555
3605static inline void mm_cid_put(struct mm_struct *mm)3556static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
3606{3557{
3607 int cid;3558 struct mm_struct *mm = t->mm;
3608 3559
3609 lockdep_assert_irqs_disabled();3560 if (cid >= max_cids)
3610 cid = mm_cid_pcpu_unset(mm);3561 return false;
3611 if (cid == MM_CID_UNSET)3562 if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
3612 return;3563 return false;
3613 __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));3564 t->mm_cid = t->last_mm_cid = cid;
3565 __this_cpu_write(mm->pcpu_cid->cid, cid);
3566 return true;
3614}3567}
3615 3568
3616static inline int __mm_cid_try_get(struct task_struct *t, struct mm_struct *mm)3569static inline bool mm_cid_get(struct task_struct *t)
3617{3570{
3618 struct cpumask *cidmask = mm_cidmask(mm);3571 struct mm_struct *mm = t->mm;
3619 struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;3572 unsigned int max_cids;
3620 int cid, max_nr_cid, allowed_max_nr_cid;
3621 3573
3622 /*3574 max_cids = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
3623 * After shrinking the number of threads or reducing the number
3624 * of allowed cpus, reduce the value of max_nr_cid so expansion
3625 * of cid allocation will preserve cache locality if the number
3626 * of threads or allowed cpus increase again.
3627 */
3628 max_nr_cid = atomic_read(&mm->max_nr_cid);
3629 while ((allowed_max_nr_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed),
3630 atomic_read(&mm->mm_users))),
3631 max_nr_cid > allowed_max_nr_cid) {
3632 /* atomic_try_cmpxchg loads previous mm->max_nr_cid into max_nr_cid. */
3633 if (atomic_try_cmpxchg(&mm->max_nr_cid, &max_nr_cid, allowed_max_nr_cid)) {
3634 max_nr_cid = allowed_max_nr_cid;
3635 break;
3636 }
3637 }
3638 /* Try to re-use recent cid. This improves cache locality. */
3639 cid = __this_cpu_read(pcpu_cid->recent_cid);
3640 if (!mm_cid_is_unset(cid) && cid < max_nr_cid &&
3641 !cpumask_test_and_set_cpu(cid, cidmask))
3642 return cid;
3643 /*
3644 * Expand cid allocation if the maximum number of concurrency
3645 * IDs allocated (max_nr_cid) is below the number cpus allowed
3646 * and number of threads. Expanding cid allocation as much as
3647 * possible improves cache locality.
3648 */
3649 cid = max_nr_cid;
3650 while (cid < READ_ONCE(mm->nr_cpus_allowed) && cid < atomic_read(&mm->mm_users)) {
3651 /* atomic_try_cmpxchg loads previous mm->max_nr_cid into cid. */
3652 if (!atomic_try_cmpxchg(&mm->max_nr_cid, &cid, cid + 1))
3653 continue;
3654 if (!cpumask_test_and_set_cpu(cid, cidmask))
3655 return cid;
3656 }
3657 /*
3658 * Find the first available concurrency id.
3659 * Retry finding first zero bit if the mask is temporarily
3660 * filled. This only happens during concurrent remote-clear
3661 * which owns a cid without holding a rq lock.
3662 */
3663 for (;;) {
3664 cid = cpumask_first_zero(cidmask);
3665 if (cid < READ_ONCE(mm->nr_cpus_allowed))
3666 break;
3667 cpu_relax();
3668 }
3669 if (cpumask_test_and_set_cpu(cid, cidmask))
3670 return -1;
3671 3575
3672 return cid;3576 /* Try to reuse the last CID of this task */
3673}3577 if (__mm_cid_get(t, t->last_mm_cid, max_cids))
3578 return true;
3674 3579
3675/*3580 /* Try to reuse the last CID of this mm on this CPU */
3676 * Save a snapshot of the current runqueue time of this cpu3581 if (__mm_cid_get(t, __this_cpu_read(mm->pcpu_cid->cid), max_cids))
3677 * with the per-cpu cid value, allowing to estimate how recently it was used.3582 return true;
3678 */
3679static inline void mm_cid_snapshot_time(struct rq *rq, struct mm_struct *mm)
3680{
3681 struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(rq));
3682 3583
3683 lockdep_assert_rq_held(rq);3584 /* Try the first zero bit in the cidmask. */
3684 WRITE_ONCE(pcpu_cid->time, rq->clock);3585 return __mm_cid_get(t, cpumask_first_zero(mm_cidmask(mm)), max_cids);
3685}3586}
3686 3587
3687static inline int __mm_cid_get(struct rq *rq, struct task_struct *t,3588static inline void mm_cid_select(struct task_struct *t)
3688 struct mm_struct *mm)
3689{3589{
3690 int cid;
3691
3692 /*3590 /*
3693 * All allocations (even those using the cid_lock) are lock-free. If3591 * mm_cid_get() can fail when the maximum CID, which is determined
3694 * use_cid_lock is set, hold the cid_lock to perform cid allocation to3592 * by min(mm->nr_cpus_allowed, mm->mm_users) changes concurrently.
3695 * guarantee forward progress.3593 * That's a transient failure as there cannot be more tasks
3594 * concurrently on a CPU (or about to be scheduled in) than that.
3696 */3595 */
3697 if (!READ_ONCE(use_cid_lock)) {3596 for (;;) {
3698 cid = __mm_cid_try_get(t, mm);3597 if (mm_cid_get(t))
3699 if (cid >= 0)3598 break;
3700 goto end;
3701 raw_spin_lock(&cid_lock);
3702 } else {
3703 raw_spin_lock(&cid_lock);
3704 cid = __mm_cid_try_get(t, mm);
3705 if (cid >= 0)
3706 goto unlock;
3707 }
3708
3709 /*
3710 * cid concurrently allocated. Retry while forcing following
3711 * allocations to use the cid_lock to ensure forward progress.
3712 */
3713 WRITE_ONCE(use_cid_lock, 1);
3714 /*
3715 * Set use_cid_lock before allocation. Only care about program order
3716 * because this is only required for forward progress.
3717 */
3718 barrier();
3719 /*
3720 * Retry until it succeeds. It is guaranteed to eventually succeed once
3721 * all newcoming allocations observe the use_cid_lock flag set.
3722 */
3723 do {
3724 cid = __mm_cid_try_get(t, mm);
3725 cpu_relax();
3726 } while (cid < 0);
3727 /*
3728 * Allocate before clearing use_cid_lock. Only care about
3729 * program order because this is for forward progress.
3730 */
3731 barrier();
3732 WRITE_ONCE(use_cid_lock, 0);
3733unlock:
3734 raw_spin_unlock(&cid_lock);
3735end:
3736 mm_cid_snapshot_time(rq, mm);
3737
3738 return cid;
3739}
3740
3741static inline int mm_cid_get(struct rq *rq, struct task_struct *t,
3742 struct mm_struct *mm)
3743{
3744 struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
3745 int cid;
3746
3747 lockdep_assert_rq_held(rq);
3748 cid = __this_cpu_read(pcpu_cid->cid);
3749 if (mm_cid_is_valid(cid)) {
3750 mm_cid_snapshot_time(rq, mm);
3751 return cid;
3752 }
3753 if (mm_cid_is_lazy_put(cid)) {
3754 if (try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
3755 __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
3756 }3599 }
3757 cid = __mm_cid_get(rq, t, mm);
3758 __this_cpu_write(pcpu_cid->cid, cid);
3759 __this_cpu_write(pcpu_cid->recent_cid, cid);
3760
3761 return cid;
3762}3600}
3763 3601
3764static inline void switch_mm_cid(struct rq *rq,3602static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
3765 struct task_struct *prev,
3766 struct task_struct *next)
3767{3603{
3768 /*
3769 * Provide a memory barrier between rq->curr store and load of
3770 * {prev,next}->mm->pcpu_cid[cpu] on rq->curr->mm transition.
3771 *
3772 * Should be adapted if context_switch() is modified.
3773 */
3774 if (!next->mm) { // to kernel
3775 /*
3776 * user -> kernel transition does not guarantee a barrier, but
3777 * we can use the fact that it performs an atomic operation in
3778 * mmgrab().
3779 */
3780 if (prev->mm) // from user
3781 smp_mb__after_mmgrab();
3782 /*
3783 * kernel -> kernel transition does not change rq->curr->mm
3784 * state. It stays NULL.
3785 */
3786 } else { // to user
3787 /*
3788 * kernel -> user transition does not provide a barrier
3789 * between rq->curr store and load of {prev,next}->mm->pcpu_cid[cpu].
3790 * Provide it here.
3791 */
3792 if (!prev->mm) { // from kernel
3793 smp_mb();
3794 } else { // from user
3795 /*
3796 * user->user transition relies on an implicit
3797 * memory barrier in switch_mm() when
3798 * current->mm changes. If the architecture
3799 * switch_mm() does not have an implicit memory
3800 * barrier, it is emitted here. If current->mm
3801 * is unchanged, no barrier is needed.
3802 */
3803 smp_mb__after_switch_mm();
3804 }
3805 }
3806 if (prev->mm_cid_active) {3604 if (prev->mm_cid_active) {
3807 mm_cid_snapshot_time(rq, prev->mm);3605 if (prev->mm_cid != MM_CID_UNSET)
3808 mm_cid_put_lazy(prev);3606 cpumask_clear_cpu(prev->mm_cid, mm_cidmask(prev->mm));
3809 prev->mm_cid = -1;3607 prev->mm_cid = MM_CID_UNSET;
3810 }3608 }
3609
3811 if (next->mm_cid_active) {3610 if (next->mm_cid_active) {
3812 next->last_mm_cid = next->mm_cid = mm_cid_get(rq, next, next->mm);3611 mm_cid_select(next);
3813 rseq_sched_set_task_mm_cid(next, next->mm_cid);3612 rseq_sched_set_task_mm_cid(next, next->mm_cid);
3814 }3613 }
3815}3614}
3816 3615
3817#else /* !CONFIG_SCHED_MM_CID: */3616#else /* !CONFIG_SCHED_MM_CID: */
3818static inline void switch_mm_cid(struct rq *rq, struct task_struct *prev, struct task_struct *next) { }
3819static inline void sched_mm_cid_migrate_from(struct task_struct *t) { }
3820static inline void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) { }
3821static inline void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) { }
3822static inline void init_sched_mm_cid(struct task_struct *t) { }3617static inline void init_sched_mm_cid(struct task_struct *t) { }
3618static inline void mm_cid_select(struct task_struct *t) { }
3619static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
3823#endif /* !CONFIG_SCHED_MM_CID */3620#endif /* !CONFIG_SCHED_MM_CID */
3824 3621
3825extern u64 avg_vruntime(struct cfs_rq *cfs_rq);3622extern u64 avg_vruntime(struct cfs_rq *cfs_rq);