History

sched/mmcid: Revert the complex CID management

The CID management is a complex beast, which affects both scheduling and task migration. The compaction mechanism forces random tasks of a process into task work on exit to user space causing latency spikes. Revert back to the initial simple bitmap allocating mechanics, which are known to have scalability issues as that allows to gradually build up a replacement functionality in a reviewable way.

Thomas Gleixner committed 11 months ago

Commit 77d7dc81 parent 80adacc4 files · +64 −800Browse files

Unified · Split

4 files
  1. include/linux/mm_types.h +2 −51
  2. kernel/fork.c +2 −3
  3. kernel/sched/core.c +17 −500
  4. kernel/sched/sched.h +43 −246
include/linux/mm_types.h+2−51
@@ -922,13 +922,9 @@ struct vm_area_struct { · 921 unmodified lines
922922 #define vma_policy(vma) NULL
923923 #endif
924924
925−#ifdef CONFIG_SCHED_MM_CID
926925 struct mm_cid {
927− u64 time;
928− int cid;
929− int recent_cid;
926+ unsigned int cid;
930927 };
931−#endif
932928
933929 /*
934930 * Opaque type representing current mm_struct flag state. Must be accessed via
@@ -1000,12 +996,6 @@ struct mm_struct { · 65 unmodified lines
1000996 * runqueue locks.
1001997 */
1002998 struct mm_cid __percpu *pcpu_cid;
1003− /*
1004− * @mm_cid_next_scan: Next mm_cid scan (in jiffies).
1005− *
1006− * When the next mm_cid scan is due (in jiffies).
1007− */
1008− unsigned long mm_cid_next_scan;
1009999 /**
10101000 * @nr_cpus_allowed: Number of CPUs allowed for mm.
10111001 *
@@ -1013,14 +1003,6 @@ struct mm_struct { · 1 unmodified line
10131003 * threads allowed CPUs.
10141004 */
10151005 unsigned int nr_cpus_allowed;
1016− /**
1017− * @max_nr_cid: Maximum number of allowed concurrency
1018− * IDs allocated.
1019− *
1020− * Track the highest number of allowed concurrency IDs
1021− * allocated for the mm.
1022− */
1023− atomic_t max_nr_cid;
10241006 /**
10251007 * @cpus_allowed_lock: Lock protecting mm cpus_allowed.
10261008 *
@@ -1371,35 +1353,7 @@ static inline void vma_iter_init(struct vma_iterator *vmi, · 344 unmodified lines
13711353
13721354 #ifdef CONFIG_SCHED_MM_CID
13731355
1374−enum mm_cid_state {
1375− MM_CID_UNSET = -1U, /* Unset state has lazy_put flag set. */
1376− MM_CID_LAZY_PUT = (1U << 31),
1377−};
1378−
1379−static inline bool mm_cid_is_unset(int cid)
1380−{
1381− return cid == MM_CID_UNSET;
1382−}
1383−
1384−static inline bool mm_cid_is_lazy_put(int cid)
1385−{
1386− return !mm_cid_is_unset(cid) && (cid & MM_CID_LAZY_PUT);
1387−}
1388−
1389−static inline bool mm_cid_is_valid(int cid)
1390−{
1391− return !(cid & MM_CID_LAZY_PUT);
1392−}
1393−
1394−static inline int mm_cid_set_lazy_put(int cid)
1395−{
1396− return cid | MM_CID_LAZY_PUT;
1397−}
1398−
1399−static inline int mm_cid_clear_lazy_put(int cid)
1400−{
1401− return cid & ~MM_CID_LAZY_PUT;
1402−}
1356+#define MM_CID_UNSET (~0U)
14031357
14041358 /*
14051359 * mm_cpus_allowed: Union of all mm's threads allowed CPUs.
@@ -1432,11 +1386,8 @@ static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p) · 26 unmodified lines
14321386 struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);
14331387
14341388 pcpu_cid->cid = MM_CID_UNSET;
1435− pcpu_cid->recent_cid = MM_CID_UNSET;
1436− pcpu_cid->time = 0;
14371389 }
14381390 mm->nr_cpus_allowed = p->nr_cpus_allowed;
1439− atomic_set(&mm->max_nr_cid, 0);
14401391 raw_spin_lock_init(&mm->cpus_allowed_lock);
14411392 cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
14421393 cpumask_clear(mm_cidmask(mm));
kernel/fork.c+2−3
@@ -955,10 +955,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) · 954 unmodified lines
955955 #endif
956956
957957 #ifdef CONFIG_SCHED_MM_CID
958− tsk->mm_cid = -1;
959− tsk->last_mm_cid = -1;
958+ tsk->mm_cid = MM_CID_UNSET;
959+ tsk->last_mm_cid = MM_CID_UNSET;
960960 tsk->mm_cid_active = 0;
961− tsk->migrate_from_cpu = -1;
962961 #endif
963962 return tsk;
964963
kernel/sched/core.c+17−500
@@ -2128,8 +2128,6 @@ void activate_task(struct rq *rq, struct task_struct *p, int flags) · 2127 unmodified lines
21282128 {
21292129 if (task_on_rq_migrating(p))
21302130 flags |= ENQUEUE_MIGRATED;
2131− if (flags & ENQUEUE_MIGRATED)
2132− sched_mm_cid_migrate_to(rq, p);
21332131
21342132 enqueue_task(rq, p, flags);
21352133
@@ -3329,7 +3327,6 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) · 1193 unmodified lines
33293327 if (p->sched_class->migrate_task_rq)
33303328 p->sched_class->migrate_task_rq(p, new_cpu);
33313329 p->se.nr_migrations++;
3332− sched_mm_cid_migrate_from(p);
33333330 perf_event_task_migrate(p);
33343331 }
33353332
@@ -5280,9 +5277,6 @@ context_switch(struct rq *rq, struct task_struct *prev, · 1944 unmodified lines
52805277 *
52815278 * kernel -> user switch + mmdrop_lazy_tlb() active
52825279 * user -> user switch
5283− *
5284− * switch_mm_cid() needs to be updated if the barriers provided
5285− * by context_switch() are modified.
52865280 */
52875281 if (!next->mm) { // to kernel
52885282 enter_lazy_tlb(prev->active_mm, next);
@@ -5312,8 +5306,7 @@ context_switch(struct rq *rq, struct task_struct *prev, · 23 unmodified lines
53125306 }
53135307 }
53145308
5315− /* switch_mm_cid() requires the memory barriers above. */
5316− switch_mm_cid(rq, prev, next);
5309+ switch_mm_cid(prev, next);
53175310
53185311 /*
53195312 * Tell rseq that the task was scheduled in. Must be after
@@ -5604,7 +5597,6 @@ void sched_tick(void) · 284 unmodified lines
56045597 resched_latency = cpu_resched_latency(rq);
56055598 calc_global_load_tick(rq);
56065599 sched_core_tick(rq);
5607− task_tick_mm_cid(rq, donor);
56085600 scx_tick(rq);
56095601
56105602 rq_unlock(rq, &rf);
@@ -10376,522 +10368,47 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count) · 4765 unmodified lines
1037610368 }
1037710369
1037810370 #ifdef CONFIG_SCHED_MM_CID
10379−
1038010371 /*
10381− * @cid_lock: Guarantee forward-progress of cid allocation.
10382− *
10383− * Concurrency ID allocation within a bitmap is mostly lock-free. The cid_lock
10384− * is only used when contention is detected by the lock-free allocation so
10385− * forward progress can be guaranteed.
10386− */
10387−DEFINE_RAW_SPINLOCK(cid_lock);
10388−
10389−/*
10390− * @use_cid_lock: Select cid allocation behavior: lock-free vs spinlock.
10391− *
10392− * When @use_cid_lock is 0, the cid allocation is lock-free. When contention is
10393− * detected, it is set to 1 to ensure that all newly coming allocations are
10394− * serialized by @cid_lock until the allocation which detected contention
10395− * completes and sets @use_cid_lock back to 0. This guarantees forward progress
10396− * of a cid allocation.
10397− */
10398−int use_cid_lock;
10399−
10400−/*
10401− * mm_cid remote-clear implements a lock-free algorithm to clear per-mm/cpu cid
10402− * concurrently with respect to the execution of the source runqueue context
10403− * switch.
10404− *
10405− * There is one basic properties we want to guarantee here:
10406− *
10407− * (1) Remote-clear should _never_ mark a per-cpu cid UNSET when it is actively
10408− * used by a task. That would lead to concurrent allocation of the cid and
10409− * userspace corruption.
10410− *
10411− * Provide this guarantee by introducing a Dekker memory ordering to guarantee
10412− * that a pair of loads observe at least one of a pair of stores, which can be
10413− * shown as:
10414− *
10415− * X = Y = 0
10416− *
10417− * w[X]=1 w[Y]=1
10418− * MB MB
10419− * r[Y]=y r[X]=x
10420− *
10421− * Which guarantees that x==0 && y==0 is impossible. But rather than using
10422− * values 0 and 1, this algorithm cares about specific state transitions of the
10423− * runqueue current task (as updated by the scheduler context switch), and the
10424− * per-mm/cpu cid value.
10425− *
10426− * Let's introduce task (Y) which has task->mm == mm and task (N) which has
10427− * task->mm != mm for the rest of the discussion. There are two scheduler state
10428− * transitions on context switch we care about:
10429− *
10430− * (TSA) Store to rq->curr with transition from (N) to (Y)
10431− *
10432− * (TSB) Store to rq->curr with transition from (Y) to (N)
10433− *
10434− * On the remote-clear side, there is one transition we care about:
10435− *
10436− * (TMA) cmpxchg to *pcpu_cid to set the LAZY flag
10437− *
10438− * There is also a transition to UNSET state which can be performed from all
10439− * sides (scheduler, remote-clear). It is always performed with a cmpxchg which
10440− * guarantees that only a single thread will succeed:
10441− *
10442− * (TMB) cmpxchg to *pcpu_cid to mark UNSET
10443− *
10444− * Just to be clear, what we do _not_ want to happen is a transition to UNSET
10445− * when a thread is actively using the cid (property (1)).
10446− *
10447− * Let's looks at the relevant combinations of TSA/TSB, and TMA transitions.
10448− *
10449− * Scenario A) (TSA)+(TMA) (from next task perspective)
10450− *
10451− * CPU0 CPU1
10452− *
10453− * Context switch CS-1 Remote-clear
10454− * - store to rq->curr: (N)->(Y) (TSA) - cmpxchg to *pcpu_id to LAZY (TMA)
10455− * (implied barrier after cmpxchg)
10456− * - switch_mm_cid()
10457− * - memory barrier (see switch_mm_cid()
10458− * comment explaining how this barrier
10459− * is combined with other scheduler
10460− * barriers)
10461− * - mm_cid_get (next)
10462− * - READ_ONCE(*pcpu_cid) - rcu_dereference(src_rq->curr)
10463− *
10464− * This Dekker ensures that either task (Y) is observed by the
10465− * rcu_dereference() or the LAZY flag is observed by READ_ONCE(), or both are
10466− * observed.
10467− *
10468− * If task (Y) store is observed by rcu_dereference(), it means that there is
10469− * still an active task on the cpu. Remote-clear will therefore not transition
10470− * to UNSET, which fulfills property (1).
10471− *
10472− * If task (Y) is not observed, but the lazy flag is observed by READ_ONCE(),
10473− * it will move its state to UNSET, which clears the percpu cid perhaps
10474− * uselessly (which is not an issue for correctness). Because task (Y) is not
10475− * observed, CPU1 can move ahead to set the state to UNSET. Because moving
10476− * state to UNSET is done with a cmpxchg expecting that the old state has the
10477− * LAZY flag set, only one thread will successfully UNSET.
10478− *
10479− * If both states (LAZY flag and task (Y)) are observed, the thread on CPU0
10480− * will observe the LAZY flag and transition to UNSET (perhaps uselessly), and
10481− * CPU1 will observe task (Y) and do nothing more, which is fine.
10482− *
10483− * What we are effectively preventing with this Dekker is a scenario where
10484− * neither LAZY flag nor store (Y) are observed, which would fail property (1)
10485− * because this would UNSET a cid which is actively used.
10372+ * When a task exits, the MM CID held by the task is not longer required as
10373+ * the task cannot return to user space.
1048610374 */
10487−
10488−void sched_mm_cid_migrate_from(struct task_struct *t)
10489−{
10490− t->migrate_from_cpu = task_cpu(t);
10491−}
10492−
10493−static
10494−int __sched_mm_cid_migrate_from_fetch_cid(struct rq *src_rq,
10495− struct task_struct *t,
10496− struct mm_cid *src_pcpu_cid)
10497−{
10498− struct mm_struct *mm = t->mm;
10499− struct task_struct *src_task;
10500− int src_cid, last_mm_cid;
10501−
10502− if (!mm)
10503− return -1;
10504−
10505− last_mm_cid = t->last_mm_cid;
10506− /*
10507− * If the migrated task has no last cid, or if the current
10508− * task on src rq uses the cid, it means the source cid does not need
10509− * to be moved to the destination cpu.
10510− */
10511− if (last_mm_cid == -1)
10512− return -1;
10513− src_cid = READ_ONCE(src_pcpu_cid->cid);
10514− if (!mm_cid_is_valid(src_cid) || last_mm_cid != src_cid)
10515− return -1;
10516−
10517− /*
10518− * If we observe an active task using the mm on this rq, it means we
10519− * are not the last task to be migrated from this cpu for this mm, so
10520− * there is no need to move src_cid to the destination cpu.
10521− */
10522− guard(rcu)();
10523− src_task = rcu_dereference(src_rq->curr);
10524− if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) {
10525− t->last_mm_cid = -1;
10526− return -1;
10527− }
10528−
10529− return src_cid;
10530−}
10531−
10532−static
10533−int __sched_mm_cid_migrate_from_try_steal_cid(struct rq *src_rq,
10534− struct task_struct *t,
10535− struct mm_cid *src_pcpu_cid,
10536− int src_cid)
10537−{
10538− struct task_struct *src_task;
10539− struct mm_struct *mm = t->mm;
10540− int lazy_cid;
10541−
10542− if (src_cid == -1)
10543− return -1;
10544−
10545− /*
10546− * Attempt to clear the source cpu cid to move it to the destination
10547− * cpu.
10548− */
10549− lazy_cid = mm_cid_set_lazy_put(src_cid);
10550− if (!try_cmpxchg(&src_pcpu_cid->cid, &src_cid, lazy_cid))
10551− return -1;
10552−
10553− /*
10554− * The implicit barrier after cmpxchg per-mm/cpu cid before loading
10555− * rq->curr->mm matches the scheduler barrier in context_switch()
10556− * between store to rq->curr and load of prev and next task's
10557− * per-mm/cpu cid.
10558− *
10559− * The implicit barrier after cmpxchg per-mm/cpu cid before loading
10560− * rq->curr->mm_cid_active matches the barrier in
10561− * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and
10562− * sched_mm_cid_after_execve() between store to t->mm_cid_active and
10563− * load of per-mm/cpu cid.
10564− */
10565−
10566− /*
10567− * If we observe an active task using the mm on this rq after setting
10568− * the lazy-put flag, this task will be responsible for transitioning
10569− * from lazy-put flag set to MM_CID_UNSET.
10570− */
10571− scoped_guard (rcu) {
10572− src_task = rcu_dereference(src_rq->curr);
10573− if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) {
10574− /*
10575− * We observed an active task for this mm, there is therefore
10576− * no point in moving this cid to the destination cpu.
10577− */
10578− t->last_mm_cid = -1;
10579− return -1;
10580− }
10581− }
10582−
10583− /*
10584− * The src_cid is unused, so it can be unset.
10585− */
10586− if (!try_cmpxchg(&src_pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
10587− return -1;
10588− WRITE_ONCE(src_pcpu_cid->recent_cid, MM_CID_UNSET);
10589− return src_cid;
10590−}
10591−
10592−/*
10593− * Migration to dst cpu. Called with dst_rq lock held.
10594− * Interrupts are disabled, which keeps the window of cid ownership without the
10595− * source rq lock held small.
10596− */
10597−void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t)
10598−{
10599− struct mm_cid *src_pcpu_cid, *dst_pcpu_cid;
10600− struct mm_struct *mm = t->mm;
10601− int src_cid, src_cpu;
10602− bool dst_cid_is_set;
10603− struct rq *src_rq;
10604−
10605− lockdep_assert_rq_held(dst_rq);
10606−
10607− if (!mm)
10608− return;
10609− src_cpu = t->migrate_from_cpu;
10610− if (src_cpu == -1) {
10611− t->last_mm_cid = -1;
10612− return;
10613− }
10614− /*
10615− * Move the src cid if the dst cid is unset. This keeps id
10616− * allocation closest to 0 in cases where few threads migrate around
10617− * many CPUs.
10618− *
10619− * If destination cid or recent cid is already set, we may have
10620− * to just clear the src cid to ensure compactness in frequent
10621− * migrations scenarios.
10622− *
10623− * It is not useful to clear the src cid when the number of threads is
10624− * greater or equal to the number of allowed CPUs, because user-space
10625− * can expect that the number of allowed cids can reach the number of
10626− * allowed CPUs.
10627− */
10628− dst_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(dst_rq));
10629− dst_cid_is_set = !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->cid)) ||
10630− !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->recent_cid));
10631− if (dst_cid_is_set && atomic_read(&mm->mm_users) >= READ_ONCE(mm->nr_cpus_allowed))
10632− return;
10633− src_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, src_cpu);
10634− src_rq = cpu_rq(src_cpu);
10635− src_cid = __sched_mm_cid_migrate_from_fetch_cid(src_rq, t, src_pcpu_cid);
10636− if (src_cid == -1)
10637− return;
10638− src_cid = __sched_mm_cid_migrate_from_try_steal_cid(src_rq, t, src_pcpu_cid,
10639− src_cid);
10640− if (src_cid == -1)
10641− return;
10642− if (dst_cid_is_set) {
10643− __mm_cid_put(mm, src_cid);
10644− return;
10645− }
10646− /* Move src_cid to dst cpu. */
10647− mm_cid_snapshot_time(dst_rq, mm);
10648− WRITE_ONCE(dst_pcpu_cid->cid, src_cid);
10649− WRITE_ONCE(dst_pcpu_cid->recent_cid, src_cid);
10650−}
10651−
10652−static void sched_mm_cid_remote_clear(struct mm_struct *mm, struct mm_cid *pcpu_cid,
10653− int cpu)
10654−{
10655− struct rq *rq = cpu_rq(cpu);
10656− struct task_struct *t;
10657− int cid, lazy_cid;
10658−
10659− cid = READ_ONCE(pcpu_cid->cid);
10660− if (!mm_cid_is_valid(cid))
10661− return;
10662−
10663− /*
10664− * Clear the cpu cid if it is set to keep cid allocation compact. If
10665− * there happens to be other tasks left on the source cpu using this
10666− * mm, the next task using this mm will reallocate its cid on context
10667− * switch.
10668− */
10669− lazy_cid = mm_cid_set_lazy_put(cid);
10670− if (!try_cmpxchg(&pcpu_cid->cid, &cid, lazy_cid))
10671− return;
10672−
10673− /*
10674− * The implicit barrier after cmpxchg per-mm/cpu cid before loading
10675− * rq->curr->mm matches the scheduler barrier in context_switch()
10676− * between store to rq->curr and load of prev and next task's
10677− * per-mm/cpu cid.
10678− *
10679− * The implicit barrier after cmpxchg per-mm/cpu cid before loading
10680− * rq->curr->mm_cid_active matches the barrier in
10681− * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and
10682− * sched_mm_cid_after_execve() between store to t->mm_cid_active and
10683− * load of per-mm/cpu cid.
10684− */
10685−
10686− /*
10687− * If we observe an active task using the mm on this rq after setting
10688− * the lazy-put flag, that task will be responsible for transitioning
10689− * from lazy-put flag set to MM_CID_UNSET.
10690− */
10691− scoped_guard (rcu) {
10692− t = rcu_dereference(rq->curr);
10693− if (READ_ONCE(t->mm_cid_active) && t->mm == mm)
10694− return;
10695− }
10696−
10697− /*
10698− * The cid is unused, so it can be unset.
10699− * Disable interrupts to keep the window of cid ownership without rq
10700− * lock small.
10701− */
10702− scoped_guard (irqsave) {
10703− if (try_cmpxchg(&pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
10704− __mm_cid_put(mm, cid);
10705− }
10706−}
10707−
10708−static void sched_mm_cid_remote_clear_old(struct mm_struct *mm, int cpu)
10709−{
10710− struct rq *rq = cpu_rq(cpu);
10711− struct mm_cid *pcpu_cid;
10712− struct task_struct *curr;
10713− u64 rq_clock;
10714−
10715− /*
10716− * rq->clock load is racy on 32-bit but one spurious clear once in a
10717− * while is irrelevant.
10718− */
10719− rq_clock = READ_ONCE(rq->clock);
10720− pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu);
10721−
10722− /*
10723− * In order to take care of infrequently scheduled tasks, bump the time
10724− * snapshot associated with this cid if an active task using the mm is
10725− * observed on this rq.
10726− */
10727− scoped_guard (rcu) {
10728− curr = rcu_dereference(rq->curr);
10729− if (READ_ONCE(curr->mm_cid_active) && curr->mm == mm) {
10730− WRITE_ONCE(pcpu_cid->time, rq_clock);
10731− return;
10732− }
10733− }
10734−
10735− if (rq_clock < pcpu_cid->time + SCHED_MM_CID_PERIOD_NS)
10736− return;
10737− sched_mm_cid_remote_clear(mm, pcpu_cid, cpu);
10738−}
10739−
10740−static void sched_mm_cid_remote_clear_weight(struct mm_struct *mm, int cpu,
10741− int weight)
10742−{
10743− struct mm_cid *pcpu_cid;
10744− int cid;
10745−
10746− pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu);
10747− cid = READ_ONCE(pcpu_cid->cid);
10748− if (!mm_cid_is_valid(cid) || cid < weight)
10749− return;
10750− sched_mm_cid_remote_clear(mm, pcpu_cid, cpu);
10751−}
10752−
10753−static void task_mm_cid_work(struct callback_head *work)
10754−{
10755− unsigned long now = jiffies, old_scan, next_scan;
10756− struct task_struct *t = current;
10757− struct cpumask *cidmask;
10758− struct mm_struct *mm;
10759− int weight, cpu;
10760−
10761− WARN_ON_ONCE(t != container_of(work, struct task_struct, cid_work));
10762−
10763− work->next = work; /* Prevent double-add */
10764− if (t->flags & PF_EXITING)
10765− return;
10766− mm = t->mm;
10767− if (!mm)
10768− return;
10769− old_scan = READ_ONCE(mm->mm_cid_next_scan);
10770− next_scan = now + msecs_to_jiffies(MM_CID_SCAN_DELAY);
10771− if (!old_scan) {
10772− unsigned long res;
10773−
10774− res = cmpxchg(&mm->mm_cid_next_scan, old_scan, next_scan);
10775− if (res != old_scan)
10776− old_scan = res;
10777− else
10778− old_scan = next_scan;
10779− }
10780− if (time_before(now, old_scan))
10781− return;
10782− if (!try_cmpxchg(&mm->mm_cid_next_scan, &old_scan, next_scan))
10783− return;
10784− cidmask = mm_cidmask(mm);
10785− /* Clear cids that were not recently used. */
10786− for_each_possible_cpu(cpu)
10787− sched_mm_cid_remote_clear_old(mm, cpu);
10788− weight = cpumask_weight(cidmask);
10789− /*
10790− * Clear cids that are greater or equal to the cidmask weight to
10791− * recompact it.
10792− */
10793− for_each_possible_cpu(cpu)
10794− sched_mm_cid_remote_clear_weight(mm, cpu, weight);
10795−}
10796−
10797−void init_sched_mm_cid(struct task_struct *t)
10798−{
10799− struct mm_struct *mm = t->mm;
10800− int mm_users = 0;
10801−
10802− if (mm) {
10803− mm_users = atomic_read(&mm->mm_users);
10804− if (mm_users == 1)
10805− mm->mm_cid_next_scan = jiffies + msecs_to_jiffies(MM_CID_SCAN_DELAY);
10806− }
10807− t->cid_work.next = &t->cid_work; /* Protect against double add */
10808− init_task_work(&t->cid_work, task_mm_cid_work);
10809−}
10810−
10811−void task_tick_mm_cid(struct rq *rq, struct task_struct *curr)
10812−{
10813− struct callback_head *work = &curr->cid_work;
10814− unsigned long now = jiffies;
10815−
10816− if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) ||
10817− work->next != work)
10818− return;
10819− if (time_before(now, READ_ONCE(curr->mm->mm_cid_next_scan)))
10820− return;
10821−
10822− /* No page allocation under rq lock */
10823− task_work_add(curr, work, TWA_RESUME);
10824−}
10825−
1082610375 void sched_mm_cid_exit_signals(struct task_struct *t)
1082710376 {
1082810377 struct mm_struct *mm = t->mm;
10829− struct rq *rq;
1083010378
10831− if (!mm)
10379+ if (!mm || !t->mm_cid_active)
1083210380 return;
1083310381
10834− preempt_disable();
10835− rq = this_rq();
10836− guard(rq_lock_irqsave)(rq);
10837− preempt_enable_no_resched(); /* holding spinlock */
10838− WRITE_ONCE(t->mm_cid_active, 0);
10839− /*
10840− * Store t->mm_cid_active before loading per-mm/cpu cid.
10841− * Matches barrier in sched_mm_cid_remote_clear_old().
10842− */
10843− smp_mb();
10844− mm_cid_put(mm);
10845− t->last_mm_cid = t->mm_cid = -1;
10382+ guard(preempt)();
10383+ t->mm_cid_active = 0;
10384+ if (t->mm_cid != MM_CID_UNSET) {
10385+ cpumask_clear_cpu(t->mm_cid, mm_cidmask(mm));
10386+ t->mm_cid = MM_CID_UNSET;
10387+ }
1084610388 }
1084710389
10390+/* Deactivate MM CID allocation across execve() */
1084810391 void sched_mm_cid_before_execve(struct task_struct *t)
1084910392 {
10850− struct mm_struct *mm = t->mm;
10851− struct rq *rq;
10852−
10853− if (!mm)
10854− return;
10855−
10856− preempt_disable();
10857− rq = this_rq();
10858− guard(rq_lock_irqsave)(rq);
10859− preempt_enable_no_resched(); /* holding spinlock */
10860− WRITE_ONCE(t->mm_cid_active, 0);
10861− /*
10862− * Store t->mm_cid_active before loading per-mm/cpu cid.
10863− * Matches barrier in sched_mm_cid_remote_clear_old().
10864− */
10865− smp_mb();
10866− mm_cid_put(mm);
10867− t->last_mm_cid = t->mm_cid = -1;
10393+ sched_mm_cid_exit_signals(t);
1086810394 }
1086910395
10396+/* Reactivate MM CID after successful execve() */
1087010397 void sched_mm_cid_after_execve(struct task_struct *t)
1087110398 {
1087210399 struct mm_struct *mm = t->mm;
10873− struct rq *rq;
1087410400
1087510401 if (!mm)
1087610402 return;
1087710403
10878− preempt_disable();
10879− rq = this_rq();
10880− scoped_guard (rq_lock_irqsave, rq) {
10881− preempt_enable_no_resched(); /* holding spinlock */
10882− WRITE_ONCE(t->mm_cid_active, 1);
10883− /*
10884− * Store t->mm_cid_active before loading per-mm/cpu cid.
10885− * Matches barrier in sched_mm_cid_remote_clear_old().
10886− */
10887− smp_mb();
10888− t->last_mm_cid = t->mm_cid = mm_cid_get(rq, t, mm);
10889− }
10404+ guard(preempt)();
10405+ t->mm_cid_active = 1;
10406+ mm_cid_select(t);
1089010407 }
1089110408
1089210409 void sched_mm_cid_fork(struct task_struct *t)
1089310410 {
10894− WARN_ON_ONCE(!t->mm || t->mm_cid != -1);
10411+ WARN_ON_ONCE(!t->mm || t->mm_cid != MM_CID_UNSET);
1089510412 t->mm_cid_active = 1;
1089610413 }
1089710414 #endif /* CONFIG_SCHED_MM_CID */
kernel/sched/sched.h+43−246
@@ -3540,286 +3540,83 @@ extern void sched_dynamic_update(int mode); · 3539 unmodified lines
35403540 extern const char *preempt_modes[];
35413541
35423542 #ifdef CONFIG_SCHED_MM_CID
3543−
3544−#define SCHED_MM_CID_PERIOD_NS (100ULL * 1000000) /* 100ms */
3545−#define MM_CID_SCAN_DELAY 100 /* 100ms */
3546−
3547−extern raw_spinlock_t cid_lock;
3548−extern int use_cid_lock;
3549−
3550−extern void sched_mm_cid_migrate_from(struct task_struct *t);
3551−extern void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t);
3552−extern void task_tick_mm_cid(struct rq *rq, struct task_struct *curr);
3553−extern void init_sched_mm_cid(struct task_struct *t);
3554−
3555−static inline void __mm_cid_put(struct mm_struct *mm, int cid)
3556−{
3557− if (cid < 0)
3558− return;
3559− cpumask_clear_cpu(cid, mm_cidmask(mm));
3560−}
3561−
3562−/*
3563− * The per-mm/cpu cid can have the MM_CID_LAZY_PUT flag set or transition to
3564− * the MM_CID_UNSET state without holding the rq lock, but the rq lock needs to
3565− * be held to transition to other states.
3566− *
3567− * State transitions synchronized with cmpxchg or try_cmpxchg need to be
3568− * consistent across CPUs, which prevents use of this_cpu_cmpxchg.
3569− */
3570−static inline void mm_cid_put_lazy(struct task_struct *t)
3543+static inline void init_sched_mm_cid(struct task_struct *t)
35713544 {
35723545 struct mm_struct *mm = t->mm;
3573− struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
3574− int cid;
3546+ unsigned int max_cid;
35753547
3576− lockdep_assert_irqs_disabled();
3577− cid = __this_cpu_read(pcpu_cid->cid);
3578− if (!mm_cid_is_lazy_put(cid) ||
3579− !try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
3548+ if (!mm)
35803549 return;
3581− __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
3582−}
35833550
3584−static inline int mm_cid_pcpu_unset(struct mm_struct *mm)
3585−{
3586− struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
3587− int cid, res;
3588−
3589− lockdep_assert_irqs_disabled();
3590− cid = __this_cpu_read(pcpu_cid->cid);
3591− for (;;) {
3592− if (mm_cid_is_unset(cid))
3593− return MM_CID_UNSET;
3594− /*
3595− * Attempt transition from valid or lazy-put to unset.
3596− */
3597− res = cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, cid, MM_CID_UNSET);
3598− if (res == cid)
3599− break;
3600− cid = res;
3601− }
3602− return cid;
3551+ /* Preset last_mm_cid */
3552+ max_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
3553+ t->last_mm_cid = max_cid - 1;
36033554 }
36043555
3605−static inline void mm_cid_put(struct mm_struct *mm)
3556+static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
36063557 {
3607− int cid;
3558+ struct mm_struct *mm = t->mm;
36083559
3609− lockdep_assert_irqs_disabled();
3610− cid = mm_cid_pcpu_unset(mm);
3611− if (cid == MM_CID_UNSET)
3612− return;
3613− __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
3560+ if (cid >= max_cids)
3561+ return false;
3562+ if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
3563+ return false;
3564+ t->mm_cid = t->last_mm_cid = cid;
3565+ __this_cpu_write(mm->pcpu_cid->cid, cid);
3566+ return true;
36143567 }
36153568
3616−static inline int __mm_cid_try_get(struct task_struct *t, struct mm_struct *mm)
3569+static inline bool mm_cid_get(struct task_struct *t)
36173570 {
3618− struct cpumask *cidmask = mm_cidmask(mm);
3619− struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
3620− int cid, max_nr_cid, allowed_max_nr_cid;
3571+ struct mm_struct *mm = t->mm;
3572+ unsigned int max_cids;
36213573
3622− /*
3623− * After shrinking the number of threads or reducing the number
3624− * of allowed cpus, reduce the value of max_nr_cid so expansion
3625− * of cid allocation will preserve cache locality if the number
3626− * of threads or allowed cpus increase again.
3627− */
3628− max_nr_cid = atomic_read(&mm->max_nr_cid);
3629− while ((allowed_max_nr_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed),
3630− atomic_read(&mm->mm_users))),
3631− max_nr_cid > allowed_max_nr_cid) {
3632− /* atomic_try_cmpxchg loads previous mm->max_nr_cid into max_nr_cid. */
3633− if (atomic_try_cmpxchg(&mm->max_nr_cid, &max_nr_cid, allowed_max_nr_cid)) {
3634− max_nr_cid = allowed_max_nr_cid;
3635− break;
3636− }
3637− }
3638− /* Try to re-use recent cid. This improves cache locality. */
3639− cid = __this_cpu_read(pcpu_cid->recent_cid);
3640− if (!mm_cid_is_unset(cid) && cid < max_nr_cid &&
3641− !cpumask_test_and_set_cpu(cid, cidmask))
3642− return cid;
3643− /*
3644− * Expand cid allocation if the maximum number of concurrency
3645− * IDs allocated (max_nr_cid) is below the number cpus allowed
3646− * and number of threads. Expanding cid allocation as much as
3647− * possible improves cache locality.
3648− */
3649− cid = max_nr_cid;
3650− while (cid < READ_ONCE(mm->nr_cpus_allowed) && cid < atomic_read(&mm->mm_users)) {
3651− /* atomic_try_cmpxchg loads previous mm->max_nr_cid into cid. */
3652− if (!atomic_try_cmpxchg(&mm->max_nr_cid, &cid, cid + 1))
3653− continue;
3654− if (!cpumask_test_and_set_cpu(cid, cidmask))
3655− return cid;
3656− }
3657− /*
3658− * Find the first available concurrency id.
3659− * Retry finding first zero bit if the mask is temporarily
3660− * filled. This only happens during concurrent remote-clear
3661− * which owns a cid without holding a rq lock.
3662− */
3663− for (;;) {
3664− cid = cpumask_first_zero(cidmask);
3665− if (cid < READ_ONCE(mm->nr_cpus_allowed))
3666− break;
3667− cpu_relax();
3668− }
3669− if (cpumask_test_and_set_cpu(cid, cidmask))
3670− return -1;
3574+ max_cids = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
36713575
3672− return cid;
3673−}
3576+ /* Try to reuse the last CID of this task */
3577+ if (__mm_cid_get(t, t->last_mm_cid, max_cids))
3578+ return true;
36743579
3675−/*
3676− * Save a snapshot of the current runqueue time of this cpu
3677− * with the per-cpu cid value, allowing to estimate how recently it was used.
3678− */
3679−static inline void mm_cid_snapshot_time(struct rq *rq, struct mm_struct *mm)
3680−{
3681− struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(rq));
3580+ /* Try to reuse the last CID of this mm on this CPU */
3581+ if (__mm_cid_get(t, __this_cpu_read(mm->pcpu_cid->cid), max_cids))
3582+ return true;
36823583
3683− lockdep_assert_rq_held(rq);
3684− WRITE_ONCE(pcpu_cid->time, rq->clock);
3584+ /* Try the first zero bit in the cidmask. */
3585+ return __mm_cid_get(t, cpumask_first_zero(mm_cidmask(mm)), max_cids);
36853586 }
36863587
3687−static inline int __mm_cid_get(struct rq *rq, struct task_struct *t,
3688− struct mm_struct *mm)
3588+static inline void mm_cid_select(struct task_struct *t)
36893589 {
3690− int cid;
3691−
36923590 /*
3693− * All allocations (even those using the cid_lock) are lock-free. If
3694− * use_cid_lock is set, hold the cid_lock to perform cid allocation to
3695− * guarantee forward progress.
3591+ * mm_cid_get() can fail when the maximum CID, which is determined
3592+ * by min(mm->nr_cpus_allowed, mm->mm_users) changes concurrently.
3593+ * That's a transient failure as there cannot be more tasks
3594+ * concurrently on a CPU (or about to be scheduled in) than that.
36963595 */
3697− if (!READ_ONCE(use_cid_lock)) {
3698− cid = __mm_cid_try_get(t, mm);
3699− if (cid >= 0)
3700− goto end;
3701− raw_spin_lock(&cid_lock);
3702− } else {
3703− raw_spin_lock(&cid_lock);
3704− cid = __mm_cid_try_get(t, mm);
3705− if (cid >= 0)
3706− goto unlock;
3707− }
3708−
3709− /*
3710− * cid concurrently allocated. Retry while forcing following
3711− * allocations to use the cid_lock to ensure forward progress.
3712− */
3713− WRITE_ONCE(use_cid_lock, 1);
3714− /*
3715− * Set use_cid_lock before allocation. Only care about program order
3716− * because this is only required for forward progress.
3717− */
3718− barrier();
3719− /*
3720− * Retry until it succeeds. It is guaranteed to eventually succeed once
3721− * all newcoming allocations observe the use_cid_lock flag set.
3722− */
3723− do {
3724− cid = __mm_cid_try_get(t, mm);
3725− cpu_relax();
3726− } while (cid < 0);
3727− /*
3728− * Allocate before clearing use_cid_lock. Only care about
3729− * program order because this is for forward progress.
3730− */
3731− barrier();
3732− WRITE_ONCE(use_cid_lock, 0);
3733−unlock:
3734− raw_spin_unlock(&cid_lock);
3735−end:
3736− mm_cid_snapshot_time(rq, mm);
3737−
3738− return cid;
3739−}
3740−
3741−static inline int mm_cid_get(struct rq *rq, struct task_struct *t,
3742− struct mm_struct *mm)
3743−{
3744− struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
3745− int cid;
3746−
3747− lockdep_assert_rq_held(rq);
3748− cid = __this_cpu_read(pcpu_cid->cid);
3749− if (mm_cid_is_valid(cid)) {
3750− mm_cid_snapshot_time(rq, mm);
3751− return cid;
3752− }
3753− if (mm_cid_is_lazy_put(cid)) {
3754− if (try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
3755− __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
3596+ for (;;) {
3597+ if (mm_cid_get(t))
3598+ break;
37563599 }
3757− cid = __mm_cid_get(rq, t, mm);
3758− __this_cpu_write(pcpu_cid->cid, cid);
3759− __this_cpu_write(pcpu_cid->recent_cid, cid);
3760−
3761− return cid;
37623600 }
37633601
3764−static inline void switch_mm_cid(struct rq *rq,
3765− struct task_struct *prev,
3766− struct task_struct *next)
3602+static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
37673603 {
3768− /*
3769− * Provide a memory barrier between rq->curr store and load of
3770− * {prev,next}->mm->pcpu_cid[cpu] on rq->curr->mm transition.
3771− *
3772− * Should be adapted if context_switch() is modified.
3773− */
3774− if (!next->mm) { // to kernel
3775− /*
3776− * user -> kernel transition does not guarantee a barrier, but
3777− * we can use the fact that it performs an atomic operation in
3778− * mmgrab().
3779− */
3780− if (prev->mm) // from user
3781− smp_mb__after_mmgrab();
3782− /*
3783− * kernel -> kernel transition does not change rq->curr->mm
3784− * state. It stays NULL.
3785− */
3786− } else { // to user
3787− /*
3788− * kernel -> user transition does not provide a barrier
3789− * between rq->curr store and load of {prev,next}->mm->pcpu_cid[cpu].
3790− * Provide it here.
3791− */
3792− if (!prev->mm) { // from kernel
3793− smp_mb();
3794− } else { // from user
3795− /*
3796− * user->user transition relies on an implicit
3797− * memory barrier in switch_mm() when
3798− * current->mm changes. If the architecture
3799− * switch_mm() does not have an implicit memory
3800− * barrier, it is emitted here. If current->mm
3801− * is unchanged, no barrier is needed.
3802− */
3803− smp_mb__after_switch_mm();
3804− }
3805− }
38063604 if (prev->mm_cid_active) {
3807− mm_cid_snapshot_time(rq, prev->mm);
3808− mm_cid_put_lazy(prev);
3809− prev->mm_cid = -1;
3605+ if (prev->mm_cid != MM_CID_UNSET)
3606+ cpumask_clear_cpu(prev->mm_cid, mm_cidmask(prev->mm));
3607+ prev->mm_cid = MM_CID_UNSET;
38103608 }
3609+
38113610 if (next->mm_cid_active) {
3812− next->last_mm_cid = next->mm_cid = mm_cid_get(rq, next, next->mm);
3611+ mm_cid_select(next);
38133612 rseq_sched_set_task_mm_cid(next, next->mm_cid);
38143613 }
38153614 }
38163615
38173616 #else /* !CONFIG_SCHED_MM_CID: */
3818−static inline void switch_mm_cid(struct rq *rq, struct task_struct *prev, struct task_struct *next) { }
3819−static inline void sched_mm_cid_migrate_from(struct task_struct *t) { }
3820−static inline void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) { }
3821−static inline void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) { }
38223617 static inline void init_sched_mm_cid(struct task_struct *t) { }
3618+static inline void mm_cid_select(struct task_struct *t) { }
3619+static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
38233620 #endif /* !CONFIG_SCHED_MM_CID */
38243621
38253622 extern u64 avg_vruntime(struct cfs_rq *cfs_rq);