History

tools/sched_ext: Order single-cid cmask helpers as (cid, mask)

The BPF arena single-cid cmask helpers take the cmask first and the cid second. Reorder them to (cid, mask) to match the kernel-side helpers and the test_bit(nr, addr), cpumask_test_cpu(cpu, mask) convention. Range and iteration helpers keep (mask, start).

Tejun Heo committed 5 months ago

Commit a83f9ed1 parent 41de4c72 files · +36 −36Browse files

Unified · Split

2 files
  1. tools/sched_ext/include/scx/cid.bpf.h +30 −30
  2. tools/sched_ext/scx_qmap.bpf.c +6 −6
tools/sched_ext/include/scx/cid.bpf.h+30−30
@@ -40,12 +40,12 @@ · 39 unmodified lines
40 */40 */
41#define CMASK_NR_WORDS(nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1))41#define CMASK_NR_WORDS(nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1))
42 42
43static __always_inline bool __cmask_contains(const struct scx_cmask __arena *m, u32 cid)43static __always_inline bool __cmask_contains(u32 cid, const struct scx_cmask __arena *m)
44{44{
45 return cid >= m->base && cid < m->base + m->nr_cids;45 return cid >= m->base && cid < m->base + m->nr_cids;
46}46}
47 47
48static __always_inline u64 __arena *__cmask_word(const struct scx_cmask __arena *m, u32 cid)48static __always_inline u64 __arena *__cmask_word(u32 cid, const struct scx_cmask __arena *m)
49{49{
50 return (u64 __arena *)&m->bits[cid / 64 - m->base / 64];50 return (u64 __arena *)&m->bits[cid / 64 - m->base / 64];
51}51}
@@ -122,11 +122,11 @@ static __always_inline void cmask_reframe(struct scx_cmask __arena *m, u32 base, · 70 unmodified lines
122 m->nr_cids = nr_cids;122 m->nr_cids = nr_cids;
123}123}
124 124
125static __always_inline bool cmask_test(const struct scx_cmask __arena *m, u32 cid)125static __always_inline bool cmask_test(u32 cid, const struct scx_cmask __arena *m)
126{126{
127 if (!__cmask_contains(m, cid))127 if (!__cmask_contains(cid, m))
128 return false;128 return false;
129 return *__cmask_word(m, cid) & BIT_U64(cid & 63);129 return *__cmask_word(cid, m) & BIT_U64(cid & 63);
130}130}
131 131
132/*132/*
@@ -140,15 +140,15 @@ static __always_inline bool cmask_test(const struct scx_cmask __arena *m, u32 ci · 7 unmodified lines
140 */140 */
141#define CMASK_CAS_TRIES (1U << 23)141#define CMASK_CAS_TRIES (1U << 23)
142 142
143static __always_inline void cmask_set(struct scx_cmask __arena *m, u32 cid)143static __always_inline void cmask_set(u32 cid, struct scx_cmask __arena *m)
144{144{
145 u64 __arena *w;145 u64 __arena *w;
146 u64 bit, old, new;146 u64 bit, old, new;
147 u32 i;147 u32 i;
148 148
149 if (!__cmask_contains(m, cid))149 if (!__cmask_contains(cid, m))
150 return;150 return;
151 w = __cmask_word(m, cid);151 w = __cmask_word(cid, m);
152 bit = BIT_U64(cid & 63);152 bit = BIT_U64(cid & 63);
153 bpf_for(i, 0, CMASK_CAS_TRIES) {153 bpf_for(i, 0, CMASK_CAS_TRIES) {
154 old = *w;154 old = *w;
@@ -161,15 +161,15 @@ static __always_inline void cmask_set(struct scx_cmask __arena *m, u32 cid) · 6 unmodified lines
161 scx_bpf_error("cmask_set CAS exhausted at cid %u", cid);161 scx_bpf_error("cmask_set CAS exhausted at cid %u", cid);
162}162}
163 163
164static __always_inline void cmask_clear(struct scx_cmask __arena *m, u32 cid)164static __always_inline void cmask_clear(u32 cid, struct scx_cmask __arena *m)
165{165{
166 u64 __arena *w;166 u64 __arena *w;
167 u64 bit, old, new;167 u64 bit, old, new;
168 u32 i;168 u32 i;
169 169
170 if (!__cmask_contains(m, cid))170 if (!__cmask_contains(cid, m))
171 return;171 return;
172 w = __cmask_word(m, cid);172 w = __cmask_word(cid, m);
173 bit = BIT_U64(cid & 63);173 bit = BIT_U64(cid & 63);
174 bpf_for(i, 0, CMASK_CAS_TRIES) {174 bpf_for(i, 0, CMASK_CAS_TRIES) {
175 old = *w;175 old = *w;
@@ -182,15 +182,15 @@ static __always_inline void cmask_clear(struct scx_cmask __arena *m, u32 cid) · 6 unmodified lines
182 scx_bpf_error("cmask_clear CAS exhausted at cid %u", cid);182 scx_bpf_error("cmask_clear CAS exhausted at cid %u", cid);
183}183}
184 184
185static __always_inline bool cmask_test_and_set(struct scx_cmask __arena *m, u32 cid)185static __always_inline bool cmask_test_and_set(u32 cid, struct scx_cmask __arena *m)
186{186{
187 u64 __arena *w;187 u64 __arena *w;
188 u64 bit, old, new;188 u64 bit, old, new;
189 u32 i;189 u32 i;
190 190
191 if (!__cmask_contains(m, cid))191 if (!__cmask_contains(cid, m))
192 return false;192 return false;
193 w = __cmask_word(m, cid);193 w = __cmask_word(cid, m);
194 bit = BIT_U64(cid & 63);194 bit = BIT_U64(cid & 63);
195 bpf_for(i, 0, CMASK_CAS_TRIES) {195 bpf_for(i, 0, CMASK_CAS_TRIES) {
196 old = *w;196 old = *w;
@@ -204,15 +204,15 @@ static __always_inline bool cmask_test_and_set(struct scx_cmask __arena *m, u32 · 7 unmodified lines
204 return false;204 return false;
205}205}
206 206
207static __always_inline bool cmask_test_and_clear(struct scx_cmask __arena *m, u32 cid)207static __always_inline bool cmask_test_and_clear(u32 cid, struct scx_cmask __arena *m)
208{208{
209 u64 __arena *w;209 u64 __arena *w;
210 u64 bit, old, new;210 u64 bit, old, new;
211 u32 i;211 u32 i;
212 212
213 if (!__cmask_contains(m, cid))213 if (!__cmask_contains(cid, m))
214 return false;214 return false;
215 w = __cmask_word(m, cid);215 w = __cmask_word(cid, m);
216 bit = BIT_U64(cid & 63);216 bit = BIT_U64(cid & 63);
217 bpf_for(i, 0, CMASK_CAS_TRIES) {217 bpf_for(i, 0, CMASK_CAS_TRIES) {
218 old = *w;218 old = *w;
@@ -226,43 +226,43 @@ static __always_inline bool cmask_test_and_clear(struct scx_cmask __arena *m, u3 · 7 unmodified lines
226 return false;226 return false;
227}227}
228 228
229static __always_inline void __cmask_set(struct scx_cmask __arena *m, u32 cid)229static __always_inline void __cmask_set(u32 cid, struct scx_cmask __arena *m)
230{230{
231 if (!__cmask_contains(m, cid))231 if (!__cmask_contains(cid, m))
232 return;232 return;
233 *__cmask_word(m, cid) |= BIT_U64(cid & 63);233 *__cmask_word(cid, m) |= BIT_U64(cid & 63);
234}234}
235 235
236static __always_inline void __cmask_clear(struct scx_cmask __arena *m, u32 cid)236static __always_inline void __cmask_clear(u32 cid, struct scx_cmask __arena *m)
237{237{
238 if (!__cmask_contains(m, cid))238 if (!__cmask_contains(cid, m))
239 return;239 return;
240 *__cmask_word(m, cid) &= ~BIT_U64(cid & 63);240 *__cmask_word(cid, m) &= ~BIT_U64(cid & 63);
241}241}
242 242
243static __always_inline bool __cmask_test_and_set(struct scx_cmask __arena *m, u32 cid)243static __always_inline bool __cmask_test_and_set(u32 cid, struct scx_cmask __arena *m)
244{244{
245 u64 bit = BIT_U64(cid & 63);245 u64 bit = BIT_U64(cid & 63);
246 u64 __arena *w;246 u64 __arena *w;
247 u64 prev;247 u64 prev;
248 248
249 if (!__cmask_contains(m, cid))249 if (!__cmask_contains(cid, m))
250 return false;250 return false;
251 w = __cmask_word(m, cid);251 w = __cmask_word(cid, m);
252 prev = *w & bit;252 prev = *w & bit;
253 *w |= bit;253 *w |= bit;
254 return prev;254 return prev;
255}255}
256 256
257static __always_inline bool __cmask_test_and_clear(struct scx_cmask __arena *m, u32 cid)257static __always_inline bool __cmask_test_and_clear(u32 cid, struct scx_cmask __arena *m)
258{258{
259 u64 bit = BIT_U64(cid & 63);259 u64 bit = BIT_U64(cid & 63);
260 u64 __arena *w;260 u64 __arena *w;
261 u64 prev;261 u64 prev;
262 262
263 if (!__cmask_contains(m, cid))263 if (!__cmask_contains(cid, m))
264 return false;264 return false;
265 w = __cmask_word(m, cid);265 w = __cmask_word(cid, m);
266 prev = *w & bit;266 prev = *w & bit;
267 *w &= ~bit;267 *w &= ~bit;
268 return prev;268 return prev;
@@ -671,7 +671,7 @@ static __always_inline void cmask_from_cpumask(struct scx_cmask __arena *m, · 402 unmodified lines
671 continue;671 continue;
672 cid = scx_bpf_cpu_to_cid(cpu);672 cid = scx_bpf_cpu_to_cid(cpu);
673 if (cid >= 0)673 if (cid >= 0)
674 __cmask_set(m, cid);674 __cmask_set(cid, m);
675 }675 }
676}676}
677 677
tools/sched_ext/scx_qmap.bpf.c+6−6
@@ -214,7 +214,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid, · 213 unmodified lines
214 if (!always_enq_immed && p->nr_cpus_allowed == 1)214 if (!always_enq_immed && p->nr_cpus_allowed == 1)
215 return prev_cid;215 return prev_cid;
216 216
217 if (cmask_test_and_clear(qa_idle_cids, prev_cid))217 if (cmask_test_and_clear(prev_cid, qa_idle_cids))
218 return prev_cid;218 return prev_cid;
219 219
220 cid = prev_cid;220 cid = prev_cid;
@@ -224,7 +224,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid, · 3 unmodified lines
224 barrier_var(cid);224 barrier_var(cid);
225 if (cid >= nr_cids)225 if (cid >= nr_cids)
226 return -1;226 return -1;
227 if (cmask_test_and_clear(qa_idle_cids, cid))227 if (cmask_test_and_clear(cid, qa_idle_cids))
228 return cid;228 return cid;
229 }229 }
230 return -1;230 return -1;
@@ -534,7 +534,7 @@ static bool dispatch_highpri(bool from_timer) · 303 unmodified lines
534 if (!(taskc = lookup_task_ctx(p)))534 if (!(taskc = lookup_task_ctx(p)))
535 return false;535 return false;
536 536
537 if (cmask_test(&taskc->cpus_allowed, this_cid))537 if (cmask_test(this_cid, &taskc->cpus_allowed))
538 cid = this_cid;538 cid = this_cid;
539 else539 else
540 cid = cmask_next_set_wrap(&taskc->cpus_allowed,540 cid = cmask_next_set_wrap(&taskc->cpus_allowed,
@@ -656,7 +656,7 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev) · 115 unmodified lines
656 * document this class of issue -- other schedulers656 * document this class of issue -- other schedulers
657 * seeing similar warnings can use this as a reference.657 * seeing similar warnings can use this as a reference.
658 */658 */
659 if (!cmask_test(&taskc->cpus_allowed, cid))659 if (!cmask_test(cid, &taskc->cpus_allowed))
660 scx_bpf_kick_cid(scx_bpf_task_cid(p), 0);660 scx_bpf_kick_cid(scx_bpf_task_cid(p), 0);
661 661
662 batch--;662 batch--;
@@ -913,9 +913,9 @@ void BPF_STRUCT_OPS(qmap_update_idle, s32 cid, bool idle) · 250 unmodified lines
913{913{
914 QMAP_TOUCH_ARENA();914 QMAP_TOUCH_ARENA();
915 if (idle)915 if (idle)
916 cmask_set(qa_idle_cids, cid);916 cmask_set(cid, qa_idle_cids);
917 else917 else
918 cmask_clear(qa_idle_cids, cid);918 cmask_clear(cid, qa_idle_cids);
919}919}
920 920
921void BPF_STRUCT_OPS(qmap_set_cmask, struct task_struct *p,921void BPF_STRUCT_OPS(qmap_set_cmask, struct task_struct *p,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t