History

tools/sched_ext: Order single-cid cmask helpers as (cid, mask)

The BPF arena single-cid cmask helpers take the cmask first and the cid second. Reorder them to (cid, mask) to match the kernel-side helpers and the test_bit(nr, addr), cpumask_test_cpu(cpu, mask) convention. Range and iteration helpers keep (mask, start).

Tejun Heo committed 5 months ago

Commit a83f9ed1 parent 41de4c72 files · +36 −36Browse files

Unified · Split

2 files
  1. tools/sched_ext/include/scx/cid.bpf.h +30 −30
  2. tools/sched_ext/scx_qmap.bpf.c +6 −6
tools/sched_ext/include/scx/cid.bpf.h+30−30
@@ -40,12 +40,12 @@ · 39 unmodified lines
4040 */
4141 #define CMASK_NR_WORDS(nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1))
4242
43−static __always_inline bool __cmask_contains(const struct scx_cmask __arena *m, u32 cid)
43+static __always_inline bool __cmask_contains(u32 cid, const struct scx_cmask __arena *m)
4444 {
4545 return cid >= m->base && cid < m->base + m->nr_cids;
4646 }
4747
48−static __always_inline u64 __arena *__cmask_word(const struct scx_cmask __arena *m, u32 cid)
48+static __always_inline u64 __arena *__cmask_word(u32 cid, const struct scx_cmask __arena *m)
4949 {
5050 return (u64 __arena *)&m->bits[cid / 64 - m->base / 64];
5151 }
@@ -122,11 +122,11 @@ static __always_inline void cmask_reframe(struct scx_cmask __arena *m, u32 base, · 70 unmodified lines
122122 m->nr_cids = nr_cids;
123123 }
124124
125−static __always_inline bool cmask_test(const struct scx_cmask __arena *m, u32 cid)
125+static __always_inline bool cmask_test(u32 cid, const struct scx_cmask __arena *m)
126126 {
127− if (!__cmask_contains(m, cid))
127+ if (!__cmask_contains(cid, m))
128128 return false;
129− return *__cmask_word(m, cid) & BIT_U64(cid & 63);
129+ return *__cmask_word(cid, m) & BIT_U64(cid & 63);
130130 }
131131
132132 /*
@@ -140,15 +140,15 @@ static __always_inline bool cmask_test(const struct scx_cmask __arena *m, u32 ci · 7 unmodified lines
140140 */
141141 #define CMASK_CAS_TRIES (1U << 23)
142142
143−static __always_inline void cmask_set(struct scx_cmask __arena *m, u32 cid)
143+static __always_inline void cmask_set(u32 cid, struct scx_cmask __arena *m)
144144 {
145145 u64 __arena *w;
146146 u64 bit, old, new;
147147 u32 i;
148148
149− if (!__cmask_contains(m, cid))
149+ if (!__cmask_contains(cid, m))
150150 return;
151− w = __cmask_word(m, cid);
151+ w = __cmask_word(cid, m);
152152 bit = BIT_U64(cid & 63);
153153 bpf_for(i, 0, CMASK_CAS_TRIES) {
154154 old = *w;
@@ -161,15 +161,15 @@ static __always_inline void cmask_set(struct scx_cmask __arena *m, u32 cid) · 6 unmodified lines
161161 scx_bpf_error("cmask_set CAS exhausted at cid %u", cid);
162162 }
163163
164−static __always_inline void cmask_clear(struct scx_cmask __arena *m, u32 cid)
164+static __always_inline void cmask_clear(u32 cid, struct scx_cmask __arena *m)
165165 {
166166 u64 __arena *w;
167167 u64 bit, old, new;
168168 u32 i;
169169
170− if (!__cmask_contains(m, cid))
170+ if (!__cmask_contains(cid, m))
171171 return;
172− w = __cmask_word(m, cid);
172+ w = __cmask_word(cid, m);
173173 bit = BIT_U64(cid & 63);
174174 bpf_for(i, 0, CMASK_CAS_TRIES) {
175175 old = *w;
@@ -182,15 +182,15 @@ static __always_inline void cmask_clear(struct scx_cmask __arena *m, u32 cid) · 6 unmodified lines
182182 scx_bpf_error("cmask_clear CAS exhausted at cid %u", cid);
183183 }
184184
185−static __always_inline bool cmask_test_and_set(struct scx_cmask __arena *m, u32 cid)
185+static __always_inline bool cmask_test_and_set(u32 cid, struct scx_cmask __arena *m)
186186 {
187187 u64 __arena *w;
188188 u64 bit, old, new;
189189 u32 i;
190190
191− if (!__cmask_contains(m, cid))
191+ if (!__cmask_contains(cid, m))
192192 return false;
193− w = __cmask_word(m, cid);
193+ w = __cmask_word(cid, m);
194194 bit = BIT_U64(cid & 63);
195195 bpf_for(i, 0, CMASK_CAS_TRIES) {
196196 old = *w;
@@ -204,15 +204,15 @@ static __always_inline bool cmask_test_and_set(struct scx_cmask __arena *m, u32 · 7 unmodified lines
204204 return false;
205205 }
206206
207−static __always_inline bool cmask_test_and_clear(struct scx_cmask __arena *m, u32 cid)
207+static __always_inline bool cmask_test_and_clear(u32 cid, struct scx_cmask __arena *m)
208208 {
209209 u64 __arena *w;
210210 u64 bit, old, new;
211211 u32 i;
212212
213− if (!__cmask_contains(m, cid))
213+ if (!__cmask_contains(cid, m))
214214 return false;
215− w = __cmask_word(m, cid);
215+ w = __cmask_word(cid, m);
216216 bit = BIT_U64(cid & 63);
217217 bpf_for(i, 0, CMASK_CAS_TRIES) {
218218 old = *w;
@@ -226,43 +226,43 @@ static __always_inline bool cmask_test_and_clear(struct scx_cmask __arena *m, u3 · 7 unmodified lines
226226 return false;
227227 }
228228
229−static __always_inline void __cmask_set(struct scx_cmask __arena *m, u32 cid)
229+static __always_inline void __cmask_set(u32 cid, struct scx_cmask __arena *m)
230230 {
231− if (!__cmask_contains(m, cid))
231+ if (!__cmask_contains(cid, m))
232232 return;
233− *__cmask_word(m, cid) |= BIT_U64(cid & 63);
233+ *__cmask_word(cid, m) |= BIT_U64(cid & 63);
234234 }
235235
236−static __always_inline void __cmask_clear(struct scx_cmask __arena *m, u32 cid)
236+static __always_inline void __cmask_clear(u32 cid, struct scx_cmask __arena *m)
237237 {
238− if (!__cmask_contains(m, cid))
238+ if (!__cmask_contains(cid, m))
239239 return;
240− *__cmask_word(m, cid) &= ~BIT_U64(cid & 63);
240+ *__cmask_word(cid, m) &= ~BIT_U64(cid & 63);
241241 }
242242
243−static __always_inline bool __cmask_test_and_set(struct scx_cmask __arena *m, u32 cid)
243+static __always_inline bool __cmask_test_and_set(u32 cid, struct scx_cmask __arena *m)
244244 {
245245 u64 bit = BIT_U64(cid & 63);
246246 u64 __arena *w;
247247 u64 prev;
248248
249− if (!__cmask_contains(m, cid))
249+ if (!__cmask_contains(cid, m))
250250 return false;
251− w = __cmask_word(m, cid);
251+ w = __cmask_word(cid, m);
252252 prev = *w & bit;
253253 *w |= bit;
254254 return prev;
255255 }
256256
257−static __always_inline bool __cmask_test_and_clear(struct scx_cmask __arena *m, u32 cid)
257+static __always_inline bool __cmask_test_and_clear(u32 cid, struct scx_cmask __arena *m)
258258 {
259259 u64 bit = BIT_U64(cid & 63);
260260 u64 __arena *w;
261261 u64 prev;
262262
263− if (!__cmask_contains(m, cid))
263+ if (!__cmask_contains(cid, m))
264264 return false;
265− w = __cmask_word(m, cid);
265+ w = __cmask_word(cid, m);
266266 prev = *w & bit;
267267 *w &= ~bit;
268268 return prev;
@@ -671,7 +671,7 @@ static __always_inline void cmask_from_cpumask(struct scx_cmask __arena *m, · 402 unmodified lines
671671 continue;
672672 cid = scx_bpf_cpu_to_cid(cpu);
673673 if (cid >= 0)
674− __cmask_set(m, cid);
674+ __cmask_set(cid, m);
675675 }
676676 }
677677
tools/sched_ext/scx_qmap.bpf.c+6−6
@@ -214,7 +214,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid, · 213 unmodified lines
214214 if (!always_enq_immed && p->nr_cpus_allowed == 1)
215215 return prev_cid;
216216
217− if (cmask_test_and_clear(qa_idle_cids, prev_cid))
217+ if (cmask_test_and_clear(prev_cid, qa_idle_cids))
218218 return prev_cid;
219219
220220 cid = prev_cid;
@@ -224,7 +224,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid, · 3 unmodified lines
224224 barrier_var(cid);
225225 if (cid >= nr_cids)
226226 return -1;
227− if (cmask_test_and_clear(qa_idle_cids, cid))
227+ if (cmask_test_and_clear(cid, qa_idle_cids))
228228 return cid;
229229 }
230230 return -1;
@@ -534,7 +534,7 @@ static bool dispatch_highpri(bool from_timer) · 303 unmodified lines
534534 if (!(taskc = lookup_task_ctx(p)))
535535 return false;
536536
537− if (cmask_test(&taskc->cpus_allowed, this_cid))
537+ if (cmask_test(this_cid, &taskc->cpus_allowed))
538538 cid = this_cid;
539539 else
540540 cid = cmask_next_set_wrap(&taskc->cpus_allowed,
@@ -656,7 +656,7 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev) · 115 unmodified lines
656656 * document this class of issue -- other schedulers
657657 * seeing similar warnings can use this as a reference.
658658 */
659− if (!cmask_test(&taskc->cpus_allowed, cid))
659+ if (!cmask_test(cid, &taskc->cpus_allowed))
660660 scx_bpf_kick_cid(scx_bpf_task_cid(p), 0);
661661
662662 batch--;
@@ -913,9 +913,9 @@ void BPF_STRUCT_OPS(qmap_update_idle, s32 cid, bool idle) · 250 unmodified lines
913913 {
914914 QMAP_TOUCH_ARENA();
915915 if (idle)
916− cmask_set(qa_idle_cids, cid);
916+ cmask_set(cid, qa_idle_cids);
917917 else
918− cmask_clear(qa_idle_cids, cid);
918+ cmask_clear(cid, qa_idle_cids);
919919 }
920920
921921 void BPF_STRUCT_OPS(qmap_set_cmask, struct task_struct *p,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t