The BPF arena single-cid cmask helpers take the cmask first and the cid second. Reorder them to (cid, mask) to match the kernel-side helpers and the test_bit(nr, addr), cpumask_test_cpu(cpu, mask) convention. Range and iteration helpers keep (mask, start).
Unified · Split
| @@ -40,12 +40,12 @@ · 39 unmodified lines | |||
| 40 | */ | 40 | */ |
| 41 | #define CMASK_NR_WORDS(nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1)) | 41 | #define CMASK_NR_WORDS(nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1)) |
| 42 | 42 | ||
| 43 | static __always_inline bool __cmask_contains(const struct scx_cmask __arena *m, u32 cid) | 43 | static __always_inline bool __cmask_contains(u32 cid, const struct scx_cmask __arena *m) |
| 44 | { | 44 | { |
| 45 | return cid >= m->base && cid < m->base + m->nr_cids; | 45 | return cid >= m->base && cid < m->base + m->nr_cids; |
| 46 | } | 46 | } |
| 47 | 47 | ||
| 48 | static __always_inline u64 __arena *__cmask_word(const struct scx_cmask __arena *m, u32 cid) | 48 | static __always_inline u64 __arena *__cmask_word(u32 cid, const struct scx_cmask __arena *m) |
| 49 | { | 49 | { |
| 50 | return (u64 __arena *)&m->bits[cid / 64 - m->base / 64]; | 50 | return (u64 __arena *)&m->bits[cid / 64 - m->base / 64]; |
| 51 | } | 51 | } |
| @@ -122,11 +122,11 @@ static __always_inline void cmask_reframe(struct scx_cmask __arena *m, u32 base, · 70 unmodified lines | |||
| 122 | m->nr_cids = nr_cids; | 122 | m->nr_cids = nr_cids; |
| 123 | } | 123 | } |
| 124 | 124 | ||
| 125 | static __always_inline bool cmask_test(const struct scx_cmask __arena *m, u32 cid) | 125 | static __always_inline bool cmask_test(u32 cid, const struct scx_cmask __arena *m) |
| 126 | { | 126 | { |
| 127 | if (!__cmask_contains(m, cid)) | 127 | if (!__cmask_contains(cid, m)) |
| 128 | return false; | 128 | return false; |
| 129 | return *__cmask_word(m, cid) & BIT_U64(cid & 63); | 129 | return *__cmask_word(cid, m) & BIT_U64(cid & 63); |
| 130 | } | 130 | } |
| 131 | 131 | ||
| 132 | /* | 132 | /* |
| @@ -140,15 +140,15 @@ static __always_inline bool cmask_test(const struct scx_cmask __arena *m, u32 ci · 7 unmodified lines | |||
| 140 | */ | 140 | */ |
| 141 | #define CMASK_CAS_TRIES (1U << 23) | 141 | #define CMASK_CAS_TRIES (1U << 23) |
| 142 | 142 | ||
| 143 | static __always_inline void cmask_set(struct scx_cmask __arena *m, u32 cid) | 143 | static __always_inline void cmask_set(u32 cid, struct scx_cmask __arena *m) |
| 144 | { | 144 | { |
| 145 | u64 __arena *w; | 145 | u64 __arena *w; |
| 146 | u64 bit, old, new; | 146 | u64 bit, old, new; |
| 147 | u32 i; | 147 | u32 i; |
| 148 | 148 | ||
| 149 | if (!__cmask_contains(m, cid)) | 149 | if (!__cmask_contains(cid, m)) |
| 150 | return; | 150 | return; |
| 151 | w = __cmask_word(m, cid); | 151 | w = __cmask_word(cid, m); |
| 152 | bit = BIT_U64(cid & 63); | 152 | bit = BIT_U64(cid & 63); |
| 153 | bpf_for(i, 0, CMASK_CAS_TRIES) { | 153 | bpf_for(i, 0, CMASK_CAS_TRIES) { |
| 154 | old = *w; | 154 | old = *w; |
| @@ -161,15 +161,15 @@ static __always_inline void cmask_set(struct scx_cmask __arena *m, u32 cid) · 6 unmodified lines | |||
| 161 | scx_bpf_error("cmask_set CAS exhausted at cid %u", cid); | 161 | scx_bpf_error("cmask_set CAS exhausted at cid %u", cid); |
| 162 | } | 162 | } |
| 163 | 163 | ||
| 164 | static __always_inline void cmask_clear(struct scx_cmask __arena *m, u32 cid) | 164 | static __always_inline void cmask_clear(u32 cid, struct scx_cmask __arena *m) |
| 165 | { | 165 | { |
| 166 | u64 __arena *w; | 166 | u64 __arena *w; |
| 167 | u64 bit, old, new; | 167 | u64 bit, old, new; |
| 168 | u32 i; | 168 | u32 i; |
| 169 | 169 | ||
| 170 | if (!__cmask_contains(m, cid)) | 170 | if (!__cmask_contains(cid, m)) |
| 171 | return; | 171 | return; |
| 172 | w = __cmask_word(m, cid); | 172 | w = __cmask_word(cid, m); |
| 173 | bit = BIT_U64(cid & 63); | 173 | bit = BIT_U64(cid & 63); |
| 174 | bpf_for(i, 0, CMASK_CAS_TRIES) { | 174 | bpf_for(i, 0, CMASK_CAS_TRIES) { |
| 175 | old = *w; | 175 | old = *w; |
| @@ -182,15 +182,15 @@ static __always_inline void cmask_clear(struct scx_cmask __arena *m, u32 cid) · 6 unmodified lines | |||
| 182 | scx_bpf_error("cmask_clear CAS exhausted at cid %u", cid); | 182 | scx_bpf_error("cmask_clear CAS exhausted at cid %u", cid); |
| 183 | } | 183 | } |
| 184 | 184 | ||
| 185 | static __always_inline bool cmask_test_and_set(struct scx_cmask __arena *m, u32 cid) | 185 | static __always_inline bool cmask_test_and_set(u32 cid, struct scx_cmask __arena *m) |
| 186 | { | 186 | { |
| 187 | u64 __arena *w; | 187 | u64 __arena *w; |
| 188 | u64 bit, old, new; | 188 | u64 bit, old, new; |
| 189 | u32 i; | 189 | u32 i; |
| 190 | 190 | ||
| 191 | if (!__cmask_contains(m, cid)) | 191 | if (!__cmask_contains(cid, m)) |
| 192 | return false; | 192 | return false; |
| 193 | w = __cmask_word(m, cid); | 193 | w = __cmask_word(cid, m); |
| 194 | bit = BIT_U64(cid & 63); | 194 | bit = BIT_U64(cid & 63); |
| 195 | bpf_for(i, 0, CMASK_CAS_TRIES) { | 195 | bpf_for(i, 0, CMASK_CAS_TRIES) { |
| 196 | old = *w; | 196 | old = *w; |
| @@ -204,15 +204,15 @@ static __always_inline bool cmask_test_and_set(struct scx_cmask __arena *m, u32 · 7 unmodified lines | |||
| 204 | return false; | 204 | return false; |
| 205 | } | 205 | } |
| 206 | 206 | ||
| 207 | static __always_inline bool cmask_test_and_clear(struct scx_cmask __arena *m, u32 cid) | 207 | static __always_inline bool cmask_test_and_clear(u32 cid, struct scx_cmask __arena *m) |
| 208 | { | 208 | { |
| 209 | u64 __arena *w; | 209 | u64 __arena *w; |
| 210 | u64 bit, old, new; | 210 | u64 bit, old, new; |
| 211 | u32 i; | 211 | u32 i; |
| 212 | 212 | ||
| 213 | if (!__cmask_contains(m, cid)) | 213 | if (!__cmask_contains(cid, m)) |
| 214 | return false; | 214 | return false; |
| 215 | w = __cmask_word(m, cid); | 215 | w = __cmask_word(cid, m); |
| 216 | bit = BIT_U64(cid & 63); | 216 | bit = BIT_U64(cid & 63); |
| 217 | bpf_for(i, 0, CMASK_CAS_TRIES) { | 217 | bpf_for(i, 0, CMASK_CAS_TRIES) { |
| 218 | old = *w; | 218 | old = *w; |
| @@ -226,43 +226,43 @@ static __always_inline bool cmask_test_and_clear(struct scx_cmask __arena *m, u3 · 7 unmodified lines | |||
| 226 | return false; | 226 | return false; |
| 227 | } | 227 | } |
| 228 | 228 | ||
| 229 | static __always_inline void __cmask_set(struct scx_cmask __arena *m, u32 cid) | 229 | static __always_inline void __cmask_set(u32 cid, struct scx_cmask __arena *m) |
| 230 | { | 230 | { |
| 231 | if (!__cmask_contains(m, cid)) | 231 | if (!__cmask_contains(cid, m)) |
| 232 | return; | 232 | return; |
| 233 | *__cmask_word(m, cid) |= BIT_U64(cid & 63); | 233 | *__cmask_word(cid, m) |= BIT_U64(cid & 63); |
| 234 | } | 234 | } |
| 235 | 235 | ||
| 236 | static __always_inline void __cmask_clear(struct scx_cmask __arena *m, u32 cid) | 236 | static __always_inline void __cmask_clear(u32 cid, struct scx_cmask __arena *m) |
| 237 | { | 237 | { |
| 238 | if (!__cmask_contains(m, cid)) | 238 | if (!__cmask_contains(cid, m)) |
| 239 | return; | 239 | return; |
| 240 | *__cmask_word(m, cid) &= ~BIT_U64(cid & 63); | 240 | *__cmask_word(cid, m) &= ~BIT_U64(cid & 63); |
| 241 | } | 241 | } |
| 242 | 242 | ||
| 243 | static __always_inline bool __cmask_test_and_set(struct scx_cmask __arena *m, u32 cid) | 243 | static __always_inline bool __cmask_test_and_set(u32 cid, struct scx_cmask __arena *m) |
| 244 | { | 244 | { |
| 245 | u64 bit = BIT_U64(cid & 63); | 245 | u64 bit = BIT_U64(cid & 63); |
| 246 | u64 __arena *w; | 246 | u64 __arena *w; |
| 247 | u64 prev; | 247 | u64 prev; |
| 248 | 248 | ||
| 249 | if (!__cmask_contains(m, cid)) | 249 | if (!__cmask_contains(cid, m)) |
| 250 | return false; | 250 | return false; |
| 251 | w = __cmask_word(m, cid); | 251 | w = __cmask_word(cid, m); |
| 252 | prev = *w & bit; | 252 | prev = *w & bit; |
| 253 | *w |= bit; | 253 | *w |= bit; |
| 254 | return prev; | 254 | return prev; |
| 255 | } | 255 | } |
| 256 | 256 | ||
| 257 | static __always_inline bool __cmask_test_and_clear(struct scx_cmask __arena *m, u32 cid) | 257 | static __always_inline bool __cmask_test_and_clear(u32 cid, struct scx_cmask __arena *m) |
| 258 | { | 258 | { |
| 259 | u64 bit = BIT_U64(cid & 63); | 259 | u64 bit = BIT_U64(cid & 63); |
| 260 | u64 __arena *w; | 260 | u64 __arena *w; |
| 261 | u64 prev; | 261 | u64 prev; |
| 262 | 262 | ||
| 263 | if (!__cmask_contains(m, cid)) | 263 | if (!__cmask_contains(cid, m)) |
| 264 | return false; | 264 | return false; |
| 265 | w = __cmask_word(m, cid); | 265 | w = __cmask_word(cid, m); |
| 266 | prev = *w & bit; | 266 | prev = *w & bit; |
| 267 | *w &= ~bit; | 267 | *w &= ~bit; |
| 268 | return prev; | 268 | return prev; |
| @@ -671,7 +671,7 @@ static __always_inline void cmask_from_cpumask(struct scx_cmask __arena *m, · 402 unmodified lines | |||
| 671 | continue; | 671 | continue; |
| 672 | cid = scx_bpf_cpu_to_cid(cpu); | 672 | cid = scx_bpf_cpu_to_cid(cpu); |
| 673 | if (cid >= 0) | 673 | if (cid >= 0) |
| 674 | __cmask_set(m, cid); | 674 | __cmask_set(cid, m); |
| 675 | } | 675 | } |
| 676 | } | 676 | } |
| 677 | 677 | ||
| @@ -214,7 +214,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid, · 213 unmodified lines | |||
| 214 | if (!always_enq_immed && p->nr_cpus_allowed == 1) | 214 | if (!always_enq_immed && p->nr_cpus_allowed == 1) |
| 215 | return prev_cid; | 215 | return prev_cid; |
| 216 | 216 | ||
| 217 | if (cmask_test_and_clear(qa_idle_cids, prev_cid)) | 217 | if (cmask_test_and_clear(prev_cid, qa_idle_cids)) |
| 218 | return prev_cid; | 218 | return prev_cid; |
| 219 | 219 | ||
| 220 | cid = prev_cid; | 220 | cid = prev_cid; |
| @@ -224,7 +224,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid, · 3 unmodified lines | |||
| 224 | barrier_var(cid); | 224 | barrier_var(cid); |
| 225 | if (cid >= nr_cids) | 225 | if (cid >= nr_cids) |
| 226 | return -1; | 226 | return -1; |
| 227 | if (cmask_test_and_clear(qa_idle_cids, cid)) | 227 | if (cmask_test_and_clear(cid, qa_idle_cids)) |
| 228 | return cid; | 228 | return cid; |
| 229 | } | 229 | } |
| 230 | return -1; | 230 | return -1; |
| @@ -534,7 +534,7 @@ static bool dispatch_highpri(bool from_timer) · 303 unmodified lines | |||
| 534 | if (!(taskc = lookup_task_ctx(p))) | 534 | if (!(taskc = lookup_task_ctx(p))) |
| 535 | return false; | 535 | return false; |
| 536 | 536 | ||
| 537 | if (cmask_test(&taskc->cpus_allowed, this_cid)) | 537 | if (cmask_test(this_cid, &taskc->cpus_allowed)) |
| 538 | cid = this_cid; | 538 | cid = this_cid; |
| 539 | else | 539 | else |
| 540 | cid = cmask_next_set_wrap(&taskc->cpus_allowed, | 540 | cid = cmask_next_set_wrap(&taskc->cpus_allowed, |
| @@ -656,7 +656,7 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev) · 115 unmodified lines | |||
| 656 | * document this class of issue -- other schedulers | 656 | * document this class of issue -- other schedulers |
| 657 | * seeing similar warnings can use this as a reference. | 657 | * seeing similar warnings can use this as a reference. |
| 658 | */ | 658 | */ |
| 659 | if (!cmask_test(&taskc->cpus_allowed, cid)) | 659 | if (!cmask_test(cid, &taskc->cpus_allowed)) |
| 660 | scx_bpf_kick_cid(scx_bpf_task_cid(p), 0); | 660 | scx_bpf_kick_cid(scx_bpf_task_cid(p), 0); |
| 661 | 661 | ||
| 662 | batch--; | 662 | batch--; |
| @@ -913,9 +913,9 @@ void BPF_STRUCT_OPS(qmap_update_idle, s32 cid, bool idle) · 250 unmodified lines | |||
| 913 | { | 913 | { |
| 914 | QMAP_TOUCH_ARENA(); | 914 | QMAP_TOUCH_ARENA(); |
| 915 | if (idle) | 915 | if (idle) |
| 916 | cmask_set(qa_idle_cids, cid); | 916 | cmask_set(cid, qa_idle_cids); |
| 917 | else | 917 | else |
| 918 | cmask_clear(qa_idle_cids, cid); | 918 | cmask_clear(cid, qa_idle_cids); |
| 919 | } | 919 | } |
| 920 | 920 | ||
| 921 | void BPF_STRUCT_OPS(qmap_set_cmask, struct task_struct *p, | 921 | void BPF_STRUCT_OPS(qmap_set_cmask, struct task_struct *p, |