The BPF arena single-cid cmask helpers take the cmask first and the cid second. Reorder them to (cid, mask) to match the kernel-side helpers and the test_bit(nr, addr), cpumask_test_cpu(cpu, mask) convention. Range and iteration helpers keep (mask, start).
Unified · Split
| @@ -40,12 +40,12 @@ · 39 unmodified lines | ||
| 40 | 40 | */ |
| 41 | 41 | #define CMASK_NR_WORDS(nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1)) |
| 42 | 42 | |
| 43 | −static __always_inline bool __cmask_contains(const struct scx_cmask __arena *m, u32 cid) | |
| 43 | +static __always_inline bool __cmask_contains(u32 cid, const struct scx_cmask __arena *m) | |
| 44 | 44 | { |
| 45 | 45 | return cid >= m->base && cid < m->base + m->nr_cids; |
| 46 | 46 | } |
| 47 | 47 | |
| 48 | −static __always_inline u64 __arena *__cmask_word(const struct scx_cmask __arena *m, u32 cid) | |
| 48 | +static __always_inline u64 __arena *__cmask_word(u32 cid, const struct scx_cmask __arena *m) | |
| 49 | 49 | { |
| 50 | 50 | return (u64 __arena *)&m->bits[cid / 64 - m->base / 64]; |
| 51 | 51 | } |
| @@ -122,11 +122,11 @@ static __always_inline void cmask_reframe(struct scx_cmask __arena *m, u32 base, · 70 unmodified lines | ||
| 122 | 122 | m->nr_cids = nr_cids; |
| 123 | 123 | } |
| 124 | 124 | |
| 125 | −static __always_inline bool cmask_test(const struct scx_cmask __arena *m, u32 cid) | |
| 125 | +static __always_inline bool cmask_test(u32 cid, const struct scx_cmask __arena *m) | |
| 126 | 126 | { |
| 127 | − if (!__cmask_contains(m, cid)) | |
| 127 | + if (!__cmask_contains(cid, m)) | |
| 128 | 128 | return false; |
| 129 | − return *__cmask_word(m, cid) & BIT_U64(cid & 63); | |
| 129 | + return *__cmask_word(cid, m) & BIT_U64(cid & 63); | |
| 130 | 130 | } |
| 131 | 131 | |
| 132 | 132 | /* |
| @@ -140,15 +140,15 @@ static __always_inline bool cmask_test(const struct scx_cmask __arena *m, u32 ci · 7 unmodified lines | ||
| 140 | 140 | */ |
| 141 | 141 | #define CMASK_CAS_TRIES (1U << 23) |
| 142 | 142 | |
| 143 | −static __always_inline void cmask_set(struct scx_cmask __arena *m, u32 cid) | |
| 143 | +static __always_inline void cmask_set(u32 cid, struct scx_cmask __arena *m) | |
| 144 | 144 | { |
| 145 | 145 | u64 __arena *w; |
| 146 | 146 | u64 bit, old, new; |
| 147 | 147 | u32 i; |
| 148 | 148 | |
| 149 | − if (!__cmask_contains(m, cid)) | |
| 149 | + if (!__cmask_contains(cid, m)) | |
| 150 | 150 | return; |
| 151 | − w = __cmask_word(m, cid); | |
| 151 | + w = __cmask_word(cid, m); | |
| 152 | 152 | bit = BIT_U64(cid & 63); |
| 153 | 153 | bpf_for(i, 0, CMASK_CAS_TRIES) { |
| 154 | 154 | old = *w; |
| @@ -161,15 +161,15 @@ static __always_inline void cmask_set(struct scx_cmask __arena *m, u32 cid) · 6 unmodified lines | ||
| 161 | 161 | scx_bpf_error("cmask_set CAS exhausted at cid %u", cid); |
| 162 | 162 | } |
| 163 | 163 | |
| 164 | −static __always_inline void cmask_clear(struct scx_cmask __arena *m, u32 cid) | |
| 164 | +static __always_inline void cmask_clear(u32 cid, struct scx_cmask __arena *m) | |
| 165 | 165 | { |
| 166 | 166 | u64 __arena *w; |
| 167 | 167 | u64 bit, old, new; |
| 168 | 168 | u32 i; |
| 169 | 169 | |
| 170 | − if (!__cmask_contains(m, cid)) | |
| 170 | + if (!__cmask_contains(cid, m)) | |
| 171 | 171 | return; |
| 172 | − w = __cmask_word(m, cid); | |
| 172 | + w = __cmask_word(cid, m); | |
| 173 | 173 | bit = BIT_U64(cid & 63); |
| 174 | 174 | bpf_for(i, 0, CMASK_CAS_TRIES) { |
| 175 | 175 | old = *w; |
| @@ -182,15 +182,15 @@ static __always_inline void cmask_clear(struct scx_cmask __arena *m, u32 cid) · 6 unmodified lines | ||
| 182 | 182 | scx_bpf_error("cmask_clear CAS exhausted at cid %u", cid); |
| 183 | 183 | } |
| 184 | 184 | |
| 185 | −static __always_inline bool cmask_test_and_set(struct scx_cmask __arena *m, u32 cid) | |
| 185 | +static __always_inline bool cmask_test_and_set(u32 cid, struct scx_cmask __arena *m) | |
| 186 | 186 | { |
| 187 | 187 | u64 __arena *w; |
| 188 | 188 | u64 bit, old, new; |
| 189 | 189 | u32 i; |
| 190 | 190 | |
| 191 | − if (!__cmask_contains(m, cid)) | |
| 191 | + if (!__cmask_contains(cid, m)) | |
| 192 | 192 | return false; |
| 193 | − w = __cmask_word(m, cid); | |
| 193 | + w = __cmask_word(cid, m); | |
| 194 | 194 | bit = BIT_U64(cid & 63); |
| 195 | 195 | bpf_for(i, 0, CMASK_CAS_TRIES) { |
| 196 | 196 | old = *w; |
| @@ -204,15 +204,15 @@ static __always_inline bool cmask_test_and_set(struct scx_cmask __arena *m, u32 · 7 unmodified lines | ||
| 204 | 204 | return false; |
| 205 | 205 | } |
| 206 | 206 | |
| 207 | −static __always_inline bool cmask_test_and_clear(struct scx_cmask __arena *m, u32 cid) | |
| 207 | +static __always_inline bool cmask_test_and_clear(u32 cid, struct scx_cmask __arena *m) | |
| 208 | 208 | { |
| 209 | 209 | u64 __arena *w; |
| 210 | 210 | u64 bit, old, new; |
| 211 | 211 | u32 i; |
| 212 | 212 | |
| 213 | − if (!__cmask_contains(m, cid)) | |
| 213 | + if (!__cmask_contains(cid, m)) | |
| 214 | 214 | return false; |
| 215 | − w = __cmask_word(m, cid); | |
| 215 | + w = __cmask_word(cid, m); | |
| 216 | 216 | bit = BIT_U64(cid & 63); |
| 217 | 217 | bpf_for(i, 0, CMASK_CAS_TRIES) { |
| 218 | 218 | old = *w; |
| @@ -226,43 +226,43 @@ static __always_inline bool cmask_test_and_clear(struct scx_cmask __arena *m, u3 · 7 unmodified lines | ||
| 226 | 226 | return false; |
| 227 | 227 | } |
| 228 | 228 | |
| 229 | −static __always_inline void __cmask_set(struct scx_cmask __arena *m, u32 cid) | |
| 229 | +static __always_inline void __cmask_set(u32 cid, struct scx_cmask __arena *m) | |
| 230 | 230 | { |
| 231 | − if (!__cmask_contains(m, cid)) | |
| 231 | + if (!__cmask_contains(cid, m)) | |
| 232 | 232 | return; |
| 233 | − *__cmask_word(m, cid) |= BIT_U64(cid & 63); | |
| 233 | + *__cmask_word(cid, m) |= BIT_U64(cid & 63); | |
| 234 | 234 | } |
| 235 | 235 | |
| 236 | −static __always_inline void __cmask_clear(struct scx_cmask __arena *m, u32 cid) | |
| 236 | +static __always_inline void __cmask_clear(u32 cid, struct scx_cmask __arena *m) | |
| 237 | 237 | { |
| 238 | − if (!__cmask_contains(m, cid)) | |
| 238 | + if (!__cmask_contains(cid, m)) | |
| 239 | 239 | return; |
| 240 | − *__cmask_word(m, cid) &= ~BIT_U64(cid & 63); | |
| 240 | + *__cmask_word(cid, m) &= ~BIT_U64(cid & 63); | |
| 241 | 241 | } |
| 242 | 242 | |
| 243 | −static __always_inline bool __cmask_test_and_set(struct scx_cmask __arena *m, u32 cid) | |
| 243 | +static __always_inline bool __cmask_test_and_set(u32 cid, struct scx_cmask __arena *m) | |
| 244 | 244 | { |
| 245 | 245 | u64 bit = BIT_U64(cid & 63); |
| 246 | 246 | u64 __arena *w; |
| 247 | 247 | u64 prev; |
| 248 | 248 | |
| 249 | − if (!__cmask_contains(m, cid)) | |
| 249 | + if (!__cmask_contains(cid, m)) | |
| 250 | 250 | return false; |
| 251 | − w = __cmask_word(m, cid); | |
| 251 | + w = __cmask_word(cid, m); | |
| 252 | 252 | prev = *w & bit; |
| 253 | 253 | *w |= bit; |
| 254 | 254 | return prev; |
| 255 | 255 | } |
| 256 | 256 | |
| 257 | −static __always_inline bool __cmask_test_and_clear(struct scx_cmask __arena *m, u32 cid) | |
| 257 | +static __always_inline bool __cmask_test_and_clear(u32 cid, struct scx_cmask __arena *m) | |
| 258 | 258 | { |
| 259 | 259 | u64 bit = BIT_U64(cid & 63); |
| 260 | 260 | u64 __arena *w; |
| 261 | 261 | u64 prev; |
| 262 | 262 | |
| 263 | − if (!__cmask_contains(m, cid)) | |
| 263 | + if (!__cmask_contains(cid, m)) | |
| 264 | 264 | return false; |
| 265 | − w = __cmask_word(m, cid); | |
| 265 | + w = __cmask_word(cid, m); | |
| 266 | 266 | prev = *w & bit; |
| 267 | 267 | *w &= ~bit; |
| 268 | 268 | return prev; |
| @@ -671,7 +671,7 @@ static __always_inline void cmask_from_cpumask(struct scx_cmask __arena *m, · 402 unmodified lines | ||
| 671 | 671 | continue; |
| 672 | 672 | cid = scx_bpf_cpu_to_cid(cpu); |
| 673 | 673 | if (cid >= 0) |
| 674 | − __cmask_set(m, cid); | |
| 674 | + __cmask_set(cid, m); | |
| 675 | 675 | } |
| 676 | 676 | } |
| 677 | 677 | |
| @@ -214,7 +214,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid, · 213 unmodified lines | ||
| 214 | 214 | if (!always_enq_immed && p->nr_cpus_allowed == 1) |
| 215 | 215 | return prev_cid; |
| 216 | 216 | |
| 217 | − if (cmask_test_and_clear(qa_idle_cids, prev_cid)) | |
| 217 | + if (cmask_test_and_clear(prev_cid, qa_idle_cids)) | |
| 218 | 218 | return prev_cid; |
| 219 | 219 | |
| 220 | 220 | cid = prev_cid; |
| @@ -224,7 +224,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid, · 3 unmodified lines | ||
| 224 | 224 | barrier_var(cid); |
| 225 | 225 | if (cid >= nr_cids) |
| 226 | 226 | return -1; |
| 227 | − if (cmask_test_and_clear(qa_idle_cids, cid)) | |
| 227 | + if (cmask_test_and_clear(cid, qa_idle_cids)) | |
| 228 | 228 | return cid; |
| 229 | 229 | } |
| 230 | 230 | return -1; |
| @@ -534,7 +534,7 @@ static bool dispatch_highpri(bool from_timer) · 303 unmodified lines | ||
| 534 | 534 | if (!(taskc = lookup_task_ctx(p))) |
| 535 | 535 | return false; |
| 536 | 536 | |
| 537 | − if (cmask_test(&taskc->cpus_allowed, this_cid)) | |
| 537 | + if (cmask_test(this_cid, &taskc->cpus_allowed)) | |
| 538 | 538 | cid = this_cid; |
| 539 | 539 | else |
| 540 | 540 | cid = cmask_next_set_wrap(&taskc->cpus_allowed, |
| @@ -656,7 +656,7 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev) · 115 unmodified lines | ||
| 656 | 656 | * document this class of issue -- other schedulers |
| 657 | 657 | * seeing similar warnings can use this as a reference. |
| 658 | 658 | */ |
| 659 | − if (!cmask_test(&taskc->cpus_allowed, cid)) | |
| 659 | + if (!cmask_test(cid, &taskc->cpus_allowed)) | |
| 660 | 660 | scx_bpf_kick_cid(scx_bpf_task_cid(p), 0); |
| 661 | 661 | |
| 662 | 662 | batch--; |
| @@ -913,9 +913,9 @@ void BPF_STRUCT_OPS(qmap_update_idle, s32 cid, bool idle) · 250 unmodified lines | ||
| 913 | 913 | { |
| 914 | 914 | QMAP_TOUCH_ARENA(); |
| 915 | 915 | if (idle) |
| 916 | − cmask_set(qa_idle_cids, cid); | |
| 916 | + cmask_set(cid, qa_idle_cids); | |
| 917 | 917 | else |
| 918 | − cmask_clear(qa_idle_cids, cid); | |
| 918 | + cmask_clear(cid, qa_idle_cids); | |
| 919 | 919 | } |
| 920 | 920 | |
| 921 | 921 | void BPF_STRUCT_OPS(qmap_set_cmask, struct task_struct *p, |