History

smp: Cleanup smp_call_function*()

Get rid of the __call_single_node union and cleanup the API a little to avoid external code relying on the structure layout as much.

Peter Zijlstra committed 6 years ago

Commit 545b8c81 parent 7a9f50a13 files · +60 −94Browse files

Showing the changes to kernel (3 of 13 files). Show all

Unified · Split

3 files
  1. kernel/debug/debug_core.c +3 −3
  2. kernel/sched/core.c +2 −10
  3. kernel/smp.c +25 −25
kernel/debug/debug_core.c+3−3
@@ -225,8 +225,6 @@ NOKPROBE_SYMBOL(kgdb_skipexception); · 224 unmodified lines
225225 * Default (weak) implementation for kgdb_roundup_cpus
226226 */
227227
228−static DEFINE_PER_CPU(call_single_data_t, kgdb_roundup_csd);
229−
230228 void __weak kgdb_call_nmi_hook(void *ignored)
231229 {
232230 /*
@@ -241,6 +239,9 @@ void __weak kgdb_call_nmi_hook(void *ignored) · 8 unmodified lines
241239 }
242240 NOKPROBE_SYMBOL(kgdb_call_nmi_hook);
243241
242+static DEFINE_PER_CPU(call_single_data_t, kgdb_roundup_csd) =
243+ CSD_INIT(kgdb_call_nmi_hook, NULL);
244+
244245 void __weak kgdb_roundup_cpus(void)
245246 {
246247 call_single_data_t *csd;
@@ -267,7 +268,6 @@ void __weak kgdb_roundup_cpus(void) · 20 unmodified lines
267268 continue;
268269 kgdb_info[cpu].rounding_up = true;
269270
270− csd->func = kgdb_call_nmi_hook;
271271 ret = smp_call_function_single_async(cpu, csd);
272272 if (ret)
273273 kgdb_info[cpu].rounding_up = false;
kernel/sched/core.c+2−10
@@ -320,14 +320,6 @@ void update_rq_clock(struct rq *rq) · 319 unmodified lines
320320 update_rq_clock_task(rq, delta);
321321 }
322322
323−static inline void
324−rq_csd_init(struct rq *rq, call_single_data_t *csd, smp_call_func_t func)
325−{
326− csd->flags = 0;
327− csd->func = func;
328− csd->info = rq;
329−}
330−
331323 #ifdef CONFIG_SCHED_HRTICK
332324 /*
333325 * Use HR-timers to deliver accurate preemption points.
@@ -428,7 +420,7 @@ void hrtick_start(struct rq *rq, u64 delay) · 94 unmodified lines
428420 static void hrtick_rq_init(struct rq *rq)
429421 {
430422 #ifdef CONFIG_SMP
431− rq_csd_init(rq, &rq->hrtick_csd, __hrtick_start);
423+ INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq);
432424 #endif
433425 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
434426 rq->hrtick_timer.function = hrtick;
@@ -7774,7 +7766,7 @@ void __init sched_init(void) · 7339 unmodified lines
77747766 rq->last_blocked_load_update_tick = jiffies;
77757767 atomic_set(&rq->nohz_flags, 0);
77767768
7777− rq_csd_init(rq, &rq->nohz_csd, nohz_csd_func);
7769+ INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq);
77787770 #endif
77797771 #ifdef CONFIG_HOTPLUG_CPU
77807772 rcuwait_init(&rq->hotplug_wait);
kernel/smp.c+25−25
@@ -27,7 +27,7 @@ · 26 unmodified lines
2727 #include "smpboot.h"
2828 #include "sched/smp.h"
2929
30−#define CSD_TYPE(_csd) ((_csd)->flags & CSD_FLAG_TYPE_MASK)
30+#define CSD_TYPE(_csd) ((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK)
3131
3232 struct call_function_data {
3333 call_single_data_t __percpu *csd;
@@ -146,7 +146,7 @@ static __always_inline bool csd_lock_wait_toolong(call_single_data_t *csd, u64 t · 112 unmodified lines
146146 bool firsttime;
147147 u64 ts2, ts_delta;
148148 call_single_data_t *cpu_cur_csd;
149− unsigned int flags = READ_ONCE(csd->flags);
149+ unsigned int flags = READ_ONCE(csd->node.u_flags);
150150
151151 if (!(flags & CSD_FLAG_LOCK)) {
152152 if (!unlikely(*bug_id))
@@ -224,14 +224,14 @@ static void csd_lock_record(call_single_data_t *csd) · 71 unmodified lines
224224
225225 static __always_inline void csd_lock_wait(call_single_data_t *csd)
226226 {
227− smp_cond_load_acquire(&csd->flags, !(VAL & CSD_FLAG_LOCK));
227+ smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
228228 }
229229 #endif
230230
231231 static __always_inline void csd_lock(call_single_data_t *csd)
232232 {
233233 csd_lock_wait(csd);
234− csd->flags |= CSD_FLAG_LOCK;
234+ csd->node.u_flags |= CSD_FLAG_LOCK;
235235
236236 /*
237237 * prevent CPU from reordering the above assignment
@@ -243,12 +243,12 @@ static __always_inline void csd_lock(call_single_data_t *csd) · 5 unmodified lines
243243
244244 static __always_inline void csd_unlock(call_single_data_t *csd)
245245 {
246− WARN_ON(!(csd->flags & CSD_FLAG_LOCK));
246+ WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK));
247247
248248 /*
249249 * ensure we're all done before releasing data:
250250 */
251− smp_store_release(&csd->flags, 0);
251+ smp_store_release(&csd->node.u_flags, 0);
252252 }
253253
254254 static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
@@ -300,7 +300,7 @@ static int generic_exec_single(int cpu, call_single_data_t *csd) · 45 unmodified lines
300300 return -ENXIO;
301301 }
302302
303− __smp_call_single_queue(cpu, &csd->llist);
303+ __smp_call_single_queue(cpu, &csd->node.llist);
304304
305305 return 0;
306306 }
@@ -353,7 +353,7 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) · 46 unmodified lines
353353 * We don't have to use the _safe() variant here
354354 * because we are not invoking the IPI handlers yet.
355355 */
356− llist_for_each_entry(csd, entry, llist) {
356+ llist_for_each_entry(csd, entry, node.llist) {
357357 switch (CSD_TYPE(csd)) {
358358 case CSD_TYPE_ASYNC:
359359 case CSD_TYPE_SYNC:
@@ -378,16 +378,16 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) · 18 unmodified lines
378378 * First; run all SYNC callbacks, people are waiting for us.
379379 */
380380 prev = NULL;
381− llist_for_each_entry_safe(csd, csd_next, entry, llist) {
381+ llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {
382382 /* Do we wait until *after* callback? */
383383 if (CSD_TYPE(csd) == CSD_TYPE_SYNC) {
384384 smp_call_func_t func = csd->func;
385385 void *info = csd->info;
386386
387387 if (prev) {
388− prev->next = &csd_next->llist;
388+ prev->next = &csd_next->node.llist;
389389 } else {
390− entry = &csd_next->llist;
390+ entry = &csd_next->node.llist;
391391 }
392392
393393 csd_lock_record(csd);
@@ -395,7 +395,7 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) · 1 unmodified line
395395 csd_unlock(csd);
396396 csd_lock_record(NULL);
397397 } else {
398− prev = &csd->llist;
398+ prev = &csd->node.llist;
399399 }
400400 }
401401
@@ -406,14 +406,14 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) · 4 unmodified lines
406406 * Second; run all !SYNC callbacks.
407407 */
408408 prev = NULL;
409− llist_for_each_entry_safe(csd, csd_next, entry, llist) {
409+ llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {
410410 int type = CSD_TYPE(csd);
411411
412412 if (type != CSD_TYPE_TTWU) {
413413 if (prev) {
414− prev->next = &csd_next->llist;
414+ prev->next = &csd_next->node.llist;
415415 } else {
416− entry = &csd_next->llist;
416+ entry = &csd_next->node.llist;
417417 }
418418
419419 if (type == CSD_TYPE_ASYNC) {
@@ -429,7 +429,7 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) · 9 unmodified lines
429429 }
430430
431431 } else {
432− prev = &csd->llist;
432+ prev = &csd->node.llist;
433433 }
434434 }
435435
@@ -465,7 +465,7 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, · 29 unmodified lines
465465 {
466466 call_single_data_t *csd;
467467 call_single_data_t csd_stack = {
468− .flags = CSD_FLAG_LOCK | CSD_TYPE_SYNC,
468+ .node = { .u_flags = CSD_FLAG_LOCK | CSD_TYPE_SYNC, },
469469 };
470470 int this_cpu;
471471 int err;
@@ -502,8 +502,8 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, · 30 unmodified lines
502502 csd->func = func;
503503 csd->info = info;
504504 #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
505− csd->src = smp_processor_id();
506− csd->dst = cpu;
505+ csd->node.src = smp_processor_id();
506+ csd->node.dst = cpu;
507507 #endif
508508
509509 err = generic_exec_single(cpu, csd);
@@ -544,12 +544,12 @@ int smp_call_function_single_async(int cpu, call_single_data_t *csd) · 34 unmodified lines
544544
545545 preempt_disable();
546546
547− if (csd->flags & CSD_FLAG_LOCK) {
547+ if (csd->node.u_flags & CSD_FLAG_LOCK) {
548548 err = -EBUSY;
549549 goto out;
550550 }
551551
552− csd->flags = CSD_FLAG_LOCK;
552+ csd->node.u_flags = CSD_FLAG_LOCK;
553553 smp_wmb();
554554
555555 err = generic_exec_single(cpu, csd);
@@ -667,14 +667,14 @@ static void smp_call_function_many_cond(const struct cpumask *mask, · 111 unmodified lines
667667
668668 csd_lock(csd);
669669 if (wait)
670− csd->flags |= CSD_TYPE_SYNC;
670+ csd->node.u_flags |= CSD_TYPE_SYNC;
671671 csd->func = func;
672672 csd->info = info;
673673 #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
674− csd->src = smp_processor_id();
675− csd->dst = cpu;
674+ csd->node.src = smp_processor_id();
675+ csd->node.dst = cpu;
676676 #endif
677− if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
677+ if (llist_add(&csd->node.llist, &per_cpu(call_single_queue, cpu)))
678678 __cpumask_set_cpu(cpu, cfd->cpumask_ipi);
679679 }
680680

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t