History

smp: Optimize send_call_function_single_ipi()

Just like the ttwu_queue_remote() IPI, make use of _TIF_POLLING_NRFLAG to avoid sending IPIs to idle CPUs. [ mingo: Fix UP build bug. ]

Peter Zijlstra committed 6 years ago

Commit b2a02fc1 parent afaa6534 files · +34 −4Browse files

Showing the changes to kernel (4 of 4 files). Show all

Unified · Split

4 files
  1. kernel/sched/core.c +10 −0
  2. kernel/sched/idle.c +5 −0
  3. kernel/sched/sched.h +4 −3
  4. kernel/smp.c +15 −1
kernel/sched/core.c+10−0
@@ -2296,6 +2296,16 @@ static void wake_csd_func(void *info) · 2295 unmodified lines
2296 sched_ttwu_pending();2296 sched_ttwu_pending();
2297}2297}
2298 2298
2299void send_call_function_single_ipi(int cpu)
2300{
2301 struct rq *rq = cpu_rq(cpu);
2302
2303 if (!set_nr_if_polling(rq->idle))
2304 arch_send_call_function_single_ipi(cpu);
2305 else
2306 trace_sched_wake_idle_without_ipi(cpu);
2307}
2308
2299/*2309/*
2300 * Queue a task on the target CPUs wake_list and wake the CPU via IPI if2310 * Queue a task on the target CPUs wake_list and wake the CPU via IPI if
2301 * necessary. The wakee CPU on receipt of the IPI will queue the task2311 * necessary. The wakee CPU on receipt of the IPI will queue the task
kernel/sched/idle.c+5−0
@@ -289,6 +289,11 @@ static void do_idle(void) · 288 unmodified lines
289 */289 */
290 smp_mb__after_atomic();290 smp_mb__after_atomic();
291 291
292 /*
293 * RCU relies on this call to be done outside of an RCU read-side
294 * critical section.
295 */
296 flush_smp_call_function_from_idle();
292 sched_ttwu_pending();297 sched_ttwu_pending();
293 schedule_idle();298 schedule_idle();
294 299
kernel/sched/sched.h+4−3
@@ -1506,11 +1506,12 @@ static inline void unregister_sched_domain_sysctl(void) · 1505 unmodified lines
1506}1506}
1507#endif1507#endif
1508 1508
1509#else1509extern void flush_smp_call_function_from_idle(void);
1510 1510
1511#else /* !CONFIG_SMP: */
1512static inline void flush_smp_call_function_from_idle(void) { }
1511static inline void sched_ttwu_pending(void) { }1513static inline void sched_ttwu_pending(void) { }
1512 1514#endif
1513#endif /* CONFIG_SMP */
1514 1515
1515#include "stats.h"1516#include "stats.h"
1516#include "autogroup.h"1517#include "autogroup.h"
kernel/smp.c+15−1
@@ -135,6 +135,8 @@ static __always_inline void csd_unlock(call_single_data_t *csd) · 134 unmodified lines
135 135
136static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);136static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
137 137
138extern void send_call_function_single_ipi(int cpu);
139
138/*140/*
139 * Insert a previously allocated call_single_data_t element141 * Insert a previously allocated call_single_data_t element
140 * for execution on the given CPU. data must already have142 * for execution on the given CPU. data must already have
@@ -178,7 +180,7 @@ static int generic_exec_single(int cpu, call_single_data_t *csd, · 37 unmodified lines
178 * equipped to do the right thing...180 * equipped to do the right thing...
179 */181 */
180 if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))182 if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
181 arch_send_call_function_single_ipi(cpu);183 send_call_function_single_ipi(cpu);
182 184
183 return 0;185 return 0;
184}186}
@@ -278,6 +280,18 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) · 93 unmodified lines
278 }280 }
279}281}
280 282
283void flush_smp_call_function_from_idle(void)
284{
285 unsigned long flags;
286
287 if (llist_empty(this_cpu_ptr(&call_single_queue)))
288 return;
289
290 local_irq_save(flags);
291 flush_smp_call_function_queue(true);
292 local_irq_restore(flags);
293}
294
281/*295/*
282 * smp_call_function_single - Run a function on a specific CPU296 * smp_call_function_single - Run a function on a specific CPU
283 * @func: The function to run. This must be fast and non-blocking.297 * @func: The function to run. This must be fast and non-blocking.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t