History

smp: Optimize send_call_function_single_ipi()

Just like the ttwu_queue_remote() IPI, make use of _TIF_POLLING_NRFLAG to avoid sending IPIs to idle CPUs. [ mingo: Fix UP build bug. ]

Peter Zijlstra committed 6 years ago

Commit b2a02fc1 parent afaa6534 files · +34 −4Browse files

Showing the changes to kernel (4 of 4 files). Show all

Unified · Split

4 files
  1. kernel/sched/core.c +10 −0
  2. kernel/sched/idle.c +5 −0
  3. kernel/sched/sched.h +4 −3
  4. kernel/smp.c +15 −1
kernel/sched/core.c+10−0
@@ -2296,6 +2296,16 @@ static void wake_csd_func(void *info) · 2295 unmodified lines
22962296 sched_ttwu_pending();
22972297 }
22982298
2299+void send_call_function_single_ipi(int cpu)
2300+{
2301+ struct rq *rq = cpu_rq(cpu);
2302+
2303+ if (!set_nr_if_polling(rq->idle))
2304+ arch_send_call_function_single_ipi(cpu);
2305+ else
2306+ trace_sched_wake_idle_without_ipi(cpu);
2307+}
2308+
22992309 /*
23002310 * Queue a task on the target CPUs wake_list and wake the CPU via IPI if
23012311 * necessary. The wakee CPU on receipt of the IPI will queue the task
kernel/sched/idle.c+5−0
@@ -289,6 +289,11 @@ static void do_idle(void) · 288 unmodified lines
289289 */
290290 smp_mb__after_atomic();
291291
292+ /*
293+ * RCU relies on this call to be done outside of an RCU read-side
294+ * critical section.
295+ */
296+ flush_smp_call_function_from_idle();
292297 sched_ttwu_pending();
293298 schedule_idle();
294299
kernel/sched/sched.h+4−3
@@ -1506,11 +1506,12 @@ static inline void unregister_sched_domain_sysctl(void) · 1505 unmodified lines
15061506 }
15071507 #endif
15081508
1509−#else
1509+extern void flush_smp_call_function_from_idle(void);
15101510
1511+#else /* !CONFIG_SMP: */
1512+static inline void flush_smp_call_function_from_idle(void) { }
15111513 static inline void sched_ttwu_pending(void) { }
1512−
1513−#endif /* CONFIG_SMP */
1514+#endif
15141515
15151516 #include "stats.h"
15161517 #include "autogroup.h"
kernel/smp.c+15−1
@@ -135,6 +135,8 @@ static __always_inline void csd_unlock(call_single_data_t *csd) · 134 unmodified lines
135135
136136 static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
137137
138+extern void send_call_function_single_ipi(int cpu);
139+
138140 /*
139141 * Insert a previously allocated call_single_data_t element
140142 * for execution on the given CPU. data must already have
@@ -178,7 +180,7 @@ static int generic_exec_single(int cpu, call_single_data_t *csd, · 37 unmodified lines
178180 * equipped to do the right thing...
179181 */
180182 if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
181− arch_send_call_function_single_ipi(cpu);
183+ send_call_function_single_ipi(cpu);
182184
183185 return 0;
184186 }
@@ -278,6 +280,18 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) · 93 unmodified lines
278280 }
279281 }
280282
283+void flush_smp_call_function_from_idle(void)
284+{
285+ unsigned long flags;
286+
287+ if (llist_empty(this_cpu_ptr(&call_single_queue)))
288+ return;
289+
290+ local_irq_save(flags);
291+ flush_smp_call_function_queue(true);
292+ local_irq_restore(flags);
293+}
294+
281295 /*
282296 * smp_call_function_single - Run a function on a specific CPU
283297 * @func: The function to run. This must be fast and non-blocking.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t