Just like the ttwu_queue_remote() IPI, make use of _TIF_POLLING_NRFLAG to avoid sending IPIs to idle CPUs. [ mingo: Fix UP build bug. ]
Unified · Split
| @@ -2296,6 +2296,16 @@ static void wake_csd_func(void *info) · 2295 unmodified lines | ||
| 2296 | 2296 | sched_ttwu_pending(); |
| 2297 | 2297 | } |
| 2298 | 2298 | |
| 2299 | +void send_call_function_single_ipi(int cpu) | |
| 2300 | +{ | |
| 2301 | + struct rq *rq = cpu_rq(cpu); | |
| 2302 | + | |
| 2303 | + if (!set_nr_if_polling(rq->idle)) | |
| 2304 | + arch_send_call_function_single_ipi(cpu); | |
| 2305 | + else | |
| 2306 | + trace_sched_wake_idle_without_ipi(cpu); | |
| 2307 | +} | |
| 2308 | + | |
| 2299 | 2309 | /* |
| 2300 | 2310 | * Queue a task on the target CPUs wake_list and wake the CPU via IPI if |
| 2301 | 2311 | * necessary. The wakee CPU on receipt of the IPI will queue the task |
| @@ -289,6 +289,11 @@ static void do_idle(void) · 288 unmodified lines | ||
| 289 | 289 | */ |
| 290 | 290 | smp_mb__after_atomic(); |
| 291 | 291 | |
| 292 | + /* | |
| 293 | + * RCU relies on this call to be done outside of an RCU read-side | |
| 294 | + * critical section. | |
| 295 | + */ | |
| 296 | + flush_smp_call_function_from_idle(); | |
| 292 | 297 | sched_ttwu_pending(); |
| 293 | 298 | schedule_idle(); |
| 294 | 299 | |
| @@ -1506,11 +1506,12 @@ static inline void unregister_sched_domain_sysctl(void) · 1505 unmodified lines | ||
| 1506 | 1506 | } |
| 1507 | 1507 | #endif |
| 1508 | 1508 | |
| 1509 | −#else | |
| 1509 | +extern void flush_smp_call_function_from_idle(void); | |
| 1510 | 1510 | |
| 1511 | +#else /* !CONFIG_SMP: */ | |
| 1512 | +static inline void flush_smp_call_function_from_idle(void) { } | |
| 1511 | 1513 | static inline void sched_ttwu_pending(void) { } |
| 1512 | − | |
| 1513 | −#endif /* CONFIG_SMP */ | |
| 1514 | +#endif | |
| 1514 | 1515 | |
| 1515 | 1516 | #include "stats.h" |
| 1516 | 1517 | #include "autogroup.h" |
| @@ -135,6 +135,8 @@ static __always_inline void csd_unlock(call_single_data_t *csd) · 134 unmodified lines | ||
| 135 | 135 | |
| 136 | 136 | static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data); |
| 137 | 137 | |
| 138 | +extern void send_call_function_single_ipi(int cpu); | |
| 139 | + | |
| 138 | 140 | /* |
| 139 | 141 | * Insert a previously allocated call_single_data_t element |
| 140 | 142 | * for execution on the given CPU. data must already have |
| @@ -178,7 +180,7 @@ static int generic_exec_single(int cpu, call_single_data_t *csd, · 37 unmodified lines | ||
| 178 | 180 | * equipped to do the right thing... |
| 179 | 181 | */ |
| 180 | 182 | if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu))) |
| 181 | − arch_send_call_function_single_ipi(cpu); | |
| 183 | + send_call_function_single_ipi(cpu); | |
| 182 | 184 | |
| 183 | 185 | return 0; |
| 184 | 186 | } |
| @@ -278,6 +280,18 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) · 93 unmodified lines | ||
| 278 | 280 | } |
| 279 | 281 | } |
| 280 | 282 | |
| 283 | +void flush_smp_call_function_from_idle(void) | |
| 284 | +{ | |
| 285 | + unsigned long flags; | |
| 286 | + | |
| 287 | + if (llist_empty(this_cpu_ptr(&call_single_queue))) | |
| 288 | + return; | |
| 289 | + | |
| 290 | + local_irq_save(flags); | |
| 291 | + flush_smp_call_function_queue(true); | |
| 292 | + local_irq_restore(flags); | |
| 293 | +} | |
| 294 | + | |
| 281 | 295 | /* |
| 282 | 296 | * smp_call_function_single - Run a function on a specific CPU |
| 283 | 297 | * @func: The function to run. This must be fast and non-blocking. |