Just like the ttwu_queue_remote() IPI, make use of _TIF_POLLING_NRFLAG to avoid sending IPIs to idle CPUs. [ mingo: Fix UP build bug. ]
Showing the changes to kernel (4 of 4 files). Show all
Unified · Split
| @@ -2296,6 +2296,16 @@ static void wake_csd_func(void *info) · 2295 unmodified lines | |||
| 2296 | sched_ttwu_pending(); | 2296 | sched_ttwu_pending(); |
| 2297 | } | 2297 | } |
| 2298 | 2298 | ||
| 2299 | void send_call_function_single_ipi(int cpu) | ||
| 2300 | { | ||
| 2301 | struct rq *rq = cpu_rq(cpu); | ||
| 2302 | |||
| 2303 | if (!set_nr_if_polling(rq->idle)) | ||
| 2304 | arch_send_call_function_single_ipi(cpu); | ||
| 2305 | else | ||
| 2306 | trace_sched_wake_idle_without_ipi(cpu); | ||
| 2307 | } | ||
| 2308 | |||
| 2299 | /* | 2309 | /* |
| 2300 | * Queue a task on the target CPUs wake_list and wake the CPU via IPI if | 2310 | * Queue a task on the target CPUs wake_list and wake the CPU via IPI if |
| 2301 | * necessary. The wakee CPU on receipt of the IPI will queue the task | 2311 | * necessary. The wakee CPU on receipt of the IPI will queue the task |
| @@ -289,6 +289,11 @@ static void do_idle(void) · 288 unmodified lines | |||
| 289 | */ | 289 | */ |
| 290 | smp_mb__after_atomic(); | 290 | smp_mb__after_atomic(); |
| 291 | 291 | ||
| 292 | /* | ||
| 293 | * RCU relies on this call to be done outside of an RCU read-side | ||
| 294 | * critical section. | ||
| 295 | */ | ||
| 296 | flush_smp_call_function_from_idle(); | ||
| 292 | sched_ttwu_pending(); | 297 | sched_ttwu_pending(); |
| 293 | schedule_idle(); | 298 | schedule_idle(); |
| 294 | 299 | ||
| @@ -1506,11 +1506,12 @@ static inline void unregister_sched_domain_sysctl(void) · 1505 unmodified lines | |||
| 1506 | } | 1506 | } |
| 1507 | #endif | 1507 | #endif |
| 1508 | 1508 | ||
| 1509 | #else | 1509 | extern void flush_smp_call_function_from_idle(void); |
| 1510 | 1510 | ||
| 1511 | #else /* !CONFIG_SMP: */ | ||
| 1512 | static inline void flush_smp_call_function_from_idle(void) { } | ||
| 1511 | static inline void sched_ttwu_pending(void) { } | 1513 | static inline void sched_ttwu_pending(void) { } |
| 1512 | 1514 | #endif | |
| 1513 | #endif /* CONFIG_SMP */ | ||
| 1514 | 1515 | ||
| 1515 | #include "stats.h" | 1516 | #include "stats.h" |
| 1516 | #include "autogroup.h" | 1517 | #include "autogroup.h" |
| @@ -135,6 +135,8 @@ static __always_inline void csd_unlock(call_single_data_t *csd) · 134 unmodified lines | |||
| 135 | 135 | ||
| 136 | static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data); | 136 | static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data); |
| 137 | 137 | ||
| 138 | extern void send_call_function_single_ipi(int cpu); | ||
| 139 | |||
| 138 | /* | 140 | /* |
| 139 | * Insert a previously allocated call_single_data_t element | 141 | * Insert a previously allocated call_single_data_t element |
| 140 | * for execution on the given CPU. data must already have | 142 | * for execution on the given CPU. data must already have |
| @@ -178,7 +180,7 @@ static int generic_exec_single(int cpu, call_single_data_t *csd, · 37 unmodified lines | |||
| 178 | * equipped to do the right thing... | 180 | * equipped to do the right thing... |
| 179 | */ | 181 | */ |
| 180 | if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu))) | 182 | if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu))) |
| 181 | arch_send_call_function_single_ipi(cpu); | 183 | send_call_function_single_ipi(cpu); |
| 182 | 184 | ||
| 183 | return 0; | 185 | return 0; |
| 184 | } | 186 | } |
| @@ -278,6 +280,18 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) · 93 unmodified lines | |||
| 278 | } | 280 | } |
| 279 | } | 281 | } |
| 280 | 282 | ||
| 283 | void flush_smp_call_function_from_idle(void) | ||
| 284 | { | ||
| 285 | unsigned long flags; | ||
| 286 | |||
| 287 | if (llist_empty(this_cpu_ptr(&call_single_queue))) | ||
| 288 | return; | ||
| 289 | |||
| 290 | local_irq_save(flags); | ||
| 291 | flush_smp_call_function_queue(true); | ||
| 292 | local_irq_restore(flags); | ||
| 293 | } | ||
| 294 | |||
| 281 | /* | 295 | /* |
| 282 | * smp_call_function_single - Run a function on a specific CPU | 296 | * smp_call_function_single - Run a function on a specific CPU |
| 283 | * @func: The function to run. This must be fast and non-blocking. | 297 | * @func: The function to run. This must be fast and non-blocking. |