Workqueue now automatically marks per-cpu work items that hog CPU for too long as CPU_INTENSIVE, which excludes them from concurrency management and prevents stalling other concurrency-managed work items. If a work function keeps running over the thershold, it likely needs to be switched to use an unbound workqueue. This patch adds a debug mechanism which tracks the work functions which trigger the automatic CPU_INTENSIVE mechanism and report them using pr_warn() with exponential backoff. v3: Documentation update. v2: Drop bouncing to kthread_worker for printing messages. It was to avoid introducing circular locking dependency through printk but not effective as it still had pool lock -> wci_lock -> printk -> pool lock loop. Let's just print directly using printk_deferred().
Showing the changes to kernel/workqueue.c (1 of 3 files). Show all
Unified · Split
| @@ -948,6 +948,98 @@ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) · 947 unmodified lines | |||
| 948 | pool->nr_running++; | 948 | pool->nr_running++; |
| 949 | } | 949 | } |
| 950 | 950 | ||
| 951 | #ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT | ||
| 952 | |||
| 953 | /* | ||
| 954 | * Concurrency-managed per-cpu work items that hog CPU for longer than | ||
| 955 | * wq_cpu_intensive_thresh_us trigger the automatic CPU_INTENSIVE mechanism, | ||
| 956 | * which prevents them from stalling other concurrency-managed work items. If a | ||
| 957 | * work function keeps triggering this mechanism, it's likely that the work item | ||
| 958 | * should be using an unbound workqueue instead. | ||
| 959 | * | ||
| 960 | * wq_cpu_intensive_report() tracks work functions which trigger such conditions | ||
| 961 | * and report them so that they can be examined and converted to use unbound | ||
| 962 | * workqueues as appropriate. To avoid flooding the console, each violating work | ||
| 963 | * function is tracked and reported with exponential backoff. | ||
| 964 | */ | ||
| 965 | #define WCI_MAX_ENTS 128 | ||
| 966 | |||
| 967 | struct wci_ent { | ||
| 968 | work_func_t func; | ||
| 969 | atomic64_t cnt; | ||
| 970 | struct hlist_node hash_node; | ||
| 971 | }; | ||
| 972 | |||
| 973 | static struct wci_ent wci_ents[WCI_MAX_ENTS]; | ||
| 974 | static int wci_nr_ents; | ||
| 975 | static DEFINE_RAW_SPINLOCK(wci_lock); | ||
| 976 | static DEFINE_HASHTABLE(wci_hash, ilog2(WCI_MAX_ENTS)); | ||
| 977 | |||
| 978 | static struct wci_ent *wci_find_ent(work_func_t func) | ||
| 979 | { | ||
| 980 | struct wci_ent *ent; | ||
| 981 | |||
| 982 | hash_for_each_possible_rcu(wci_hash, ent, hash_node, | ||
| 983 | (unsigned long)func) { | ||
| 984 | if (ent->func == func) | ||
| 985 | return ent; | ||
| 986 | } | ||
| 987 | return NULL; | ||
| 988 | } | ||
| 989 | |||
| 990 | static void wq_cpu_intensive_report(work_func_t func) | ||
| 991 | { | ||
| 992 | struct wci_ent *ent; | ||
| 993 | |||
| 994 | restart: | ||
| 995 | ent = wci_find_ent(func); | ||
| 996 | if (ent) { | ||
| 997 | u64 cnt; | ||
| 998 | |||
| 999 | /* | ||
| 1000 | * Start reporting from the fourth time and back off | ||
| 1001 | * exponentially. | ||
| 1002 | */ | ||
| 1003 | cnt = atomic64_inc_return_relaxed(&ent->cnt); | ||
| 1004 | if (cnt >= 4 && is_power_of_2(cnt)) | ||
| 1005 | printk_deferred(KERN_WARNING "workqueue: %ps hogged CPU for >%luus %llu times, consider switching to WQ_UNBOUND\n", | ||
| 1006 | ent->func, wq_cpu_intensive_thresh_us, | ||
| 1007 | atomic64_read(&ent->cnt)); | ||
| 1008 | return; | ||
| 1009 | } | ||
| 1010 | |||
| 1011 | /* | ||
| 1012 | * @func is a new violation. Allocate a new entry for it. If wcn_ents[] | ||
| 1013 | * is exhausted, something went really wrong and we probably made enough | ||
| 1014 | * noise already. | ||
| 1015 | */ | ||
| 1016 | if (wci_nr_ents >= WCI_MAX_ENTS) | ||
| 1017 | return; | ||
| 1018 | |||
| 1019 | raw_spin_lock(&wci_lock); | ||
| 1020 | |||
| 1021 | if (wci_nr_ents >= WCI_MAX_ENTS) { | ||
| 1022 | raw_spin_unlock(&wci_lock); | ||
| 1023 | return; | ||
| 1024 | } | ||
| 1025 | |||
| 1026 | if (wci_find_ent(func)) { | ||
| 1027 | raw_spin_unlock(&wci_lock); | ||
| 1028 | goto restart; | ||
| 1029 | } | ||
| 1030 | |||
| 1031 | ent = &wci_ents[wci_nr_ents++]; | ||
| 1032 | ent->func = func; | ||
| 1033 | atomic64_set(&ent->cnt, 1); | ||
| 1034 | hash_add_rcu(wci_hash, &ent->hash_node, (unsigned long)func); | ||
| 1035 | |||
| 1036 | raw_spin_unlock(&wci_lock); | ||
| 1037 | } | ||
| 1038 | |||
| 1039 | #else /* CONFIG_WQ_CPU_INTENSIVE_REPORT */ | ||
| 1040 | static void wq_cpu_intensive_report(work_func_t func) {} | ||
| 1041 | #endif /* CONFIG_WQ_CPU_INTENSIVE_REPORT */ | ||
| 1042 | |||
| 951 | /** | 1043 | /** |
| 952 | * wq_worker_running - a worker is running again | 1044 | * wq_worker_running - a worker is running again |
| 953 | * @task: task waking up | 1045 | * @task: task waking up |
| @@ -1057,6 +1149,7 @@ void wq_worker_tick(struct task_struct *task) · 103 unmodified lines | |||
| 1057 | raw_spin_lock(&pool->lock); | 1149 | raw_spin_lock(&pool->lock); |
| 1058 | 1150 | ||
| 1059 | worker_set_flags(worker, WORKER_CPU_INTENSIVE); | 1151 | worker_set_flags(worker, WORKER_CPU_INTENSIVE); |
| 1152 | wq_cpu_intensive_report(worker->current_func); | ||
| 1060 | pwq->stats[PWQ_STAT_CPU_INTENSIVE]++; | 1153 | pwq->stats[PWQ_STAT_CPU_INTENSIVE]++; |
| 1061 | 1154 | ||
| 1062 | if (need_more_worker(pool)) { | 1155 | if (need_more_worker(pool)) { |