All cpumasks are assumed to have cpu 0 permanently set on UP, so it can't be used to signify whether there's something to be done for the CPU. workqueue was using cpumask to track which CPU requested rescuer assistance and this led rescuer thread to think there always are pending mayday requests on UP, which resulted in infinite busy loops. This patch fixes the problem by introducing mayday_mask_t and associated helpers which wrap cpumask on SMP and emulates its behavior using bitops and unsigned long on UP.
Showing the changes to kernel/workqueue.c (1 of 1 file). Show all
Unified · Split
| @@ -185,6 +185,27 @@ struct wq_flusher { · 184 unmodified lines | ||
| 185 | 185 | struct completion done; /* flush completion */ |
| 186 | 186 | }; |
| 187 | 187 | |
| 188 | +/* | |
| 189 | + * All cpumasks are assumed to be always set on UP and thus can't be | |
| 190 | + * used to determine whether there's something to be done. | |
| 191 | + */ | |
| 192 | +#ifdef CONFIG_SMP | |
| 193 | +typedef cpumask_var_t mayday_mask_t; | |
| 194 | +#define mayday_test_and_set_cpu(cpu, mask) \ | |
| 195 | + cpumask_test_and_set_cpu((cpu), (mask)) | |
| 196 | +#define mayday_clear_cpu(cpu, mask) cpumask_clear_cpu((cpu), (mask)) | |
| 197 | +#define for_each_mayday_cpu(cpu, mask) for_each_cpu((cpu), (mask)) | |
| 198 | +#define alloc_mayday_mask(maskp, gfp) alloc_cpumask_var((maskp), (gfp)) | |
| 199 | +#define free_mayday_mask(mask) free_cpumask_var((mask)) | |
| 200 | +#else | |
| 201 | +typedef unsigned long mayday_mask_t; | |
| 202 | +#define mayday_test_and_set_cpu(cpu, mask) test_and_set_bit(0, &(mask)) | |
| 203 | +#define mayday_clear_cpu(cpu, mask) clear_bit(0, &(mask)) | |
| 204 | +#define for_each_mayday_cpu(cpu, mask) if ((cpu) = 0, (mask)) | |
| 205 | +#define alloc_mayday_mask(maskp, gfp) true | |
| 206 | +#define free_mayday_mask(mask) do { } while (0) | |
| 207 | +#endif | |
| 208 | + | |
| 188 | 209 | /* |
| 189 | 210 | * The externally visible workqueue abstraction is an array of |
| 190 | 211 | * per-CPU workqueues: |
| @@ -206,7 +227,7 @@ struct workqueue_struct { · 15 unmodified lines | ||
| 206 | 227 | struct list_head flusher_queue; /* F: flush waiters */ |
| 207 | 228 | struct list_head flusher_overflow; /* F: flush overflow list */ |
| 208 | 229 | |
| 209 | − cpumask_var_t mayday_mask; /* cpus requesting rescue */ | |
| 230 | + mayday_mask_t mayday_mask; /* cpus requesting rescue */ | |
| 210 | 231 | struct worker *rescuer; /* I: rescue worker */ |
| 211 | 232 | |
| 212 | 233 | int saved_max_active; /* W: saved cwq max_active */ |
| @@ -1387,7 +1408,7 @@ static bool send_mayday(struct work_struct *work) · 1174 unmodified lines | ||
| 1387 | 1408 | /* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */ |
| 1388 | 1409 | if (cpu == WORK_CPU_UNBOUND) |
| 1389 | 1410 | cpu = 0; |
| 1390 | − if (!cpumask_test_and_set_cpu(cpu, wq->mayday_mask)) | |
| 1411 | + if (!mayday_test_and_set_cpu(cpu, wq->mayday_mask)) | |
| 1391 | 1412 | wake_up_process(wq->rescuer->task); |
| 1392 | 1413 | return true; |
| 1393 | 1414 | } |
| @@ -1915,14 +1936,14 @@ repeat: · 521 unmodified lines | ||
| 1915 | 1936 | * See whether any cpu is asking for help. Unbounded |
| 1916 | 1937 | * workqueues use cpu 0 in mayday_mask for CPU_UNBOUND. |
| 1917 | 1938 | */ |
| 1918 | − for_each_cpu(cpu, wq->mayday_mask) { | |
| 1939 | + for_each_mayday_cpu(cpu, wq->mayday_mask) { | |
| 1919 | 1940 | unsigned int tcpu = is_unbound ? WORK_CPU_UNBOUND : cpu; |
| 1920 | 1941 | struct cpu_workqueue_struct *cwq = get_cwq(tcpu, wq); |
| 1921 | 1942 | struct global_cwq *gcwq = cwq->gcwq; |
| 1922 | 1943 | struct work_struct *work, *n; |
| 1923 | 1944 | |
| 1924 | 1945 | __set_current_state(TASK_RUNNING); |
| 1925 | − cpumask_clear_cpu(cpu, wq->mayday_mask); | |
| 1946 | + mayday_clear_cpu(cpu, wq->mayday_mask); | |
| 1926 | 1947 | |
| 1927 | 1948 | /* migrate to the target cpu if possible */ |
| 1928 | 1949 | rescuer->gcwq = gcwq; |
| @@ -2724,7 +2745,7 @@ struct workqueue_struct *__alloc_workqueue_key(const char *name, · 795 unmodified lines | ||
| 2724 | 2745 | if (flags & WQ_RESCUER) { |
| 2725 | 2746 | struct worker *rescuer; |
| 2726 | 2747 | |
| 2727 | − if (!alloc_cpumask_var(&wq->mayday_mask, GFP_KERNEL)) | |
| 2748 | + if (!alloc_mayday_mask(&wq->mayday_mask, GFP_KERNEL)) | |
| 2728 | 2749 | goto err; |
| 2729 | 2750 | |
| 2730 | 2751 | wq->rescuer = rescuer = alloc_worker(); |
| @@ -2759,7 +2780,7 @@ struct workqueue_struct *__alloc_workqueue_key(const char *name, · 28 unmodified lines | ||
| 2759 | 2780 | err: |
| 2760 | 2781 | if (wq) { |
| 2761 | 2782 | free_cwqs(wq); |
| 2762 | − free_cpumask_var(wq->mayday_mask); | |
| 2783 | + free_mayday_mask(wq->mayday_mask); | |
| 2763 | 2784 | kfree(wq->rescuer); |
| 2764 | 2785 | kfree(wq); |
| 2765 | 2786 | } |
| @@ -2800,7 +2821,7 @@ void destroy_workqueue(struct workqueue_struct *wq) · 34 unmodified lines | ||
| 2800 | 2821 | |
| 2801 | 2822 | if (wq->flags & WQ_RESCUER) { |
| 2802 | 2823 | kthread_stop(wq->rescuer->task); |
| 2803 | − free_cpumask_var(wq->mayday_mask); | |
| 2824 | + free_mayday_mask(wq->mayday_mask); | |
| 2804 | 2825 | } |
| 2805 | 2826 | |
| 2806 | 2827 | free_cwqs(wq); |