History

Merge branch 'for-6.10' into test-merge-for-6.10

Tejun Heo committed 2 years ago

Commit a2a58902 parents 8c06da6 51da7f64 files · +327 −193Browse files

Showing the changes to kernel/workqueue.c (1 of 4 files). Show all

Unified · Split

kernel/workqueue.c+275−178
@@ -99,6 +99,7 @@ enum worker_flags { · 98 unmodified lines
9999
100100 enum work_cancel_flags {
101101 WORK_CANCEL_DELAYED = 1 << 0, /* canceling a delayed_work */
102+ WORK_CANCEL_DISABLE = 1 << 1, /* canceling to disable */
102103 };
103104
104105 enum wq_internal_consts {
@@ -392,6 +393,12 @@ struct wq_pod_type { · 287 unmodified lines
392393 int *cpu_pod; /* cpu -> pod */
393394 };
394395
396+struct work_offq_data {
397+ u32 pool_id;
398+ u32 disable;
399+ u32 flags;
400+};
401+
395402 static const char *wq_affn_names[WQ_AFFN_NR_TYPES] = {
396403 [WQ_AFFN_DFL] = "default",
397404 [WQ_AFFN_CPU] = "cpu",
@@ -489,12 +496,6 @@ static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS]; · 91 unmodified lines
489496 /* I: attributes used when instantiating ordered pools on demand */
490497 static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
491498
492−/*
493− * Used to synchronize multiple cancel_sync attempts on the same work item. See
494− * work_grab_pending() and __cancel_work_sync().
495− */
496−static DECLARE_WAIT_QUEUE_HEAD(wq_cancel_waitq);
497−
498499 /*
499500 * I: kthread_worker to release pwq's. pwq release needs to be bounced to a
500501 * process context while holding a pool lock. Bounce to a dedicated kthread
@@ -763,6 +764,11 @@ static int work_next_color(int color) · 262 unmodified lines
763764 return (color + 1) % WORK_NR_COLORS;
764765 }
765766
767+static unsigned long pool_offq_flags(struct worker_pool *pool)
768+{
769+ return (pool->flags & POOL_BH) ? WORK_OFFQ_BH : 0;
770+}
771+
766772 /*
767773 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
768774 * contain the pointer to the queued pwq. Once execution starts, the flag
@@ -776,11 +782,6 @@ static int work_next_color(int color) · 7 unmodified lines
776782 * corresponding to a work. Pool is available once the work has been
777783 * queued anywhere after initialization until it is sync canceled. pwq is
778784 * available only while the work item is queued.
779− *
780− * %WORK_OFFQ_CANCELING is used to mark a work item which is being
781− * canceled. While being canceled, a work item may have its PENDING set
782− * but stay off timer and worklist for arbitrarily long and nobody should
783− * try to steal the PENDING bit.
784785 */
785786 static inline void set_work_data(struct work_struct *work, unsigned long data)
786787 {
@@ -892,36 +893,26 @@ static struct worker_pool *get_work_pool(struct work_struct *work) · 105 unmodified lines
892893 return idr_find(&worker_pool_idr, pool_id);
893894 }
894895
895−/**
896− * get_work_pool_id - return the worker pool ID a given work is associated with
897− * @work: the work item of interest
898− *
899− * Return: The worker_pool ID @work was last associated with.
900− * %WORK_OFFQ_POOL_NONE if none.
901− */
902−static int get_work_pool_id(struct work_struct *work)
896+static unsigned long shift_and_mask(unsigned long v, u32 shift, u32 bits)
903897 {
904− unsigned long data = atomic_long_read(&work->data);
905−
906− if (data & WORK_STRUCT_PWQ)
907− return work_struct_pwq(data)->pool->id;
908−
909− return data >> WORK_OFFQ_POOL_SHIFT;
898+ return (v >> shift) & ((1 << bits) - 1);
910899 }
911900
912−static void mark_work_canceling(struct work_struct *work)
901+static void work_offqd_unpack(struct work_offq_data *offqd, unsigned long data)
913902 {
914− unsigned long pool_id = get_work_pool_id(work);
903+ WARN_ON_ONCE(data & WORK_STRUCT_PWQ);
915904
916− pool_id <<= WORK_OFFQ_POOL_SHIFT;
917− set_work_data(work, pool_id | WORK_STRUCT_PENDING | WORK_OFFQ_CANCELING);
905+ offqd->pool_id = shift_and_mask(data, WORK_OFFQ_POOL_SHIFT,
906+ WORK_OFFQ_POOL_BITS);
907+ offqd->disable = shift_and_mask(data, WORK_OFFQ_DISABLE_SHIFT,
908+ WORK_OFFQ_DISABLE_BITS);
909+ offqd->flags = data & WORK_OFFQ_FLAG_MASK;
918910 }
919911
920−static bool work_is_canceling(struct work_struct *work)
912+static unsigned long work_offqd_pack_flags(struct work_offq_data *offqd)
921913 {
922− unsigned long data = atomic_long_read(&work->data);
923−
924− return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
914+ return ((unsigned long)offqd->disable << WORK_OFFQ_DISABLE_SHIFT) |
915+ ((unsigned long)offqd->flags);
925916 }
926917
927918 /*
@@ -2067,8 +2058,6 @@ out_put: · 1139 unmodified lines
20672058 * 1 if @work was pending and we successfully stole PENDING
20682059 * 0 if @work was idle and we claimed PENDING
20692060 * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
2070− * -ENOENT if someone else is canceling @work, this state may persist
2071− * for arbitrarily long
20722061 * ======== ================================================================
20732062 *
20742063 * Note:
@@ -2151,7 +2140,8 @@ static int try_to_grab_pending(struct work_struct *work, u32 cflags, · 76 unmodified lines
21512140 * this destroys work->data needed by the next step, stash it.
21522141 */
21532142 work_data = *work_data_bits(work);
2154− set_work_pool_and_keep_pending(work, pool->id, 0);
2143+ set_work_pool_and_keep_pending(work, pool->id,
2144+ pool_offq_flags(pool));
21552145
21562146 /* must be the last step, see the function comment */
21572147 pwq_dec_nr_in_flight(pwq, work_data);
@@ -2164,26 +2154,9 @@ static int try_to_grab_pending(struct work_struct *work, u32 cflags, · 6 unmodified lines
21642154 fail:
21652155 rcu_read_unlock();
21662156 local_irq_restore(*irq_flags);
2167− if (work_is_canceling(work))
2168− return -ENOENT;
2169− cpu_relax();
21702157 return -EAGAIN;
21712158 }
21722159
2173−struct cwt_wait {
2174− wait_queue_entry_t wait;
2175− struct work_struct *work;
2176−};
2177−
2178−static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
2179−{
2180− struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);
2181−
2182− if (cwait->work != key)
2183− return 0;
2184− return autoremove_wake_function(wait, mode, sync, key);
2185−}
2186−
21872160 /**
21882161 * work_grab_pending - steal work item from worklist and disable irq
21892162 * @work: work item to steal
@@ -2193,7 +2166,7 @@ static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *k · 3 unmodified lines
21932166 * Grab PENDING bit of @work. @work can be in any stable state - idle, on timer
21942167 * or on worklist.
21952168 *
2196− * Must be called in process context. IRQ is disabled on return with IRQ state
2169+ * Can be called from any context. IRQ is disabled on return with IRQ state
21972170 * stored in *@irq_flags. The caller is responsible for re-enabling it using
21982171 * local_irq_restore().
21992172 *
@@ -2202,41 +2175,14 @@ static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *k · 2 unmodified lines
22022175 static bool work_grab_pending(struct work_struct *work, u32 cflags,
22032176 unsigned long *irq_flags)
22042177 {
2205− struct cwt_wait cwait;
22062178 int ret;
22072179
2208− might_sleep();
2209−repeat:
2210− ret = try_to_grab_pending(work, cflags, irq_flags);
2211− if (likely(ret >= 0))
2212− return ret;
2213− if (ret != -ENOENT)
2214− goto repeat;
2215−
2216− /*
2217− * Someone is already canceling. Wait for it to finish. flush_work()
2218− * doesn't work for PREEMPT_NONE because we may get woken up between
2219− * @work's completion and the other canceling task resuming and clearing
2220− * CANCELING - flush_work() will return false immediately as @work is no
2221− * longer busy, try_to_grab_pending() will return -ENOENT as @work is
2222− * still being canceled and the other canceling task won't be able to
2223− * clear CANCELING as we're hogging the CPU.
2224− *
2225− * Let's wait for completion using a waitqueue. As this may lead to the
2226− * thundering herd problem, use a custom wake function which matches
2227− * @work along with exclusive wait and wakeup.
2228− */
2229− init_wait(&cwait.wait);
2230− cwait.wait.func = cwt_wakefn;
2231− cwait.work = work;
2232−
2233− prepare_to_wait_exclusive(&wq_cancel_waitq, &cwait.wait,
2234− TASK_UNINTERRUPTIBLE);
2235− if (work_is_canceling(work))
2236− schedule();
2237− finish_wait(&wq_cancel_waitq, &cwait.wait);
2238−
2239− goto repeat;
2180+ while (true) {
2181+ ret = try_to_grab_pending(work, cflags, irq_flags);
2182+ if (ret >= 0)
2183+ return ret;
2184+ cpu_relax();
2185+ }
22402186 }
22412187
22422188 /**
@@ -2422,6 +2368,21 @@ out: · 179 unmodified lines
24222368 rcu_read_unlock();
24232369 }
24242370
2371+static bool clear_pending_if_disabled(struct work_struct *work)
2372+{
2373+ unsigned long data = *work_data_bits(work);
2374+ struct work_offq_data offqd;
2375+
2376+ if (likely((data & WORK_STRUCT_PWQ) ||
2377+ !(data & WORK_OFFQ_DISABLE_MASK)))
2378+ return false;
2379+
2380+ work_offqd_unpack(&offqd, data);
2381+ set_work_pool_and_clear_pending(work, offqd.pool_id,
2382+ work_offqd_pack_flags(&offqd));
2383+ return true;
2384+}
2385+
24252386 /**
24262387 * queue_work_on - queue work on specific cpu
24272388 * @cpu: CPU number to execute work on
@@ -2444,7 +2405,8 @@ bool queue_work_on(int cpu, struct workqueue_struct *wq, · 16 unmodified lines
24442405
24452406 local_irq_save(irq_flags);
24462407
2447− if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
2408+ if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2409+ !clear_pending_if_disabled(work)) {
24482410 __queue_work(cpu, wq, work);
24492411 ret = true;
24502412 }
@@ -2522,7 +2484,8 @@ bool queue_work_node(int node, struct workqueue_struct *wq, · 71 unmodified lines
25222484
25232485 local_irq_save(irq_flags);
25242486
2525− if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
2487+ if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2488+ !clear_pending_if_disabled(work)) {
25262489 int cpu = select_numa_node_cpu(node);
25272490
25282491 __queue_work(cpu, wq, work);
@@ -2604,7 +2567,8 @@ bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq, · 75 unmodified lines
26042567 /* read the comment in __queue_work() */
26052568 local_irq_save(irq_flags);
26062569
2607− if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
2570+ if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2571+ !clear_pending_if_disabled(work)) {
26082572 __queue_delayed_work(cpu, wq, dwork, delay);
26092573 ret = true;
26102574 }
@@ -2636,19 +2600,14 @@ bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq, · 25 unmodified lines
26362600 struct delayed_work *dwork, unsigned long delay)
26372601 {
26382602 unsigned long irq_flags;
2639− int ret;
2603+ bool ret;
26402604
2641− do {
2642− ret = try_to_grab_pending(&dwork->work, WORK_CANCEL_DELAYED,
2643− &irq_flags);
2644− } while (unlikely(ret == -EAGAIN));
2605+ ret = work_grab_pending(&dwork->work, WORK_CANCEL_DELAYED, &irq_flags);
26452606
2646− if (likely(ret >= 0)) {
2607+ if (!clear_pending_if_disabled(&dwork->work))
26472608 __queue_delayed_work(cpu, wq, dwork, delay);
2648− local_irq_restore(irq_flags);
2649− }
26502609
2651− /* -ENOENT from try_to_grab_pending() becomes %true */
2610+ local_irq_restore(irq_flags);
26522611 return ret;
26532612 }
26542613 EXPORT_SYMBOL_GPL(mod_delayed_work_on);
@@ -2677,7 +2636,12 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork) · 22 unmodified lines
26772636 {
26782637 struct work_struct *work = &rwork->work;
26792638
2680− if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
2639+ /*
2640+ * rcu_work can't be canceled or disabled. Warn if the user reached
2641+ * inside @rwork and disabled the inner work.
2642+ */
2643+ if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2644+ !WARN_ON_ONCE(clear_pending_if_disabled(work))) {
26812645 rwork->wq = wq;
26822646 call_rcu_hurry(&rwork->rcu, rcu_work_rcufn);
26832647 return true;
@@ -2953,7 +2917,7 @@ static void idle_worker_timeout(struct timer_list *t) · 269 unmodified lines
29532917 unsigned long expires;
29542918
29552919 /* idle_list is kept in LIFO order, check the last one */
2956− worker = list_entry(pool->idle_list.prev, struct worker, entry);
2920+ worker = list_last_entry(&pool->idle_list, struct worker, entry);
29572921 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
29582922 do_cull = !time_before(jiffies, expires);
29592923
@@ -2995,7 +2959,7 @@ static void idle_cull_fn(struct work_struct *work) · 35 unmodified lines
29952959 struct worker *worker;
29962960 unsigned long expires;
29972961
2998− worker = list_entry(pool->idle_list.prev, struct worker, entry);
2962+ worker = list_last_entry(&pool->idle_list, struct worker, entry);
29992963 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
30002964
30012965 if (time_before(jiffies, expires)) {
@@ -3230,7 +3194,7 @@ __acquires(&pool->lock) · 228 unmodified lines
32303194 * PENDING and queued state changes happen together while IRQ is
32313195 * disabled.
32323196 */
3233− set_work_pool_and_clear_pending(work, pool->id, 0);
3197+ set_work_pool_and_clear_pending(work, pool->id, pool_offq_flags(pool));
32343198
32353199 pwq->stats[PWQ_STAT_STARTED]++;
32363200 raw_spin_unlock_irq(&pool->lock);
@@ -3700,7 +3664,7 @@ void workqueue_softirq_dead(unsigned int cpu) · 463 unmodified lines
37003664 if (!need_more_worker(pool))
37013665 continue;
37023666
3703− INIT_WORK(&dead_work.work, drain_dead_softirq_workfn);
3667+ INIT_WORK_ONSTACK(&dead_work.work, drain_dead_softirq_workfn);
37043668 dead_work.pool = pool;
37053669 init_completion(&dead_work.done);
37063670
@@ -3710,6 +3674,7 @@ void workqueue_softirq_dead(unsigned int cpu) · 3 unmodified lines
37103674 queue_work(system_bh_wq, &dead_work.work);
37113675
37123676 wait_for_completion(&dead_work.done);
3677+ destroy_work_on_stack(&dead_work.work);
37133678 }
37143679 }
37153680
@@ -4154,8 +4119,6 @@ static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr, · 438 unmodified lines
41544119 struct pool_workqueue *pwq;
41554120 struct workqueue_struct *wq;
41564121
4157− might_sleep();
4158−
41594122 rcu_read_lock();
41604123 pool = get_work_pool(work);
41614124 if (!pool) {
@@ -4207,6 +4170,7 @@ already_gone: · 45 unmodified lines
42074170 static bool __flush_work(struct work_struct *work, bool from_cancel)
42084171 {
42094172 struct wq_barrier barr;
4173+ unsigned long data;
42104174
42114175 if (WARN_ON(!wq_online))
42124176 return false;
@@ -4214,13 +4178,41 @@ static bool __flush_work(struct work_struct *work, bool from_cancel) · 1 unmodified line
42144178 if (WARN_ON(!work->func))
42154179 return false;
42164180
4217− if (start_flush_work(work, &barr, from_cancel)) {
4218− wait_for_completion(&barr.done);
4219− destroy_work_on_stack(&barr.work);
4220− return true;
4221− } else {
4181+ if (!start_flush_work(work, &barr, from_cancel))
42224182 return false;
4183+
4184+ /*
4185+ * start_flush_work() returned %true. If @from_cancel is set, we know
4186+ * that @work must have been executing during start_flush_work() and
4187+ * can't currently be queued. Its data must contain OFFQ bits. If @work
4188+ * was queued on a BH workqueue, we also know that it was running in the
4189+ * BH context and thus can be busy-waited.
4190+ */
4191+ data = *work_data_bits(work);
4192+ if (from_cancel &&
4193+ !WARN_ON_ONCE(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_BH)) {
4194+ /*
4195+ * On RT, prevent a live lock when %current preempted soft
4196+ * interrupt processing or prevents ksoftirqd from running by
4197+ * keeping flipping BH. If the BH work item runs on a different
4198+ * CPU then this has no effect other than doing the BH
4199+ * disable/enable dance for nothing. This is copied from
4200+ * kernel/softirq.c::tasklet_unlock_spin_wait().
4201+ */
4202+ while (!try_wait_for_completion(&barr.done)) {
4203+ if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
4204+ local_bh_disable();
4205+ local_bh_enable();
4206+ } else {
4207+ cpu_relax();
4208+ }
4209+ }
4210+ } else {
4211+ wait_for_completion(&barr.done);
42234212 }
4213+
4214+ destroy_work_on_stack(&barr.work);
4215+ return true;
42244216 }
42254217
42264218 /**
@@ -4236,6 +4228,7 @@ static bool __flush_work(struct work_struct *work, bool from_cancel) · 9 unmodified lines
42364228 */
42374229 bool flush_work(struct work_struct *work)
42384230 {
4231+ might_sleep();
42394232 return __flush_work(work, false);
42404233 }
42414234 EXPORT_SYMBOL_GPL(flush_work);
@@ -4282,32 +4275,53 @@ bool flush_rcu_work(struct rcu_work *rwork) · 40 unmodified lines
42824275 }
42834276 EXPORT_SYMBOL(flush_rcu_work);
42844277
4278+static void work_offqd_disable(struct work_offq_data *offqd)
4279+{
4280+ const unsigned long max = (1lu << WORK_OFFQ_DISABLE_BITS) - 1;
4281+
4282+ if (likely(offqd->disable < max))
4283+ offqd->disable++;
4284+ else
4285+ WARN_ONCE(true, "workqueue: work disable count overflowed\n");
4286+}
4287+
4288+static void work_offqd_enable(struct work_offq_data *offqd)
4289+{
4290+ if (likely(offqd->disable > 0))
4291+ offqd->disable--;
4292+ else
4293+ WARN_ONCE(true, "workqueue: work disable count underflowed\n");
4294+}
4295+
42854296 static bool __cancel_work(struct work_struct *work, u32 cflags)
42864297 {
4298+ struct work_offq_data offqd;
42874299 unsigned long irq_flags;
42884300 int ret;
42894301
4290− do {
4291− ret = try_to_grab_pending(work, cflags, &irq_flags);
4292− } while (unlikely(ret == -EAGAIN));
4302+ ret = work_grab_pending(work, cflags, &irq_flags);
42934303
4294− if (unlikely(ret < 0))
4295− return false;
4304+ work_offqd_unpack(&offqd, *work_data_bits(work));
42964305
4297− set_work_pool_and_clear_pending(work, get_work_pool_id(work), 0);
4306+ if (cflags & WORK_CANCEL_DISABLE)
4307+ work_offqd_disable(&offqd);
4308+
4309+ set_work_pool_and_clear_pending(work, offqd.pool_id,
4310+ work_offqd_pack_flags(&offqd));
42984311 local_irq_restore(irq_flags);
42994312 return ret;
43004313 }
43014314
43024315 static bool __cancel_work_sync(struct work_struct *work, u32 cflags)
43034316 {
4304− unsigned long irq_flags;
43054317 bool ret;
43064318
4307− /* claim @work and tell other tasks trying to grab @work to back off */
4308− ret = work_grab_pending(work, cflags, &irq_flags);
4309− mark_work_canceling(work);
4310− local_irq_restore(irq_flags);
4319+ ret = __cancel_work(work, cflags | WORK_CANCEL_DISABLE);
4320+
4321+ if (*work_data_bits(work) & WORK_OFFQ_BH)
4322+ WARN_ON_ONCE(in_hardirq());
4323+ else
4324+ might_sleep();
43114325
43124326 /*
43134327 * Skip __flush_work() during early boot when we know that @work isn't
@@ -4316,15 +4330,8 @@ static bool __cancel_work_sync(struct work_struct *work, u32 cflags) · 2 unmodified lines
43164330 if (wq_online)
43174331 __flush_work(work, true);
43184332
4319− /*
4320− * smp_mb() at the end of set_work_pool_and_clear_pending() is paired
4321− * with prepare_to_wait() above so that either waitqueue_active() is
4322− * visible here or !work_is_canceling() is visible there.
4323− */
4324− set_work_pool_and_clear_pending(work, WORK_OFFQ_POOL_NONE, 0);
4325−
4326− if (waitqueue_active(&wq_cancel_waitq))
4327− __wake_up(&wq_cancel_waitq, TASK_NORMAL, 1, work);
4333+ if (!(cflags & WORK_CANCEL_DISABLE))
4334+ enable_work(work);
43284335
43294336 return ret;
43304337 }
@@ -4342,19 +4349,19 @@ EXPORT_SYMBOL(cancel_work); · 11 unmodified lines
43424349 * cancel_work_sync - cancel a work and wait for it to finish
43434350 * @work: the work to cancel
43444351 *
4345− * Cancel @work and wait for its execution to finish. This function
4346− * can be used even if the work re-queues itself or migrates to
4347− * another workqueue. On return from this function, @work is
4348− * guaranteed to be not pending or executing on any CPU.
4352+ * Cancel @work and wait for its execution to finish. This function can be used
4353+ * even if the work re-queues itself or migrates to another workqueue. On return
4354+ * from this function, @work is guaranteed to be not pending or executing on any
4355+ * CPU as long as there aren't racing enqueues.
43494356 *
4350− * cancel_work_sync(&delayed_work->work) must not be used for
4351− * delayed_work's. Use cancel_delayed_work_sync() instead.
4357+ * cancel_work_sync(&delayed_work->work) must not be used for delayed_work's.
4358+ * Use cancel_delayed_work_sync() instead.
43524359 *
4353− * The caller must ensure that the workqueue on which @work was last
4354− * queued can't be destroyed before this function returns.
4360+ * Must be called from a sleepable context if @work was last queued on a non-BH
4361+ * workqueue. Can also be called from non-hardirq atomic contexts including BH
4362+ * if @work was last queued on a BH workqueue.
43554363 *
4356− * Return:
4357− * %true if @work was pending, %false otherwise.
4364+ * Returns %true if @work was pending, %false otherwise.
43584365 */
43594366 bool cancel_work_sync(struct work_struct *work)
43604367 {
@@ -4399,6 +4406,108 @@ bool cancel_delayed_work_sync(struct delayed_work *dwork) · 38 unmodified lines
43994406 }
44004407 EXPORT_SYMBOL(cancel_delayed_work_sync);
44014408
4409+/**
4410+ * disable_work - Disable and cancel a work item
4411+ * @work: work item to disable
4412+ *
4413+ * Disable @work by incrementing its disable count and cancel it if currently
4414+ * pending. As long as the disable count is non-zero, any attempt to queue @work
4415+ * will fail and return %false. The maximum supported disable depth is 2 to the
4416+ * power of %WORK_OFFQ_DISABLE_BITS, currently 65536.
4417+ *
4418+ * Can be called from any context. Returns %true if @work was pending, %false
4419+ * otherwise.
4420+ */
4421+bool disable_work(struct work_struct *work)
4422+{
4423+ return __cancel_work(work, WORK_CANCEL_DISABLE);
4424+}
4425+EXPORT_SYMBOL_GPL(disable_work);
4426+
4427+/**
4428+ * disable_work_sync - Disable, cancel and drain a work item
4429+ * @work: work item to disable
4430+ *
4431+ * Similar to disable_work() but also wait for @work to finish if currently
4432+ * executing.
4433+ *
4434+ * Must be called from a sleepable context if @work was last queued on a non-BH
4435+ * workqueue. Can also be called from non-hardirq atomic contexts including BH
4436+ * if @work was last queued on a BH workqueue.
4437+ *
4438+ * Returns %true if @work was pending, %false otherwise.
4439+ */
4440+bool disable_work_sync(struct work_struct *work)
4441+{
4442+ return __cancel_work_sync(work, WORK_CANCEL_DISABLE);
4443+}
4444+EXPORT_SYMBOL_GPL(disable_work_sync);
4445+
4446+/**
4447+ * enable_work - Enable a work item
4448+ * @work: work item to enable
4449+ *
4450+ * Undo disable_work[_sync]() by decrementing @work's disable count. @work can
4451+ * only be queued if its disable count is 0.
4452+ *
4453+ * Can be called from any context. Returns %true if the disable count reached 0.
4454+ * Otherwise, %false.
4455+ */
4456+bool enable_work(struct work_struct *work)
4457+{
4458+ struct work_offq_data offqd;
4459+ unsigned long irq_flags;
4460+
4461+ work_grab_pending(work, 0, &irq_flags);
4462+
4463+ work_offqd_unpack(&offqd, *work_data_bits(work));
4464+ work_offqd_enable(&offqd);
4465+ set_work_pool_and_clear_pending(work, offqd.pool_id,
4466+ work_offqd_pack_flags(&offqd));
4467+ local_irq_restore(irq_flags);
4468+
4469+ return !offqd.disable;
4470+}
4471+EXPORT_SYMBOL_GPL(enable_work);
4472+
4473+/**
4474+ * disable_delayed_work - Disable and cancel a delayed work item
4475+ * @dwork: delayed work item to disable
4476+ *
4477+ * disable_work() for delayed work items.
4478+ */
4479+bool disable_delayed_work(struct delayed_work *dwork)
4480+{
4481+ return __cancel_work(&dwork->work,
4482+ WORK_CANCEL_DELAYED | WORK_CANCEL_DISABLE);
4483+}
4484+EXPORT_SYMBOL_GPL(disable_delayed_work);
4485+
4486+/**
4487+ * disable_delayed_work_sync - Disable, cancel and drain a delayed work item
4488+ * @dwork: delayed work item to disable
4489+ *
4490+ * disable_work_sync() for delayed work items.
4491+ */
4492+bool disable_delayed_work_sync(struct delayed_work *dwork)
4493+{
4494+ return __cancel_work_sync(&dwork->work,
4495+ WORK_CANCEL_DELAYED | WORK_CANCEL_DISABLE);
4496+}
4497+EXPORT_SYMBOL_GPL(disable_delayed_work_sync);
4498+
4499+/**
4500+ * enable_delayed_work - Enable a delayed work item
4501+ * @dwork: delayed work item to enable
4502+ *
4503+ * enable_work() for delayed work items.
4504+ */
4505+bool enable_delayed_work(struct delayed_work *dwork)
4506+{
4507+ return enable_work(&dwork->work);
4508+}
4509+EXPORT_SYMBOL_GPL(enable_delayed_work);
4510+
44024511 /**
44034512 * schedule_on_each_cpu - execute a function synchronously on each online CPU
44044513 * @func: the function to call
@@ -4530,6 +4639,8 @@ static void wqattrs_clear_for_pool(struct workqueue_attrs *attrs) · 125 unmodified lines
45304639 {
45314640 attrs->affn_scope = WQ_AFFN_NR_TYPES;
45324641 attrs->ordered = false;
4642+ if (attrs->affn_strict)
4643+ cpumask_copy(attrs->cpumask, cpu_possible_mask);
45334644 }
45344645
45354646 /* hash value of the content of @attr */
@@ -4538,11 +4649,12 @@ static u32 wqattrs_hash(const struct workqueue_attrs *attrs) · 2 unmodified lines
45384649 u32 hash = 0;
45394650
45404651 hash = jhash_1word(attrs->nice, hash);
4541− hash = jhash(cpumask_bits(attrs->cpumask),
4542− BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
4652+ hash = jhash_1word(attrs->affn_strict, hash);
45434653 hash = jhash(cpumask_bits(attrs->__pod_cpumask),
45444654 BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
4545− hash = jhash_1word(attrs->affn_strict, hash);
4655+ if (!attrs->affn_strict)
4656+ hash = jhash(cpumask_bits(attrs->cpumask),
4657+ BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
45464658 return hash;
45474659 }
45484660
@@ -4552,11 +4664,11 @@ static bool wqattrs_equal(const struct workqueue_attrs *a, · 3 unmodified lines
45524664 {
45534665 if (a->nice != b->nice)
45544666 return false;
4555− if (!cpumask_equal(a->cpumask, b->cpumask))
4667+ if (a->affn_strict != b->affn_strict)
45564668 return false;
45574669 if (!cpumask_equal(a->__pod_cpumask, b->__pod_cpumask))
45584670 return false;
4559− if (a->affn_strict != b->affn_strict)
4671+ if (!a->affn_strict && !cpumask_equal(a->cpumask, b->cpumask))
45604672 return false;
45614673 return true;
45624674 }
@@ -7148,25 +7260,27 @@ static ssize_t __wq_cpumask_show(struct device *dev, · 2585 unmodified lines
71487260 return written;
71497261 }
71507262
7151−static ssize_t wq_unbound_cpumask_show(struct device *dev,
7263+static ssize_t cpumask_requested_show(struct device *dev,
71527264 struct device_attribute *attr, char *buf)
71537265 {
7154− return __wq_cpumask_show(dev, attr, buf, wq_unbound_cpumask);
7266+ return __wq_cpumask_show(dev, attr, buf, wq_requested_unbound_cpumask);
71557267 }
7268+static DEVICE_ATTR_RO(cpumask_requested);
71567269
7157−static ssize_t wq_requested_cpumask_show(struct device *dev,
7270+static ssize_t cpumask_isolated_show(struct device *dev,
71587271 struct device_attribute *attr, char *buf)
71597272 {
7160− return __wq_cpumask_show(dev, attr, buf, wq_requested_unbound_cpumask);
7273+ return __wq_cpumask_show(dev, attr, buf, wq_isolated_cpumask);
71617274 }
7275+static DEVICE_ATTR_RO(cpumask_isolated);
71627276
7163−static ssize_t wq_isolated_cpumask_show(struct device *dev,
7277+static ssize_t cpumask_show(struct device *dev,
71647278 struct device_attribute *attr, char *buf)
71657279 {
7166− return __wq_cpumask_show(dev, attr, buf, wq_isolated_cpumask);
7280+ return __wq_cpumask_show(dev, attr, buf, wq_unbound_cpumask);
71677281 }
71687282
7169−static ssize_t wq_unbound_cpumask_store(struct device *dev,
7283+static ssize_t cpumask_store(struct device *dev,
71707284 struct device_attribute *attr, const char *buf, size_t count)
71717285 {
71727286 cpumask_var_t cpumask;
@@ -7182,36 +7296,19 @@ static ssize_t wq_unbound_cpumask_store(struct device *dev, · 9 unmodified lines
71827296 free_cpumask_var(cpumask);
71837297 return ret ? ret : count;
71847298 }
7299+static DEVICE_ATTR_RW(cpumask);
71857300
7186−static struct device_attribute wq_sysfs_cpumask_attrs[] = {
7187− __ATTR(cpumask, 0644, wq_unbound_cpumask_show,
7188− wq_unbound_cpumask_store),
7189− __ATTR(cpumask_requested, 0444, wq_requested_cpumask_show, NULL),
7190− __ATTR(cpumask_isolated, 0444, wq_isolated_cpumask_show, NULL),
7191− __ATTR_NULL,
7301+static struct attribute *wq_sysfs_cpumask_attrs[] = {
7302+ &dev_attr_cpumask.attr,
7303+ &dev_attr_cpumask_requested.attr,
7304+ &dev_attr_cpumask_isolated.attr,
7305+ NULL,
71927306 };
7307+ATTRIBUTE_GROUPS(wq_sysfs_cpumask);
71937308
71947309 static int __init wq_sysfs_init(void)
71957310 {
7196− struct device *dev_root;
7197− int err;
7198−
7199− err = subsys_virtual_register(&wq_subsys, NULL);
7200− if (err)
7201− return err;
7202−
7203− dev_root = bus_get_dev_root(&wq_subsys);
7204− if (dev_root) {
7205− struct device_attribute *attr;
7206−
7207− for (attr = wq_sysfs_cpumask_attrs; attr->attr.name; attr++) {
7208− err = device_create_file(dev_root, attr);
7209− if (err)
7210− break;
7211− }
7212− put_device(dev_root);
7213− }
7214− return err;
7311+ return subsys_virtual_register(&wq_subsys, wq_sysfs_cpumask_groups);
72157312 }
72167313 core_initcall(wq_sysfs_init);
72177314