History

Merge branch 'for-6.10' into test-merge-for-6.10

Tejun Heo committed 2 years ago

Commit a2a58902 parents 8c06da6 51da7f64 files · +327 −193Browse files

Showing the changes to kernel/workqueue.c (1 of 4 files). Show all

Unified · Split

kernel/workqueue.c+275−178
@@ -99,6 +99,7 @@ enum worker_flags { · 98 unmodified lines
99 99
100enum work_cancel_flags {100enum work_cancel_flags {
101 WORK_CANCEL_DELAYED = 1 << 0, /* canceling a delayed_work */101 WORK_CANCEL_DELAYED = 1 << 0, /* canceling a delayed_work */
102 WORK_CANCEL_DISABLE = 1 << 1, /* canceling to disable */
102};103};
103 104
104enum wq_internal_consts {105enum wq_internal_consts {
@@ -392,6 +393,12 @@ struct wq_pod_type { · 287 unmodified lines
392 int *cpu_pod; /* cpu -> pod */393 int *cpu_pod; /* cpu -> pod */
393};394};
394 395
396struct work_offq_data {
397 u32 pool_id;
398 u32 disable;
399 u32 flags;
400};
401
395static const char *wq_affn_names[WQ_AFFN_NR_TYPES] = {402static const char *wq_affn_names[WQ_AFFN_NR_TYPES] = {
396 [WQ_AFFN_DFL] = "default",403 [WQ_AFFN_DFL] = "default",
397 [WQ_AFFN_CPU] = "cpu",404 [WQ_AFFN_CPU] = "cpu",
@@ -489,12 +496,6 @@ static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS]; · 91 unmodified lines
489/* I: attributes used when instantiating ordered pools on demand */496/* I: attributes used when instantiating ordered pools on demand */
490static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];497static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
491 498
492/*
493 * Used to synchronize multiple cancel_sync attempts on the same work item. See
494 * work_grab_pending() and __cancel_work_sync().
495 */
496static DECLARE_WAIT_QUEUE_HEAD(wq_cancel_waitq);
497
498/*499/*
499 * I: kthread_worker to release pwq's. pwq release needs to be bounced to a500 * I: kthread_worker to release pwq's. pwq release needs to be bounced to a
500 * process context while holding a pool lock. Bounce to a dedicated kthread501 * process context while holding a pool lock. Bounce to a dedicated kthread
@@ -763,6 +764,11 @@ static int work_next_color(int color) · 262 unmodified lines
763 return (color + 1) % WORK_NR_COLORS;764 return (color + 1) % WORK_NR_COLORS;
764}765}
765 766
767static unsigned long pool_offq_flags(struct worker_pool *pool)
768{
769 return (pool->flags & POOL_BH) ? WORK_OFFQ_BH : 0;
770}
771
766/*772/*
767 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data773 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
768 * contain the pointer to the queued pwq. Once execution starts, the flag774 * contain the pointer to the queued pwq. Once execution starts, the flag
@@ -776,11 +782,6 @@ static int work_next_color(int color) · 7 unmodified lines
776 * corresponding to a work. Pool is available once the work has been782 * corresponding to a work. Pool is available once the work has been
777 * queued anywhere after initialization until it is sync canceled. pwq is783 * queued anywhere after initialization until it is sync canceled. pwq is
778 * available only while the work item is queued.784 * available only while the work item is queued.
779 *
780 * %WORK_OFFQ_CANCELING is used to mark a work item which is being
781 * canceled. While being canceled, a work item may have its PENDING set
782 * but stay off timer and worklist for arbitrarily long and nobody should
783 * try to steal the PENDING bit.
784 */785 */
785static inline void set_work_data(struct work_struct *work, unsigned long data)786static inline void set_work_data(struct work_struct *work, unsigned long data)
786{787{
@@ -892,36 +893,26 @@ static struct worker_pool *get_work_pool(struct work_struct *work) · 105 unmodified lines
892 return idr_find(&worker_pool_idr, pool_id);893 return idr_find(&worker_pool_idr, pool_id);
893}894}
894 895
895/**896static unsigned long shift_and_mask(unsigned long v, u32 shift, u32 bits)
896 * get_work_pool_id - return the worker pool ID a given work is associated with
897 * @work: the work item of interest
898 *
899 * Return: The worker_pool ID @work was last associated with.
900 * %WORK_OFFQ_POOL_NONE if none.
901 */
902static int get_work_pool_id(struct work_struct *work)
903{897{
904 unsigned long data = atomic_long_read(&work->data);898 return (v >> shift) & ((1 << bits) - 1);
905
906 if (data & WORK_STRUCT_PWQ)
907 return work_struct_pwq(data)->pool->id;
908
909 return data >> WORK_OFFQ_POOL_SHIFT;
910}899}
911 900
912static void mark_work_canceling(struct work_struct *work)901static void work_offqd_unpack(struct work_offq_data *offqd, unsigned long data)
913{902{
914 unsigned long pool_id = get_work_pool_id(work);903 WARN_ON_ONCE(data & WORK_STRUCT_PWQ);
915 904
916 pool_id <<= WORK_OFFQ_POOL_SHIFT;905 offqd->pool_id = shift_and_mask(data, WORK_OFFQ_POOL_SHIFT,
917 set_work_data(work, pool_id | WORK_STRUCT_PENDING | WORK_OFFQ_CANCELING);906 WORK_OFFQ_POOL_BITS);
907 offqd->disable = shift_and_mask(data, WORK_OFFQ_DISABLE_SHIFT,
908 WORK_OFFQ_DISABLE_BITS);
909 offqd->flags = data & WORK_OFFQ_FLAG_MASK;
918}910}
919 911
920static bool work_is_canceling(struct work_struct *work)912static unsigned long work_offqd_pack_flags(struct work_offq_data *offqd)
921{913{
922 unsigned long data = atomic_long_read(&work->data);914 return ((unsigned long)offqd->disable << WORK_OFFQ_DISABLE_SHIFT) |
923 915 ((unsigned long)offqd->flags);
924 return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
925}916}
926 917
927/*918/*
@@ -2067,8 +2058,6 @@ out_put: · 1139 unmodified lines
2067 * 1 if @work was pending and we successfully stole PENDING2058 * 1 if @work was pending and we successfully stole PENDING
2068 * 0 if @work was idle and we claimed PENDING2059 * 0 if @work was idle and we claimed PENDING
2069 * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry2060 * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
2070 * -ENOENT if someone else is canceling @work, this state may persist
2071 * for arbitrarily long
2072 * ======== ================================================================2061 * ======== ================================================================
2073 *2062 *
2074 * Note:2063 * Note:
@@ -2151,7 +2140,8 @@ static int try_to_grab_pending(struct work_struct *work, u32 cflags, · 76 unmodified lines
2151 * this destroys work->data needed by the next step, stash it.2140 * this destroys work->data needed by the next step, stash it.
2152 */2141 */
2153 work_data = *work_data_bits(work);2142 work_data = *work_data_bits(work);
2154 set_work_pool_and_keep_pending(work, pool->id, 0);2143 set_work_pool_and_keep_pending(work, pool->id,
2144 pool_offq_flags(pool));
2155 2145
2156 /* must be the last step, see the function comment */2146 /* must be the last step, see the function comment */
2157 pwq_dec_nr_in_flight(pwq, work_data);2147 pwq_dec_nr_in_flight(pwq, work_data);
@@ -2164,26 +2154,9 @@ static int try_to_grab_pending(struct work_struct *work, u32 cflags, · 6 unmodified lines
2164fail:2154fail:
2165 rcu_read_unlock();2155 rcu_read_unlock();
2166 local_irq_restore(*irq_flags);2156 local_irq_restore(*irq_flags);
2167 if (work_is_canceling(work))
2168 return -ENOENT;
2169 cpu_relax();
2170 return -EAGAIN;2157 return -EAGAIN;
2171}2158}
2172 2159
2173struct cwt_wait {
2174 wait_queue_entry_t wait;
2175 struct work_struct *work;
2176};
2177
2178static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
2179{
2180 struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);
2181
2182 if (cwait->work != key)
2183 return 0;
2184 return autoremove_wake_function(wait, mode, sync, key);
2185}
2186
2187/**2160/**
2188 * work_grab_pending - steal work item from worklist and disable irq2161 * work_grab_pending - steal work item from worklist and disable irq
2189 * @work: work item to steal2162 * @work: work item to steal
@@ -2193,7 +2166,7 @@ static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *k · 3 unmodified lines
2193 * Grab PENDING bit of @work. @work can be in any stable state - idle, on timer2166 * Grab PENDING bit of @work. @work can be in any stable state - idle, on timer
2194 * or on worklist.2167 * or on worklist.
2195 *2168 *
2196 * Must be called in process context. IRQ is disabled on return with IRQ state2169 * Can be called from any context. IRQ is disabled on return with IRQ state
2197 * stored in *@irq_flags. The caller is responsible for re-enabling it using2170 * stored in *@irq_flags. The caller is responsible for re-enabling it using
2198 * local_irq_restore().2171 * local_irq_restore().
2199 *2172 *
@@ -2202,41 +2175,14 @@ static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *k · 2 unmodified lines
2202static bool work_grab_pending(struct work_struct *work, u32 cflags,2175static bool work_grab_pending(struct work_struct *work, u32 cflags,
2203 unsigned long *irq_flags)2176 unsigned long *irq_flags)
2204{2177{
2205 struct cwt_wait cwait;
2206 int ret;2178 int ret;
2207 2179
2208 might_sleep();2180 while (true) {
2209repeat:2181 ret = try_to_grab_pending(work, cflags, irq_flags);
2210 ret = try_to_grab_pending(work, cflags, irq_flags);2182 if (ret >= 0)
2211 if (likely(ret >= 0))2183 return ret;
2212 return ret;2184 cpu_relax();
2213 if (ret != -ENOENT)2185 }
2214 goto repeat;
2215
2216 /*
2217 * Someone is already canceling. Wait for it to finish. flush_work()
2218 * doesn't work for PREEMPT_NONE because we may get woken up between
2219 * @work's completion and the other canceling task resuming and clearing
2220 * CANCELING - flush_work() will return false immediately as @work is no
2221 * longer busy, try_to_grab_pending() will return -ENOENT as @work is
2222 * still being canceled and the other canceling task won't be able to
2223 * clear CANCELING as we're hogging the CPU.
2224 *
2225 * Let's wait for completion using a waitqueue. As this may lead to the
2226 * thundering herd problem, use a custom wake function which matches
2227 * @work along with exclusive wait and wakeup.
2228 */
2229 init_wait(&cwait.wait);
2230 cwait.wait.func = cwt_wakefn;
2231 cwait.work = work;
2232
2233 prepare_to_wait_exclusive(&wq_cancel_waitq, &cwait.wait,
2234 TASK_UNINTERRUPTIBLE);
2235 if (work_is_canceling(work))
2236 schedule();
2237 finish_wait(&wq_cancel_waitq, &cwait.wait);
2238
2239 goto repeat;
2240}2186}
2241 2187
2242/**2188/**
@@ -2422,6 +2368,21 @@ out: · 179 unmodified lines
2422 rcu_read_unlock();2368 rcu_read_unlock();
2423}2369}
2424 2370
2371static bool clear_pending_if_disabled(struct work_struct *work)
2372{
2373 unsigned long data = *work_data_bits(work);
2374 struct work_offq_data offqd;
2375
2376 if (likely((data & WORK_STRUCT_PWQ) ||
2377 !(data & WORK_OFFQ_DISABLE_MASK)))
2378 return false;
2379
2380 work_offqd_unpack(&offqd, data);
2381 set_work_pool_and_clear_pending(work, offqd.pool_id,
2382 work_offqd_pack_flags(&offqd));
2383 return true;
2384}
2385
2425/**2386/**
2426 * queue_work_on - queue work on specific cpu2387 * queue_work_on - queue work on specific cpu
2427 * @cpu: CPU number to execute work on2388 * @cpu: CPU number to execute work on
@@ -2444,7 +2405,8 @@ bool queue_work_on(int cpu, struct workqueue_struct *wq, · 16 unmodified lines
2444 2405
2445 local_irq_save(irq_flags);2406 local_irq_save(irq_flags);
2446 2407
2447 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {2408 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2409 !clear_pending_if_disabled(work)) {
2448 __queue_work(cpu, wq, work);2410 __queue_work(cpu, wq, work);
2449 ret = true;2411 ret = true;
2450 }2412 }
@@ -2522,7 +2484,8 @@ bool queue_work_node(int node, struct workqueue_struct *wq, · 71 unmodified lines
2522 2484
2523 local_irq_save(irq_flags);2485 local_irq_save(irq_flags);
2524 2486
2525 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {2487 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2488 !clear_pending_if_disabled(work)) {
2526 int cpu = select_numa_node_cpu(node);2489 int cpu = select_numa_node_cpu(node);
2527 2490
2528 __queue_work(cpu, wq, work);2491 __queue_work(cpu, wq, work);
@@ -2604,7 +2567,8 @@ bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq, · 75 unmodified lines
2604 /* read the comment in __queue_work() */2567 /* read the comment in __queue_work() */
2605 local_irq_save(irq_flags);2568 local_irq_save(irq_flags);
2606 2569
2607 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {2570 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2571 !clear_pending_if_disabled(work)) {
2608 __queue_delayed_work(cpu, wq, dwork, delay);2572 __queue_delayed_work(cpu, wq, dwork, delay);
2609 ret = true;2573 ret = true;
2610 }2574 }
@@ -2636,19 +2600,14 @@ bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq, · 25 unmodified lines
2636 struct delayed_work *dwork, unsigned long delay)2600 struct delayed_work *dwork, unsigned long delay)
2637{2601{
2638 unsigned long irq_flags;2602 unsigned long irq_flags;
2639 int ret;2603 bool ret;
2640 2604
2641 do {2605 ret = work_grab_pending(&dwork->work, WORK_CANCEL_DELAYED, &irq_flags);
2642 ret = try_to_grab_pending(&dwork->work, WORK_CANCEL_DELAYED,
2643 &irq_flags);
2644 } while (unlikely(ret == -EAGAIN));
2645 2606
2646 if (likely(ret >= 0)) {2607 if (!clear_pending_if_disabled(&dwork->work))
2647 __queue_delayed_work(cpu, wq, dwork, delay);2608 __queue_delayed_work(cpu, wq, dwork, delay);
2648 local_irq_restore(irq_flags);
2649 }
2650 2609
2651 /* -ENOENT from try_to_grab_pending() becomes %true */2610 local_irq_restore(irq_flags);
2652 return ret;2611 return ret;
2653}2612}
2654EXPORT_SYMBOL_GPL(mod_delayed_work_on);2613EXPORT_SYMBOL_GPL(mod_delayed_work_on);
@@ -2677,7 +2636,12 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork) · 22 unmodified lines
2677{2636{
2678 struct work_struct *work = &rwork->work;2637 struct work_struct *work = &rwork->work;
2679 2638
2680 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {2639 /*
2640 * rcu_work can't be canceled or disabled. Warn if the user reached
2641 * inside @rwork and disabled the inner work.
2642 */
2643 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2644 !WARN_ON_ONCE(clear_pending_if_disabled(work))) {
2681 rwork->wq = wq;2645 rwork->wq = wq;
2682 call_rcu_hurry(&rwork->rcu, rcu_work_rcufn);2646 call_rcu_hurry(&rwork->rcu, rcu_work_rcufn);
2683 return true;2647 return true;
@@ -2953,7 +2917,7 @@ static void idle_worker_timeout(struct timer_list *t) · 269 unmodified lines
2953 unsigned long expires;2917 unsigned long expires;
2954 2918
2955 /* idle_list is kept in LIFO order, check the last one */2919 /* idle_list is kept in LIFO order, check the last one */
2956 worker = list_entry(pool->idle_list.prev, struct worker, entry);2920 worker = list_last_entry(&pool->idle_list, struct worker, entry);
2957 expires = worker->last_active + IDLE_WORKER_TIMEOUT;2921 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2958 do_cull = !time_before(jiffies, expires);2922 do_cull = !time_before(jiffies, expires);
2959 2923
@@ -2995,7 +2959,7 @@ static void idle_cull_fn(struct work_struct *work) · 35 unmodified lines
2995 struct worker *worker;2959 struct worker *worker;
2996 unsigned long expires;2960 unsigned long expires;
2997 2961
2998 worker = list_entry(pool->idle_list.prev, struct worker, entry);2962 worker = list_last_entry(&pool->idle_list, struct worker, entry);
2999 expires = worker->last_active + IDLE_WORKER_TIMEOUT;2963 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
3000 2964
3001 if (time_before(jiffies, expires)) {2965 if (time_before(jiffies, expires)) {
@@ -3230,7 +3194,7 @@ __acquires(&pool->lock) · 228 unmodified lines
3230 * PENDING and queued state changes happen together while IRQ is3194 * PENDING and queued state changes happen together while IRQ is
3231 * disabled.3195 * disabled.
3232 */3196 */
3233 set_work_pool_and_clear_pending(work, pool->id, 0);3197 set_work_pool_and_clear_pending(work, pool->id, pool_offq_flags(pool));
3234 3198
3235 pwq->stats[PWQ_STAT_STARTED]++;3199 pwq->stats[PWQ_STAT_STARTED]++;
3236 raw_spin_unlock_irq(&pool->lock);3200 raw_spin_unlock_irq(&pool->lock);
@@ -3700,7 +3664,7 @@ void workqueue_softirq_dead(unsigned int cpu) · 463 unmodified lines
3700 if (!need_more_worker(pool))3664 if (!need_more_worker(pool))
3701 continue;3665 continue;
3702 3666
3703 INIT_WORK(&dead_work.work, drain_dead_softirq_workfn);3667 INIT_WORK_ONSTACK(&dead_work.work, drain_dead_softirq_workfn);
3704 dead_work.pool = pool;3668 dead_work.pool = pool;
3705 init_completion(&dead_work.done);3669 init_completion(&dead_work.done);
3706 3670
@@ -3710,6 +3674,7 @@ void workqueue_softirq_dead(unsigned int cpu) · 3 unmodified lines
3710 queue_work(system_bh_wq, &dead_work.work);3674 queue_work(system_bh_wq, &dead_work.work);
3711 3675
3712 wait_for_completion(&dead_work.done);3676 wait_for_completion(&dead_work.done);
3677 destroy_work_on_stack(&dead_work.work);
3713 }3678 }
3714}3679}
3715 3680
@@ -4154,8 +4119,6 @@ static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr, · 438 unmodified lines
4154 struct pool_workqueue *pwq;4119 struct pool_workqueue *pwq;
4155 struct workqueue_struct *wq;4120 struct workqueue_struct *wq;
4156 4121
4157 might_sleep();
4158
4159 rcu_read_lock();4122 rcu_read_lock();
4160 pool = get_work_pool(work);4123 pool = get_work_pool(work);
4161 if (!pool) {4124 if (!pool) {
@@ -4207,6 +4170,7 @@ already_gone: · 45 unmodified lines
4207static bool __flush_work(struct work_struct *work, bool from_cancel)4170static bool __flush_work(struct work_struct *work, bool from_cancel)
4208{4171{
4209 struct wq_barrier barr;4172 struct wq_barrier barr;
4173 unsigned long data;
4210 4174
4211 if (WARN_ON(!wq_online))4175 if (WARN_ON(!wq_online))
4212 return false;4176 return false;
@@ -4214,13 +4178,41 @@ static bool __flush_work(struct work_struct *work, bool from_cancel) · 1 unmodified line
4214 if (WARN_ON(!work->func))4178 if (WARN_ON(!work->func))
4215 return false;4179 return false;
4216 4180
4217 if (start_flush_work(work, &barr, from_cancel)) {4181 if (!start_flush_work(work, &barr, from_cancel))
4218 wait_for_completion(&barr.done);
4219 destroy_work_on_stack(&barr.work);
4220 return true;
4221 } else {
4222 return false;4182 return false;
4183
4184 /*
4185 * start_flush_work() returned %true. If @from_cancel is set, we know
4186 * that @work must have been executing during start_flush_work() and
4187 * can't currently be queued. Its data must contain OFFQ bits. If @work
4188 * was queued on a BH workqueue, we also know that it was running in the
4189 * BH context and thus can be busy-waited.
4190 */
4191 data = *work_data_bits(work);
4192 if (from_cancel &&
4193 !WARN_ON_ONCE(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_BH)) {
4194 /*
4195 * On RT, prevent a live lock when %current preempted soft
4196 * interrupt processing or prevents ksoftirqd from running by
4197 * keeping flipping BH. If the BH work item runs on a different
4198 * CPU then this has no effect other than doing the BH
4199 * disable/enable dance for nothing. This is copied from
4200 * kernel/softirq.c::tasklet_unlock_spin_wait().
4201 */
4202 while (!try_wait_for_completion(&barr.done)) {
4203 if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
4204 local_bh_disable();
4205 local_bh_enable();
4206 } else {
4207 cpu_relax();
4208 }
4209 }
4210 } else {
4211 wait_for_completion(&barr.done);
4223 }4212 }
4213
4214 destroy_work_on_stack(&barr.work);
4215 return true;
4224}4216}
4225 4217
4226/**4218/**
@@ -4236,6 +4228,7 @@ static bool __flush_work(struct work_struct *work, bool from_cancel) · 9 unmodified lines
4236 */4228 */
4237bool flush_work(struct work_struct *work)4229bool flush_work(struct work_struct *work)
4238{4230{
4231 might_sleep();
4239 return __flush_work(work, false);4232 return __flush_work(work, false);
4240}4233}
4241EXPORT_SYMBOL_GPL(flush_work);4234EXPORT_SYMBOL_GPL(flush_work);
@@ -4282,32 +4275,53 @@ bool flush_rcu_work(struct rcu_work *rwork) · 40 unmodified lines
4282}4275}
4283EXPORT_SYMBOL(flush_rcu_work);4276EXPORT_SYMBOL(flush_rcu_work);
4284 4277
4278static void work_offqd_disable(struct work_offq_data *offqd)
4279{
4280 const unsigned long max = (1lu << WORK_OFFQ_DISABLE_BITS) - 1;
4281
4282 if (likely(offqd->disable < max))
4283 offqd->disable++;
4284 else
4285 WARN_ONCE(true, "workqueue: work disable count overflowed\n");
4286}
4287
4288static void work_offqd_enable(struct work_offq_data *offqd)
4289{
4290 if (likely(offqd->disable > 0))
4291 offqd->disable--;
4292 else
4293 WARN_ONCE(true, "workqueue: work disable count underflowed\n");
4294}
4295
4285static bool __cancel_work(struct work_struct *work, u32 cflags)4296static bool __cancel_work(struct work_struct *work, u32 cflags)
4286{4297{
4298 struct work_offq_data offqd;
4287 unsigned long irq_flags;4299 unsigned long irq_flags;
4288 int ret;4300 int ret;
4289 4301
4290 do {4302 ret = work_grab_pending(work, cflags, &irq_flags);
4291 ret = try_to_grab_pending(work, cflags, &irq_flags);
4292 } while (unlikely(ret == -EAGAIN));
4293 4303
4294 if (unlikely(ret < 0))4304 work_offqd_unpack(&offqd, *work_data_bits(work));
4295 return false;
4296 4305
4297 set_work_pool_and_clear_pending(work, get_work_pool_id(work), 0);4306 if (cflags & WORK_CANCEL_DISABLE)
4307 work_offqd_disable(&offqd);
4308
4309 set_work_pool_and_clear_pending(work, offqd.pool_id,
4310 work_offqd_pack_flags(&offqd));
4298 local_irq_restore(irq_flags);4311 local_irq_restore(irq_flags);
4299 return ret;4312 return ret;
4300}4313}
4301 4314
4302static bool __cancel_work_sync(struct work_struct *work, u32 cflags)4315static bool __cancel_work_sync(struct work_struct *work, u32 cflags)
4303{4316{
4304 unsigned long irq_flags;
4305 bool ret;4317 bool ret;
4306 4318
4307 /* claim @work and tell other tasks trying to grab @work to back off */4319 ret = __cancel_work(work, cflags | WORK_CANCEL_DISABLE);
4308 ret = work_grab_pending(work, cflags, &irq_flags);4320
4309 mark_work_canceling(work);4321 if (*work_data_bits(work) & WORK_OFFQ_BH)
4310 local_irq_restore(irq_flags);4322 WARN_ON_ONCE(in_hardirq());
4323 else
4324 might_sleep();
4311 4325
4312 /*4326 /*
4313 * Skip __flush_work() during early boot when we know that @work isn't4327 * Skip __flush_work() during early boot when we know that @work isn't
@@ -4316,15 +4330,8 @@ static bool __cancel_work_sync(struct work_struct *work, u32 cflags) · 2 unmodified lines
4316 if (wq_online)4330 if (wq_online)
4317 __flush_work(work, true);4331 __flush_work(work, true);
4318 4332
4319 /*4333 if (!(cflags & WORK_CANCEL_DISABLE))
4320 * smp_mb() at the end of set_work_pool_and_clear_pending() is paired4334 enable_work(work);
4321 * with prepare_to_wait() above so that either waitqueue_active() is
4322 * visible here or !work_is_canceling() is visible there.
4323 */
4324 set_work_pool_and_clear_pending(work, WORK_OFFQ_POOL_NONE, 0);
4325
4326 if (waitqueue_active(&wq_cancel_waitq))
4327 __wake_up(&wq_cancel_waitq, TASK_NORMAL, 1, work);
4328 4335
4329 return ret;4336 return ret;
4330}4337}
@@ -4342,19 +4349,19 @@ EXPORT_SYMBOL(cancel_work); · 11 unmodified lines
4342 * cancel_work_sync - cancel a work and wait for it to finish4349 * cancel_work_sync - cancel a work and wait for it to finish
4343 * @work: the work to cancel4350 * @work: the work to cancel
4344 *4351 *
4345 * Cancel @work and wait for its execution to finish. This function4352 * Cancel @work and wait for its execution to finish. This function can be used
4346 * can be used even if the work re-queues itself or migrates to4353 * even if the work re-queues itself or migrates to another workqueue. On return
4347 * another workqueue. On return from this function, @work is4354 * from this function, @work is guaranteed to be not pending or executing on any
4348 * guaranteed to be not pending or executing on any CPU.4355 * CPU as long as there aren't racing enqueues.
4349 *4356 *
4350 * cancel_work_sync(&delayed_work->work) must not be used for4357 * cancel_work_sync(&delayed_work->work) must not be used for delayed_work's.
4351 * delayed_work's. Use cancel_delayed_work_sync() instead.4358 * Use cancel_delayed_work_sync() instead.
4352 *4359 *
4353 * The caller must ensure that the workqueue on which @work was last4360 * Must be called from a sleepable context if @work was last queued on a non-BH
4354 * queued can't be destroyed before this function returns.4361 * workqueue. Can also be called from non-hardirq atomic contexts including BH
4362 * if @work was last queued on a BH workqueue.
4355 *4363 *
4356 * Return:4364 * Returns %true if @work was pending, %false otherwise.
4357 * %true if @work was pending, %false otherwise.
4358 */4365 */
4359bool cancel_work_sync(struct work_struct *work)4366bool cancel_work_sync(struct work_struct *work)
4360{4367{
@@ -4399,6 +4406,108 @@ bool cancel_delayed_work_sync(struct delayed_work *dwork) · 38 unmodified lines
4399}4406}
4400EXPORT_SYMBOL(cancel_delayed_work_sync);4407EXPORT_SYMBOL(cancel_delayed_work_sync);
4401 4408
4409/**
4410 * disable_work - Disable and cancel a work item
4411 * @work: work item to disable
4412 *
4413 * Disable @work by incrementing its disable count and cancel it if currently
4414 * pending. As long as the disable count is non-zero, any attempt to queue @work
4415 * will fail and return %false. The maximum supported disable depth is 2 to the
4416 * power of %WORK_OFFQ_DISABLE_BITS, currently 65536.
4417 *
4418 * Can be called from any context. Returns %true if @work was pending, %false
4419 * otherwise.
4420 */
4421bool disable_work(struct work_struct *work)
4422{
4423 return __cancel_work(work, WORK_CANCEL_DISABLE);
4424}
4425EXPORT_SYMBOL_GPL(disable_work);
4426
4427/**
4428 * disable_work_sync - Disable, cancel and drain a work item
4429 * @work: work item to disable
4430 *
4431 * Similar to disable_work() but also wait for @work to finish if currently
4432 * executing.
4433 *
4434 * Must be called from a sleepable context if @work was last queued on a non-BH
4435 * workqueue. Can also be called from non-hardirq atomic contexts including BH
4436 * if @work was last queued on a BH workqueue.
4437 *
4438 * Returns %true if @work was pending, %false otherwise.
4439 */
4440bool disable_work_sync(struct work_struct *work)
4441{
4442 return __cancel_work_sync(work, WORK_CANCEL_DISABLE);
4443}
4444EXPORT_SYMBOL_GPL(disable_work_sync);
4445
4446/**
4447 * enable_work - Enable a work item
4448 * @work: work item to enable
4449 *
4450 * Undo disable_work[_sync]() by decrementing @work's disable count. @work can
4451 * only be queued if its disable count is 0.
4452 *
4453 * Can be called from any context. Returns %true if the disable count reached 0.
4454 * Otherwise, %false.
4455 */
4456bool enable_work(struct work_struct *work)
4457{
4458 struct work_offq_data offqd;
4459 unsigned long irq_flags;
4460
4461 work_grab_pending(work, 0, &irq_flags);
4462
4463 work_offqd_unpack(&offqd, *work_data_bits(work));
4464 work_offqd_enable(&offqd);
4465 set_work_pool_and_clear_pending(work, offqd.pool_id,
4466 work_offqd_pack_flags(&offqd));
4467 local_irq_restore(irq_flags);
4468
4469 return !offqd.disable;
4470}
4471EXPORT_SYMBOL_GPL(enable_work);
4472
4473/**
4474 * disable_delayed_work - Disable and cancel a delayed work item
4475 * @dwork: delayed work item to disable
4476 *
4477 * disable_work() for delayed work items.
4478 */
4479bool disable_delayed_work(struct delayed_work *dwork)
4480{
4481 return __cancel_work(&dwork->work,
4482 WORK_CANCEL_DELAYED | WORK_CANCEL_DISABLE);
4483}
4484EXPORT_SYMBOL_GPL(disable_delayed_work);
4485
4486/**
4487 * disable_delayed_work_sync - Disable, cancel and drain a delayed work item
4488 * @dwork: delayed work item to disable
4489 *
4490 * disable_work_sync() for delayed work items.
4491 */
4492bool disable_delayed_work_sync(struct delayed_work *dwork)
4493{
4494 return __cancel_work_sync(&dwork->work,
4495 WORK_CANCEL_DELAYED | WORK_CANCEL_DISABLE);
4496}
4497EXPORT_SYMBOL_GPL(disable_delayed_work_sync);
4498
4499/**
4500 * enable_delayed_work - Enable a delayed work item
4501 * @dwork: delayed work item to enable
4502 *
4503 * enable_work() for delayed work items.
4504 */
4505bool enable_delayed_work(struct delayed_work *dwork)
4506{
4507 return enable_work(&dwork->work);
4508}
4509EXPORT_SYMBOL_GPL(enable_delayed_work);
4510
4402/**4511/**
4403 * schedule_on_each_cpu - execute a function synchronously on each online CPU4512 * schedule_on_each_cpu - execute a function synchronously on each online CPU
4404 * @func: the function to call4513 * @func: the function to call
@@ -4530,6 +4639,8 @@ static void wqattrs_clear_for_pool(struct workqueue_attrs *attrs) · 125 unmodified lines
4530{4639{
4531 attrs->affn_scope = WQ_AFFN_NR_TYPES;4640 attrs->affn_scope = WQ_AFFN_NR_TYPES;
4532 attrs->ordered = false;4641 attrs->ordered = false;
4642 if (attrs->affn_strict)
4643 cpumask_copy(attrs->cpumask, cpu_possible_mask);
4533}4644}
4534 4645
4535/* hash value of the content of @attr */4646/* hash value of the content of @attr */
@@ -4538,11 +4649,12 @@ static u32 wqattrs_hash(const struct workqueue_attrs *attrs) · 2 unmodified lines
4538 u32 hash = 0;4649 u32 hash = 0;
4539 4650
4540 hash = jhash_1word(attrs->nice, hash);4651 hash = jhash_1word(attrs->nice, hash);
4541 hash = jhash(cpumask_bits(attrs->cpumask),4652 hash = jhash_1word(attrs->affn_strict, hash);
4542 BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
4543 hash = jhash(cpumask_bits(attrs->__pod_cpumask),4653 hash = jhash(cpumask_bits(attrs->__pod_cpumask),
4544 BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);4654 BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
4545 hash = jhash_1word(attrs->affn_strict, hash);4655 if (!attrs->affn_strict)
4656 hash = jhash(cpumask_bits(attrs->cpumask),
4657 BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
4546 return hash;4658 return hash;
4547}4659}
4548 4660
@@ -4552,11 +4664,11 @@ static bool wqattrs_equal(const struct workqueue_attrs *a, · 3 unmodified lines
4552{4664{
4553 if (a->nice != b->nice)4665 if (a->nice != b->nice)
4554 return false;4666 return false;
4555 if (!cpumask_equal(a->cpumask, b->cpumask))4667 if (a->affn_strict != b->affn_strict)
4556 return false;4668 return false;
4557 if (!cpumask_equal(a->__pod_cpumask, b->__pod_cpumask))4669 if (!cpumask_equal(a->__pod_cpumask, b->__pod_cpumask))
4558 return false;4670 return false;
4559 if (a->affn_strict != b->affn_strict)4671 if (!a->affn_strict && !cpumask_equal(a->cpumask, b->cpumask))
4560 return false;4672 return false;
4561 return true;4673 return true;
4562}4674}
@@ -7148,25 +7260,27 @@ static ssize_t __wq_cpumask_show(struct device *dev, · 2585 unmodified lines
7148 return written;7260 return written;
7149}7261}
7150 7262
7151static ssize_t wq_unbound_cpumask_show(struct device *dev,7263static ssize_t cpumask_requested_show(struct device *dev,
7152 struct device_attribute *attr, char *buf)7264 struct device_attribute *attr, char *buf)
7153{7265{
7154 return __wq_cpumask_show(dev, attr, buf, wq_unbound_cpumask);7266 return __wq_cpumask_show(dev, attr, buf, wq_requested_unbound_cpumask);
7155}7267}
7268static DEVICE_ATTR_RO(cpumask_requested);
7156 7269
7157static ssize_t wq_requested_cpumask_show(struct device *dev,7270static ssize_t cpumask_isolated_show(struct device *dev,
7158 struct device_attribute *attr, char *buf)7271 struct device_attribute *attr, char *buf)
7159{7272{
7160 return __wq_cpumask_show(dev, attr, buf, wq_requested_unbound_cpumask);7273 return __wq_cpumask_show(dev, attr, buf, wq_isolated_cpumask);
7161}7274}
7275static DEVICE_ATTR_RO(cpumask_isolated);
7162 7276
7163static ssize_t wq_isolated_cpumask_show(struct device *dev,7277static ssize_t cpumask_show(struct device *dev,
7164 struct device_attribute *attr, char *buf)7278 struct device_attribute *attr, char *buf)
7165{7279{
7166 return __wq_cpumask_show(dev, attr, buf, wq_isolated_cpumask);7280 return __wq_cpumask_show(dev, attr, buf, wq_unbound_cpumask);
7167}7281}
7168 7282
7169static ssize_t wq_unbound_cpumask_store(struct device *dev,7283static ssize_t cpumask_store(struct device *dev,
7170 struct device_attribute *attr, const char *buf, size_t count)7284 struct device_attribute *attr, const char *buf, size_t count)
7171{7285{
7172 cpumask_var_t cpumask;7286 cpumask_var_t cpumask;
@@ -7182,36 +7296,19 @@ static ssize_t wq_unbound_cpumask_store(struct device *dev, · 9 unmodified lines
7182 free_cpumask_var(cpumask);7296 free_cpumask_var(cpumask);
7183 return ret ? ret : count;7297 return ret ? ret : count;
7184}7298}
7299static DEVICE_ATTR_RW(cpumask);
7185 7300
7186static struct device_attribute wq_sysfs_cpumask_attrs[] = {7301static struct attribute *wq_sysfs_cpumask_attrs[] = {
7187 __ATTR(cpumask, 0644, wq_unbound_cpumask_show,7302 &dev_attr_cpumask.attr,
7188 wq_unbound_cpumask_store),7303 &dev_attr_cpumask_requested.attr,
7189 __ATTR(cpumask_requested, 0444, wq_requested_cpumask_show, NULL),7304 &dev_attr_cpumask_isolated.attr,
7190 __ATTR(cpumask_isolated, 0444, wq_isolated_cpumask_show, NULL),7305 NULL,
7191 __ATTR_NULL,
7192};7306};
7307ATTRIBUTE_GROUPS(wq_sysfs_cpumask);
7193 7308
7194static int __init wq_sysfs_init(void)7309static int __init wq_sysfs_init(void)
7195{7310{
7196 struct device *dev_root;7311 return subsys_virtual_register(&wq_subsys, wq_sysfs_cpumask_groups);
7197 int err;
7198
7199 err = subsys_virtual_register(&wq_subsys, NULL);
7200 if (err)
7201 return err;
7202
7203 dev_root = bus_get_dev_root(&wq_subsys);
7204 if (dev_root) {
7205 struct device_attribute *attr;
7206
7207 for (attr = wq_sysfs_cpumask_attrs; attr->attr.name; attr++) {
7208 err = device_create_file(dev_root, attr);
7209 if (err)
7210 break;
7211 }
7212 put_device(dev_root);
7213 }
7214 return err;
7215}7312}
7216core_initcall(wq_sysfs_init);7313core_initcall(wq_sysfs_init);
7217 7314