History

Merge branch 'locking/urgent' into locking/core, to pick up dependent commits

We are applying further, lower-prio fixes on top of two ww_mutex fixes in locking/urgent.

Ingo Molnar committed 5 years ago

Commit 01438742 parents 4faf62b bee6457691 files · +5814 −3820Browse files

Showing the changes to kernel (11 of 691 files). Show all

Unified · Split

11 files
  1. kernel/bpf/btf.c +0 −2
  2. kernel/bpf/core.c +6 −0
  3. kernel/bpf/verifier.c +58 −33
  4. kernel/events/core.c +38 −4
  5. kernel/fork.c +9 −0
  6. kernel/irq/irqdomain.c +4 −5
  7. kernel/locking/mutex.c +14 −11
  8. kernel/sched/core.c +62 −64
  9. kernel/sched/membarrier.c +1 −3
  10. kernel/sys.c +1 −1
  11. kernel/time/hrtimer.c +39 −21
kernel/bpf/btf.c+0−2
@@ -4321,8 +4321,6 @@ btf_get_prog_ctx_type(struct bpf_verifier_log *log, struct btf *btf, · 4320 unmodified lines
43214321 * is not supported yet.
43224322 * BPF_PROG_TYPE_RAW_TRACEPOINT is fine.
43234323 */
4324− if (log->level & BPF_LOG_LEVEL)
4325− bpf_log(log, "arg#%d type is not a struct\n", arg);
43264324 return NULL;
43274325 }
43284326 tname = btf_name_by_offset(btf, t->name_off);
kernel/bpf/core.c+6−0
@@ -1118,6 +1118,8 @@ static void bpf_prog_clone_free(struct bpf_prog *fp) · 1117 unmodified lines
11181118 * clone is guaranteed to not be locked.
11191119 */
11201120 fp->aux = NULL;
1121+ fp->stats = NULL;
1122+ fp->active = NULL;
11211123 __bpf_prog_free(fp);
11221124 }
11231125
@@ -2342,6 +2344,10 @@ bool __weak bpf_helper_changes_pkt_data(void *func) · 1218 unmodified lines
23422344 /* Return TRUE if the JIT backend wants verifier to enable sub-register usage
23432345 * analysis code and wants explicit zero extension inserted by verifier.
23442346 * Otherwise, return FALSE.
2347+ *
2348+ * The verifier inserts an explicit zero extension after BPF_CMPXCHGs even if
2349+ * you don't override this. JITs that don't want these extra insns can detect
2350+ * them using insn_is_zext.
23452351 */
23462352 bool __weak bpf_jit_needs_zext(void)
23472353 {
kernel/bpf/verifier.c+58−33
@@ -504,6 +504,13 @@ static bool is_ptr_cast_function(enum bpf_func_id func_id) · 503 unmodified lines
504504 func_id == BPF_FUNC_skc_to_tcp_request_sock;
505505 }
506506
507+static bool is_cmpxchg_insn(const struct bpf_insn *insn)
508+{
509+ return BPF_CLASS(insn->code) == BPF_STX &&
510+ BPF_MODE(insn->code) == BPF_ATOMIC &&
511+ insn->imm == BPF_CMPXCHG;
512+}
513+
507514 /* string representation of 'enum bpf_reg_type' */
508515 static const char * const reg_type_str[] = {
509516 [NOT_INIT] = "?",
@@ -1120,7 +1127,7 @@ static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) · 610 unmodified lines
11201127 reg->type = PTR_TO_RDWR_BUF;
11211128 break;
11221129 default:
1123− WARN_ON("unknown nullable register type");
1130+ WARN_ONCE(1, "unknown nullable register type");
11241131 }
11251132 }
11261133
@@ -1703,7 +1710,11 @@ static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, · 576 unmodified lines
17031710 }
17041711
17051712 if (class == BPF_STX) {
1706− if (reg->type != SCALAR_VALUE)
1713+ /* BPF_STX (including atomic variants) has multiple source
1714+ * operands, one of which is a ptr. Check whether the caller is
1715+ * asking about it.
1716+ */
1717+ if (t == SRC_OP && reg->type != SCALAR_VALUE)
17071718 return true;
17081719 return BPF_SIZE(code) == BPF_DW;
17091720 }
@@ -1735,22 +1746,38 @@ static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, · 25 unmodified lines
17351746 return true;
17361747 }
17371748
1738−/* Return TRUE if INSN doesn't have explicit value define. */
1739−static bool insn_no_def(struct bpf_insn *insn)
1749+/* Return the regno defined by the insn, or -1. */
1750+static int insn_def_regno(const struct bpf_insn *insn)
17401751 {
1741− u8 class = BPF_CLASS(insn->code);
1742−
1743− return (class == BPF_JMP || class == BPF_JMP32 ||
1744− class == BPF_STX || class == BPF_ST);
1752+ switch (BPF_CLASS(insn->code)) {
1753+ case BPF_JMP:
1754+ case BPF_JMP32:
1755+ case BPF_ST:
1756+ return -1;
1757+ case BPF_STX:
1758+ if (BPF_MODE(insn->code) == BPF_ATOMIC &&
1759+ (insn->imm & BPF_FETCH)) {
1760+ if (insn->imm == BPF_CMPXCHG)
1761+ return BPF_REG_0;
1762+ else
1763+ return insn->src_reg;
1764+ } else {
1765+ return -1;
1766+ }
1767+ default:
1768+ return insn->dst_reg;
1769+ }
17451770 }
17461771
17471772 /* Return TRUE if INSN has defined any 32-bit value explicitly. */
17481773 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn)
17491774 {
1750− if (insn_no_def(insn))
1775+ int dst_reg = insn_def_regno(insn);
1776+
1777+ if (dst_reg == -1)
17511778 return false;
17521779
1753− return !is_reg64(env, insn, insn->dst_reg, NULL, DST_OP);
1780+ return !is_reg64(env, insn, dst_reg, NULL, DST_OP);
17541781 }
17551782
17561783 static void mark_insn_zext(struct bpf_verifier_env *env,
@@ -11006,9 +11033,10 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 9249 unmodified lines
1100611033 for (i = 0; i < len; i++) {
1100711034 int adj_idx = i + delta;
1100811035 struct bpf_insn insn;
11009− u8 load_reg;
11036+ int load_reg;
1101011037
1101111038 insn = insns[adj_idx];
11039+ load_reg = insn_def_regno(&insn);
1101211040 if (!aux[adj_idx].zext_dst) {
1101311041 u8 code, class;
1101411042 u32 imm_rnd;
@@ -11018,14 +11046,14 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 3 unmodified lines
1101811046
1101911047 code = insn.code;
1102011048 class = BPF_CLASS(code);
11021− if (insn_no_def(&insn))
11049+ if (load_reg == -1)
1102211050 continue;
1102311051
1102411052 /* NOTE: arg "reg" (the fourth one) is only used for
11025− * BPF_STX which has been ruled out in above
11026− * check, it is safe to pass NULL here.
11053+ * BPF_STX + SRC_OP, so it is safe to pass NULL
11054+ * here.
1102711055 */
11028− if (is_reg64(env, &insn, insn.dst_reg, NULL, DST_OP)) {
11056+ if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) {
1102911057 if (class == BPF_LD &&
1103011058 BPF_MODE(code) == BPF_IMM)
1103111059 i++;
@@ -11040,31 +11068,28 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 8 unmodified lines
1104011068 imm_rnd = get_random_int();
1104111069 rnd_hi32_patch[0] = insn;
1104211070 rnd_hi32_patch[1].imm = imm_rnd;
11043− rnd_hi32_patch[3].dst_reg = insn.dst_reg;
11071+ rnd_hi32_patch[3].dst_reg = load_reg;
1104411072 patch = rnd_hi32_patch;
1104511073 patch_len = 4;
1104611074 goto apply_patch_buffer;
1104711075 }
1104811076
11049− if (!bpf_jit_needs_zext())
11077+ /* Add in an zero-extend instruction if a) the JIT has requested
11078+ * it or b) it's a CMPXCHG.
11079+ *
11080+ * The latter is because: BPF_CMPXCHG always loads a value into
11081+ * R0, therefore always zero-extends. However some archs'
11082+ * equivalent instruction only does this load when the
11083+ * comparison is successful. This detail of CMPXCHG is
11084+ * orthogonal to the general zero-extension behaviour of the
11085+ * CPU, so it's treated independently of bpf_jit_needs_zext.
11086+ */
11087+ if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn))
1105011088 continue;
1105111089
11052− /* zext_dst means that we want to zero-extend whatever register
11053− * the insn defines, which is dst_reg most of the time, with
11054− * the notable exception of BPF_STX + BPF_ATOMIC + BPF_FETCH.
11055− */
11056− if (BPF_CLASS(insn.code) == BPF_STX &&
11057− BPF_MODE(insn.code) == BPF_ATOMIC) {
11058− /* BPF_STX + BPF_ATOMIC insns without BPF_FETCH do not
11059− * define any registers, therefore zext_dst cannot be
11060− * set.
11061− */
11062− if (WARN_ON(!(insn.imm & BPF_FETCH)))
11063− return -EINVAL;
11064− load_reg = insn.imm == BPF_CMPXCHG ? BPF_REG_0
11065− : insn.src_reg;
11066− } else {
11067− load_reg = insn.dst_reg;
11090+ if (WARN_ON(load_reg == -1)) {
11091+ verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n");
11092+ return -EFAULT;
1106811093 }
1106911094
1107011095 zext_patch[0] = insn;
kernel/events/core.c+38−4
@@ -386,6 +386,7 @@ static DEFINE_MUTEX(perf_sched_mutex); · 385 unmodified lines
386386 static atomic_t perf_sched_count;
387387
388388 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
389+static DEFINE_PER_CPU(int, perf_sched_cb_usages);
389390 static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
390391
391392 static atomic_t nr_mmap_events __read_mostly;
@@ -3461,11 +3462,16 @@ unlock: · 3069 unmodified lines
34613462 }
34623463 }
34633464
3465+static DEFINE_PER_CPU(struct list_head, sched_cb_list);
3466+
34643467 void perf_sched_cb_dec(struct pmu *pmu)
34653468 {
34663469 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
34673470
3468− --cpuctx->sched_cb_usage;
3471+ this_cpu_dec(perf_sched_cb_usages);
3472+
3473+ if (!--cpuctx->sched_cb_usage)
3474+ list_del(&cpuctx->sched_cb_entry);
34693475 }
34703476
34713477
@@ -3473,7 +3479,10 @@ void perf_sched_cb_inc(struct pmu *pmu) · 1 unmodified line
34733479 {
34743480 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
34753481
3476− cpuctx->sched_cb_usage++;
3482+ if (!cpuctx->sched_cb_usage++)
3483+ list_add(&cpuctx->sched_cb_entry, this_cpu_ptr(&sched_cb_list));
3484+
3485+ this_cpu_inc(perf_sched_cb_usages);
34773486 }
34783487
34793488 /*
@@ -3502,6 +3511,24 @@ static void __perf_pmu_sched_task(struct perf_cpu_context *cpuctx, bool sched_in · 22 unmodified lines
35023511 perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
35033512 }
35043513
3514+static void perf_pmu_sched_task(struct task_struct *prev,
3515+ struct task_struct *next,
3516+ bool sched_in)
3517+{
3518+ struct perf_cpu_context *cpuctx;
3519+
3520+ if (prev == next)
3521+ return;
3522+
3523+ list_for_each_entry(cpuctx, this_cpu_ptr(&sched_cb_list), sched_cb_entry) {
3524+ /* will be handled in perf_event_context_sched_in/out */
3525+ if (cpuctx->task_ctx)
3526+ continue;
3527+
3528+ __perf_pmu_sched_task(cpuctx, sched_in);
3529+ }
3530+}
3531+
35053532 static void perf_event_switch(struct task_struct *task,
35063533 struct task_struct *next_prev, bool sched_in);
35073534
@@ -3524,6 +3551,9 @@ void __perf_event_task_sched_out(struct task_struct *task, · 16 unmodified lines
35243551 {
35253552 int ctxn;
35263553
3554+ if (__this_cpu_read(perf_sched_cb_usages))
3555+ perf_pmu_sched_task(task, next, false);
3556+
35273557 if (atomic_read(&nr_switch_events))
35283558 perf_event_switch(task, next, false);
35293559
@@ -3832,6 +3862,9 @@ void __perf_event_task_sched_in(struct task_struct *prev, · 302 unmodified lines
38323862
38333863 if (atomic_read(&nr_switch_events))
38343864 perf_event_switch(task, prev, true);
3865+
3866+ if (__this_cpu_read(perf_sched_cb_usages))
3867+ perf_pmu_sched_task(prev, task, true);
38353868 }
38363869
38373870 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
@@ -4656,7 +4689,7 @@ static void unaccount_event(struct perf_event *event) · 818 unmodified lines
46564689 if (event->parent)
46574690 return;
46584691
4659− if (event->attach_state & PERF_ATTACH_TASK)
4692+ if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
46604693 dec = true;
46614694 if (event->attr.mmap || event->attr.mmap_data)
46624695 atomic_dec(&nr_mmap_events);
@@ -11175,7 +11208,7 @@ static void account_event(struct perf_event *event) · 6512 unmodified lines
1117511208 if (event->parent)
1117611209 return;
1117711210
11178− if (event->attach_state & PERF_ATTACH_TASK)
11211+ if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
1117911212 inc = true;
1118011213 if (event->attr.mmap || event->attr.mmap_data)
1118111214 atomic_inc(&nr_mmap_events);
@@ -12972,6 +13005,7 @@ static void __init perf_event_init_all_cpus(void) · 1790 unmodified lines
1297213005 #ifdef CONFIG_CGROUP_PERF
1297313006 INIT_LIST_HEAD(&per_cpu(cgrp_cpuctx_list, cpu));
1297413007 #endif
13008+ INIT_LIST_HEAD(&per_cpu(sched_cb_list, cpu));
1297513009 }
1297613010 }
1297713011
kernel/fork.c+9−0
@@ -994,6 +994,13 @@ static void mm_init_owner(struct mm_struct *mm, struct task_struct *p) · 993 unmodified lines
994994 #endif
995995 }
996996
997+static void mm_init_pasid(struct mm_struct *mm)
998+{
999+#ifdef CONFIG_IOMMU_SUPPORT
1000+ mm->pasid = INIT_PASID;
1001+#endif
1002+}
1003+
9971004 static void mm_init_uprobes_state(struct mm_struct *mm)
9981005 {
9991006 #ifdef CONFIG_UPROBES
@@ -1024,6 +1031,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, · 24 unmodified lines
10241031 mm_init_cpumask(mm);
10251032 mm_init_aio(mm);
10261033 mm_init_owner(mm, p);
1034+ mm_init_pasid(mm);
10271035 RCU_INIT_POINTER(mm->exe_file, NULL);
10281036 mmu_notifier_subscriptions_init(mm);
10291037 init_tlb_flush_pending(mm);
@@ -2436,6 +2444,7 @@ struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node) · 1406 unmodified lines
24362444 if (!IS_ERR(tsk)) {
24372445 sigfillset(&tsk->blocked);
24382446 sigdelsetmask(&tsk->blocked, sigmask(SIGKILL));
2447+ tsk->flags |= PF_NOFREEZE;
24392448 }
24402449 return tsk;
24412450 }
kernel/irq/irqdomain.c+4−5
@@ -1898,16 +1898,15 @@ DEFINE_SHOW_ATTRIBUTE(irq_domain_debug); · 1897 unmodified lines
18981898
18991899 static void debugfs_add_domain_dir(struct irq_domain *d)
19001900 {
1901− if (!d->name || !domain_dir || d->debugfs_file)
1901+ if (!d->name || !domain_dir)
19021902 return;
1903− d->debugfs_file = debugfs_create_file(d->name, 0444, domain_dir, d,
1904− &irq_domain_debug_fops);
1903+ debugfs_create_file(d->name, 0444, domain_dir, d,
1904+ &irq_domain_debug_fops);
19051905 }
19061906
19071907 static void debugfs_remove_domain_dir(struct irq_domain *d)
19081908 {
1909− debugfs_remove(d->debugfs_file);
1910− d->debugfs_file = NULL;
1909+ debugfs_remove(debugfs_lookup(d->name, domain_dir));
19111910 }
19121911
19131912 void __init irq_domain_debugfs_init(struct dentry *root)
kernel/locking/mutex.c+14−11
@@ -626,7 +626,7 @@ static inline int mutex_can_spin_on_owner(struct mutex *lock) · 625 unmodified lines
626626 */
627627 static __always_inline bool
628628 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
629− const bool use_ww_ctx, struct mutex_waiter *waiter)
629+ struct mutex_waiter *waiter)
630630 {
631631 if (!waiter) {
632632 /*
@@ -702,7 +702,7 @@ fail: · 69 unmodified lines
702702 #else
703703 static __always_inline bool
704704 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
705− const bool use_ww_ctx, struct mutex_waiter *waiter)
705+ struct mutex_waiter *waiter)
706706 {
707707 return false;
708708 }
@@ -922,6 +922,9 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, · 213 unmodified lines
922922 struct ww_mutex *ww;
923923 int ret;
924924
925+ if (!use_ww_ctx)
926+ ww_ctx = NULL;
927+
925928 might_sleep();
926929
927930 #ifdef CONFIG_DEBUG_MUTEXES
@@ -929,7 +932,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, · 1 unmodified line
929932 #endif
930933
931934 ww = container_of(lock, struct ww_mutex, base);
932− if (use_ww_ctx && ww_ctx) {
935+ if (ww_ctx) {
933936 if (unlikely(ww_ctx == READ_ONCE(ww->ctx)))
934937 return -EALREADY;
935938
@@ -946,10 +949,10 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, · 10 unmodified lines
946949 mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
947950
948951 if (__mutex_trylock(lock) ||
949− mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx, NULL)) {
952+ mutex_optimistic_spin(lock, ww_ctx, NULL)) {
950953 /* got the lock, yay! */
951954 lock_acquired(&lock->dep_map, ip);
952− if (use_ww_ctx && ww_ctx)
955+ if (ww_ctx)
953956 ww_mutex_set_context_fastpath(ww, ww_ctx);
954957 preempt_enable();
955958 return 0;
@@ -960,7 +963,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, · 4 unmodified lines
960963 * After waiting to acquire the wait_lock, try again.
961964 */
962965 if (__mutex_trylock(lock)) {
963− if (use_ww_ctx && ww_ctx)
966+ if (ww_ctx)
964967 __ww_mutex_check_waiters(lock, ww_ctx);
965968
966969 goto skip_wait;
@@ -1013,7 +1016,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, · 46 unmodified lines
10131016 goto err;
10141017 }
10151018
1016− if (use_ww_ctx && ww_ctx) {
1019+ if (ww_ctx) {
10171020 ret = __ww_mutex_check_kill(lock, &waiter, ww_ctx);
10181021 if (ret)
10191022 goto err;
@@ -1026,7 +1029,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, · 6 unmodified lines
10261029 * ww_mutex needs to always recheck its position since its waiter
10271030 * list is not FIFO ordered.
10281031 */
1029− if ((use_ww_ctx && ww_ctx) || !first) {
1032+ if (ww_ctx || !first) {
10301033 first = __mutex_waiter_is_first(lock, &waiter);
10311034 if (first)
10321035 __mutex_set_flag(lock, MUTEX_FLAG_HANDOFF);
@@ -1039,7 +1042,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, · 6 unmodified lines
10391042 * or we must see its unlock and acquire.
10401043 */
10411044 if (__mutex_trylock(lock) ||
1042− (first && mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx, &waiter)))
1045+ (first && mutex_optimistic_spin(lock, ww_ctx, &waiter)))
10431046 break;
10441047
10451048 spin_lock(&lock->wait_lock);
@@ -1048,7 +1051,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, · 2 unmodified lines
10481051 acquired:
10491052 __set_current_state(TASK_RUNNING);
10501053
1051− if (use_ww_ctx && ww_ctx) {
1054+ if (ww_ctx) {
10521055 /*
10531056 * Wound-Wait; we stole the lock (!first_waiter), check the
10541057 * waiters as anyone might want to wound us.
@@ -1068,7 +1071,7 @@ skip_wait: · 13 unmodified lines
10681071 /* got the lock - cleanup and rejoice! */
10691072 lock_acquired(&lock->dep_map, ip);
10701073
1071− if (use_ww_ctx && ww_ctx)
1074+ if (ww_ctx)
10721075 ww_mutex_lock_acquired(ww, ww_ctx);
10731076
10741077 spin_unlock(&lock->wait_lock);
kernel/sched/core.c+62−64
@@ -1862,8 +1862,13 @@ struct migration_arg { · 1861 unmodified lines
18621862 struct set_affinity_pending *pending;
18631863 };
18641864
1865+/*
1866+ * @refs: number of wait_for_completion()
1867+ * @stop_pending: is @stop_work in use
1868+ */
18651869 struct set_affinity_pending {
18661870 refcount_t refs;
1871+ unsigned int stop_pending;
18671872 struct completion done;
18681873 struct cpu_stop_work stop_work;
18691874 struct migration_arg arg;
@@ -1898,8 +1903,8 @@ static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, · 28 unmodified lines
18981903 */
18991904 static int migration_cpu_stop(void *data)
19001905 {
1901− struct set_affinity_pending *pending;
19021906 struct migration_arg *arg = data;
1907+ struct set_affinity_pending *pending = arg->pending;
19031908 struct task_struct *p = arg->task;
19041909 int dest_cpu = arg->dest_cpu;
19051910 struct rq *rq = this_rq();
@@ -1921,7 +1926,6 @@ static int migration_cpu_stop(void *data) · 15 unmodified lines
19211926 raw_spin_lock(&p->pi_lock);
19221927 rq_lock(rq, &rf);
19231928
1924− pending = p->migration_pending;
19251929 /*
19261930 * If task_rq(p) != rq, it cannot be migrated here, because we're
19271931 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
@@ -1932,21 +1936,14 @@ static int migration_cpu_stop(void *data) · 4 unmodified lines
19321936 goto out;
19331937
19341938 if (pending) {
1935− p->migration_pending = NULL;
1939+ if (p->migration_pending == pending)
1940+ p->migration_pending = NULL;
19361941 complete = true;
19371942 }
19381943
1939− /* migrate_enable() -- we must not race against SCA */
19401944 if (dest_cpu < 0) {
1941− /*
1942− * When this was migrate_enable() but we no longer
1943− * have a @pending, a concurrent SCA 'fixed' things
1944− * and we should be valid again. Nothing to do.
1945− */
1946− if (!pending) {
1947− WARN_ON_ONCE(!cpumask_test_cpu(task_cpu(p), &p->cpus_mask));
1945+ if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask))
19481946 goto out;
1949− }
19501947
19511948 dest_cpu = cpumask_any_distribute(&p->cpus_mask);
19521949 }
@@ -1956,7 +1953,14 @@ static int migration_cpu_stop(void *data) · 3 unmodified lines
19561953 else
19571954 p->wake_cpu = dest_cpu;
19581955
1959− } else if (dest_cpu < 0 || pending) {
1956+ /*
1957+ * XXX __migrate_task() can fail, at which point we might end
1958+ * up running on a dodgy CPU, AFAICT this can only happen
1959+ * during CPU hotplug, at which point we'll get pushed out
1960+ * anyway, so it's probably not a big deal.
1961+ */
1962+
1963+ } else if (pending) {
19601964 /*
19611965 * This happens when we get migrated between migrate_enable()'s
19621966 * preempt_enable() and scheduling the stopper task. At that
@@ -1971,43 +1975,32 @@ static int migration_cpu_stop(void *data) · 8 unmodified lines
19711975 * ->pi_lock, so the allowed mask is stable - if it got
19721976 * somewhere allowed, we're done.
19731977 */
1974− if (pending && cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) {
1975− p->migration_pending = NULL;
1978+ if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) {
1979+ if (p->migration_pending == pending)
1980+ p->migration_pending = NULL;
19761981 complete = true;
19771982 goto out;
19781983 }
19791984
1980− /*
1981− * When this was migrate_enable() but we no longer have an
1982− * @pending, a concurrent SCA 'fixed' things and we should be
1983− * valid again. Nothing to do.
1984− */
1985− if (!pending) {
1986− WARN_ON_ONCE(!cpumask_test_cpu(task_cpu(p), &p->cpus_mask));
1987− goto out;
1988− }
1989−
19901985 /*
19911986 * When migrate_enable() hits a rq mis-match we can't reliably
19921987 * determine is_migration_disabled() and so have to chase after
19931988 * it.
19941989 */
1990+ WARN_ON_ONCE(!pending->stop_pending);
19951991 task_rq_unlock(rq, p, &rf);
19961992 stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop,
19971993 &pending->arg, &pending->stop_work);
19981994 return 0;
19991995 }
20001996 out:
1997+ if (pending)
1998+ pending->stop_pending = false;
20011999 task_rq_unlock(rq, p, &rf);
20022000
20032001 if (complete)
20042002 complete_all(&pending->done);
20052003
2006− /* For pending->{arg,stop_work} */
2007− pending = arg->pending;
2008− if (pending && refcount_dec_and_test(&pending->refs))
2009− wake_up_var(&pending->refs);
2010−
20112004 return 0;
20122005 }
20132006
@@ -2194,11 +2187,7 @@ static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flag · 180 unmodified lines
21942187 int dest_cpu, unsigned int flags)
21952188 {
21962189 struct set_affinity_pending my_pending = { }, *pending = NULL;
2197− struct migration_arg arg = {
2198− .task = p,
2199− .dest_cpu = dest_cpu,
2200− };
2201− bool complete = false;
2190+ bool stop_pending, complete = false;
22022191
22032192 /* Can the task run on the task's current CPU? If so, we're done */
22042193 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) {
@@ -2210,12 +2199,16 @@ static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flag · 5 unmodified lines
22102199 push_task = get_task_struct(p);
22112200 }
22122201
2202+ /*
2203+ * If there are pending waiters, but no pending stop_work,
2204+ * then complete now.
2205+ */
22132206 pending = p->migration_pending;
2214− if (pending) {
2215− refcount_inc(&pending->refs);
2207+ if (pending && !pending->stop_pending) {
22162208 p->migration_pending = NULL;
22172209 complete = true;
22182210 }
2211+
22192212 task_rq_unlock(rq, p, rf);
22202213
22212214 if (push_task) {
@@ -2224,7 +2217,7 @@ static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flag · 2 unmodified lines
22242217 }
22252218
22262219 if (complete)
2227− goto do_complete;
2220+ complete_all(&pending->done);
22282221
22292222 return 0;
22302223 }
@@ -2235,6 +2228,12 @@ static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flag · 4 unmodified lines
22352228 /* Install the request */
22362229 refcount_set(&my_pending.refs, 1);
22372230 init_completion(&my_pending.done);
2231+ my_pending.arg = (struct migration_arg) {
2232+ .task = p,
2233+ .dest_cpu = -1, /* any */
2234+ .pending = &my_pending,
2235+ };
2236+
22382237 p->migration_pending = &my_pending;
22392238 } else {
22402239 pending = p->migration_pending;
@@ -2259,45 +2258,41 @@ static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flag · 18 unmodified lines
22592258 return -EINVAL;
22602259 }
22612260
2262− if (flags & SCA_MIGRATE_ENABLE) {
2263−
2264− refcount_inc(&pending->refs); /* pending->{arg,stop_work} */
2265− p->migration_flags &= ~MDF_PUSH;
2266− task_rq_unlock(rq, p, rf);
2267−
2268− pending->arg = (struct migration_arg) {
2269− .task = p,
2270− .dest_cpu = -1,
2271− .pending = pending,
2272− };
2273−
2274− stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop,
2275− &pending->arg, &pending->stop_work);
2276−
2277− return 0;
2278− }
2279−
22802261 if (task_running(rq, p) || p->state == TASK_WAKING) {
22812262 /*
2282− * Lessen races (and headaches) by delegating
2283− * is_migration_disabled(p) checks to the stopper, which will
2284− * run on the same CPU as said p.
2263+ * MIGRATE_ENABLE gets here because 'p == current', but for
2264+ * anything else we cannot do is_migration_disabled(), punt
2265+ * and have the stopper function handle it all race-free.
22852266 */
2267+ stop_pending = pending->stop_pending;
2268+ if (!stop_pending)
2269+ pending->stop_pending = true;
2270+
2271+ if (flags & SCA_MIGRATE_ENABLE)
2272+ p->migration_flags &= ~MDF_PUSH;
2273+
22862274 task_rq_unlock(rq, p, rf);
2287− stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
22882275
2276+ if (!stop_pending) {
2277+ stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop,
2278+ &pending->arg, &pending->stop_work);
2279+ }
2280+
2281+ if (flags & SCA_MIGRATE_ENABLE)
2282+ return 0;
22892283 } else {
22902284
22912285 if (!is_migration_disabled(p)) {
22922286 if (task_on_rq_queued(p))
22932287 rq = move_queued_task(rq, rf, p, dest_cpu);
22942288
2295− p->migration_pending = NULL;
2296− complete = true;
2289+ if (!pending->stop_pending) {
2290+ p->migration_pending = NULL;
2291+ complete = true;
2292+ }
22972293 }
22982294 task_rq_unlock(rq, p, rf);
22992295
2300−do_complete:
23012296 if (complete)
23022297 complete_all(&pending->done);
23032298 }
@@ -2305,7 +2300,7 @@ do_complete: · 1 unmodified line
23052300 wait_for_completion(&pending->done);
23062301
23072302 if (refcount_dec_and_test(&pending->refs))
2308− wake_up_var(&pending->refs);
2303+ wake_up_var(&pending->refs); /* No UaF, just an address */
23092304
23102305 /*
23112306 * Block the original owner of &pending until all subsequent callers
@@ -2313,6 +2308,9 @@ do_complete: · 1 unmodified line
23132308 */
23142309 wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs));
23152310
2311+ /* ARGH */
2312+ WARN_ON_ONCE(my_pending.stop_pending);
2313+
23162314 return 0;
23172315 }
23182316
kernel/sched/membarrier.c+1−3
@@ -471,9 +471,7 @@ static int sync_runqueues_membarrier_state(struct mm_struct *mm) · 470 unmodified lines
471471 }
472472 rcu_read_unlock();
473473
474− preempt_disable();
475− smp_call_function_many(tmpmask, ipi_sync_rq_state, mm, 1);
476− preempt_enable();
474+ on_each_cpu_mask(tmpmask, ipi_sync_rq_state, mm, true);
477475
478476 free_cpumask_var(tmpmask);
479477 cpus_read_unlock();
kernel/sys.c+1−1
@@ -2079,7 +2079,7 @@ static int prctl_set_auxv(struct mm_struct *mm, unsigned long addr, · 2078 unmodified lines
20792079 * up to the caller to provide sane values here, otherwise userspace
20802080 * tools which use this vector might be unhappy.
20812081 */
2082− unsigned long user_auxv[AT_VECTOR_SIZE];
2082+ unsigned long user_auxv[AT_VECTOR_SIZE] = {};
20832083
20842084 if (len > sizeof(user_auxv))
20852085 return -EINVAL;
kernel/time/hrtimer.c+39−21
@@ -546,8 +546,11 @@ static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base, · 545 unmodified lines
546546 }
547547
548548 /*
549− * Recomputes cpu_base::*next_timer and returns the earliest expires_next but
550− * does not set cpu_base::*expires_next, that is done by hrtimer_reprogram.
549+ * Recomputes cpu_base::*next_timer and returns the earliest expires_next
550+ * but does not set cpu_base::*expires_next, that is done by
551+ * hrtimer[_force]_reprogram and hrtimer_interrupt only. When updating
552+ * cpu_base::*expires_next right away, reprogramming logic would no longer
553+ * work.
551554 *
552555 * When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases,
553556 * those timers will get run whenever the softirq gets handled, at the end of
@@ -588,6 +591,37 @@ __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_ · 34 unmodified lines
588591 return expires_next;
589592 }
590593
594+static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
595+{
596+ ktime_t expires_next, soft = KTIME_MAX;
597+
598+ /*
599+ * If the soft interrupt has already been activated, ignore the
600+ * soft bases. They will be handled in the already raised soft
601+ * interrupt.
602+ */
603+ if (!cpu_base->softirq_activated) {
604+ soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
605+ /*
606+ * Update the soft expiry time. clock_settime() might have
607+ * affected it.
608+ */
609+ cpu_base->softirq_expires_next = soft;
610+ }
611+
612+ expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);
613+ /*
614+ * If a softirq timer is expiring first, update cpu_base->next_timer
615+ * and program the hardware with the soft expiry time.
616+ */
617+ if (expires_next > soft) {
618+ cpu_base->next_timer = cpu_base->softirq_next_timer;
619+ expires_next = soft;
620+ }
621+
622+ return expires_next;
623+}
624+
591625 static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
592626 {
593627 ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
@@ -628,23 +662,7 @@ hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal) · 34 unmodified lines
628662 {
629663 ktime_t expires_next;
630664
631− /*
632− * Find the current next expiration time.
633− */
634− expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
635−
636− if (cpu_base->next_timer && cpu_base->next_timer->is_soft) {
637− /*
638− * When the softirq is activated, hrtimer has to be
639− * programmed with the first hard hrtimer because soft
640− * timer interrupt could occur too late.
641− */
642− if (cpu_base->softirq_activated)
643− expires_next = __hrtimer_get_next_event(cpu_base,
644− HRTIMER_ACTIVE_HARD);
645− else
646− cpu_base->softirq_expires_next = expires_next;
647− }
665+ expires_next = hrtimer_update_next_event(cpu_base);
648666
649667 if (skip_equal && expires_next == cpu_base->expires_next)
650668 return;
@@ -1644,8 +1662,8 @@ retry: · 993 unmodified lines
16441662
16451663 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
16461664
1647− /* Reevaluate the clock bases for the next expiry */
1648− expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
1665+ /* Reevaluate the clock bases for the [soft] next expiry */
1666+ expires_next = hrtimer_update_next_event(cpu_base);
16491667 /*
16501668 * Store the new expiry value so the migration code can verify
16511669 * against it.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t