lael-labs/linux

kernel/bpf/btf.c+0−2

 		 * is not supported yet.
 		 * BPF_PROG_TYPE_RAW_TRACEPOINT is fine.
 		 */
-		if (log->level & BPF_LOG_LEVEL)
-			bpf_log(log, "arg#%d type is not a struct\n", arg);
 		return NULL;
 	}
 	tname = btf_name_by_offset(btf, t->name_off);

kernel/bpf/core.c+6−0

 	 * clone is guaranteed to not be locked.
 	 */
 	fp->aux = NULL;
+	fp->stats = NULL;
+	fp->active = NULL;
 	__bpf_prog_free(fp);
 }
 /* Return TRUE if the JIT backend wants verifier to enable sub-register usage
  * analysis code and wants explicit zero extension inserted by verifier.
  * Otherwise, return FALSE.
+ *
+ * The verifier inserts an explicit zero extension after BPF_CMPXCHGs even if
+ * you don't override this. JITs that don't want these extra insns can detect
+ * them using insn_is_zext.
  */
 bool __weak bpf_jit_needs_zext(void)
 {

kernel/bpf/verifier.c+58−33

@@ -504,6 +504,13 @@ static bool is_ptr_cast_function(enum bpf_func_id func_id) · 503 unmodified lines
504	func_id == BPF_FUNC_skc_to_tcp_request_sock;	504	func_id == BPF_FUNC_skc_to_tcp_request_sock;
505	}	505	}
506		506
		507	static bool is_cmpxchg_insn(const struct bpf_insn *insn)
		508	{
		509	return BPF_CLASS(insn->code) == BPF_STX &&
		510	BPF_MODE(insn->code) == BPF_ATOMIC &&
		511	insn->imm == BPF_CMPXCHG;
		512	}
		513
507	/* string representation of 'enum bpf_reg_type' */	514	/* string representation of 'enum bpf_reg_type' */
508	static const char * const reg_type_str[] = {	515	static const char * const reg_type_str[] = {
509	[NOT_INIT] = "?",	516	[NOT_INIT] = "?",
@@ -1120,7 +1127,7 @@ static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) · 610 unmodified lines
1120	reg->type = PTR_TO_RDWR_BUF;	1127	reg->type = PTR_TO_RDWR_BUF;
1121	break;	1128	break;
1122	default:	1129	default:
1123	WARN_ON("unknown nullable register type");	1130	WARN_ONCE(1, "unknown nullable register type");
1124	}	1131	}
1125	}	1132	}
1126		1133
@@ -1703,7 +1710,11 @@ static bool is_reg64(struct bpf_verifier_env env, struct bpf_insn insn, · 576 unmodified lines
1703	}	1710	}
1704		1711
1705	if (class == BPF_STX) {	1712	if (class == BPF_STX) {
1706	if (reg->type != SCALAR_VALUE)	1713	/* BPF_STX (including atomic variants) has multiple source
		1714	* operands, one of which is a ptr. Check whether the caller is
		1715	* asking about it.
		1716	*/
		1717	if (t == SRC_OP && reg->type != SCALAR_VALUE)
1707	return true;	1718	return true;
1708	return BPF_SIZE(code) == BPF_DW;	1719	return BPF_SIZE(code) == BPF_DW;
1709	}	1720	}
@@ -1735,22 +1746,38 @@ static bool is_reg64(struct bpf_verifier_env env, struct bpf_insn insn, · 25 unmodified lines
1735	return true;	1746	return true;
1736	}	1747	}
1737		1748
1738	/* Return TRUE if INSN doesn't have explicit value define. */	1749	/* Return the regno defined by the insn, or -1. */
1739	static bool insn_no_def(struct bpf_insn *insn)	1750	static int insn_def_regno(const struct bpf_insn *insn)
1740	{	1751	{
1741	u8 class = BPF_CLASS(insn->code);	1752	switch (BPF_CLASS(insn->code)) {
1742		1753	case BPF_JMP:
1743	return (class == BPF_JMP \|\| class == BPF_JMP32 \|\|	1754	case BPF_JMP32:
1744	class == BPF_STX \|\| class == BPF_ST);	1755	case BPF_ST:
		1756	return -1;
		1757	case BPF_STX:
		1758	if (BPF_MODE(insn->code) == BPF_ATOMIC &&
		1759	(insn->imm & BPF_FETCH)) {
		1760	if (insn->imm == BPF_CMPXCHG)
		1761	return BPF_REG_0;
		1762	else
		1763	return insn->src_reg;
		1764	} else {
		1765	return -1;
		1766	}
		1767	default:
		1768	return insn->dst_reg;
		1769	}
1745	}	1770	}
1746		1771
1747	/* Return TRUE if INSN has defined any 32-bit value explicitly. */	1772	/* Return TRUE if INSN has defined any 32-bit value explicitly. */
1748	static bool insn_has_def32(struct bpf_verifier_env env, struct bpf_insn insn)	1773	static bool insn_has_def32(struct bpf_verifier_env env, struct bpf_insn insn)
1749	{	1774	{
1750	if (insn_no_def(insn))	1775	int dst_reg = insn_def_regno(insn);
		1776
		1777	if (dst_reg == -1)
1751	return false;	1778	return false;
1752		1779
1753	return !is_reg64(env, insn, insn->dst_reg, NULL, DST_OP);	1780	return !is_reg64(env, insn, dst_reg, NULL, DST_OP);
1754	}	1781	}
1755		1782
1756	static void mark_insn_zext(struct bpf_verifier_env *env,	1783	static void mark_insn_zext(struct bpf_verifier_env *env,
@@ -11006,9 +11033,10 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 9249 unmodified lines
11006	for (i = 0; i < len; i++) {	11033	for (i = 0; i < len; i++) {
11007	int adj_idx = i + delta;	11034	int adj_idx = i + delta;
11008	struct bpf_insn insn;	11035	struct bpf_insn insn;
11009	u8 load_reg;	11036	int load_reg;
11010		11037
11011	insn = insns[adj_idx];	11038	insn = insns[adj_idx];
		11039	load_reg = insn_def_regno(&insn);
11012	if (!aux[adj_idx].zext_dst) {	11040	if (!aux[adj_idx].zext_dst) {
11013	u8 code, class;	11041	u8 code, class;
11014	u32 imm_rnd;	11042	u32 imm_rnd;
@@ -11018,14 +11046,14 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 3 unmodified lines
11018		11046
11019	code = insn.code;	11047	code = insn.code;
11020	class = BPF_CLASS(code);	11048	class = BPF_CLASS(code);
11021	if (insn_no_def(&insn))	11049	if (load_reg == -1)
11022	continue;	11050	continue;
11023		11051
11024	/* NOTE: arg "reg" (the fourth one) is only used for	11052	/* NOTE: arg "reg" (the fourth one) is only used for
11025	* BPF_STX which has been ruled out in above	11053	* BPF_STX + SRC_OP, so it is safe to pass NULL
11026	* check, it is safe to pass NULL here.	11054	* here.
11027	*/	11055	*/
11028	if (is_reg64(env, &insn, insn.dst_reg, NULL, DST_OP)) {	11056	if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) {
11029	if (class == BPF_LD &&	11057	if (class == BPF_LD &&
11030	BPF_MODE(code) == BPF_IMM)	11058	BPF_MODE(code) == BPF_IMM)
11031	i++;	11059	i++;
@@ -11040,31 +11068,28 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 8 unmodified lines
11040	imm_rnd = get_random_int();	11068	imm_rnd = get_random_int();
11041	rnd_hi32_patch[0] = insn;	11069	rnd_hi32_patch[0] = insn;
11042	rnd_hi32_patch[1].imm = imm_rnd;	11070	rnd_hi32_patch[1].imm = imm_rnd;
11043	rnd_hi32_patch[3].dst_reg = insn.dst_reg;	11071	rnd_hi32_patch[3].dst_reg = load_reg;
11044	patch = rnd_hi32_patch;	11072	patch = rnd_hi32_patch;
11045	patch_len = 4;	11073	patch_len = 4;
11046	goto apply_patch_buffer;	11074	goto apply_patch_buffer;
11047	}	11075	}
11048		11076
11049	if (!bpf_jit_needs_zext())	11077	/* Add in an zero-extend instruction if a) the JIT has requested
		11078	* it or b) it's a CMPXCHG.
		11079	*
		11080	* The latter is because: BPF_CMPXCHG always loads a value into
		11081	* R0, therefore always zero-extends. However some archs'
		11082	* equivalent instruction only does this load when the
		11083	* comparison is successful. This detail of CMPXCHG is
		11084	* orthogonal to the general zero-extension behaviour of the
		11085	* CPU, so it's treated independently of bpf_jit_needs_zext.
		11086	*/
		11087	if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn))
11050	continue;	11088	continue;
11051		11089
11052	/* zext_dst means that we want to zero-extend whatever register	11090	if (WARN_ON(load_reg == -1)) {
11053	* the insn defines, which is dst_reg most of the time, with	11091	verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n");
11054	* the notable exception of BPF_STX + BPF_ATOMIC + BPF_FETCH.	11092	return -EFAULT;
11055	*/
11056	if (BPF_CLASS(insn.code) == BPF_STX &&
11057	BPF_MODE(insn.code) == BPF_ATOMIC) {
11058	/* BPF_STX + BPF_ATOMIC insns without BPF_FETCH do not
11059	* define any registers, therefore zext_dst cannot be
11060	* set.
11061	*/
11062	if (WARN_ON(!(insn.imm & BPF_FETCH)))
11063	return -EINVAL;
11064	load_reg = insn.imm == BPF_CMPXCHG ? BPF_REG_0
11065	: insn.src_reg;
11066	} else {
11067	load_reg = insn.dst_reg;
11068	}	11093	}
11069		11094
11070	zext_patch[0] = insn;	11095	zext_patch[0] = insn;

kernel/events/core.c+38−4

 static atomic_t perf_sched_count;
 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
+static DEFINE_PER_CPU(int, perf_sched_cb_usages);
 static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
 static atomic_t nr_mmap_events __read_mostly;
 	}
 }
+static DEFINE_PER_CPU(struct list_head, sched_cb_list);
 void perf_sched_cb_dec(struct pmu *pmu)
 {
 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
-	--cpuctx->sched_cb_usage;
+	this_cpu_dec(perf_sched_cb_usages);
+	if (!--cpuctx->sched_cb_usage)
+		list_del(&cpuctx->sched_cb_entry);
 }
 {
 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
-	cpuctx->sched_cb_usage++;
+	if (!cpuctx->sched_cb_usage++)
+		list_add(&cpuctx->sched_cb_entry, this_cpu_ptr(&sched_cb_list));
+	this_cpu_inc(perf_sched_cb_usages);
 }
 /*
 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
 }
+static void perf_pmu_sched_task(struct task_struct *prev,
+				struct task_struct *next,
+				bool sched_in)
+{
+	struct perf_cpu_context *cpuctx;
+	if (prev == next)
+		return;
+	list_for_each_entry(cpuctx, this_cpu_ptr(&sched_cb_list), sched_cb_entry) {
+		/* will be handled in perf_event_context_sched_in/out */
+		if (cpuctx->task_ctx)
+			continue;
+		__perf_pmu_sched_task(cpuctx, sched_in);
+	}
+}
 static void perf_event_switch(struct task_struct *task,
 			      struct task_struct *next_prev, bool sched_in);
 {
 	int ctxn;
+	if (__this_cpu_read(perf_sched_cb_usages))
+		perf_pmu_sched_task(task, next, false);
 	if (atomic_read(&nr_switch_events))
 		perf_event_switch(task, next, false);
 	if (atomic_read(&nr_switch_events))
 		perf_event_switch(task, prev, true);
+	if (__this_cpu_read(perf_sched_cb_usages))
+		perf_pmu_sched_task(prev, task, true);
 }
 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
 	if (event->parent)
 		return;
-	if (event->attach_state & PERF_ATTACH_TASK)
+	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
 		dec = true;
 	if (event->attr.mmap || event->attr.mmap_data)
 		atomic_dec(&nr_mmap_events);
 	if (event->parent)
 		return;
-	if (event->attach_state & PERF_ATTACH_TASK)
+	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
 		inc = true;
 	if (event->attr.mmap || event->attr.mmap_data)
 		atomic_inc(&nr_mmap_events);
 #ifdef CONFIG_CGROUP_PERF
 		INIT_LIST_HEAD(&per_cpu(cgrp_cpuctx_list, cpu));
 #endif
+		INIT_LIST_HEAD(&per_cpu(sched_cb_list, cpu));
 	}
 }

kernel/fork.c+9−0

 #endif
 }
+static void mm_init_pasid(struct mm_struct *mm)
+{
+#ifdef CONFIG_IOMMU_SUPPORT
+	mm->pasid = INIT_PASID;
+#endif
+}
 static void mm_init_uprobes_state(struct mm_struct *mm)
 {
 #ifdef CONFIG_UPROBES
 	mm_init_cpumask(mm);
 	mm_init_aio(mm);
 	mm_init_owner(mm, p);
+	mm_init_pasid(mm);
 	RCU_INIT_POINTER(mm->exe_file, NULL);
 	mmu_notifier_subscriptions_init(mm);
 	init_tlb_flush_pending(mm);
 	if (!IS_ERR(tsk)) {
 		sigfillset(&tsk->blocked);
 		sigdelsetmask(&tsk->blocked, sigmask(SIGKILL));
+		tsk->flags |= PF_NOFREEZE;
 	}
 	return tsk;
 }

kernel/irq/irqdomain.c+4−5

 static void debugfs_add_domain_dir(struct irq_domain *d)
 {
-	if (!d->name || !domain_dir || d->debugfs_file)
+	if (!d->name || !domain_dir)
 		return;
-	d->debugfs_file = debugfs_create_file(d->name, 0444, domain_dir, d,
+	debugfs_create_file(d->name, 0444, domain_dir, d,
-					      &irq_domain_debug_fops);
+			    &irq_domain_debug_fops);
 }
 static void debugfs_remove_domain_dir(struct irq_domain *d)
 {
-	debugfs_remove(d->debugfs_file);
+	debugfs_remove(debugfs_lookup(d->name, domain_dir));
-	d->debugfs_file = NULL;
 }
 void __init irq_domain_debugfs_init(struct dentry *root)

kernel/locking/mutex.c+14−11

  */
 static __always_inline bool
 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
-		      const bool use_ww_ctx, struct mutex_waiter *waiter)
+		      struct mutex_waiter *waiter)
 {
 	if (!waiter) {
 		/*
 #else
 static __always_inline bool
 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
-		      const bool use_ww_ctx, struct mutex_waiter *waiter)
+		      struct mutex_waiter *waiter)
 {
 	return false;
 }
 	struct ww_mutex *ww;
 	int ret;
+	if (!use_ww_ctx)
+		ww_ctx = NULL;
 	might_sleep();
 #ifdef CONFIG_DEBUG_MUTEXES
 #endif
 	ww = container_of(lock, struct ww_mutex, base);
-	if (use_ww_ctx && ww_ctx) {
+	if (ww_ctx) {
 		if (unlikely(ww_ctx == READ_ONCE(ww->ctx)))
 			return -EALREADY;
 	mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
 	if (__mutex_trylock(lock) ||
-	    mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx, NULL)) {
+	    mutex_optimistic_spin(lock, ww_ctx, NULL)) {
 		/* got the lock, yay! */
 		lock_acquired(&lock->dep_map, ip);
-		if (use_ww_ctx && ww_ctx)
+		if (ww_ctx)
 			ww_mutex_set_context_fastpath(ww, ww_ctx);
 		preempt_enable();
 		return 0;
 	 * After waiting to acquire the wait_lock, try again.
 	 */
 	if (__mutex_trylock(lock)) {
-		if (use_ww_ctx && ww_ctx)
+		if (ww_ctx)
 			__ww_mutex_check_waiters(lock, ww_ctx);
 		goto skip_wait;
 			goto err;
 		}
-		if (use_ww_ctx && ww_ctx) {
+		if (ww_ctx) {
 			ret = __ww_mutex_check_kill(lock, &waiter, ww_ctx);
 			if (ret)
 				goto err;
 		 * ww_mutex needs to always recheck its position since its waiter
 		 * list is not FIFO ordered.
 		 */
-		if ((use_ww_ctx && ww_ctx) || !first) {
+		if (ww_ctx || !first) {
 			first = __mutex_waiter_is_first(lock, &waiter);
 			if (first)
 				__mutex_set_flag(lock, MUTEX_FLAG_HANDOFF);
 		 * or we must see its unlock and acquire.
 		 */
 		if (__mutex_trylock(lock) ||
-		    (first && mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx, &waiter)))
+		    (first && mutex_optimistic_spin(lock, ww_ctx, &waiter)))
 			break;
 		spin_lock(&lock->wait_lock);
 acquired:
 	__set_current_state(TASK_RUNNING);
-	if (use_ww_ctx && ww_ctx) {
+	if (ww_ctx) {
 		/*
 		 * Wound-Wait; we stole the lock (!first_waiter), check the
 		 * waiters as anyone might want to wound us.
 	/* got the lock - cleanup and rejoice! */
 	lock_acquired(&lock->dep_map, ip);
-	if (use_ww_ctx && ww_ctx)
+	if (ww_ctx)
 		ww_mutex_lock_acquired(ww, ww_ctx);
 	spin_unlock(&lock->wait_lock);

kernel/sched/core.c+62−64

 	struct set_affinity_pending	*pending;
 };
+/*
+ * @refs: number of wait_for_completion()
+ * @stop_pending: is @stop_work in use
+ */
 struct set_affinity_pending {
 	refcount_t		refs;
+	unsigned int		stop_pending;
 	struct completion	done;
 	struct cpu_stop_work	stop_work;
 	struct migration_arg	arg;
  */
 static int migration_cpu_stop(void *data)
 {
-	struct set_affinity_pending *pending;
 	struct migration_arg *arg = data;
+	struct set_affinity_pending *pending = arg->pending;
 	struct task_struct *p = arg->task;
 	int dest_cpu = arg->dest_cpu;
 	struct rq *rq = this_rq();
 	raw_spin_lock(&p->pi_lock);
 	rq_lock(rq, &rf);
-	pending = p->migration_pending;
 	/*
 	 * If task_rq(p) != rq, it cannot be migrated here, because we're
 	 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
 			goto out;
 		if (pending) {
-			p->migration_pending = NULL;
+			if (p->migration_pending == pending)
+				p->migration_pending = NULL;
 			complete = true;
 		}
-		/* migrate_enable() --  we must not race against SCA */
 		if (dest_cpu < 0) {
-			/*
+			if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask))
-			 * When this was migrate_enable() but we no longer
-			 * have a @pending, a concurrent SCA 'fixed' things
-			 * and we should be valid again. Nothing to do.
-			 */
-			if (!pending) {
-				WARN_ON_ONCE(!cpumask_test_cpu(task_cpu(p), &p->cpus_mask));
 				goto out;
-			}
 			dest_cpu = cpumask_any_distribute(&p->cpus_mask);
 		}
 		else
 			p->wake_cpu = dest_cpu;
-	} else if (dest_cpu < 0 || pending) {
+		/*
+		 * XXX __migrate_task() can fail, at which point we might end
+		 * up running on a dodgy CPU, AFAICT this can only happen
+		 * during CPU hotplug, at which point we'll get pushed out
+		 * anyway, so it's probably not a big deal.
+		 */
+	} else if (pending) {
 		/*
 		 * This happens when we get migrated between migrate_enable()'s
 		 * preempt_enable() and scheduling the stopper task. At that
 		 * ->pi_lock, so the allowed mask is stable - if it got
 		 * somewhere allowed, we're done.
 		 */
-		if (pending && cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) {
+		if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) {
-			p->migration_pending = NULL;
+			if (p->migration_pending == pending)
+				p->migration_pending = NULL;
 			complete = true;
 			goto out;
 		}
-		/*
-		 * When this was migrate_enable() but we no longer have an
-		 * @pending, a concurrent SCA 'fixed' things and we should be
-		 * valid again. Nothing to do.
-		 */
-		if (!pending) {
-			WARN_ON_ONCE(!cpumask_test_cpu(task_cpu(p), &p->cpus_mask));
-			goto out;
-		}
 		/*
 		 * When migrate_enable() hits a rq mis-match we can't reliably
 		 * determine is_migration_disabled() and so have to chase after
 		 * it.
 		 */
+		WARN_ON_ONCE(!pending->stop_pending);
 		task_rq_unlock(rq, p, &rf);
 		stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop,
 				    &pending->arg, &pending->stop_work);
 		return 0;
 	}
 out:
+	if (pending)
+		pending->stop_pending = false;
 	task_rq_unlock(rq, p, &rf);
 	if (complete)
 		complete_all(&pending->done);
-	/* For pending->{arg,stop_work} */
-	pending = arg->pending;
-	if (pending && refcount_dec_and_test(&pending->refs))
-		wake_up_var(&pending->refs);
 	return 0;
 }
 			    int dest_cpu, unsigned int flags)
 {
 	struct set_affinity_pending my_pending = { }, *pending = NULL;
-	struct migration_arg arg = {
+	bool stop_pending, complete = false;
-		.task = p,
-		.dest_cpu = dest_cpu,
-	};
-	bool complete = false;
 	/* Can the task run on the task's current CPU? If so, we're done */
 	if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) {
 			push_task = get_task_struct(p);
 		}
+		/*
+		 * If there are pending waiters, but no pending stop_work,
+		 * then complete now.
+		 */
 		pending = p->migration_pending;
-		if (pending) {
+		if (pending && !pending->stop_pending) {
-			refcount_inc(&pending->refs);
 			p->migration_pending = NULL;
 			complete = true;
 		}
 		task_rq_unlock(rq, p, rf);
 		if (push_task) {
 		}
 		if (complete)
-			goto do_complete;
+			complete_all(&pending->done);
 		return 0;
 	}
 			/* Install the request */
 			refcount_set(&my_pending.refs, 1);
 			init_completion(&my_pending.done);
+			my_pending.arg = (struct migration_arg) {
+				.task = p,
+				.dest_cpu = -1,		/* any */
+				.pending = &my_pending,
+			};
 			p->migration_pending = &my_pending;
 		} else {
 			pending = p->migration_pending;
 		return -EINVAL;
 	}
-	if (flags & SCA_MIGRATE_ENABLE) {
-		refcount_inc(&pending->refs); /* pending->{arg,stop_work} */
-		p->migration_flags &= ~MDF_PUSH;
-		task_rq_unlock(rq, p, rf);
-		pending->arg = (struct migration_arg) {
-			.task = p,
-			.dest_cpu = -1,
-			.pending = pending,
-		};
-		stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop,
-				    &pending->arg, &pending->stop_work);
-		return 0;
-	}
 	if (task_running(rq, p) || p->state == TASK_WAKING) {
 		/*
-		 * Lessen races (and headaches) by delegating
+		 * MIGRATE_ENABLE gets here because 'p == current', but for
-		 * is_migration_disabled(p) checks to the stopper, which will
+		 * anything else we cannot do is_migration_disabled(), punt
-		 * run on the same CPU as said p.
+		 * and have the stopper function handle it all race-free.
 		 */
+		stop_pending = pending->stop_pending;
+		if (!stop_pending)
+			pending->stop_pending = true;
+		if (flags & SCA_MIGRATE_ENABLE)
+			p->migration_flags &= ~MDF_PUSH;
 		task_rq_unlock(rq, p, rf);
-		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
+		if (!stop_pending) {
+			stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop,
+					    &pending->arg, &pending->stop_work);
+		}
+		if (flags & SCA_MIGRATE_ENABLE)
+			return 0;
 	} else {
 		if (!is_migration_disabled(p)) {
 			if (task_on_rq_queued(p))
 				rq = move_queued_task(rq, rf, p, dest_cpu);
-			p->migration_pending = NULL;
+			if (!pending->stop_pending) {
-			complete = true;
+				p->migration_pending = NULL;
+				complete = true;
+			}
 		}
 		task_rq_unlock(rq, p, rf);
-do_complete:
 		if (complete)
 			complete_all(&pending->done);
 	}
 	wait_for_completion(&pending->done);
 	if (refcount_dec_and_test(&pending->refs))
-		wake_up_var(&pending->refs);
+		wake_up_var(&pending->refs); /* No UaF, just an address */
 	/*
 	 * Block the original owner of &pending until all subsequent callers
 	 */
 	wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs));
+	/* ARGH */
+	WARN_ON_ONCE(my_pending.stop_pending);
 	return 0;
 }

kernel/sched/membarrier.c+1−3

@@ -471,9 +471,7 @@ static int sync_runqueues_membarrier_state(struct mm_struct *mm) · 470 unmodified lines
471	}	471	}
472	rcu_read_unlock();	472	rcu_read_unlock();
473		473
474	preempt_disable();	474	on_each_cpu_mask(tmpmask, ipi_sync_rq_state, mm, true);
475	smp_call_function_many(tmpmask, ipi_sync_rq_state, mm, 1);
476	preempt_enable();
477		475
478	free_cpumask_var(tmpmask);	476	free_cpumask_var(tmpmask);
479	cpus_read_unlock();	477	cpus_read_unlock();

kernel/sys.c+1−1

@@ -2079,7 +2079,7 @@ static int prctl_set_auxv(struct mm_struct *mm, unsigned long addr, · 2078 unmodified lines
2079	* up to the caller to provide sane values here, otherwise userspace	2079	* up to the caller to provide sane values here, otherwise userspace
2080	* tools which use this vector might be unhappy.	2080	* tools which use this vector might be unhappy.
2081	*/	2081	*/
2082	unsigned long user_auxv[AT_VECTOR_SIZE];	2082	unsigned long user_auxv[AT_VECTOR_SIZE] = {};
2083		2083
2084	if (len > sizeof(user_auxv))	2084	if (len > sizeof(user_auxv))
2085	return -EINVAL;	2085	return -EINVAL;

kernel/time/hrtimer.c+39−21

 }
 /*
- * Recomputes cpu_base::*next_timer and returns the earliest expires_next but
+ * Recomputes cpu_base::*next_timer and returns the earliest expires_next
- * does not set cpu_base::*expires_next, that is done by hrtimer_reprogram.
+ * but does not set cpu_base::*expires_next, that is done by
+ * hrtimer[_force]_reprogram and hrtimer_interrupt only. When updating
+ * cpu_base::*expires_next right away, reprogramming logic would no longer
+ * work.
  *
  * When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases,
  * those timers will get run whenever the softirq gets handled, at the end of
 	return expires_next;
 }
+static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
+{
+	ktime_t expires_next, soft = KTIME_MAX;
+	/*
+	 * If the soft interrupt has already been activated, ignore the
+	 * soft bases. They will be handled in the already raised soft
+	 * interrupt.
+	 */
+	if (!cpu_base->softirq_activated) {
+		soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
+		/*
+		 * Update the soft expiry time. clock_settime() might have
+		 * affected it.
+		 */
+		cpu_base->softirq_expires_next = soft;
+	}
+	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);
+	/*
+	 * If a softirq timer is expiring first, update cpu_base->next_timer
+	 * and program the hardware with the soft expiry time.
+	 */
+	if (expires_next > soft) {
+		cpu_base->next_timer = cpu_base->softirq_next_timer;
+		expires_next = soft;
+	}
+	return expires_next;
+}
 static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
 {
 	ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
 {
 	ktime_t expires_next;
-	/*
+	expires_next = hrtimer_update_next_event(cpu_base);
-	 * Find the current next expiration time.
-	 */
-	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
-	if (cpu_base->next_timer && cpu_base->next_timer->is_soft) {
-		/*
-		 * When the softirq is activated, hrtimer has to be
-		 * programmed with the first hard hrtimer because soft
-		 * timer interrupt could occur too late.
-		 */
-		if (cpu_base->softirq_activated)
-			expires_next = __hrtimer_get_next_event(cpu_base,
-								HRTIMER_ACTIVE_HARD);
-		else
-			cpu_base->softirq_expires_next = expires_next;
-	}
 	if (skip_equal && expires_next == cpu_base->expires_next)
 		return;
 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
-	/* Reevaluate the clock bases for the next expiry */
+	/* Reevaluate the clock bases for the [soft] next expiry */
-	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
+	expires_next = hrtimer_update_next_event(cpu_base);
 	/*
 	 * Store the new expiry value so the migration code can verify
 	 * against it.

Merge branch 'locking/urgent' into locking/core, to pick up dependent commits

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code