// SPDX-License-Identifier: GPL-2.0-or-later #include <linux/slab.h>#include <linux/sched/rt.h>#include <linux/sched/task.h> #include "futex.h"#include "../locking/rtmutex_common.h" /* * PI code: */int refill_pi_state_cache(void){	struct futex_pi_state *pi_state; 	if (likely(current->futex.pi_state_cache))		return 0; 	pi_state = kzalloc_obj(*pi_state); 	if (!pi_state)		return -ENOMEM; 	INIT_LIST_HEAD(&pi_state->list);	/* pi_mutex gets initialized later */	pi_state->owner = NULL;	refcount_set(&pi_state->refcount, 1);	pi_state->key = FUTEX_KEY_INIT; 	current->futex.pi_state_cache = pi_state; 	return 0;} static struct futex_pi_state *alloc_pi_state(void){	struct futex_pi_state *pi_state = current->futex.pi_state_cache; 	WARN_ON(!pi_state);	current->futex.pi_state_cache = NULL; 	return pi_state;} static void pi_state_update_owner(struct futex_pi_state *pi_state,				  struct task_struct *new_owner){	struct task_struct *old_owner = pi_state->owner; 	lockdep_assert_held(&pi_state->pi_mutex.wait_lock); 	if (old_owner) {		raw_spin_lock(&old_owner->pi_lock);		WARN_ON(list_empty(&pi_state->list));		list_del_init(&pi_state->list);		raw_spin_unlock(&old_owner->pi_lock);	} 	if (new_owner) {		raw_spin_lock(&new_owner->pi_lock);		WARN_ON(!list_empty(&pi_state->list));		list_add(&pi_state->list, &new_owner->futex.pi_state_list);		pi_state->owner = new_owner;		raw_spin_unlock(&new_owner->pi_lock);	}} void get_pi_state(struct futex_pi_state *pi_state){	WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount));} /* * Drops a reference to the pi_state object and frees or caches it * when the last reference is gone. */void put_pi_state(struct futex_pi_state *pi_state){	if (!pi_state)		return; 	if (!refcount_dec_and_test(&pi_state->refcount))		return; 	/*	 * If pi_state->owner is NULL, the owner is most probably dying	 * and has cleaned up the pi_state already	 */	if (pi_state->owner) {		unsigned long flags; 		raw_spin_lock_irqsave(&pi_state->pi_mutex.wait_lock, flags);		pi_state_update_owner(pi_state, NULL);		rt_mutex_proxy_unlock(&pi_state->pi_mutex);		raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags);	} 	if (current->futex.pi_state_cache) {		kfree(pi_state);	} else {		/*		 * pi_state->list is already empty.		 * clear pi_state->owner.		 * refcount is at 0 - put it back to 1.		 */		pi_state->owner = NULL;		refcount_set(&pi_state->refcount, 1);		current->futex.pi_state_cache = pi_state;	}} /* * We need to check the following states: * *      Waiter | pi_state | pi->owner | uTID      | uODIED | ? * * [1]  NULL   | ---      | ---       | 0         | 0/1    | Valid * [2]  NULL   | ---      | ---       | >0        | 0/1    | Valid * * [3]  Found  | NULL     | --        | Any       | 0/1    | Invalid * * [4]  Found  | Found    | NULL      | 0         | 1      | Valid * [5]  Found  | Found    | NULL      | >0        | 1      | Invalid * * [6]  Found  | Found    | task      | 0         | 1      | Valid * * [7]  Found  | Found    | NULL      | Any       | 0      | Invalid * * [8]  Found  | Found    | task      | ==taskTID | 0/1    | Valid * [9]  Found  | Found    | task      | 0         | 0      | Invalid * [10] Found  | Found    | task      | !=taskTID | 0/1    | Invalid * * [1]	Indicates that the kernel can acquire the futex atomically. We *	came here due to a stale FUTEX_WAITERS/FUTEX_OWNER_DIED bit. * * [2]	Valid, if TID does not belong to a kernel thread. If no matching *      thread is found then it indicates that the owner TID has died. * * [3]	Invalid. The waiter is queued on a non PI futex * * [4]	Valid state after exit_robust_list(), which sets the user space *	value to FUTEX_WAITERS | FUTEX_OWNER_DIED. * * [5]	The user space value got manipulated between exit_robust_list() *	and exit_pi_state_list() * * [6]	Valid state after exit_pi_state_list() which sets the new owner in *	the pi_state but cannot access the user space value. * * [7]	pi_state->owner can only be NULL when the OWNER_DIED bit is set. * * [8]	Owner and user space value match * * [9]	There is no transient state which sets the user space TID to 0 *	except exit_robust_list(), but this is indicated by the *	FUTEX_OWNER_DIED bit. See [4] * * [10] There is no transient state which leaves owner and user space *	TID out of sync. Except one error case where the kernel is denied *	write access to the user address, see fixup_pi_state_owner(). * * * Serialization and lifetime rules: * * hb->lock: * *	hb -> futex_q, relation *	futex_q -> pi_state, relation * *	(cannot be raw because hb can contain arbitrary amount *	 of futex_q's) * * pi_mutex->wait_lock: * *	{uval, pi_state} * *	(and pi_mutex 'obviously') * * p->pi_lock: * *	p->futex.pi_state_list -> pi_state->list, relation *	pi_mutex->owner -> pi_state->owner, relation * * pi_state->refcount: * *	pi_state lifetime * * * Lock order: * *   hb->lock *     pi_mutex->wait_lock *       p->pi_lock * * Futex kernel state: * * The kernel tracks the task state in p::futex::state to protect against exit() * and exec(). The states are: * * - FUTEX_STATE_OK when the task is alive and waiters can be attached * * - FUTEX_STATE_EXITING when the task cleans up the robust list and PI *   state. Concurrent waiters cannot attach anymore and have to wait until the *   cleanup is finished to re-evaluate the potential changes caused by the *   robust list and PI state cleanups. * * - FUTEX_STATE_DEAD when the task has cleaned up the robust list. This state *   is set independent of exit() or exec(). In the exit() case the task is *   gone. In the exec() case this ensures that nothing can attach to the task *   after cleaning up the robust list and PI state before it has switched to *   the new mm. From a futex point of view the task is dead until it sets the *   state to FUTEX_STATE_OK again after switching to the new mm. * * The valid state transitions for exit(): * *   FUTEX_STATE_OK -> FUTEX_STATE_EXITING -> FUTEX_STATE_DEAD * * The valid state transitions for exec(): * *   FUTEX_STATE_OK -> FUTEX_STATE_EXITING -> FUTEX_STATE_DEAD -> FUTEX_STATE_OK * * The state has two related locks: * * 1) p::pi_lock * *    p::pi_lock has to be taken by the waiter when evaluating the state to *    protect against a concurrent exit/exec cleanup by the owner. If the state *    is OK then the waiter can be attached to the owner while still holding *    pi_lock. * *    The cleanup code has to hold it for all state transitions to ensure that *    the stores to the state cannot be reordered against previous stores on *    which the waiter correctness depends on. * * 2) p::futex::exit_mutex * *    The mutex is acquired when the cleanup starts and released at the end. It *    obviously is not serializing the owner's cleanup against itself. It is *    used to avoid a live lock caused by a waiter preempting the owner's *    cleanup. Such a waiter would busy loop forever waiting for the owner to *    finish the cleanup. * *    To prevent this, waiters have to drop all locks when observing *    FUTEX_STATE_EXITING and block on the mutex. When the owner releases the *    mutex after finishing the cleanup the waiters make progress and *    re-evaluate the situation. */ /* * Validate that the existing waiter has a pi_state and sanity check * the pi_state against the user space value. If correct, attach to * it. */static int attach_to_pi_state(u32 __user *uaddr, u32 uval,			      struct futex_pi_state *pi_state,			      struct futex_pi_state **ps){	pid_t pid = uval & FUTEX_TID_MASK;	u32 uval2;	int ret; 	/*	 * Userspace might have messed up non-PI and PI futexes [3]	 */	if (unlikely(!pi_state))		return -EINVAL; 	/*	 * We get here with hb->lock held, and having found a	 * futex_top_waiter(). This means that futex_lock_pi() of said futex_q	 * has dropped the hb->lock in between futex_queue() and futex_unqueue_pi(),	 * which in turn means that futex_lock_pi() still has a reference on	 * our pi_state.	 *	 * The waiter holding a reference on @pi_state also protects against	 * the unlocked put_pi_state() in futex_unlock_pi(), futex_lock_pi()	 * and futex_wait_requeue_pi() as it cannot go to 0 and consequently	 * free pi_state before we can take a reference ourselves.	 */	WARN_ON(!refcount_read(&pi_state->refcount)); 	/*	 * Now that we have a pi_state, we can acquire wait_lock	 * and do the state validation.	 */	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); 	/*	 * Since {uval, pi_state} is serialized by wait_lock, and our current	 * uval was read without holding it, it can have changed. Verify it	 * still is what we expect it to be, otherwise retry the entire	 * operation.	 */	if (futex_get_value_locked(&uval2, uaddr))		goto out_efault; 	if (uval != uval2)		goto out_eagain; 	/*	 * Handle the owner died case:	 */	if (uval & FUTEX_OWNER_DIED) {		/*		 * exit_pi_state_list sets owner to NULL and wakes the		 * topmost waiter. The task which acquires the		 * pi_state->rt_mutex will fixup owner.		 */		if (!pi_state->owner) {			/*			 * No pi state owner, but the user space TID			 * is not 0. Inconsistent state. [5]			 */			if (pid)				goto out_einval;			/*			 * Take a ref on the state and return success. [4]			 */			goto out_attach;		} 		/*		 * If TID is 0, then either the dying owner has not		 * yet executed exit_pi_state_list() or some waiter		 * acquired the rtmutex in the pi state, but did not		 * yet fixup the TID in user space.		 *		 * Take a ref on the state and return success. [6]		 */		if (!pid)			goto out_attach;	} else {		/*		 * If the owner died bit is not set, then the pi_state		 * must have an owner. [7]		 */		if (!pi_state->owner)			goto out_einval;	} 	/*	 * Bail out if user space manipulated the futex value. If pi	 * state exists then the owner TID must be the same as the	 * user space TID. [9/10]	 */	if (pid != task_pid_vnr(pi_state->owner))		goto out_einval; out_attach:	get_pi_state(pi_state);	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);	*ps = pi_state;	return 0; out_einval:	ret = -EINVAL;	goto out_error; out_eagain:	ret = -EAGAIN;	goto out_error; out_efault:	ret = -EFAULT;	goto out_error; out_error:	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);	return ret;} static int handle_exit_race(u32 __user *uaddr, u32 uval){	u32 uval2; 	/*	 * Reread the user space value to handle the following situation:	 *	 * CPU0				CPU1	 *	 * sys_exit()			sys_futex()	 *  do_exit()			 futex_lock_pi()	 *                                futex_lock_pi_atomic()	 *   exit_signals(tsk)		    No waiters:	 *    tsk->flags |= PF_EXITING;	    *uaddr == 0x00000PID	 *  mm_release(tsk)		    Set waiter bit	 *   exit_robust_list(tsk) {	    *uaddr = 0x80000PID;	 *      Set owner died		    attach_to_pi_owner() {	 *    *uaddr = 0xC0000000;	     tsk = get_task(PID);	 *   }				     if (!tsk->flags & PF_EXITING) {	 *  ...				       attach();	 *  tsk->futex.state =               } else {	 *	FUTEX_STATE_DEAD;              if (tsk->futex.state !=	 *					  FUTEX_STATE_DEAD)	 *				         return -EAGAIN;	 *				       return -ESRCH; <--- FAIL	 *				     }	 *	 * Returning ESRCH unconditionally is wrong here because the	 * user space value has been changed by the exiting task.	 *	 * The same logic applies to the case where the exiting task is	 * already gone.	 */	if (futex_get_value_locked(&uval2, uaddr))		return -EFAULT; 	/* If the user space value has changed, try again. */	if (uval2 != uval)		return -EAGAIN; 	/*	 * The exiting task did not have a robust list, the robust list was	 * corrupted or the user space value in *uaddr is simply bogus.	 * Give up and tell user space.	 */	return -ESRCH;} static void __attach_to_pi_owner(struct task_struct *p, union futex_key *key,				 struct futex_pi_state **ps){	/*	 * No existing pi state. First waiter. [2]	 *	 * This creates pi_state, we have hb->lock held, this means nothing can	 * observe this state, wait_lock is irrelevant.	 */	struct futex_pi_state *pi_state = alloc_pi_state(); 	/*	 * Initialize the pi_mutex in locked state and make @p	 * the owner of it:	 */	__assume_ctx_lock(&pi_state->pi_mutex.wait_lock);	rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p); 	/* Store the key for possible exit cleanups: */	pi_state->key = *key; 	WARN_ON(!list_empty(&pi_state->list));	list_add(&pi_state->list, &p->futex.pi_state_list);	/*	 * Assignment without holding pi_state->pi_mutex.wait_lock is safe	 * because there is no concurrency as the object is not published yet.	 */	pi_state->owner = p; 	*ps = pi_state;}/* * Lookup the task for the TID provided from user space and attach to * it after doing proper sanity checks. */static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key,			      struct futex_pi_state **ps,			      struct task_struct **exiting){	pid_t pid = uval & FUTEX_TID_MASK;	struct task_struct *p; 	/*	 * We are the first waiter - try to look up the real owner and attach	 * the new pi_state to it, but bail out when TID = 0 [1]	 *	 * The !pid check is paranoid. None of the call sites should end up	 * with pid == 0, but better safe than sorry. Let the caller retry	 */	if (!pid)		return -EAGAIN;	p = find_get_task_by_vpid(pid);	if (!p)		return handle_exit_race(uaddr, uval); 	if (unlikely(p->flags & PF_KTHREAD)) {		put_task_struct(p);		return -EPERM;	} 	/*	 * We need to look at the task state to figure out whether the task is	 * exiting. To protect against the change of the task state from	 * FUTEX_STATE_OK to FUTEX_STATE_EXISTING in futex_cleanup_begin() it is	 * required to do this protected by p->pi_lock, which prevents the owner	 * from concurrently starting the exit cleanup.	 *	 * If the state is FUTEX_STATE_OK pi_lock must be held until the waiter	 * is attached to protect against a concurrent exit()/exec().	 */	raw_spin_lock_irq(&p->pi_lock); 	/* Validate that the task is ready for futex operations. */	if (unlikely(p->futex.state != FUTEX_STATE_OK)) {		/*		 * The task is on the way out. When state is FUTEX_STATE_EXITING		 * the cleanup is in progress. To avoid a live lock when the		 * waiter preempted the owner, store the task pointer in		 * @exiting and keep the reference on the task. The calling code		 * will drop all locks, block on @p::futex::exit_mutex and wait

Lines 1–500 of 1371. Load more