*/void __sched rt_mutex_adjust_pi(struct task_struct *task){ struct rt_mutex_waiter *waiter; struct rt_mutex_base *next_lock; unsigned long flags; raw_spin_lock_irqsave(&task->pi_lock, flags); waiter = task->pi_blocked_on; if (!waiter || rt_waiter_node_equal(&waiter->tree, task_to_waiter_node(task))) { raw_spin_unlock_irqrestore(&task->pi_lock, flags); return; } next_lock = waiter->lock; raw_spin_unlock_irqrestore(&task->pi_lock, flags); /* gets dropped in rt_mutex_adjust_prio_chain()! */ get_task_struct(task); rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL, next_lock, NULL, task);} /* * Performs the wakeup of the top-waiter and re-enables preemption. */void __sched rt_mutex_postunlock(struct rt_wake_q_head *wqh){ rt_mutex_wake_up_q(wqh);} #ifdef CONFIG_DEBUG_RT_MUTEXESvoid rt_mutex_debug_task_free(struct task_struct *task){ DEBUG_LOCKS_WARN_ON(!RB_EMPTY_ROOT(&task->pi_waiters.rb_root)); DEBUG_LOCKS_WARN_ON(task->pi_blocked_on);}#endif #ifdef CONFIG_PREEMPT_RT/* Mutexes */static void __mutex_rt_init_generic(struct mutex *mutex){ rt_mutex_base_init(&mutex->rtmutex); debug_check_no_locks_freed((void *)mutex, sizeof(*mutex));} static __always_inline int __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass, struct lockdep_map *nest_lock, unsigned long ip) __acquires(lock) __no_context_analysis{ int ret; might_sleep(); mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip); ret = __rt_mutex_lock(&lock->rtmutex, state); if (ret) mutex_release(&lock->dep_map, ip); else lock_acquired(&lock->dep_map, ip); return ret;} #ifdef CONFIG_DEBUG_LOCK_ALLOCvoid mutex_rt_init_lockdep(struct mutex *mutex, const char *name, struct lock_class_key *key){ __mutex_rt_init_generic(mutex); lockdep_init_map_wait(&mutex->dep_map, name, key, 0, LD_WAIT_SLEEP);}EXPORT_SYMBOL(mutex_rt_init_lockdep); void __sched mutex_lock_nested(struct mutex *lock, unsigned int subclass){ __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);}EXPORT_SYMBOL_GPL(mutex_lock_nested); void __sched _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock){ __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, nest_lock, _RET_IP_);}EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock); int __sched mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass){ return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_);}EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested); int __sched _mutex_lock_killable(struct mutex *lock, unsigned int subclass, struct lockdep_map *nest_lock){ return __mutex_lock_common(lock, TASK_KILLABLE, subclass, nest_lock, _RET_IP_);}EXPORT_SYMBOL_GPL(_mutex_lock_killable); void __sched mutex_lock_io_nested(struct mutex *lock, unsigned int subclass){ int token; might_sleep(); token = io_schedule_prepare(); __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_); io_schedule_finish(token);}EXPORT_SYMBOL_GPL(mutex_lock_io_nested); int __sched _mutex_trylock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock){ int ret; if (IS_ENABLED(CONFIG_DEBUG_RT_MUTEXES) && WARN_ON_ONCE(!in_task())) return 0; ret = __rt_mutex_trylock(&lock->rtmutex); if (ret) mutex_acquire_nest(&lock->dep_map, 0, 1, nest_lock, _RET_IP_); return ret;}EXPORT_SYMBOL_GPL(_mutex_trylock_nest_lock);#else /* CONFIG_DEBUG_LOCK_ALLOC */ void mutex_rt_init_generic(struct mutex *mutex){ __mutex_rt_init_generic(mutex);}EXPORT_SYMBOL(mutex_rt_init_generic); void __sched mutex_lock(struct mutex *lock){ __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);}EXPORT_SYMBOL(mutex_lock); int __sched mutex_lock_interruptible(struct mutex *lock){ return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);}EXPORT_SYMBOL(mutex_lock_interruptible); int __sched mutex_lock_killable(struct mutex *lock){ return __mutex_lock_common(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);}EXPORT_SYMBOL(mutex_lock_killable); void __sched mutex_lock_io(struct mutex *lock){ int token = io_schedule_prepare(); __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_); io_schedule_finish(token);}EXPORT_SYMBOL(mutex_lock_io); int __sched mutex_trylock(struct mutex *lock){ if (IS_ENABLED(CONFIG_DEBUG_RT_MUTEXES) && WARN_ON_ONCE(!in_task())) return 0; return __rt_mutex_trylock(&lock->rtmutex);}EXPORT_SYMBOL(mutex_trylock);#endif /* !CONFIG_DEBUG_LOCK_ALLOC */ void __sched mutex_unlock(struct mutex *lock) __releases(lock) __no_context_analysis{ mutex_release(&lock->dep_map, _RET_IP_); __rt_mutex_unlock(&lock->rtmutex);}EXPORT_SYMBOL(mutex_unlock); #endif /* CONFIG_PREEMPT_RT */Lines 501–683 of 683. Previous lines
*/ void __sched rt_mutex_adjust_pi(struct task_struct *task) { struct rt_mutex_waiter *waiter; struct rt_mutex_base *next_lock; unsigned long flags; raw_spin_lock_irqsave(&task->pi_lock, flags); waiter = task->pi_blocked_on; if (!waiter || rt_waiter_node_equal(&waiter->tree, task_to_waiter_node(task))) { raw_spin_unlock_irqrestore(&task->pi_lock, flags); return; } next_lock = waiter->lock; raw_spin_unlock_irqrestore(&task->pi_lock, flags); /* gets dropped in rt_mutex_adjust_prio_chain()! */ get_task_struct(task); rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL, next_lock, NULL, task); } /* * Performs the wakeup of the top-waiter and re-enables preemption. */ void __sched rt_mutex_postunlock(struct rt_wake_q_head *wqh) { rt_mutex_wake_up_q(wqh); } #ifdef CONFIG_DEBUG_RT_MUTEXES void rt_mutex_debug_task_free(struct task_struct *task) { DEBUG_LOCKS_WARN_ON(!RB_EMPTY_ROOT(&task->pi_waiters.rb_root)); DEBUG_LOCKS_WARN_ON(task->pi_blocked_on); } #endif #ifdef CONFIG_PREEMPT_RT /* Mutexes */ static void __mutex_rt_init_generic(struct mutex *mutex) { rt_mutex_base_init(&mutex->rtmutex); debug_check_no_locks_freed((void *)mutex, sizeof(*mutex)); } static __always_inline int __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass, struct lockdep_map *nest_lock, unsigned long ip) __acquires(lock) __no_context_analysis { int ret; might_sleep(); mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip); ret = __rt_mutex_lock(&lock->rtmutex, state); if (ret) mutex_release(&lock->dep_map, ip); else lock_acquired(&lock->dep_map, ip); return ret; } #ifdef CONFIG_DEBUG_LOCK_ALLOC void mutex_rt_init_lockdep(struct mutex *mutex, const char *name, struct lock_class_key *key) { __mutex_rt_init_generic(mutex); lockdep_init_map_wait(&mutex->dep_map, name, key, 0, LD_WAIT_SLEEP); } EXPORT_SYMBOL(mutex_rt_init_lockdep); void __sched mutex_lock_nested(struct mutex *lock, unsigned int subclass) { __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_); } EXPORT_SYMBOL_GPL(mutex_lock_nested); void __sched _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock) { __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, nest_lock, _RET_IP_); } EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock); int __sched mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass) { return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_); } EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested); int __sched _mutex_lock_killable(struct mutex *lock, unsigned int subclass, struct lockdep_map *nest_lock) { return __mutex_lock_common(lock, TASK_KILLABLE, subclass, nest_lock, _RET_IP_); } EXPORT_SYMBOL_GPL(_mutex_lock_killable); void __sched mutex_lock_io_nested(struct mutex *lock, unsigned int subclass) { int token; might_sleep(); token = io_schedule_prepare(); __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_); io_schedule_finish(token); } EXPORT_SYMBOL_GPL(mutex_lock_io_nested); int __sched _mutex_trylock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock) { int ret; if (IS_ENABLED(CONFIG_DEBUG_RT_MUTEXES) && WARN_ON_ONCE(!in_task())) return 0; ret = __rt_mutex_trylock(&lock->rtmutex); if (ret) mutex_acquire_nest(&lock->dep_map, 0, 1, nest_lock, _RET_IP_); return ret; } EXPORT_SYMBOL_GPL(_mutex_trylock_nest_lock); #else /* CONFIG_DEBUG_LOCK_ALLOC */ void mutex_rt_init_generic(struct mutex *mutex) { __mutex_rt_init_generic(mutex); } EXPORT_SYMBOL(mutex_rt_init_generic); void __sched mutex_lock(struct mutex *lock) { __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_); } EXPORT_SYMBOL(mutex_lock); int __sched mutex_lock_interruptible(struct mutex *lock) { return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_); } EXPORT_SYMBOL(mutex_lock_interruptible); int __sched mutex_lock_killable(struct mutex *lock) { return __mutex_lock_common(lock, TASK_KILLABLE, 0, NULL, _RET_IP_); } EXPORT_SYMBOL(mutex_lock_killable); void __sched mutex_lock_io(struct mutex *lock) { int token = io_schedule_prepare(); __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_); io_schedule_finish(token); } EXPORT_SYMBOL(mutex_lock_io); int __sched mutex_trylock(struct mutex *lock) { if (IS_ENABLED(CONFIG_DEBUG_RT_MUTEXES) && WARN_ON_ONCE(!in_task())) return 0; return __rt_mutex_trylock(&lock->rtmutex); } EXPORT_SYMBOL(mutex_trylock); #endif /* !CONFIG_DEBUG_LOCK_ALLOC */ void __sched mutex_unlock(struct mutex *lock) __releases(lock) __no_context_analysis { mutex_release(&lock->dep_map, _RET_IP_); __rt_mutex_unlock(&lock->rtmutex); } EXPORT_SYMBOL(mutex_unlock); #endif /* CONFIG_PREEMPT_RT */