History

locking/rtmutex: Make text section and inlining consistent

rtmutex is half __sched and the other half is not. If the compiler decides to not inline larger static functions then part of the code ends up in the regular text section. There are also quite some performance related small helpers which are either static or plain inline. Force inline those which make sense and mark the rest __sched.

Thomas Gleixner committed 5 years ago

Commit d7a2edb1 parent f41dcc11 file · +76 −76Browse files

Unified · Split

kernel/locking/rtmutex.c+76−76
@@ -49,7 +49,7 @@ · 48 unmodified lines
49 * set this bit before looking at the lock.49 * set this bit before looking at the lock.
50 */50 */
51 51
52static void52static __always_inline void
53rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)53rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
54{54{
55 unsigned long val = (unsigned long)owner;55 unsigned long val = (unsigned long)owner;
@@ -60,13 +60,13 @@ rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner) · 4 unmodified lines
60 WRITE_ONCE(lock->owner, (struct task_struct *)val);60 WRITE_ONCE(lock->owner, (struct task_struct *)val);
61}61}
62 62
63static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)63static __always_inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
64{64{
65 lock->owner = (struct task_struct *)65 lock->owner = (struct task_struct *)
66 ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);66 ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
67}67}
68 68
69static void fixup_rt_mutex_waiters(struct rt_mutex *lock)69static __always_inline void fixup_rt_mutex_waiters(struct rt_mutex *lock)
70{70{
71 unsigned long owner, *p = (unsigned long *) &lock->owner;71 unsigned long owner, *p = (unsigned long *) &lock->owner;
72 72
@@ -149,7 +149,7 @@ static void fixup_rt_mutex_waiters(struct rt_mutex *lock) · 76 unmodified lines
149 * all future threads that attempt to [Rmw] the lock to the slowpath. As such149 * all future threads that attempt to [Rmw] the lock to the slowpath. As such
150 * relaxed semantics suffice.150 * relaxed semantics suffice.
151 */151 */
152static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)152static __always_inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
153{153{
154 unsigned long owner, *p = (unsigned long *) &lock->owner;154 unsigned long owner, *p = (unsigned long *) &lock->owner;
155 155
@@ -165,8 +165,8 @@ static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) · 9 unmodified lines
165 * 2) Drop lock->wait_lock165 * 2) Drop lock->wait_lock
166 * 3) Try to unlock the lock with cmpxchg166 * 3) Try to unlock the lock with cmpxchg
167 */167 */
168static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,168static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
169 unsigned long flags)169 unsigned long flags)
170 __releases(lock->wait_lock)170 __releases(lock->wait_lock)
171{171{
172 struct task_struct *owner = rt_mutex_owner(lock);172 struct task_struct *owner = rt_mutex_owner(lock);
@@ -204,7 +204,7 @@ static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock, · 31 unmodified lines
204# define rt_mutex_cmpxchg_acquire(l,c,n) (0)204# define rt_mutex_cmpxchg_acquire(l,c,n) (0)
205# define rt_mutex_cmpxchg_release(l,c,n) (0)205# define rt_mutex_cmpxchg_release(l,c,n) (0)
206 206
207static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)207static __always_inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
208{208{
209 lock->owner = (struct task_struct *)209 lock->owner = (struct task_struct *)
210 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);210 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
@@ -213,8 +213,8 @@ static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) · 2 unmodified lines
213/*213/*
214 * Simple slow path only version: lock->owner is protected by lock->wait_lock.214 * Simple slow path only version: lock->owner is protected by lock->wait_lock.
215 */215 */
216static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,216static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
217 unsigned long flags)217 unsigned long flags)
218 __releases(lock->wait_lock)218 __releases(lock->wait_lock)
219{219{
220 lock->owner = NULL;220 lock->owner = NULL;
@@ -229,9 +229,8 @@ static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock, · 8 unmodified lines
229#define task_to_waiter(p) \229#define task_to_waiter(p) \
230 &(struct rt_mutex_waiter){ .prio = (p)->prio, .deadline = (p)->dl.deadline }230 &(struct rt_mutex_waiter){ .prio = (p)->prio, .deadline = (p)->dl.deadline }
231 231
232static inline int232static __always_inline int rt_mutex_waiter_less(struct rt_mutex_waiter *left,
233rt_mutex_waiter_less(struct rt_mutex_waiter *left,233 struct rt_mutex_waiter *right)
234 struct rt_mutex_waiter *right)
235{234{
236 if (left->prio < right->prio)235 if (left->prio < right->prio)
237 return 1;236 return 1;
@@ -248,9 +247,8 @@ rt_mutex_waiter_less(struct rt_mutex_waiter *left, · 10 unmodified lines
248 return 0;247 return 0;
249}248}
250 249
251static inline int250static __always_inline int rt_mutex_waiter_equal(struct rt_mutex_waiter *left,
252rt_mutex_waiter_equal(struct rt_mutex_waiter *left,251 struct rt_mutex_waiter *right)
253 struct rt_mutex_waiter *right)
254{252{
255 if (left->prio != right->prio)253 if (left->prio != right->prio)
256 return 0;254 return 0;
@@ -270,18 +268,18 @@ rt_mutex_waiter_equal(struct rt_mutex_waiter *left, · 13 unmodified lines
270#define __node_2_waiter(node) \268#define __node_2_waiter(node) \
271 rb_entry((node), struct rt_mutex_waiter, tree_entry)269 rb_entry((node), struct rt_mutex_waiter, tree_entry)
272 270
273static inline bool __waiter_less(struct rb_node *a, const struct rb_node *b)271static __always_inline bool __waiter_less(struct rb_node *a, const struct rb_node *b)
274{272{
275 return rt_mutex_waiter_less(__node_2_waiter(a), __node_2_waiter(b));273 return rt_mutex_waiter_less(__node_2_waiter(a), __node_2_waiter(b));
276}274}
277 275
278static void276static __always_inline void
279rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)277rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
280{278{
281 rb_add_cached(&waiter->tree_entry, &lock->waiters, __waiter_less);279 rb_add_cached(&waiter->tree_entry, &lock->waiters, __waiter_less);
282}280}
283 281
284static void282static __always_inline void
285rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)283rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
286{284{
287 if (RB_EMPTY_NODE(&waiter->tree_entry))285 if (RB_EMPTY_NODE(&waiter->tree_entry))
@@ -294,18 +292,19 @@ rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter) · 6 unmodified lines
294#define __node_2_pi_waiter(node) \292#define __node_2_pi_waiter(node) \
295 rb_entry((node), struct rt_mutex_waiter, pi_tree_entry)293 rb_entry((node), struct rt_mutex_waiter, pi_tree_entry)
296 294
297static inline bool __pi_waiter_less(struct rb_node *a, const struct rb_node *b)295static __always_inline bool
296__pi_waiter_less(struct rb_node *a, const struct rb_node *b)
298{297{
299 return rt_mutex_waiter_less(__node_2_pi_waiter(a), __node_2_pi_waiter(b));298 return rt_mutex_waiter_less(__node_2_pi_waiter(a), __node_2_pi_waiter(b));
300}299}
301 300
302static void301static __always_inline void
303rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)302rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
304{303{
305 rb_add_cached(&waiter->pi_tree_entry, &task->pi_waiters, __pi_waiter_less);304 rb_add_cached(&waiter->pi_tree_entry, &task->pi_waiters, __pi_waiter_less);
306}305}
307 306
308static void307static __always_inline void
309rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)308rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
310{309{
311 if (RB_EMPTY_NODE(&waiter->pi_tree_entry))310 if (RB_EMPTY_NODE(&waiter->pi_tree_entry))
@@ -315,7 +314,7 @@ rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter) · 3 unmodified lines
315 RB_CLEAR_NODE(&waiter->pi_tree_entry);314 RB_CLEAR_NODE(&waiter->pi_tree_entry);
316}315}
317 316
318static void rt_mutex_adjust_prio(struct task_struct *p)317static __always_inline void rt_mutex_adjust_prio(struct task_struct *p)
319{318{
320 struct task_struct *pi_task = NULL;319 struct task_struct *pi_task = NULL;
321 320
@@ -340,8 +339,9 @@ static void rt_mutex_adjust_prio(struct task_struct *p) · 18 unmodified lines
340 * deadlock detection is disabled independent of the detect argument339 * deadlock detection is disabled independent of the detect argument
341 * and the config settings.340 * and the config settings.
342 */341 */
343static bool rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,342static __always_inline bool
344 enum rtmutex_chainwalk chwalk)343rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,
344 enum rtmutex_chainwalk chwalk)
345{345{
346 if (IS_ENABLED(CONFIG_DEBUG_RT_MUTEX))346 if (IS_ENABLED(CONFIG_DEBUG_RT_MUTEX))
347 return waiter != NULL;347 return waiter != NULL;
@@ -353,7 +353,7 @@ static bool rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter, · 5 unmodified lines
353 */353 */
354int max_lock_depth = 1024;354int max_lock_depth = 1024;
355 355
356static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)356static __always_inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
357{357{
358 return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;358 return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;
359}359}
@@ -421,12 +421,12 @@ static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p) · 61 unmodified lines
421 * unlock(lock->wait_lock); release [L]421 * unlock(lock->wait_lock); release [L]
422 * goto again;422 * goto again;
423 */423 */
424static int rt_mutex_adjust_prio_chain(struct task_struct *task,424static int __sched rt_mutex_adjust_prio_chain(struct task_struct *task,
425 enum rtmutex_chainwalk chwalk,425 enum rtmutex_chainwalk chwalk,
426 struct rt_mutex *orig_lock,426 struct rt_mutex *orig_lock,
427 struct rt_mutex *next_lock,427 struct rt_mutex *next_lock,
428 struct rt_mutex_waiter *orig_waiter,428 struct rt_mutex_waiter *orig_waiter,
429 struct task_struct *top_task)429 struct task_struct *top_task)
430{430{
431 struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;431 struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
432 struct rt_mutex_waiter *prerequeue_top_waiter;432 struct rt_mutex_waiter *prerequeue_top_waiter;
@@ -778,8 +778,9 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 345 unmodified lines
778 * @waiter: The waiter that is queued to the lock's wait tree if the778 * @waiter: The waiter that is queued to the lock's wait tree if the
779 * callsite called task_blocked_on_lock(), otherwise NULL779 * callsite called task_blocked_on_lock(), otherwise NULL
780 */780 */
781static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,781static int __sched
782 struct rt_mutex_waiter *waiter)782try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
783 struct rt_mutex_waiter *waiter)
783{784{
784 lockdep_assert_held(&lock->wait_lock);785 lockdep_assert_held(&lock->wait_lock);
785 786
@@ -896,10 +897,10 @@ takeit: · 110 unmodified lines
896 *897 *
897 * This must be called with lock->wait_lock held and interrupts disabled898 * This must be called with lock->wait_lock held and interrupts disabled
898 */899 */
899static int task_blocks_on_rt_mutex(struct rt_mutex *lock,900static int __sched task_blocks_on_rt_mutex(struct rt_mutex *lock,
900 struct rt_mutex_waiter *waiter,901 struct rt_mutex_waiter *waiter,
901 struct task_struct *task,902 struct task_struct *task,
902 enum rtmutex_chainwalk chwalk)903 enum rtmutex_chainwalk chwalk)
903{904{
904 struct task_struct *owner = rt_mutex_owner(lock);905 struct task_struct *owner = rt_mutex_owner(lock);
905 struct rt_mutex_waiter *top_waiter = waiter;906 struct rt_mutex_waiter *top_waiter = waiter;
@@ -985,8 +986,8 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 79 unmodified lines
985 *986 *
986 * Called with lock->wait_lock held and interrupts disabled.987 * Called with lock->wait_lock held and interrupts disabled.
987 */988 */
988static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,989static void __sched mark_wakeup_next_waiter(struct wake_q_head *wake_q,
989 struct rt_mutex *lock)990 struct rt_mutex *lock)
990{991{
991 struct rt_mutex_waiter *waiter;992 struct rt_mutex_waiter *waiter;
992 993
@@ -1035,8 +1036,8 @@ static void mark_wakeup_next_waiter(struct wake_q_head *wake_q, · 42 unmodified lines
1035 * Must be called with lock->wait_lock held and interrupts disabled. I must1036 * Must be called with lock->wait_lock held and interrupts disabled. I must
1036 * have just failed to try_to_take_rt_mutex().1037 * have just failed to try_to_take_rt_mutex().
1037 */1038 */
1038static void remove_waiter(struct rt_mutex *lock,1039static void __sched remove_waiter(struct rt_mutex *lock,
1039 struct rt_mutex_waiter *waiter)1040 struct rt_mutex_waiter *waiter)
1040{1041{
1041 bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));1042 bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
1042 struct task_struct *owner = rt_mutex_owner(lock);1043 struct task_struct *owner = rt_mutex_owner(lock);
@@ -1093,7 +1094,7 @@ static void remove_waiter(struct rt_mutex *lock, · 50 unmodified lines
1093 *1094 *
1094 * Called from sched_setscheduler1095 * Called from sched_setscheduler
1095 */1096 */
1096void rt_mutex_adjust_pi(struct task_struct *task)1097void __sched rt_mutex_adjust_pi(struct task_struct *task)
1097{1098{
1098 struct rt_mutex_waiter *waiter;1099 struct rt_mutex_waiter *waiter;
1099 struct rt_mutex *next_lock;1100 struct rt_mutex *next_lock;
@@ -1116,7 +1117,7 @@ void rt_mutex_adjust_pi(struct task_struct *task) · 16 unmodified lines
1116 next_lock, NULL, task);1117 next_lock, NULL, task);
1117}1118}
1118 1119
1119void rt_mutex_init_waiter(struct rt_mutex_waiter *waiter)1120void __sched rt_mutex_init_waiter(struct rt_mutex_waiter *waiter)
1120{1121{
1121 debug_rt_mutex_init_waiter(waiter);1122 debug_rt_mutex_init_waiter(waiter);
1122 RB_CLEAR_NODE(&waiter->pi_tree_entry);1123 RB_CLEAR_NODE(&waiter->pi_tree_entry);
@@ -1134,10 +1135,9 @@ void rt_mutex_init_waiter(struct rt_mutex_waiter *waiter) · 11 unmodified lines
1134 *1135 *
1135 * Must be called with lock->wait_lock held and interrupts disabled1136 * Must be called with lock->wait_lock held and interrupts disabled
1136 */1137 */
1137static int __sched1138static int __sched __rt_mutex_slowlock(struct rt_mutex *lock, int state,
1138__rt_mutex_slowlock(struct rt_mutex *lock, int state,1139 struct hrtimer_sleeper *timeout,
1139 struct hrtimer_sleeper *timeout,1140 struct rt_mutex_waiter *waiter)
1140 struct rt_mutex_waiter *waiter)
1141{1141{
1142 int ret = 0;1142 int ret = 0;
1143 1143
@@ -1172,8 +1172,8 @@ __rt_mutex_slowlock(struct rt_mutex *lock, int state, · 28 unmodified lines
1172 return ret;1172 return ret;
1173}1173}
1174 1174
1175static void rt_mutex_handle_deadlock(int res, int detect_deadlock,1175static void __sched rt_mutex_handle_deadlock(int res, int detect_deadlock,
1176 struct rt_mutex_waiter *w)1176 struct rt_mutex_waiter *w)
1177{1177{
1178 /*1178 /*
1179 * If the result is not -EDEADLOCK or the caller requested1179 * If the result is not -EDEADLOCK or the caller requested
@@ -1195,10 +1195,9 @@ static void rt_mutex_handle_deadlock(int res, int detect_deadlock, · 15 unmodified lines
1195/*1195/*
1196 * Slow path lock function:1196 * Slow path lock function:
1197 */1197 */
1198static int __sched1198static int __sched rt_mutex_slowlock(struct rt_mutex *lock, int state,
1199rt_mutex_slowlock(struct rt_mutex *lock, int state,1199 struct hrtimer_sleeper *timeout,
1200 struct hrtimer_sleeper *timeout,1200 enum rtmutex_chainwalk chwalk)
1201 enum rtmutex_chainwalk chwalk)
1202{1201{
1203 struct rt_mutex_waiter waiter;1202 struct rt_mutex_waiter waiter;
1204 unsigned long flags;1203 unsigned long flags;
@@ -1257,7 +1256,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, · 52 unmodified lines
1257 return ret;1256 return ret;
1258}1257}
1259 1258
1260static inline int __rt_mutex_slowtrylock(struct rt_mutex *lock)1259static int __sched __rt_mutex_slowtrylock(struct rt_mutex *lock)
1261{1260{
1262 int ret = try_to_take_rt_mutex(lock, current, NULL);1261 int ret = try_to_take_rt_mutex(lock, current, NULL);
1263 1262
@@ -1273,7 +1272,7 @@ static inline int __rt_mutex_slowtrylock(struct rt_mutex *lock) · 9 unmodified lines
1273/*1272/*
1274 * Slow path try-lock function:1273 * Slow path try-lock function:
1275 */1274 */
1276static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)1275static int __sched rt_mutex_slowtrylock(struct rt_mutex *lock)
1277{1276{
1278 unsigned long flags;1277 unsigned long flags;
1279 int ret;1278 int ret;
@@ -1371,7 +1370,7 @@ static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock, · 91 unmodified lines
1371 * The atomic acquire/release ops are compiled away, when either the1370 * The atomic acquire/release ops are compiled away, when either the
1372 * architecture does not support cmpxchg or when debugging is enabled.1371 * architecture does not support cmpxchg or when debugging is enabled.
1373 */1372 */
1374static inline int1373static __always_inline int
1375rt_mutex_fastlock(struct rt_mutex *lock, int state,1374rt_mutex_fastlock(struct rt_mutex *lock, int state,
1376 int (*slowfn)(struct rt_mutex *lock, int state,1375 int (*slowfn)(struct rt_mutex *lock, int state,
1377 struct hrtimer_sleeper *timeout,1376 struct hrtimer_sleeper *timeout,
@@ -1383,7 +1382,7 @@ rt_mutex_fastlock(struct rt_mutex *lock, int state, · 5 unmodified lines
1383 return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);1382 return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);
1384}1383}
1385 1384
1386static inline int1385static __always_inline int
1387rt_mutex_fasttrylock(struct rt_mutex *lock,1386rt_mutex_fasttrylock(struct rt_mutex *lock,
1388 int (*slowfn)(struct rt_mutex *lock))1387 int (*slowfn)(struct rt_mutex *lock))
1389{1388{
@@ -1396,7 +1395,7 @@ rt_mutex_fasttrylock(struct rt_mutex *lock, · 6 unmodified lines
1396/*1395/*
1397 * Performs the wakeup of the top-waiter and re-enables preemption.1396 * Performs the wakeup of the top-waiter and re-enables preemption.
1398 */1397 */
1399void rt_mutex_postunlock(struct wake_q_head *wake_q)1398void __sched rt_mutex_postunlock(struct wake_q_head *wake_q)
1400{1399{
1401 wake_up_q(wake_q);1400 wake_up_q(wake_q);
1402 1401
@@ -1404,7 +1403,7 @@ void rt_mutex_postunlock(struct wake_q_head *wake_q) · 1 unmodified line
1404 preempt_enable();1403 preempt_enable();
1405}1404}
1406 1405
1407static inline void1406static __always_inline void
1408rt_mutex_fastunlock(struct rt_mutex *lock,1407rt_mutex_fastunlock(struct rt_mutex *lock,
1409 bool (*slowfn)(struct rt_mutex *lock,1408 bool (*slowfn)(struct rt_mutex *lock,
1410 struct wake_q_head *wqh))1409 struct wake_q_head *wqh))
@@ -1418,7 +1417,8 @@ rt_mutex_fastunlock(struct rt_mutex *lock, · 7 unmodified lines
1418 rt_mutex_postunlock(&wake_q);1417 rt_mutex_postunlock(&wake_q);
1419}1418}
1420 1419
1421static inline void __rt_mutex_lock(struct rt_mutex *lock, unsigned int subclass)1420static __always_inline void __rt_mutex_lock(struct rt_mutex *lock,
1421 unsigned int subclass)
1422{1422{
1423 might_sleep();1423 might_sleep();
1424 1424
@@ -1536,7 +1536,7 @@ EXPORT_SYMBOL_GPL(rt_mutex_unlock); · 111 unmodified lines
1536 * @wake_q: The wake queue head from which to get the next lock waiter1536 * @wake_q: The wake queue head from which to get the next lock waiter
1537 */1537 */
1538bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock,1538bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock,
1539 struct wake_q_head *wake_q)1539 struct wake_q_head *wake_q)
1540{1540{
1541 lockdep_assert_held(&lock->wait_lock);1541 lockdep_assert_held(&lock->wait_lock);
1542 1542
@@ -1583,7 +1583,7 @@ void __sched rt_mutex_futex_unlock(struct rt_mutex *lock) · 40 unmodified lines
1583 *1583 *
1584 * Initializing of a locked rt_mutex is not allowed1584 * Initializing of a locked rt_mutex is not allowed
1585 */1585 */
1586void __rt_mutex_init(struct rt_mutex *lock, const char *name,1586void __sched __rt_mutex_init(struct rt_mutex *lock, const char *name,
1587 struct lock_class_key *key)1587 struct lock_class_key *key)
1588{1588{
1589 debug_check_no_locks_freed((void *)lock, sizeof(*lock));1589 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
@@ -1607,8 +1607,8 @@ EXPORT_SYMBOL_GPL(__rt_mutex_init); · 17 unmodified lines
1607 * possible at this point because the pi_state which contains the rtmutex1607 * possible at this point because the pi_state which contains the rtmutex
1608 * is not yet visible to other tasks.1608 * is not yet visible to other tasks.
1609 */1609 */
1610void rt_mutex_init_proxy_locked(struct rt_mutex *lock,1610void __sched rt_mutex_init_proxy_locked(struct rt_mutex *lock,
1611 struct task_struct *proxy_owner)1611 struct task_struct *proxy_owner)
1612{1612{
1613 __rt_mutex_basic_init(lock);1613 __rt_mutex_basic_init(lock);
1614 rt_mutex_set_owner(lock, proxy_owner);1614 rt_mutex_set_owner(lock, proxy_owner);
@@ -1626,7 +1626,7 @@ void rt_mutex_init_proxy_locked(struct rt_mutex *lock, · 11 unmodified lines
1626 * possible because it belongs to the pi_state which is about to be freed1626 * possible because it belongs to the pi_state which is about to be freed
1627 * and it is not longer visible to other tasks.1627 * and it is not longer visible to other tasks.
1628 */1628 */
1629void rt_mutex_proxy_unlock(struct rt_mutex *lock)1629void __sched rt_mutex_proxy_unlock(struct rt_mutex *lock)
1630{1630{
1631 debug_rt_mutex_proxy_unlock(lock);1631 debug_rt_mutex_proxy_unlock(lock);
1632 rt_mutex_set_owner(lock, NULL);1632 rt_mutex_set_owner(lock, NULL);
@@ -1651,9 +1651,9 @@ void rt_mutex_proxy_unlock(struct rt_mutex *lock) · 18 unmodified lines
1651 *1651 *
1652 * Special API call for PI-futex support.1652 * Special API call for PI-futex support.
1653 */1653 */
1654int __rt_mutex_start_proxy_lock(struct rt_mutex *lock,1654int __sched __rt_mutex_start_proxy_lock(struct rt_mutex *lock,
1655 struct rt_mutex_waiter *waiter,1655 struct rt_mutex_waiter *waiter,
1656 struct task_struct *task)1656 struct task_struct *task)
1657{1657{
1658 int ret;1658 int ret;
1659 1659
@@ -1698,9 +1698,9 @@ int __rt_mutex_start_proxy_lock(struct rt_mutex *lock, · 38 unmodified lines
1698 *1698 *
1699 * Special API call for PI-futex support.1699 * Special API call for PI-futex support.
1700 */1700 */
1701int rt_mutex_start_proxy_lock(struct rt_mutex *lock,1701int __sched rt_mutex_start_proxy_lock(struct rt_mutex *lock,
1702 struct rt_mutex_waiter *waiter,1702 struct rt_mutex_waiter *waiter,
1703 struct task_struct *task)1703 struct task_struct *task)
1704{1704{
1705 int ret;1705 int ret;
1706 1706
@@ -1730,9 +1730,9 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock, · 23 unmodified lines
1730 *1730 *
1731 * Special API call for PI-futex support1731 * Special API call for PI-futex support
1732 */1732 */
1733int rt_mutex_wait_proxy_lock(struct rt_mutex *lock,1733int __sched rt_mutex_wait_proxy_lock(struct rt_mutex *lock,
1734 struct hrtimer_sleeper *to,1734 struct hrtimer_sleeper *to,
1735 struct rt_mutex_waiter *waiter)1735 struct rt_mutex_waiter *waiter)
1736{1736{
1737 int ret;1737 int ret;
1738 1738
@@ -1770,8 +1770,8 @@ int rt_mutex_wait_proxy_lock(struct rt_mutex *lock, · 31 unmodified lines
1770 *1770 *
1771 * Special API call for PI-futex support1771 * Special API call for PI-futex support
1772 */1772 */
1773bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock,1773bool __sched rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock,
1774 struct rt_mutex_waiter *waiter)1774 struct rt_mutex_waiter *waiter)
1775{1775{
1776 bool cleanup = false;1776 bool cleanup = false;
1777 1777