History

locking/rtmutex: Make text section and inlining consistent

rtmutex is half __sched and the other half is not. If the compiler decides to not inline larger static functions then part of the code ends up in the regular text section. There are also quite some performance related small helpers which are either static or plain inline. Force inline those which make sense and mark the rest __sched.

Thomas Gleixner committed 5 years ago

Commit d7a2edb1 parent f41dcc11 file · +76 −76Browse files

Unified · Split

kernel/locking/rtmutex.c+76−76
@@ -49,7 +49,7 @@ · 48 unmodified lines
4949 * set this bit before looking at the lock.
5050 */
5151
52−static void
52+static __always_inline void
5353 rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
5454 {
5555 unsigned long val = (unsigned long)owner;
@@ -60,13 +60,13 @@ rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner) · 4 unmodified lines
6060 WRITE_ONCE(lock->owner, (struct task_struct *)val);
6161 }
6262
63−static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
63+static __always_inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
6464 {
6565 lock->owner = (struct task_struct *)
6666 ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
6767 }
6868
69−static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
69+static __always_inline void fixup_rt_mutex_waiters(struct rt_mutex *lock)
7070 {
7171 unsigned long owner, *p = (unsigned long *) &lock->owner;
7272
@@ -149,7 +149,7 @@ static void fixup_rt_mutex_waiters(struct rt_mutex *lock) · 76 unmodified lines
149149 * all future threads that attempt to [Rmw] the lock to the slowpath. As such
150150 * relaxed semantics suffice.
151151 */
152−static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
152+static __always_inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
153153 {
154154 unsigned long owner, *p = (unsigned long *) &lock->owner;
155155
@@ -165,8 +165,8 @@ static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) · 9 unmodified lines
165165 * 2) Drop lock->wait_lock
166166 * 3) Try to unlock the lock with cmpxchg
167167 */
168−static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
169− unsigned long flags)
168+static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
169+ unsigned long flags)
170170 __releases(lock->wait_lock)
171171 {
172172 struct task_struct *owner = rt_mutex_owner(lock);
@@ -204,7 +204,7 @@ static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock, · 31 unmodified lines
204204 # define rt_mutex_cmpxchg_acquire(l,c,n) (0)
205205 # define rt_mutex_cmpxchg_release(l,c,n) (0)
206206
207−static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
207+static __always_inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
208208 {
209209 lock->owner = (struct task_struct *)
210210 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
@@ -213,8 +213,8 @@ static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) · 2 unmodified lines
213213 /*
214214 * Simple slow path only version: lock->owner is protected by lock->wait_lock.
215215 */
216−static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
217− unsigned long flags)
216+static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
217+ unsigned long flags)
218218 __releases(lock->wait_lock)
219219 {
220220 lock->owner = NULL;
@@ -229,9 +229,8 @@ static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock, · 8 unmodified lines
229229 #define task_to_waiter(p) \
230230 &(struct rt_mutex_waiter){ .prio = (p)->prio, .deadline = (p)->dl.deadline }
231231
232−static inline int
233−rt_mutex_waiter_less(struct rt_mutex_waiter *left,
234− struct rt_mutex_waiter *right)
232+static __always_inline int rt_mutex_waiter_less(struct rt_mutex_waiter *left,
233+ struct rt_mutex_waiter *right)
235234 {
236235 if (left->prio < right->prio)
237236 return 1;
@@ -248,9 +247,8 @@ rt_mutex_waiter_less(struct rt_mutex_waiter *left, · 10 unmodified lines
248247 return 0;
249248 }
250249
251−static inline int
252−rt_mutex_waiter_equal(struct rt_mutex_waiter *left,
253− struct rt_mutex_waiter *right)
250+static __always_inline int rt_mutex_waiter_equal(struct rt_mutex_waiter *left,
251+ struct rt_mutex_waiter *right)
254252 {
255253 if (left->prio != right->prio)
256254 return 0;
@@ -270,18 +268,18 @@ rt_mutex_waiter_equal(struct rt_mutex_waiter *left, · 13 unmodified lines
270268 #define __node_2_waiter(node) \
271269 rb_entry((node), struct rt_mutex_waiter, tree_entry)
272270
273−static inline bool __waiter_less(struct rb_node *a, const struct rb_node *b)
271+static __always_inline bool __waiter_less(struct rb_node *a, const struct rb_node *b)
274272 {
275273 return rt_mutex_waiter_less(__node_2_waiter(a), __node_2_waiter(b));
276274 }
277275
278−static void
276+static __always_inline void
279277 rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
280278 {
281279 rb_add_cached(&waiter->tree_entry, &lock->waiters, __waiter_less);
282280 }
283281
284−static void
282+static __always_inline void
285283 rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
286284 {
287285 if (RB_EMPTY_NODE(&waiter->tree_entry))
@@ -294,18 +292,19 @@ rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter) · 6 unmodified lines
294292 #define __node_2_pi_waiter(node) \
295293 rb_entry((node), struct rt_mutex_waiter, pi_tree_entry)
296294
297−static inline bool __pi_waiter_less(struct rb_node *a, const struct rb_node *b)
295+static __always_inline bool
296+__pi_waiter_less(struct rb_node *a, const struct rb_node *b)
298297 {
299298 return rt_mutex_waiter_less(__node_2_pi_waiter(a), __node_2_pi_waiter(b));
300299 }
301300
302−static void
301+static __always_inline void
303302 rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
304303 {
305304 rb_add_cached(&waiter->pi_tree_entry, &task->pi_waiters, __pi_waiter_less);
306305 }
307306
308−static void
307+static __always_inline void
309308 rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
310309 {
311310 if (RB_EMPTY_NODE(&waiter->pi_tree_entry))
@@ -315,7 +314,7 @@ rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter) · 3 unmodified lines
315314 RB_CLEAR_NODE(&waiter->pi_tree_entry);
316315 }
317316
318−static void rt_mutex_adjust_prio(struct task_struct *p)
317+static __always_inline void rt_mutex_adjust_prio(struct task_struct *p)
319318 {
320319 struct task_struct *pi_task = NULL;
321320
@@ -340,8 +339,9 @@ static void rt_mutex_adjust_prio(struct task_struct *p) · 18 unmodified lines
340339 * deadlock detection is disabled independent of the detect argument
341340 * and the config settings.
342341 */
343−static bool rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,
344− enum rtmutex_chainwalk chwalk)
342+static __always_inline bool
343+rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,
344+ enum rtmutex_chainwalk chwalk)
345345 {
346346 if (IS_ENABLED(CONFIG_DEBUG_RT_MUTEX))
347347 return waiter != NULL;
@@ -353,7 +353,7 @@ static bool rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter, · 5 unmodified lines
353353 */
354354 int max_lock_depth = 1024;
355355
356−static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
356+static __always_inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
357357 {
358358 return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;
359359 }
@@ -421,12 +421,12 @@ static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p) · 61 unmodified lines
421421 * unlock(lock->wait_lock); release [L]
422422 * goto again;
423423 */
424−static int rt_mutex_adjust_prio_chain(struct task_struct *task,
425− enum rtmutex_chainwalk chwalk,
426− struct rt_mutex *orig_lock,
427− struct rt_mutex *next_lock,
428− struct rt_mutex_waiter *orig_waiter,
429− struct task_struct *top_task)
424+static int __sched rt_mutex_adjust_prio_chain(struct task_struct *task,
425+ enum rtmutex_chainwalk chwalk,
426+ struct rt_mutex *orig_lock,
427+ struct rt_mutex *next_lock,
428+ struct rt_mutex_waiter *orig_waiter,
429+ struct task_struct *top_task)
430430 {
431431 struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
432432 struct rt_mutex_waiter *prerequeue_top_waiter;
@@ -778,8 +778,9 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 345 unmodified lines
778778 * @waiter: The waiter that is queued to the lock's wait tree if the
779779 * callsite called task_blocked_on_lock(), otherwise NULL
780780 */
781−static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
782− struct rt_mutex_waiter *waiter)
781+static int __sched
782+try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
783+ struct rt_mutex_waiter *waiter)
783784 {
784785 lockdep_assert_held(&lock->wait_lock);
785786
@@ -896,10 +897,10 @@ takeit: · 110 unmodified lines
896897 *
897898 * This must be called with lock->wait_lock held and interrupts disabled
898899 */
899−static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
900− struct rt_mutex_waiter *waiter,
901− struct task_struct *task,
902− enum rtmutex_chainwalk chwalk)
900+static int __sched task_blocks_on_rt_mutex(struct rt_mutex *lock,
901+ struct rt_mutex_waiter *waiter,
902+ struct task_struct *task,
903+ enum rtmutex_chainwalk chwalk)
903904 {
904905 struct task_struct *owner = rt_mutex_owner(lock);
905906 struct rt_mutex_waiter *top_waiter = waiter;
@@ -985,8 +986,8 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 79 unmodified lines
985986 *
986987 * Called with lock->wait_lock held and interrupts disabled.
987988 */
988−static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
989− struct rt_mutex *lock)
989+static void __sched mark_wakeup_next_waiter(struct wake_q_head *wake_q,
990+ struct rt_mutex *lock)
990991 {
991992 struct rt_mutex_waiter *waiter;
992993
@@ -1035,8 +1036,8 @@ static void mark_wakeup_next_waiter(struct wake_q_head *wake_q, · 42 unmodified lines
10351036 * Must be called with lock->wait_lock held and interrupts disabled. I must
10361037 * have just failed to try_to_take_rt_mutex().
10371038 */
1038−static void remove_waiter(struct rt_mutex *lock,
1039− struct rt_mutex_waiter *waiter)
1039+static void __sched remove_waiter(struct rt_mutex *lock,
1040+ struct rt_mutex_waiter *waiter)
10401041 {
10411042 bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
10421043 struct task_struct *owner = rt_mutex_owner(lock);
@@ -1093,7 +1094,7 @@ static void remove_waiter(struct rt_mutex *lock, · 50 unmodified lines
10931094 *
10941095 * Called from sched_setscheduler
10951096 */
1096−void rt_mutex_adjust_pi(struct task_struct *task)
1097+void __sched rt_mutex_adjust_pi(struct task_struct *task)
10971098 {
10981099 struct rt_mutex_waiter *waiter;
10991100 struct rt_mutex *next_lock;
@@ -1116,7 +1117,7 @@ void rt_mutex_adjust_pi(struct task_struct *task) · 16 unmodified lines
11161117 next_lock, NULL, task);
11171118 }
11181119
1119−void rt_mutex_init_waiter(struct rt_mutex_waiter *waiter)
1120+void __sched rt_mutex_init_waiter(struct rt_mutex_waiter *waiter)
11201121 {
11211122 debug_rt_mutex_init_waiter(waiter);
11221123 RB_CLEAR_NODE(&waiter->pi_tree_entry);
@@ -1134,10 +1135,9 @@ void rt_mutex_init_waiter(struct rt_mutex_waiter *waiter) · 11 unmodified lines
11341135 *
11351136 * Must be called with lock->wait_lock held and interrupts disabled
11361137 */
1137−static int __sched
1138−__rt_mutex_slowlock(struct rt_mutex *lock, int state,
1139− struct hrtimer_sleeper *timeout,
1140− struct rt_mutex_waiter *waiter)
1138+static int __sched __rt_mutex_slowlock(struct rt_mutex *lock, int state,
1139+ struct hrtimer_sleeper *timeout,
1140+ struct rt_mutex_waiter *waiter)
11411141 {
11421142 int ret = 0;
11431143
@@ -1172,8 +1172,8 @@ __rt_mutex_slowlock(struct rt_mutex *lock, int state, · 28 unmodified lines
11721172 return ret;
11731173 }
11741174
1175−static void rt_mutex_handle_deadlock(int res, int detect_deadlock,
1176− struct rt_mutex_waiter *w)
1175+static void __sched rt_mutex_handle_deadlock(int res, int detect_deadlock,
1176+ struct rt_mutex_waiter *w)
11771177 {
11781178 /*
11791179 * If the result is not -EDEADLOCK or the caller requested
@@ -1195,10 +1195,9 @@ static void rt_mutex_handle_deadlock(int res, int detect_deadlock, · 15 unmodified lines
11951195 /*
11961196 * Slow path lock function:
11971197 */
1198−static int __sched
1199−rt_mutex_slowlock(struct rt_mutex *lock, int state,
1200− struct hrtimer_sleeper *timeout,
1201− enum rtmutex_chainwalk chwalk)
1198+static int __sched rt_mutex_slowlock(struct rt_mutex *lock, int state,
1199+ struct hrtimer_sleeper *timeout,
1200+ enum rtmutex_chainwalk chwalk)
12021201 {
12031202 struct rt_mutex_waiter waiter;
12041203 unsigned long flags;
@@ -1257,7 +1256,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, · 52 unmodified lines
12571256 return ret;
12581257 }
12591258
1260−static inline int __rt_mutex_slowtrylock(struct rt_mutex *lock)
1259+static int __sched __rt_mutex_slowtrylock(struct rt_mutex *lock)
12611260 {
12621261 int ret = try_to_take_rt_mutex(lock, current, NULL);
12631262
@@ -1273,7 +1272,7 @@ static inline int __rt_mutex_slowtrylock(struct rt_mutex *lock) · 9 unmodified lines
12731272 /*
12741273 * Slow path try-lock function:
12751274 */
1276−static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)
1275+static int __sched rt_mutex_slowtrylock(struct rt_mutex *lock)
12771276 {
12781277 unsigned long flags;
12791278 int ret;
@@ -1371,7 +1370,7 @@ static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock, · 91 unmodified lines
13711370 * The atomic acquire/release ops are compiled away, when either the
13721371 * architecture does not support cmpxchg or when debugging is enabled.
13731372 */
1374−static inline int
1373+static __always_inline int
13751374 rt_mutex_fastlock(struct rt_mutex *lock, int state,
13761375 int (*slowfn)(struct rt_mutex *lock, int state,
13771376 struct hrtimer_sleeper *timeout,
@@ -1383,7 +1382,7 @@ rt_mutex_fastlock(struct rt_mutex *lock, int state, · 5 unmodified lines
13831382 return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);
13841383 }
13851384
1386−static inline int
1385+static __always_inline int
13871386 rt_mutex_fasttrylock(struct rt_mutex *lock,
13881387 int (*slowfn)(struct rt_mutex *lock))
13891388 {
@@ -1396,7 +1395,7 @@ rt_mutex_fasttrylock(struct rt_mutex *lock, · 6 unmodified lines
13961395 /*
13971396 * Performs the wakeup of the top-waiter and re-enables preemption.
13981397 */
1399−void rt_mutex_postunlock(struct wake_q_head *wake_q)
1398+void __sched rt_mutex_postunlock(struct wake_q_head *wake_q)
14001399 {
14011400 wake_up_q(wake_q);
14021401
@@ -1404,7 +1403,7 @@ void rt_mutex_postunlock(struct wake_q_head *wake_q) · 1 unmodified line
14041403 preempt_enable();
14051404 }
14061405
1407−static inline void
1406+static __always_inline void
14081407 rt_mutex_fastunlock(struct rt_mutex *lock,
14091408 bool (*slowfn)(struct rt_mutex *lock,
14101409 struct wake_q_head *wqh))
@@ -1418,7 +1417,8 @@ rt_mutex_fastunlock(struct rt_mutex *lock, · 7 unmodified lines
14181417 rt_mutex_postunlock(&wake_q);
14191418 }
14201419
1421−static inline void __rt_mutex_lock(struct rt_mutex *lock, unsigned int subclass)
1420+static __always_inline void __rt_mutex_lock(struct rt_mutex *lock,
1421+ unsigned int subclass)
14221422 {
14231423 might_sleep();
14241424
@@ -1536,7 +1536,7 @@ EXPORT_SYMBOL_GPL(rt_mutex_unlock); · 111 unmodified lines
15361536 * @wake_q: The wake queue head from which to get the next lock waiter
15371537 */
15381538 bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock,
1539− struct wake_q_head *wake_q)
1539+ struct wake_q_head *wake_q)
15401540 {
15411541 lockdep_assert_held(&lock->wait_lock);
15421542
@@ -1583,7 +1583,7 @@ void __sched rt_mutex_futex_unlock(struct rt_mutex *lock) · 40 unmodified lines
15831583 *
15841584 * Initializing of a locked rt_mutex is not allowed
15851585 */
1586−void __rt_mutex_init(struct rt_mutex *lock, const char *name,
1586+void __sched __rt_mutex_init(struct rt_mutex *lock, const char *name,
15871587 struct lock_class_key *key)
15881588 {
15891589 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
@@ -1607,8 +1607,8 @@ EXPORT_SYMBOL_GPL(__rt_mutex_init); · 17 unmodified lines
16071607 * possible at this point because the pi_state which contains the rtmutex
16081608 * is not yet visible to other tasks.
16091609 */
1610−void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
1611− struct task_struct *proxy_owner)
1610+void __sched rt_mutex_init_proxy_locked(struct rt_mutex *lock,
1611+ struct task_struct *proxy_owner)
16121612 {
16131613 __rt_mutex_basic_init(lock);
16141614 rt_mutex_set_owner(lock, proxy_owner);
@@ -1626,7 +1626,7 @@ void rt_mutex_init_proxy_locked(struct rt_mutex *lock, · 11 unmodified lines
16261626 * possible because it belongs to the pi_state which is about to be freed
16271627 * and it is not longer visible to other tasks.
16281628 */
1629−void rt_mutex_proxy_unlock(struct rt_mutex *lock)
1629+void __sched rt_mutex_proxy_unlock(struct rt_mutex *lock)
16301630 {
16311631 debug_rt_mutex_proxy_unlock(lock);
16321632 rt_mutex_set_owner(lock, NULL);
@@ -1651,9 +1651,9 @@ void rt_mutex_proxy_unlock(struct rt_mutex *lock) · 18 unmodified lines
16511651 *
16521652 * Special API call for PI-futex support.
16531653 */
1654−int __rt_mutex_start_proxy_lock(struct rt_mutex *lock,
1655− struct rt_mutex_waiter *waiter,
1656− struct task_struct *task)
1654+int __sched __rt_mutex_start_proxy_lock(struct rt_mutex *lock,
1655+ struct rt_mutex_waiter *waiter,
1656+ struct task_struct *task)
16571657 {
16581658 int ret;
16591659
@@ -1698,9 +1698,9 @@ int __rt_mutex_start_proxy_lock(struct rt_mutex *lock, · 38 unmodified lines
16981698 *
16991699 * Special API call for PI-futex support.
17001700 */
1701−int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
1702− struct rt_mutex_waiter *waiter,
1703− struct task_struct *task)
1701+int __sched rt_mutex_start_proxy_lock(struct rt_mutex *lock,
1702+ struct rt_mutex_waiter *waiter,
1703+ struct task_struct *task)
17041704 {
17051705 int ret;
17061706
@@ -1730,9 +1730,9 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock, · 23 unmodified lines
17301730 *
17311731 * Special API call for PI-futex support
17321732 */
1733−int rt_mutex_wait_proxy_lock(struct rt_mutex *lock,
1734− struct hrtimer_sleeper *to,
1735− struct rt_mutex_waiter *waiter)
1733+int __sched rt_mutex_wait_proxy_lock(struct rt_mutex *lock,
1734+ struct hrtimer_sleeper *to,
1735+ struct rt_mutex_waiter *waiter)
17361736 {
17371737 int ret;
17381738
@@ -1770,8 +1770,8 @@ int rt_mutex_wait_proxy_lock(struct rt_mutex *lock, · 31 unmodified lines
17701770 *
17711771 * Special API call for PI-futex support
17721772 */
1773−bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock,
1774− struct rt_mutex_waiter *waiter)
1773+bool __sched rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock,
1774+ struct rt_mutex_waiter *waiter)
17751775 {
17761776 bool cleanup = false;
17771777