History

locking/rtmutex: Provide rt_mutex_slowlock_locked()

Split the inner workings of rt_mutex_slowlock() out into a separate function, which can be reused by the upcoming RT lock substitutions, e.g. for rw_semaphores.

Thomas Gleixner committed 5 years ago

Commit ebbdc411 parent 830e6ac2 files · +59 −43Browse files

Showing the changes to kernel/locking (2 of 2 files). Show all

Unified · Split

2 files
  1. kernel/locking/rtmutex.c +58 −42
  2. kernel/locking/rtmutex_api.c +1 −1
kernel/locking/rtmutex.c+58−42
@@ -1106,7 +1106,7 @@ static void __sched remove_waiter(struct rt_mutex_base *lock, · 1105 unmodified lines
11061106 }
11071107
11081108 /**
1109− * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
1109+ * rt_mutex_slowlock_block() - Perform the wait-wake-try-to-take loop
11101110 * @lock: the rt_mutex to take
11111111 * @state: the state the task should block in (TASK_INTERRUPTIBLE
11121112 * or TASK_UNINTERRUPTIBLE)
@@ -1115,10 +1115,10 @@ static void __sched remove_waiter(struct rt_mutex_base *lock, · 2 unmodified lines
11151115 *
11161116 * Must be called with lock->wait_lock held and interrupts disabled
11171117 */
1118−static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,
1119− unsigned int state,
1120− struct hrtimer_sleeper *timeout,
1121− struct rt_mutex_waiter *waiter)
1118+static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,
1119+ unsigned int state,
1120+ struct hrtimer_sleeper *timeout,
1121+ struct rt_mutex_waiter *waiter)
11221122 {
11231123 int ret = 0;
11241124
@@ -1168,52 +1168,37 @@ static void __sched rt_mutex_handle_deadlock(int res, int detect_deadlock, · 43 unmodified lines
11681168 }
11691169 }
11701170
1171−/*
1172− * Slow path lock function:
1171+/**
1172+ * __rt_mutex_slowlock - Locking slowpath invoked with lock::wait_lock held
1173+ * @lock: The rtmutex to block lock
1174+ * @state: The task state for sleeping
1175+ * @chwalk: Indicator whether full or partial chainwalk is requested
1176+ * @waiter: Initializer waiter for blocking
11731177 */
1174−static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,
1175− unsigned int state,
1176− struct hrtimer_sleeper *timeout,
1177− enum rtmutex_chainwalk chwalk)
1178+static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,
1179+ unsigned int state,
1180+ enum rtmutex_chainwalk chwalk,
1181+ struct rt_mutex_waiter *waiter)
11781182 {
1179− struct rt_mutex_waiter waiter;
1180− unsigned long flags;
1181− int ret = 0;
1182−
1183− rt_mutex_init_waiter(&waiter);
1183+ int ret;
11841184
1185− /*
1186− * Technically we could use raw_spin_[un]lock_irq() here, but this can
1187− * be called in early boot if the cmpxchg() fast path is disabled
1188− * (debug, no architecture support). In this case we will acquire the
1189− * rtmutex with lock->wait_lock held. But we cannot unconditionally
1190− * enable interrupts in that early boot case. So we need to use the
1191− * irqsave/restore variants.
1192− */
1193− raw_spin_lock_irqsave(&lock->wait_lock, flags);
1185+ lockdep_assert_held(&lock->wait_lock);
11941186
11951187 /* Try to acquire the lock again: */
1196− if (try_to_take_rt_mutex(lock, current, NULL)) {
1197− raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1188+ if (try_to_take_rt_mutex(lock, current, NULL))
11981189 return 0;
1199− }
12001190
12011191 set_current_state(state);
12021192
1203− /* Setup the timer, when timeout != NULL */
1204− if (unlikely(timeout))
1205− hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
1206−
1207− ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk);
1193+ ret = task_blocks_on_rt_mutex(lock, waiter, current, chwalk);
12081194
12091195 if (likely(!ret))
1210− /* sleep on the mutex */
1211− ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
1196+ ret = rt_mutex_slowlock_block(lock, state, NULL, waiter);
12121197
12131198 if (unlikely(ret)) {
12141199 __set_current_state(TASK_RUNNING);
1215− remove_waiter(lock, &waiter);
1216− rt_mutex_handle_deadlock(ret, chwalk, &waiter);
1200+ remove_waiter(lock, waiter);
1201+ rt_mutex_handle_deadlock(ret, chwalk, waiter);
12171202 }
12181203
12191204 /*
@@ -1221,14 +1206,45 @@ static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock, · 1 unmodified line
12211206 * unconditionally. We might have to fix that up.
12221207 */
12231208 fixup_rt_mutex_waiters(lock);
1209+ return ret;
1210+}
12241211
1225− raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1212+static inline int __rt_mutex_slowlock_locked(struct rt_mutex_base *lock,
1213+ unsigned int state)
1214+{
1215+ struct rt_mutex_waiter waiter;
1216+ int ret;
1217+
1218+ rt_mutex_init_waiter(&waiter);
12261219
1227− /* Remove pending timer: */
1228− if (unlikely(timeout))
1229− hrtimer_cancel(&timeout->timer);
1220+ ret = __rt_mutex_slowlock(lock, state, RT_MUTEX_MIN_CHAINWALK, &waiter);
12301221
12311222 debug_rt_mutex_free_waiter(&waiter);
1223+ return ret;
1224+}
1225+
1226+/*
1227+ * rt_mutex_slowlock - Locking slowpath invoked when fast path fails
1228+ * @lock: The rtmutex to block lock
1229+ * @state: The task state for sleeping
1230+ */
1231+static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,
1232+ unsigned int state)
1233+{
1234+ unsigned long flags;
1235+ int ret;
1236+
1237+ /*
1238+ * Technically we could use raw_spin_[un]lock_irq() here, but this can
1239+ * be called in early boot if the cmpxchg() fast path is disabled
1240+ * (debug, no architecture support). In this case we will acquire the
1241+ * rtmutex with lock->wait_lock held. But we cannot unconditionally
1242+ * enable interrupts in that early boot case. So we need to use the
1243+ * irqsave/restore variants.
1244+ */
1245+ raw_spin_lock_irqsave(&lock->wait_lock, flags);
1246+ ret = __rt_mutex_slowlock_locked(lock, state);
1247+ raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
12321248
12331249 return ret;
12341250 }
@@ -1239,7 +1255,7 @@ static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock, · 4 unmodified lines
12391255 if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
12401256 return 0;
12411257
1242− return rt_mutex_slowlock(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);
1258+ return rt_mutex_slowlock(lock, state);
12431259 }
12441260
12451261 static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock)
kernel/locking/rtmutex_api.c+1−1
@@ -342,7 +342,7 @@ int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock, · 341 unmodified lines
342342 raw_spin_lock_irq(&lock->wait_lock);
343343 /* sleep on the mutex */
344344 set_current_state(TASK_INTERRUPTIBLE);
345− ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
345+ ret = rt_mutex_slowlock_block(lock, TASK_INTERRUPTIBLE, to, waiter);
346346 /*
347347 * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might
348348 * have to fix that up.