History

locking/spinlock: Provide RT variant

Provide the actual locking functions which make use of the general and spinlock specific rtmutex code.

Thomas Gleixner committed 5 years ago

Commit 0f383b61 parent 1c143c42 files · +130 −0Browse files

Showing the changes to kernel (2 of 2 files). Show all

Unified · Split

2 files
  1. kernel/locking/Makefile +1 −0
  2. kernel/locking/spinlock_rt.c +129 −0
kernel/locking/Makefile+1−0
@@ -25,6 +25,7 @@ obj-$(CONFIG_LOCK_SPIN_ON_OWNER) += osq_lock.o · 24 unmodified lines
2525 obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
2626 obj-$(CONFIG_QUEUED_SPINLOCKS) += qspinlock.o
2727 obj-$(CONFIG_RT_MUTEXES) += rtmutex_api.o
28+obj-$(CONFIG_PREEMPT_RT) += spinlock_rt.o
2829 obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o
2930 obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o
3031 obj-$(CONFIG_QUEUED_RWLOCKS) += qrwlock.o
kernel/locking/spinlock_rt.c+129−0
@@ -0,0 +1,129 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * PREEMPT_RT substitution for spin/rw_locks
4+ *
5+ * spinlocks and rwlocks on RT are based on rtmutexes, with a few twists to
6+ * resemble the non RT semantics:
7+ *
8+ * - Contrary to plain rtmutexes, spinlocks and rwlocks are state
9+ * preserving. The task state is saved before blocking on the underlying
10+ * rtmutex, and restored when the lock has been acquired. Regular wakeups
11+ * during that time are redirected to the saved state so no wake up is
12+ * missed.
13+ *
14+ * - Non RT spin/rwlocks disable preemption and eventually interrupts.
15+ * Disabling preemption has the side effect of disabling migration and
16+ * preventing RCU grace periods.
17+ *
18+ * The RT substitutions explicitly disable migration and take
19+ * rcu_read_lock() across the lock held section.
20+ */
21+#include <linux/spinlock.h>
22+#include <linux/export.h>
23+
24+#define RT_MUTEX_BUILD_SPINLOCKS
25+#include "rtmutex.c"
26+
27+static __always_inline void rtlock_lock(struct rt_mutex_base *rtm)
28+{
29+ if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
30+ rtlock_slowlock(rtm);
31+}
32+
33+static __always_inline void __rt_spin_lock(spinlock_t *lock)
34+{
35+ ___might_sleep(__FILE__, __LINE__, 0);
36+ rtlock_lock(&lock->lock);
37+ rcu_read_lock();
38+ migrate_disable();
39+}
40+
41+void __sched rt_spin_lock(spinlock_t *lock)
42+{
43+ spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
44+ __rt_spin_lock(lock);
45+}
46+EXPORT_SYMBOL(rt_spin_lock);
47+
48+#ifdef CONFIG_DEBUG_LOCK_ALLOC
49+void __sched rt_spin_lock_nested(spinlock_t *lock, int subclass)
50+{
51+ spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
52+ __rt_spin_lock(lock);
53+}
54+EXPORT_SYMBOL(rt_spin_lock_nested);
55+
56+void __sched rt_spin_lock_nest_lock(spinlock_t *lock,
57+ struct lockdep_map *nest_lock)
58+{
59+ spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
60+ __rt_spin_lock(lock);
61+}
62+EXPORT_SYMBOL(rt_spin_lock_nest_lock);
63+#endif
64+
65+void __sched rt_spin_unlock(spinlock_t *lock)
66+{
67+ spin_release(&lock->dep_map, _RET_IP_);
68+ migrate_enable();
69+ rcu_read_unlock();
70+
71+ if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL)))
72+ rt_mutex_slowunlock(&lock->lock);
73+}
74+EXPORT_SYMBOL(rt_spin_unlock);
75+
76+/*
77+ * Wait for the lock to get unlocked: instead of polling for an unlock
78+ * (like raw spinlocks do), lock and unlock, to force the kernel to
79+ * schedule if there's contention:
80+ */
81+void __sched rt_spin_lock_unlock(spinlock_t *lock)
82+{
83+ spin_lock(lock);
84+ spin_unlock(lock);
85+}
86+EXPORT_SYMBOL(rt_spin_lock_unlock);
87+
88+static __always_inline int __rt_spin_trylock(spinlock_t *lock)
89+{
90+ int ret = 1;
91+
92+ if (unlikely(!rt_mutex_cmpxchg_acquire(&lock->lock, NULL, current)))
93+ ret = rt_mutex_slowtrylock(&lock->lock);
94+
95+ if (ret) {
96+ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
97+ rcu_read_lock();
98+ migrate_disable();
99+ }
100+ return ret;
101+}
102+
103+int __sched rt_spin_trylock(spinlock_t *lock)
104+{
105+ return __rt_spin_trylock(lock);
106+}
107+EXPORT_SYMBOL(rt_spin_trylock);
108+
109+int __sched rt_spin_trylock_bh(spinlock_t *lock)
110+{
111+ int ret;
112+
113+ local_bh_disable();
114+ ret = __rt_spin_trylock(lock);
115+ if (!ret)
116+ local_bh_enable();
117+ return ret;
118+}
119+EXPORT_SYMBOL(rt_spin_trylock_bh);
120+
121+#ifdef CONFIG_DEBUG_LOCK_ALLOC
122+void __rt_spin_lock_init(spinlock_t *lock, const char *name,
123+ struct lock_class_key *key)
124+{
125+ debug_check_no_locks_freed((void *)lock, sizeof(*lock));
126+ lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG);
127+}
128+EXPORT_SYMBOL(__rt_spin_lock_init);
129+#endif