History

posix-timers: Provide a mechanism to allocate a given timer ID

Checkpoint/Restore in Userspace (CRIU) requires to reconstruct posix timers with the same timer ID on restore. It uses sys_timer_create() and relies on the monotonic increasing timer ID provided by this syscall. It creates and deletes timers until the desired ID is reached. This is can loop for a long time, when the checkpointed process had a very sparse timer ID range. It has been debated to implement a new syscall to allow the creation of timers with a given timer ID, but that's tideous due to the 32/64bit compat issues of sigevent_t and of dubious value. The restore mechanism of CRIU creates the timers in a state where all threads of the restored process are held on a barrier and cannot issue syscalls. That means the restorer task has exclusive control. This allows to address this issue with a prctl() so that the restorer thread can do: if (prctl(PR_TIMER_CREATE_RESTORE_IDS, PR_TIMER_CREATE_RESTORE_IDS_ON)) goto linear_mode; create_timers_with_explicit_ids(); prctl(PR_TIMER_CREATE_RESTORE_IDS, PR_TIMER_CREATE_RESTORE_IDS_OFF); This is backwards compatible because the prctl() fails on older kernels and CRIU can fall back to the linear timer ID mechanism. CRIU versions which do not know about the prctl() just work as before. Implement the prctl() and modify timer_create() so that it copies the requested timer ID from userspace by utilizing the existing timer_t pointer, which is used to copy out the allocated timer ID on success. If the prctl() is disabled, which it is by default, timer_create() works as before and does not try to read from the userspace pointer. There is no problem when a broken or rogue user space application enables the prctl(). If the user space pointer does not contain a valid ID, then timer_create() fails. If the data is not initialized, but constains a random valid ID, timer_create() will create that random timer ID or fail if the ID is already given out. As CRIU must use the raw syscall to avoid manipulating the internal state of the restored process, this has no library dependencies and can be adopted by CRIU right away. Recreating two timers with IDs 1000000 and 2000000 takes 1.5 seconds with the create/delete method. With the prctl() it takes 3 microseconds.

Thomas Gleixner committed 1 year ago

Commit ec2d0c01 parent 2dc4dbf5 files · +98 −26Browse files

Unified · Split

5 files
  1. include/linux/posix-timers.h +2 −0
  2. include/linux/sched/signal.h +1 −0
  3. include/uapi/linux/prctl.h +11 −0
  4. kernel/sys.c +5 −0
  5. kernel/time/posix-timers.c +79 −26
include/linux/posix-timers.h+2−0
@@ -114,6 +114,7 @@ bool posixtimer_init_sigqueue(struct sigqueue *q); · 113 unmodified lines
114void posixtimer_send_sigqueue(struct k_itimer *tmr);114void posixtimer_send_sigqueue(struct k_itimer *tmr);
115bool posixtimer_deliver_signal(struct kernel_siginfo *info, struct sigqueue *timer_sigq);115bool posixtimer_deliver_signal(struct kernel_siginfo *info, struct sigqueue *timer_sigq);
116void posixtimer_free_timer(struct k_itimer *timer);116void posixtimer_free_timer(struct k_itimer *timer);
117long posixtimer_create_prctl(unsigned long ctrl);
117 118
118/* Init task static initializer */119/* Init task static initializer */
119#define INIT_CPU_TIMERBASE(b) { \120#define INIT_CPU_TIMERBASE(b) { \
@@ -140,6 +141,7 @@ static inline void posixtimer_rearm_itimer(struct task_struct *p) { } · 20 unmodified lines
140static inline bool posixtimer_deliver_signal(struct kernel_siginfo *info,141static inline bool posixtimer_deliver_signal(struct kernel_siginfo *info,
141 struct sigqueue *timer_sigq) { return false; }142 struct sigqueue *timer_sigq) { return false; }
142static inline void posixtimer_free_timer(struct k_itimer *timer) { }143static inline void posixtimer_free_timer(struct k_itimer *timer) { }
144static inline long posixtimer_create_prctl(unsigned long ctrl) { return -EINVAL; }
143#endif145#endif
144 146
145#ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK147#ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK
include/linux/sched/signal.h+1−0
@@ -136,6 +136,7 @@ struct signal_struct { · 135 unmodified lines
136#ifdef CONFIG_POSIX_TIMERS136#ifdef CONFIG_POSIX_TIMERS
137 137
138 /* POSIX.1b Interval Timers */138 /* POSIX.1b Interval Timers */
139 unsigned int timer_create_restore_ids:1;
139 atomic_t next_posix_timer_id;140 atomic_t next_posix_timer_id;
140 struct hlist_head posix_timers;141 struct hlist_head posix_timers;
141 struct hlist_head ignored_posix_timers;142 struct hlist_head ignored_posix_timers;
include/uapi/linux/prctl.h+11−0
@@ -353,4 +353,15 @@ struct prctl_mm_map { · 352 unmodified lines
353 */353 */
354#define PR_LOCK_SHADOW_STACK_STATUS 76354#define PR_LOCK_SHADOW_STACK_STATUS 76
355 355
356/*
357 * Controls the mode of timer_create() for CRIU restore operations.
358 * Enabling this allows CRIU to restore timers with explicit IDs.
359 *
360 * Don't use for normal operations as the result might be undefined.
361 */
362#define PR_TIMER_CREATE_RESTORE_IDS 77
363# define PR_TIMER_CREATE_RESTORE_IDS_OFF 0
364# define PR_TIMER_CREATE_RESTORE_IDS_ON 1
365# define PR_TIMER_CREATE_RESTORE_IDS_GET 2
366
356#endif /* _LINUX_PRCTL_H */367#endif /* _LINUX_PRCTL_H */
kernel/sys.c+5−0
@@ -2811,6 +2811,11 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, · 2810 unmodified lines
2811 return -EINVAL;2811 return -EINVAL;
2812 error = arch_lock_shadow_stack_status(me, arg2);2812 error = arch_lock_shadow_stack_status(me, arg2);
2813 break;2813 break;
2814 case PR_TIMER_CREATE_RESTORE_IDS:
2815 if (arg3 || arg4 || arg5)
2816 return -EINVAL;
2817 error = posixtimer_create_prctl(arg2);
2818 break;
2814 default:2819 default:
2815 trace_task_prctl_unknown(option, arg2, arg3, arg4, arg5);2820 trace_task_prctl_unknown(option, arg2, arg3, arg4, arg5);
2816 error = -EINVAL;2821 error = -EINVAL;
kernel/time/posix-timers.c+79−26
@@ -19,6 +19,7 @@ · 18 unmodified lines
19#include <linux/nospec.h>19#include <linux/nospec.h>
20#include <linux/posix-clock.h>20#include <linux/posix-clock.h>
21#include <linux/posix-timers.h>21#include <linux/posix-timers.h>
22#include <linux/prctl.h>
22#include <linux/sched/task.h>23#include <linux/sched/task.h>
23#include <linux/slab.h>24#include <linux/slab.h>
24#include <linux/syscalls.h>25#include <linux/syscalls.h>
@@ -57,6 +58,8 @@ static const struct k_clock * const posix_clocks[]; · 32 unmodified lines
57static const struct k_clock *clockid_to_kclock(const clockid_t id);58static const struct k_clock *clockid_to_kclock(const clockid_t id);
58static const struct k_clock clock_realtime, clock_monotonic;59static const struct k_clock clock_realtime, clock_monotonic;
59 60
61#define TIMER_ANY_ID INT_MIN
62
60/* SIGEV_THREAD_ID cannot share a bit with the other SIGEV values. */63/* SIGEV_THREAD_ID cannot share a bit with the other SIGEV values. */
61#if SIGEV_THREAD_ID != (SIGEV_THREAD_ID & \64#if SIGEV_THREAD_ID != (SIGEV_THREAD_ID & \
62 ~(SIGEV_SIGNAL | SIGEV_NONE | SIGEV_THREAD))65 ~(SIGEV_SIGNAL | SIGEV_NONE | SIGEV_THREAD))
@@ -128,38 +131,60 @@ static bool posix_timer_hashed(struct timer_hash_bucket *bucket, struct signal_s · 65 unmodified lines
128 return false;131 return false;
129}132}
130 133
131static int posix_timer_add(struct k_itimer *timer)134static bool posix_timer_add_at(struct k_itimer *timer, struct signal_struct *sig, unsigned int id)
135{
136 struct timer_hash_bucket *bucket = hash_bucket(sig, id);
137
138 scoped_guard (spinlock, &bucket->lock) {
139 /*
140 * Validate under the lock as this could have raced against
141 * another thread ending up with the same ID, which is
142 * highly unlikely, but possible.
143 */
144 if (!posix_timer_hashed(bucket, sig, id)) {
145 /*
146 * Set the timer ID and the signal pointer to make
147 * it identifiable in the hash table. The signal
148 * pointer has bit 0 set to indicate that it is not
149 * yet fully initialized. posix_timer_hashed()
150 * masks this bit out, but the syscall lookup fails
151 * to match due to it being set. This guarantees
152 * that there can't be duplicate timer IDs handed
153 * out.
154 */
155 timer->it_id = (timer_t)id;
156 timer->it_signal = (struct signal_struct *)((unsigned long)sig | 1UL);
157 hlist_add_head_rcu(&timer->t_hash, &bucket->head);
158 return true;
159 }
160 }
161 return false;
162}
163
164static int posix_timer_add(struct k_itimer *timer, int req_id)
132{165{
133 struct signal_struct *sig = current->signal;166 struct signal_struct *sig = current->signal;
134 167
168 if (unlikely(req_id != TIMER_ANY_ID)) {
169 if (!posix_timer_add_at(timer, sig, req_id))
170 return -EBUSY;
171
172 /*
173 * Move the ID counter past the requested ID, so that after
174 * switching back to normal mode the IDs are outside of the
175 * exact allocated region. That avoids ID collisions on the
176 * next regular timer_create() invocations.
177 */
178 atomic_set(&sig->next_posix_timer_id, req_id + 1);
179 return req_id;
180 }
181
135 for (unsigned int cnt = 0; cnt <= INT_MAX; cnt++) {182 for (unsigned int cnt = 0; cnt <= INT_MAX; cnt++) {
136 /* Get the next timer ID and clamp it to positive space */183 /* Get the next timer ID and clamp it to positive space */
137 unsigned int id = atomic_fetch_inc(&sig->next_posix_timer_id) & INT_MAX;184 unsigned int id = atomic_fetch_inc(&sig->next_posix_timer_id) & INT_MAX;
138 struct timer_hash_bucket *bucket = hash_bucket(sig, id);
139 185
140 scoped_guard (spinlock, &bucket->lock) {186 if (posix_timer_add_at(timer, sig, id))
141 /*187 return id;
142 * Validate under the lock as this could have raced
143 * against another thread ending up with the same
144 * ID, which is highly unlikely, but possible.
145 */
146 if (!posix_timer_hashed(bucket, sig, id)) {
147 /*
148 * Set the timer ID and the signal pointer to make
149 * it identifiable in the hash table. The signal
150 * pointer has bit 0 set to indicate that it is not
151 * yet fully initialized. posix_timer_hashed()
152 * masks this bit out, but the syscall lookup fails
153 * to match due to it being set. This guarantees
154 * that there can't be duplicate timer IDs handed
155 * out.
156 */
157 timer->it_id = (timer_t)id;
158 timer->it_signal = (struct signal_struct *)((unsigned long)sig | 1UL);
159 hlist_add_head_rcu(&timer->t_hash, &bucket->head);
160 return id;
161 }
162 }
163 cond_resched();188 cond_resched();
164 }189 }
165 /* POSIX return code when no timer ID could be allocated */190 /* POSIX return code when no timer ID could be allocated */
@@ -364,6 +389,21 @@ static enum hrtimer_restart posix_timer_fn(struct hrtimer *timer) · 198 unmodified lines
364 return HRTIMER_NORESTART;389 return HRTIMER_NORESTART;
365}390}
366 391
392long posixtimer_create_prctl(unsigned long ctrl)
393{
394 switch (ctrl) {
395 case PR_TIMER_CREATE_RESTORE_IDS_OFF:
396 current->signal->timer_create_restore_ids = 0;
397 return 0;
398 case PR_TIMER_CREATE_RESTORE_IDS_ON:
399 current->signal->timer_create_restore_ids = 1;
400 return 0;
401 case PR_TIMER_CREATE_RESTORE_IDS_GET:
402 return current->signal->timer_create_restore_ids;
403 }
404 return -EINVAL;
405}
406
367static struct pid *good_sigevent(sigevent_t * event)407static struct pid *good_sigevent(sigevent_t * event)
368{408{
369 struct pid *pid = task_tgid(current);409 struct pid *pid = task_tgid(current);
@@ -435,6 +475,7 @@ static int do_timer_create(clockid_t which_clock, struct sigevent *event, · 65 unmodified lines
435 timer_t __user *created_timer_id)475 timer_t __user *created_timer_id)
436{476{
437 const struct k_clock *kc = clockid_to_kclock(which_clock);477 const struct k_clock *kc = clockid_to_kclock(which_clock);
478 timer_t req_id = TIMER_ANY_ID;
438 struct k_itimer *new_timer;479 struct k_itimer *new_timer;
439 int error, new_timer_id;480 int error, new_timer_id;
440 481
@@ -449,11 +490,20 @@ static int do_timer_create(clockid_t which_clock, struct sigevent *event, · 8 unmodified lines
449 490
450 spin_lock_init(&new_timer->it_lock);491 spin_lock_init(&new_timer->it_lock);
451 492
493 /* Special case for CRIU to restore timers with a given timer ID. */
494 if (unlikely(current->signal->timer_create_restore_ids)) {
495 if (copy_from_user(&req_id, created_timer_id, sizeof(req_id)))
496 return -EFAULT;
497 /* Valid IDs are 0..INT_MAX */
498 if ((unsigned int)req_id > INT_MAX)
499 return -EINVAL;
500 }
501
452 /*502 /*
453 * Add the timer to the hash table. The timer is not yet valid503 * Add the timer to the hash table. The timer is not yet valid
454 * after insertion, but has a unique ID allocated.504 * after insertion, but has a unique ID allocated.
455 */505 */
456 new_timer_id = posix_timer_add(new_timer);506 new_timer_id = posix_timer_add(new_timer, req_id);
457 if (new_timer_id < 0) {507 if (new_timer_id < 0) {
458 posixtimer_free_timer(new_timer);508 posixtimer_free_timer(new_timer);
459 return new_timer_id;509 return new_timer_id;
@@ -1041,6 +1091,9 @@ void exit_itimers(struct task_struct *tsk) · 581 unmodified lines
1041 struct hlist_node *next;1091 struct hlist_node *next;
1042 struct k_itimer *timer;1092 struct k_itimer *timer;
1043 1093
1094 /* Clear restore mode for exec() */
1095 tsk->signal->timer_create_restore_ids = 0;
1096
1044 if (hlist_empty(&tsk->signal->posix_timers))1097 if (hlist_empty(&tsk->signal->posix_timers))
1045 return;1098 return;
1046 1099