History

Merge tag 'core_entry_for_v5.17_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull thread_info flag accessor helper updates from Borislav Petkov: "Add a set of thread_info.flags accessors which snapshot it before accesing it in order to prevent any potential data races, and convert all users to those new accessors" * tag 'core_entry_for_v5.17_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: powerpc: Snapshot thread flags powerpc: Avoid discarding flags in system_call_exception() openrisc: Snapshot thread flags microblaze: Snapshot thread flags arm64: Snapshot thread flags ARM: Snapshot thread flags alpha: Snapshot thread flags sched: Snapshot thread flags entry: Snapshot thread flags x86: Snapshot thread flags thread_info: Add helpers to snapshot thread flags

Linus Torvalds committed 4 years ago

Commit 48a60bd2 parents 5ba13c1 985faa719 files · +46 −34Browse files

Unified · Split

19 files
  1. arch/alpha/kernel/signal.c +1 −1
  2. arch/arm/kernel/signal.c +1 −1
  3. arch/arm/mm/alignment.c +1 −1
  4. arch/arm64/kernel/entry-common.c +1 −1
  5. arch/arm64/kernel/ptrace.c +2 −2
  6. arch/arm64/kernel/signal.c +1 −1
  7. arch/arm64/kernel/syscall.c +2 −2
  8. arch/microblaze/kernel/signal.c +1 −1
  9. arch/openrisc/kernel/signal.c +1 −1
  10. arch/powerpc/kernel/interrupt.c +7 −8
  11. arch/powerpc/kernel/ptrace/ptrace.c +1 −2
  12. arch/x86/kernel/process.c +4 −4
  13. arch/x86/kernel/process.h +2 −2
  14. arch/x86/mm/tlb.c +1 −1
  15. include/linux/entry-kvm.h +1 −1
  16. include/linux/thread_info.h +14 −0
  17. kernel/entry/common.c +2 −2
  18. kernel/entry/kvm.c +2 −2
  19. kernel/sched/core.c +1 −1
arch/alpha/kernel/signal.c+1−1
@@ -535,6 +535,6 @@ do_work_pending(struct pt_regs *regs, unsigned long thread_flags, · 534 unmodified lines
535 }535 }
536 }536 }
537 local_irq_disable();537 local_irq_disable();
538 thread_flags = current_thread_info()->flags;538 thread_flags = read_thread_flags();
539 } while (thread_flags & _TIF_WORK_MASK);539 } while (thread_flags & _TIF_WORK_MASK);
540}540}
arch/arm/kernel/signal.c+1−1
@@ -631,7 +631,7 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) · 630 unmodified lines
631 }631 }
632 }632 }
633 local_irq_disable();633 local_irq_disable();
634 thread_flags = current_thread_info()->flags;634 thread_flags = read_thread_flags();
635 } while (thread_flags & _TIF_WORK_MASK);635 } while (thread_flags & _TIF_WORK_MASK);
636 return 0;636 return 0;
637}637}
arch/arm/mm/alignment.c+1−1
@@ -990,7 +990,7 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs) · 989 unmodified lines
990 * there is no work pending for this thread.990 * there is no work pending for this thread.
991 */991 */
992 raw_local_irq_disable();992 raw_local_irq_disable();
993 if (!(current_thread_info()->flags & _TIF_WORK_MASK))993 if (!(read_thread_flags() & _TIF_WORK_MASK))
994 set_cr(cr_no_alignment);994 set_cr(cr_no_alignment);
995 }995 }
996 996
arch/arm64/kernel/entry-common.c+1−1
@@ -129,7 +129,7 @@ static __always_inline void prepare_exit_to_user_mode(struct pt_regs *regs) · 128 unmodified lines
129 129
130 local_daif_mask();130 local_daif_mask();
131 131
132 flags = READ_ONCE(current_thread_info()->flags);132 flags = read_thread_flags();
133 if (unlikely(flags & _TIF_WORK_MASK))133 if (unlikely(flags & _TIF_WORK_MASK))
134 do_notify_resume(regs, flags);134 do_notify_resume(regs, flags);
135}135}
arch/arm64/kernel/ptrace.c+2−2
@@ -1839,7 +1839,7 @@ static void tracehook_report_syscall(struct pt_regs *regs, · 1838 unmodified lines
1839 1839
1840int syscall_trace_enter(struct pt_regs *regs)1840int syscall_trace_enter(struct pt_regs *regs)
1841{1841{
1842 unsigned long flags = READ_ONCE(current_thread_info()->flags);1842 unsigned long flags = read_thread_flags();
1843 1843
1844 if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) {1844 if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) {
1845 tracehook_report_syscall(regs, PTRACE_SYSCALL_ENTER);1845 tracehook_report_syscall(regs, PTRACE_SYSCALL_ENTER);
@@ -1862,7 +1862,7 @@ int syscall_trace_enter(struct pt_regs *regs) · 16 unmodified lines
1862 1862
1863void syscall_trace_exit(struct pt_regs *regs)1863void syscall_trace_exit(struct pt_regs *regs)
1864{1864{
1865 unsigned long flags = READ_ONCE(current_thread_info()->flags);1865 unsigned long flags = read_thread_flags();
1866 1866
1867 audit_syscall_exit(regs);1867 audit_syscall_exit(regs);
1868 1868
arch/arm64/kernel/signal.c+1−1
@@ -948,7 +948,7 @@ void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) · 947 unmodified lines
948 }948 }
949 949
950 local_daif_mask();950 local_daif_mask();
951 thread_flags = READ_ONCE(current_thread_info()->flags);951 thread_flags = read_thread_flags();
952 } while (thread_flags & _TIF_WORK_MASK);952 } while (thread_flags & _TIF_WORK_MASK);
953}953}
954 954
arch/arm64/kernel/syscall.c+2−2
@@ -81,7 +81,7 @@ void syscall_trace_exit(struct pt_regs *regs); · 80 unmodified lines
81static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,81static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
82 const syscall_fn_t syscall_table[])82 const syscall_fn_t syscall_table[])
83{83{
84 unsigned long flags = current_thread_info()->flags;84 unsigned long flags = read_thread_flags();
85 85
86 regs->orig_x0 = regs->regs[0];86 regs->orig_x0 = regs->regs[0];
87 regs->syscallno = scno;87 regs->syscallno = scno;
@@ -148,7 +148,7 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr, · 60 unmodified lines
148 */148 */
149 if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {149 if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
150 local_daif_mask();150 local_daif_mask();
151 flags = current_thread_info()->flags;151 flags = read_thread_flags();
152 if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP))152 if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP))
153 return;153 return;
154 local_daif_restore(DAIF_PROCCTX);154 local_daif_restore(DAIF_PROCCTX);
arch/microblaze/kernel/signal.c+1−1
@@ -283,7 +283,7 @@ static void do_signal(struct pt_regs *regs, int in_syscall) · 282 unmodified lines
283#ifdef DEBUG_SIG283#ifdef DEBUG_SIG
284 pr_info("do signal: %p %d\n", regs, in_syscall);284 pr_info("do signal: %p %d\n", regs, in_syscall);
285 pr_info("do signal2: %lx %lx %ld [%lx]\n", regs->pc, regs->r1,285 pr_info("do signal2: %lx %lx %ld [%lx]\n", regs->pc, regs->r1,
286 regs->r12, current_thread_info()->flags);286 regs->r12, read_thread_flags());
287#endif287#endif
288 288
289 if (get_signal(&ksig)) {289 if (get_signal(&ksig)) {
arch/openrisc/kernel/signal.c+1−1
@@ -313,7 +313,7 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) · 312 unmodified lines
313 }313 }
314 }314 }
315 local_irq_disable();315 local_irq_disable();
316 thread_flags = current_thread_info()->flags;316 thread_flags = read_thread_flags();
317 } while (thread_flags & _TIF_WORK_MASK);317 } while (thread_flags & _TIF_WORK_MASK);
318 return 0;318 return 0;
319}319}
arch/powerpc/kernel/interrupt.c+7−8
@@ -148,7 +148,7 @@ notrace long system_call_exception(long r3, long r4, long r5, · 147 unmodified lines
148 */148 */
149 if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&149 if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
150 unlikely(MSR_TM_TRANSACTIONAL(regs->msr)))150 unlikely(MSR_TM_TRANSACTIONAL(regs->msr)))
151 current_thread_info()->flags |= _TIF_RESTOREALL;151 set_bits(_TIF_RESTOREALL, &current_thread_info()->flags);
152 152
153 /*153 /*
154 * If the system call was made with a transaction active, doom it and154 * If the system call was made with a transaction active, doom it and
@@ -181,7 +181,7 @@ notrace long system_call_exception(long r3, long r4, long r5, · 26 unmodified lines
181 181
182 local_irq_enable();182 local_irq_enable();
183 183
184 if (unlikely(current_thread_info()->flags & _TIF_SYSCALL_DOTRACE)) {184 if (unlikely(read_thread_flags() & _TIF_SYSCALL_DOTRACE)) {
185 if (unlikely(trap_is_unsupported_scv(regs))) {185 if (unlikely(trap_is_unsupported_scv(regs))) {
186 /* Unsupported scv vector */186 /* Unsupported scv vector */
187 _exception(SIGILL, regs, ILL_ILLOPC, regs->nip);187 _exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
@@ -343,7 +343,7 @@ interrupt_exit_user_prepare_main(unsigned long ret, struct pt_regs *regs) · 155 unmodified lines
343 unsigned long ti_flags;343 unsigned long ti_flags;
344 344
345again:345again:
346 ti_flags = READ_ONCE(current_thread_info()->flags);346 ti_flags = read_thread_flags();
347 while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) {347 while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) {
348 local_irq_enable();348 local_irq_enable();
349 if (ti_flags & _TIF_NEED_RESCHED) {349 if (ti_flags & _TIF_NEED_RESCHED) {
@@ -359,7 +359,7 @@ again: · 9 unmodified lines
359 do_notify_resume(regs, ti_flags);359 do_notify_resume(regs, ti_flags);
360 }360 }
361 local_irq_disable();361 local_irq_disable();
362 ti_flags = READ_ONCE(current_thread_info()->flags);362 ti_flags = read_thread_flags();
363 }363 }
364 364
365 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && IS_ENABLED(CONFIG_PPC_FPU)) {365 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && IS_ENABLED(CONFIG_PPC_FPU)) {
@@ -437,7 +437,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, · 71 unmodified lines
437 /* Check whether the syscall is issued inside a restartable sequence */437 /* Check whether the syscall is issued inside a restartable sequence */
438 rseq_syscall(regs);438 rseq_syscall(regs);
439 439
440 ti_flags = current_thread_info()->flags;440 ti_flags = read_thread_flags();
441 441
442 if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) {442 if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) {
443 if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) {443 if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) {
@@ -532,8 +532,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs) · 88 unmodified lines
532 unsigned long flags;532 unsigned long flags;
533 unsigned long ret = 0;533 unsigned long ret = 0;
534 unsigned long kuap;534 unsigned long kuap;
535 bool stack_store = current_thread_info()->flags &535 bool stack_store = read_thread_flags() & _TIF_EMULATE_STACK_STORE;
536 _TIF_EMULATE_STACK_STORE;
537 536
538 if (regs_is_unrecoverable(regs))537 if (regs_is_unrecoverable(regs))
539 unrecoverable_exception(regs);538 unrecoverable_exception(regs);
@@ -554,7 +553,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs) · 14 unmodified lines
554again:553again:
555 if (IS_ENABLED(CONFIG_PREEMPT)) {554 if (IS_ENABLED(CONFIG_PREEMPT)) {
556 /* Return to preemptible kernel context */555 /* Return to preemptible kernel context */
557 if (unlikely(current_thread_info()->flags & _TIF_NEED_RESCHED)) {556 if (unlikely(read_thread_flags() & _TIF_NEED_RESCHED)) {
558 if (preempt_count() == 0)557 if (preempt_count() == 0)
559 preempt_schedule_irq();558 preempt_schedule_irq();
560 }559 }
arch/powerpc/kernel/ptrace/ptrace.c+1−2
@@ -260,8 +260,7 @@ long do_syscall_trace_enter(struct pt_regs *regs) · 259 unmodified lines
260{260{
261 u32 flags;261 u32 flags;
262 262
263 flags = READ_ONCE(current_thread_info()->flags) &263 flags = read_thread_flags() & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE);
264 (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE);
265 264
266 if (flags) {265 if (flags) {
267 int rc = tracehook_report_syscall_entry(regs);266 int rc = tracehook_report_syscall_entry(regs);
arch/x86/kernel/process.c+4−4
@@ -365,7 +365,7 @@ void arch_setup_new_exec(void) · 364 unmodified lines
365 clear_thread_flag(TIF_SSBD);365 clear_thread_flag(TIF_SSBD);
366 task_clear_spec_ssb_disable(current);366 task_clear_spec_ssb_disable(current);
367 task_clear_spec_ssb_noexec(current);367 task_clear_spec_ssb_noexec(current);
368 speculation_ctrl_update(task_thread_info(current)->flags);368 speculation_ctrl_update(read_thread_flags());
369 }369 }
370}370}
371 371
@@ -617,7 +617,7 @@ static unsigned long speculation_ctrl_update_tif(struct task_struct *tsk) · 245 unmodified lines
617 clear_tsk_thread_flag(tsk, TIF_SPEC_IB);617 clear_tsk_thread_flag(tsk, TIF_SPEC_IB);
618 }618 }
619 /* Return the updated threadinfo flags*/619 /* Return the updated threadinfo flags*/
620 return task_thread_info(tsk)->flags;620 return read_task_thread_flags(tsk);
621}621}
622 622
623void speculation_ctrl_update(unsigned long tif)623void speculation_ctrl_update(unsigned long tif)
@@ -653,8 +653,8 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p) · 29 unmodified lines
653{653{
654 unsigned long tifp, tifn;654 unsigned long tifp, tifn;
655 655
656 tifn = READ_ONCE(task_thread_info(next_p)->flags);656 tifn = read_task_thread_flags(next_p);
657 tifp = READ_ONCE(task_thread_info(prev_p)->flags);657 tifp = read_task_thread_flags(prev_p);
658 658
659 switch_to_bitmap(tifp);659 switch_to_bitmap(tifp);
660 660
arch/x86/kernel/process.h+2−2
@@ -13,8 +13,8 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p); · 12 unmodified lines
13static inline void switch_to_extra(struct task_struct *prev,13static inline void switch_to_extra(struct task_struct *prev,
14 struct task_struct *next)14 struct task_struct *next)
15{15{
16 unsigned long next_tif = task_thread_info(next)->flags;16 unsigned long next_tif = read_task_thread_flags(next);
17 unsigned long prev_tif = task_thread_info(prev)->flags;17 unsigned long prev_tif = read_task_thread_flags(prev);
18 18
19 if (IS_ENABLED(CONFIG_SMP)) {19 if (IS_ENABLED(CONFIG_SMP)) {
20 /*20 /*
arch/x86/mm/tlb.c+1−1
@@ -361,7 +361,7 @@ static void l1d_flush_evaluate(unsigned long prev_mm, unsigned long next_mm, · 360 unmodified lines
361 361
362static unsigned long mm_mangle_tif_spec_bits(struct task_struct *next)362static unsigned long mm_mangle_tif_spec_bits(struct task_struct *next)
363{363{
364 unsigned long next_tif = task_thread_info(next)->flags;364 unsigned long next_tif = read_task_thread_flags(next);
365 unsigned long spec_bits = (next_tif >> TIF_SPEC_IB) & LAST_USER_MM_SPEC_MASK;365 unsigned long spec_bits = (next_tif >> TIF_SPEC_IB) & LAST_USER_MM_SPEC_MASK;
366 366
367 /*367 /*
include/linux/entry-kvm.h+1−1
@@ -75,7 +75,7 @@ static inline void xfer_to_guest_mode_prepare(void) · 74 unmodified lines
75 */75 */
76static inline bool __xfer_to_guest_mode_work_pending(void)76static inline bool __xfer_to_guest_mode_work_pending(void)
77{77{
78 unsigned long ti_work = READ_ONCE(current_thread_info()->flags);78 unsigned long ti_work = read_thread_flags();
79 79
80 return !!(ti_work & XFER_TO_GUEST_MODE_WORK);80 return !!(ti_work & XFER_TO_GUEST_MODE_WORK);
81}81}
include/linux/thread_info.h+14−0
@@ -118,6 +118,15 @@ static inline int test_ti_thread_flag(struct thread_info *ti, int flag) · 117 unmodified lines
118 return test_bit(flag, (unsigned long *)&ti->flags);118 return test_bit(flag, (unsigned long *)&ti->flags);
119}119}
120 120
121/*
122 * This may be used in noinstr code, and needs to be __always_inline to prevent
123 * inadvertent instrumentation.
124 */
125static __always_inline unsigned long read_ti_thread_flags(struct thread_info *ti)
126{
127 return READ_ONCE(ti->flags);
128}
129
121#define set_thread_flag(flag) \130#define set_thread_flag(flag) \
122 set_ti_thread_flag(current_thread_info(), flag)131 set_ti_thread_flag(current_thread_info(), flag)
123#define clear_thread_flag(flag) \132#define clear_thread_flag(flag) \
@@ -130,6 +139,11 @@ static inline int test_ti_thread_flag(struct thread_info *ti, int flag) · 6 unmodified lines
130 test_and_clear_ti_thread_flag(current_thread_info(), flag)139 test_and_clear_ti_thread_flag(current_thread_info(), flag)
131#define test_thread_flag(flag) \140#define test_thread_flag(flag) \
132 test_ti_thread_flag(current_thread_info(), flag)141 test_ti_thread_flag(current_thread_info(), flag)
142#define read_thread_flags() \
143 read_ti_thread_flags(current_thread_info())
144
145#define read_task_thread_flags(t) \
146 read_ti_thread_flags(task_thread_info(t))
133 147
134#ifdef CONFIG_GENERIC_ENTRY148#ifdef CONFIG_GENERIC_ENTRY
135#define set_syscall_work(fl) \149#define set_syscall_work(fl) \
kernel/entry/common.c+2−2
@@ -187,7 +187,7 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, · 186 unmodified lines
187 /* Check if any of the above work has queued a deferred wakeup */187 /* Check if any of the above work has queued a deferred wakeup */
188 tick_nohz_user_enter_prepare();188 tick_nohz_user_enter_prepare();
189 189
190 ti_work = READ_ONCE(current_thread_info()->flags);190 ti_work = read_thread_flags();
191 }191 }
192 192
193 /* Return the latest work state for arch_exit_to_user_mode() */193 /* Return the latest work state for arch_exit_to_user_mode() */
@@ -196,7 +196,7 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, · 2 unmodified lines
196 196
197static void exit_to_user_mode_prepare(struct pt_regs *regs)197static void exit_to_user_mode_prepare(struct pt_regs *regs)
198{198{
199 unsigned long ti_work = READ_ONCE(current_thread_info()->flags);199 unsigned long ti_work = read_thread_flags();
200 200
201 lockdep_assert_irqs_disabled();201 lockdep_assert_irqs_disabled();
202 202
kernel/entry/kvm.c+2−2
@@ -26,7 +26,7 @@ static int xfer_to_guest_mode_work(struct kvm_vcpu *vcpu, unsigned long ti_work) · 25 unmodified lines
26 if (ret)26 if (ret)
27 return ret;27 return ret;
28 28
29 ti_work = READ_ONCE(current_thread_info()->flags);29 ti_work = read_thread_flags();
30 } while (ti_work & XFER_TO_GUEST_MODE_WORK || need_resched());30 } while (ti_work & XFER_TO_GUEST_MODE_WORK || need_resched());
31 return 0;31 return 0;
32}32}
@@ -43,7 +43,7 @@ int xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu) · 10 unmodified lines
43 * disabled in the inner loop before going into guest mode. No need43 * disabled in the inner loop before going into guest mode. No need
44 * to disable interrupts here.44 * to disable interrupts here.
45 */45 */
46 ti_work = READ_ONCE(current_thread_info()->flags);46 ti_work = read_thread_flags();
47 if (!(ti_work & XFER_TO_GUEST_MODE_WORK))47 if (!(ti_work & XFER_TO_GUEST_MODE_WORK))
48 return 0;48 return 0;
49 49
kernel/sched/core.c+1−1
@@ -8520,7 +8520,7 @@ void sched_show_task(struct task_struct *p) · 8519 unmodified lines
8520 rcu_read_unlock();8520 rcu_read_unlock();
8521 pr_cont(" stack:%5lu pid:%5d ppid:%6d flags:0x%08lx\n",8521 pr_cont(" stack:%5lu pid:%5d ppid:%6d flags:0x%08lx\n",
8522 free, task_pid_nr(p), ppid,8522 free, task_pid_nr(p), ppid,
8523 (unsigned long)task_thread_info(p)->flags);8523 read_task_thread_flags(p));
8524 8524
8525 print_worker_info(KERN_INFO, p);8525 print_worker_info(KERN_INFO, p);
8526 print_stop_info(KERN_INFO, p);8526 print_stop_info(KERN_INFO, p);