x86/asm: Use register variable to get stack pointer value

Currently we use current_stack_pointer() function to get the value of the stack pointer register. Since commit: f5caf621ee35 ("x86/asm: Fix inline asm call constraints for Clang") ... we have a stack register variable declared. It can be used instead of current_stack_pointer() function which allows to optimize away some excessive "mov %rsp, %<dst>" instructions: -mov %rsp,%rdx -sub %rdx,%rax -cmp $0x3fff,%rax -ja ffffffff810722fd <ist_begin_non_atomic+0x2d> +sub %rsp,%rax +cmp $0x3fff,%rax +ja ffffffff810722fa <ist_begin_non_atomic+0x2a> Remove current_stack_pointer(), rename __asm_call_sp to current_stack_pointer and use it instead of the removed function.

Andrey Ryabinin committed 9 years ago

Commit 196bd481 parent bc829ee5 files · +7 −18Browse files

Unified · Split

5 files

arch/x86/include/asm/asm.h +2 −2
arch/x86/include/asm/thread_info.h +0 −11
arch/x86/kernel/irq_32.c +3 −3
arch/x86/kernel/traps.c +1 −1
arch/x86/mm/tlb.c +1 −1

arch/x86/include/asm/asm.h+2−2

@@ -141,8 +141,8 @@ · 140 unmodified lines
141	* gets set up by the containing function. If you forget to do this, objtool	141	* gets set up by the containing function. If you forget to do this, objtool
142	* may print a "call without frame pointer save/setup" warning.	142	* may print a "call without frame pointer save/setup" warning.
143	*/	143	*/
144	register unsigned long __asm_call_sp asm(_ASM_SP);	144	register unsigned long current_stack_pointer asm(_ASM_SP);
145	#define ASM_CALL_CONSTRAINT "+r" (__asm_call_sp)	145	#define ASM_CALL_CONSTRAINT "+r" (current_stack_pointer)
146	#endif	146	#endif
147		147
148	#endif /* _ASM_X86_ASM_H */	148	#endif /* _ASM_X86_ASM_H */

arch/x86/include/asm/thread_info.h+0−11

@@ -158,17 +158,6 @@ struct thread_info { · 157 unmodified lines
158	*/	158	*/
159	#ifndef __ASSEMBLY__	159	#ifndef __ASSEMBLY__
160		160
161	static inline unsigned long current_stack_pointer(void)
162	{
163	unsigned long sp;
164	#ifdef CONFIG_X86_64
165	asm("mov %%rsp,%0" : "=g" (sp));
166	#else
167	asm("mov %%esp,%0" : "=g" (sp));
168	#endif
169	return sp;
170	}
171
172	/*	161	/*
173	* Walks up the stack frames to make sure that the specified object is	162	* Walks up the stack frames to make sure that the specified object is
174	* entirely contained by a single stack frame.	163	* entirely contained by a single stack frame.

arch/x86/kernel/irq_32.c+3−3

@@ -64,7 +64,7 @@ static void call_on_stack(void func, void stack) · 63 unmodified lines
64		64
65	static inline void *current_stack(void)	65	static inline void *current_stack(void)
66	{	66	{
67	return (void *)(current_stack_pointer() & ~(THREAD_SIZE - 1));	67	return (void *)(current_stack_pointer & ~(THREAD_SIZE - 1));
68	}	68	}
69		69
70	static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)	70	static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)
@@ -88,7 +88,7 @@ static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc) · 17 unmodified lines
88		88
89	/* Save the next esp at the bottom of the stack */	89	/* Save the next esp at the bottom of the stack */
90	prev_esp = (u32 *)irqstk;	90	prev_esp = (u32 *)irqstk;
91	*prev_esp = current_stack_pointer();	91	*prev_esp = current_stack_pointer;
92		92
93	if (unlikely(overflow))	93	if (unlikely(overflow))
94	call_on_stack(print_stack_overflow, isp);	94	call_on_stack(print_stack_overflow, isp);
@@ -139,7 +139,7 @@ void do_softirq_own_stack(void) · 44 unmodified lines
139		139
140	/* Push the previous esp onto the stack */	140	/* Push the previous esp onto the stack */
141	prev_esp = (u32 *)irqstk;	141	prev_esp = (u32 *)irqstk;
142	*prev_esp = current_stack_pointer();	142	*prev_esp = current_stack_pointer;
143		143
144	call_on_stack(__do_softirq, isp);	144	call_on_stack(__do_softirq, isp);
145	}	145	}

arch/x86/kernel/traps.c+1−1

@@ -142,7 +142,7 @@ void ist_begin_non_atomic(struct pt_regs *regs) · 141 unmodified lines
142	* from double_fault.	142	* from double_fault.
143	*/	143	*/
144	BUG_ON((unsigned long)(current_top_of_stack() -	144	BUG_ON((unsigned long)(current_top_of_stack() -
145	current_stack_pointer()) >= THREAD_SIZE);	145	current_stack_pointer) >= THREAD_SIZE);
146		146
147	preempt_enable_no_resched();	147	preempt_enable_no_resched();
148	}	148	}

arch/x86/mm/tlb.c+1−1

@@ -191,7 +191,7 @@ void switch_mm_irqs_off(struct mm_struct prev, struct mm_struct next, · 190 unmodified lines
191	* mapped in the new pgd, we'll double-fault. Forcibly	191	* mapped in the new pgd, we'll double-fault. Forcibly
192	* map it.	192	* map it.
193	*/	193	*/
194	unsigned int index = pgd_index(current_stack_pointer());	194	unsigned int index = pgd_index(current_stack_pointer);
195	pgd_t *pgd = next->pgd + index;	195	pgd_t *pgd = next->pgd + index;
196		196
197	if (unlikely(pgd_none(*pgd)))	197	if (unlikely(pgd_none(*pgd)))