sparc64: fix FP corruption in user copy functions

Short story: Exception handlers used by some copy_to_user() and copy_from_user() functions do not diligently clean up floating point register usage, and this can result in a user process seeing invalid values in floating point registers. This sometimes makes the process fail. Long story: Several cpu-specific (NG4, NG2, U1, U3) memcpy functions use floating point registers and VIS alignaddr/faligndata to accelerate data copying when source and dest addresses don't align well. Linux uses a lazy scheme for saving floating point registers; It is not done upon entering the kernel since it's a very expensive operation. Rather, it is done only when needed. If the kernel ends up not using FP regs during the course of some trap or system call, then it can return to user space without saving or restoring them. The various memcpy functions begin their FP code with VISEntry (or a variation thereof), which saves the FP regs. They conclude their FP code with VISExit (or a variation) which essentially marks the FP regs "clean", ie, they contain no unsaved values. fprs.FPRS_FEF is turned off so that a lazy restore will be triggered when/if the user process accesses floating point regs again. The bug is that the user copy variants of memcpy, copy_from_user() and copy_to_user(), employ an exception handling mechanism to detect faults when accessing user space addresses, and when this handler is invoked, an immediate return from the function is forced, and VISExit is not executed, thus leaving the fprs register in an indeterminate state, but often with fprs.FPRS_FEF set and one or more dirty bits. This results in a return to user space with invalid values in the FP regs, and since fprs.FPRS_FEF is on, no lazy restore occurs. This bug affects copy_to_user() and copy_from_user() for NG4, NG2, U3, and U1. All are fixed by using a new exception handler for those loads and stores that are done during the time between VISEnter and VISExit. n.b. In NG4memcpy, the problematic code can be triggered by a copy size greater than 128 bytes and an unaligned source address. This bug is known to be the cause of random user process memory corruptions while perf is running with the callgraph option (ie, perf record -g). This occurs because perf uses copy_from_user() to read user stacks, and may fault when it follows a stack frame pointer off to an invalid page. Validation checks on the stack address just obscure the underlying problem.

Rob Gardner committed 10 years ago

Commit a7c57241 parent 833526913 files · +235 −134Browse files

Unified · Split

13 files

arch/sparc/kernel/head_64.S +13 −0
arch/sparc/lib/NG2copy_from_user.S +8 −0
arch/sparc/lib/NG2copy_to_user.S +8 −0
arch/sparc/lib/NG2memcpy.S +62 −56
arch/sparc/lib/NG4copy_from_user.S +8 −0
arch/sparc/lib/NG4copy_to_user.S +8 −0
arch/sparc/lib/NG4memcpy.S +23 −17
arch/sparc/lib/U1copy_from_user.S +8 −0
arch/sparc/lib/U1copy_to_user.S +8 −0
arch/sparc/lib/U1memcpy.S +27 −21
arch/sparc/lib/U3copy_from_user.S +8 −0
arch/sparc/lib/U3copy_to_user.S +8 −0
arch/sparc/lib/U3memcpy.S +46 −40

arch/sparc/kernel/head_64.S+13−0

@@ -946,6 +946,12 @@ ENTRY(__retl_one) · 945 unmodified lines
946	946	mov 1, %o0
947	947	ENDPROC(__retl_one)
948	948
	949	+ENTRY(__retl_one_fp)
	950	+ VISExitHalf
	951	+ retl
	952	+ mov 1, %o0
	953	+ENDPROC(__retl_one_fp)
	954	+
949	955	ENTRY(__ret_one_asi)
950	956	wr %g0, ASI_AIUS, %asi
951	957	ret
@@ -958,6 +964,13 @@ ENTRY(__retl_one_asi) · 6 unmodified lines
958	964	mov 1, %o0
959	965	ENDPROC(__retl_one_asi)
960	966
	967	+ENTRY(__retl_one_asi_fp)
	968	+ wr %g0, ASI_AIUS, %asi
	969	+ VISExitHalf
	970	+ retl
	971	+ mov 1, %o0
	972	+ENDPROC(__retl_one_asi_fp)
	973	+
961	974	ENTRY(__retl_o1)
962	975	retl
963	976	mov %o1, %o0

arch/sparc/lib/NG2copy_from_user.S+8−0

@@ -11,6 +11,14 @@ · 10 unmodified lines
11	11	.text; \
12	12	.align 4;
13	13
	14	+#define EX_LD_FP(x) \
	15	+98: x; \
	16	+ .section __ex_table,"a";\
	17	+ .align 4; \
	18	+ .word 98b, __retl_one_asi_fp;\
	19	+ .text; \
	20	+ .align 4;
	21	+
14	22	#ifndef ASI_AIUS
15	23	#define ASI_AIUS 0x11
16	24	#endif

arch/sparc/lib/NG2copy_to_user.S+8−0

@@ -11,6 +11,14 @@ · 10 unmodified lines
11	11	.text; \
12	12	.align 4;
13	13
	14	+#define EX_ST_FP(x) \
	15	+98: x; \
	16	+ .section __ex_table,"a";\
	17	+ .align 4; \
	18	+ .word 98b, __retl_one_asi_fp;\
	19	+ .text; \
	20	+ .align 4;
	21	+
14	22	#ifndef ASI_AIUS
15	23	#define ASI_AIUS 0x11
16	24	#endif

arch/sparc/lib/NG2memcpy.S+62−56

@@ -34,10 +34,16 @@ · 33 unmodified lines
34	34	#ifndef EX_LD
35	35	#define EX_LD(x) x
36	36	#endif
	37	+#ifndef EX_LD_FP
	38	+#define EX_LD_FP(x) x
	39	+#endif
37	40
38	41	#ifndef EX_ST
39	42	#define EX_ST(x) x
40	43	#endif
	44	+#ifndef EX_ST_FP
	45	+#define EX_ST_FP(x) x
	46	+#endif
41	47
42	48	#ifndef EX_RETVAL
43	49	#define EX_RETVAL(x) x
@@ -134,40 +140,40 @@ · 90 unmodified lines
134	140	fsrc2 %x6, %f12; \
135	141	fsrc2 %x7, %f14;
136	142	#define FREG_LOAD_1(base, x0) \
137		− EX_LD(LOAD(ldd, base + 0x00, %x0))
	143	+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0))
138	144	#define FREG_LOAD_2(base, x0, x1) \
139		− EX_LD(LOAD(ldd, base + 0x00, %x0)); \
140		− EX_LD(LOAD(ldd, base + 0x08, %x1));
	145	+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
	146	+ EX_LD_FP(LOAD(ldd, base + 0x08, %x1));
141	147	#define FREG_LOAD_3(base, x0, x1, x2) \
142		− EX_LD(LOAD(ldd, base + 0x00, %x0)); \
143		− EX_LD(LOAD(ldd, base + 0x08, %x1)); \
144		− EX_LD(LOAD(ldd, base + 0x10, %x2));
	148	+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
	149	+ EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
	150	+ EX_LD_FP(LOAD(ldd, base + 0x10, %x2));
145	151	#define FREG_LOAD_4(base, x0, x1, x2, x3) \
146		− EX_LD(LOAD(ldd, base + 0x00, %x0)); \
147		− EX_LD(LOAD(ldd, base + 0x08, %x1)); \
148		− EX_LD(LOAD(ldd, base + 0x10, %x2)); \
149		− EX_LD(LOAD(ldd, base + 0x18, %x3));
	152	+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
	153	+ EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
	154	+ EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
	155	+ EX_LD_FP(LOAD(ldd, base + 0x18, %x3));
150	156	#define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \
151		− EX_LD(LOAD(ldd, base + 0x00, %x0)); \
152		− EX_LD(LOAD(ldd, base + 0x08, %x1)); \
153		− EX_LD(LOAD(ldd, base + 0x10, %x2)); \
154		− EX_LD(LOAD(ldd, base + 0x18, %x3)); \
155		− EX_LD(LOAD(ldd, base + 0x20, %x4));
	157	+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
	158	+ EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
	159	+ EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
	160	+ EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
	161	+ EX_LD_FP(LOAD(ldd, base + 0x20, %x4));
156	162	#define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \
157		− EX_LD(LOAD(ldd, base + 0x00, %x0)); \
158		− EX_LD(LOAD(ldd, base + 0x08, %x1)); \
159		− EX_LD(LOAD(ldd, base + 0x10, %x2)); \
160		− EX_LD(LOAD(ldd, base + 0x18, %x3)); \
161		− EX_LD(LOAD(ldd, base + 0x20, %x4)); \
162		− EX_LD(LOAD(ldd, base + 0x28, %x5));
	163	+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
	164	+ EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
	165	+ EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
	166	+ EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
	167	+ EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
	168	+ EX_LD_FP(LOAD(ldd, base + 0x28, %x5));
163	169	#define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \
164		− EX_LD(LOAD(ldd, base + 0x00, %x0)); \
165		− EX_LD(LOAD(ldd, base + 0x08, %x1)); \
166		− EX_LD(LOAD(ldd, base + 0x10, %x2)); \
167		− EX_LD(LOAD(ldd, base + 0x18, %x3)); \
168		− EX_LD(LOAD(ldd, base + 0x20, %x4)); \
169		− EX_LD(LOAD(ldd, base + 0x28, %x5)); \
170		− EX_LD(LOAD(ldd, base + 0x30, %x6));
	170	+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
	171	+ EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
	172	+ EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
	173	+ EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
	174	+ EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
	175	+ EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); \
	176	+ EX_LD_FP(LOAD(ldd, base + 0x30, %x6));
171	177
172	178	.register %g2,#scratch
173	179	.register %g3,#scratch
@@ -275,11 +281,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 101 unmodified lines
275	281	nop
276	282	/* fall through for 0 < low bits < 8 */
277	283	110: sub %o4, 64, %g2
278		− EX_LD(LOAD_BLK(%g2, %f0))
279		−1: EX_ST(STORE_INIT(%g0, %o4 + %g3))
280		− EX_LD(LOAD_BLK(%o4, %f16))
	284	+ EX_LD_FP(LOAD_BLK(%g2, %f0))
	285	+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
	286	+ EX_LD_FP(LOAD_BLK(%o4, %f16))
281	287	FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16)
282		− EX_ST(STORE_BLK(%f0, %o4 + %g3))
	288	+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
283	289	FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30)
284	290	subcc %g1, 64, %g1
285	291	add %o4, 64, %o4
@@ -290,10 +296,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 4 unmodified lines
290	296
291	297	120: sub %o4, 56, %g2
292	298	FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12)
293		−1: EX_ST(STORE_INIT(%g0, %o4 + %g3))
294		− EX_LD(LOAD_BLK(%o4, %f16))
	299	+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
	300	+ EX_LD_FP(LOAD_BLK(%o4, %f16))
295	301	FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18)
296		− EX_ST(STORE_BLK(%f0, %o4 + %g3))
	302	+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
297	303	FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30)
298	304	subcc %g1, 64, %g1
299	305	add %o4, 64, %o4
@@ -304,10 +310,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 4 unmodified lines
304	310
305	311	130: sub %o4, 48, %g2
306	312	FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10)
307		−1: EX_ST(STORE_INIT(%g0, %o4 + %g3))
308		− EX_LD(LOAD_BLK(%o4, %f16))
	313	+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
	314	+ EX_LD_FP(LOAD_BLK(%o4, %f16))
309	315	FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20)
310		− EX_ST(STORE_BLK(%f0, %o4 + %g3))
	316	+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
311	317	FREG_MOVE_6(f20, f22, f24, f26, f28, f30)
312	318	subcc %g1, 64, %g1
313	319	add %o4, 64, %o4
@@ -318,10 +324,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 4 unmodified lines
318	324
319	325	140: sub %o4, 40, %g2
320	326	FREG_LOAD_5(%g2, f0, f2, f4, f6, f8)
321		−1: EX_ST(STORE_INIT(%g0, %o4 + %g3))
322		− EX_LD(LOAD_BLK(%o4, %f16))
	327	+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
	328	+ EX_LD_FP(LOAD_BLK(%o4, %f16))
323	329	FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22)
324		− EX_ST(STORE_BLK(%f0, %o4 + %g3))
	330	+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
325	331	FREG_MOVE_5(f22, f24, f26, f28, f30)
326	332	subcc %g1, 64, %g1
327	333	add %o4, 64, %o4
@@ -332,10 +338,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 4 unmodified lines
332	338
333	339	150: sub %o4, 32, %g2
334	340	FREG_LOAD_4(%g2, f0, f2, f4, f6)
335		−1: EX_ST(STORE_INIT(%g0, %o4 + %g3))
336		− EX_LD(LOAD_BLK(%o4, %f16))
	341	+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
	342	+ EX_LD_FP(LOAD_BLK(%o4, %f16))
337	343	FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24)
338		− EX_ST(STORE_BLK(%f0, %o4 + %g3))
	344	+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
339	345	FREG_MOVE_4(f24, f26, f28, f30)
340	346	subcc %g1, 64, %g1
341	347	add %o4, 64, %o4
@@ -346,10 +352,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 4 unmodified lines
346	352
347	353	160: sub %o4, 24, %g2
348	354	FREG_LOAD_3(%g2, f0, f2, f4)
349		−1: EX_ST(STORE_INIT(%g0, %o4 + %g3))
350		− EX_LD(LOAD_BLK(%o4, %f16))
	355	+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
	356	+ EX_LD_FP(LOAD_BLK(%o4, %f16))
351	357	FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26)
352		− EX_ST(STORE_BLK(%f0, %o4 + %g3))
	358	+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
353	359	FREG_MOVE_3(f26, f28, f30)
354	360	subcc %g1, 64, %g1
355	361	add %o4, 64, %o4
@@ -360,10 +366,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 4 unmodified lines
360	366
361	367	170: sub %o4, 16, %g2
362	368	FREG_LOAD_2(%g2, f0, f2)
363		−1: EX_ST(STORE_INIT(%g0, %o4 + %g3))
364		− EX_LD(LOAD_BLK(%o4, %f16))
	369	+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
	370	+ EX_LD_FP(LOAD_BLK(%o4, %f16))
365	371	FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28)
366		− EX_ST(STORE_BLK(%f0, %o4 + %g3))
	372	+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
367	373	FREG_MOVE_2(f28, f30)
368	374	subcc %g1, 64, %g1
369	375	add %o4, 64, %o4
@@ -374,10 +380,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 4 unmodified lines
374	380
375	381	180: sub %o4, 8, %g2
376	382	FREG_LOAD_1(%g2, f0)
377		−1: EX_ST(STORE_INIT(%g0, %o4 + %g3))
378		− EX_LD(LOAD_BLK(%o4, %f16))
	383	+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
	384	+ EX_LD_FP(LOAD_BLK(%o4, %f16))
379	385	FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30)
380		− EX_ST(STORE_BLK(%f0, %o4 + %g3))
	386	+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
381	387	FREG_MOVE_1(f30)
382	388	subcc %g1, 64, %g1
383	389	add %o4, 64, %o4
@@ -387,10 +393,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 3 unmodified lines
387	393	nop
388	394
389	395	190:
390		−1: EX_ST(STORE_INIT(%g0, %o4 + %g3))
	396	+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
391	397	subcc %g1, 64, %g1
392		− EX_LD(LOAD_BLK(%o4, %f0))
393		− EX_ST(STORE_BLK(%f0, %o4 + %g3))
	398	+ EX_LD_FP(LOAD_BLK(%o4, %f0))
	399	+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
394	400	add %o4, 64, %o4
395	401	bne,pt %xcc, 1b
396	402	LOAD(prefetch, %o4 + 64, #one_read)

arch/sparc/lib/NG4copy_from_user.S+8−0

@@ -11,6 +11,14 @@ · 10 unmodified lines
11	11	.text; \
12	12	.align 4;
13	13
	14	+#define EX_LD_FP(x) \
	15	+98: x; \
	16	+ .section __ex_table,"a";\
	17	+ .align 4; \
	18	+ .word 98b, __retl_one_asi_fp;\
	19	+ .text; \
	20	+ .align 4;
	21	+
14	22	#ifndef ASI_AIUS
15	23	#define ASI_AIUS 0x11
16	24	#endif

arch/sparc/lib/NG4copy_to_user.S+8−0

@@ -11,6 +11,14 @@ · 10 unmodified lines
11	11	.text; \
12	12	.align 4;
13	13
	14	+#define EX_ST_FP(x) \
	15	+98: x; \
	16	+ .section __ex_table,"a";\
	17	+ .align 4; \
	18	+ .word 98b, __retl_one_asi_fp;\
	19	+ .text; \
	20	+ .align 4;
	21	+
14	22	#ifndef ASI_AIUS
15	23	#define ASI_AIUS 0x11
16	24	#endif

arch/sparc/lib/NG4memcpy.S+23−17

@@ -48,10 +48,16 @@ · 47 unmodified lines
48	48	#ifndef EX_LD
49	49	#define EX_LD(x) x
50	50	#endif
	51	+#ifndef EX_LD_FP
	52	+#define EX_LD_FP(x) x
	53	+#endif
51	54
52	55	#ifndef EX_ST
53	56	#define EX_ST(x) x
54	57	#endif
	58	+#ifndef EX_ST_FP
	59	+#define EX_ST_FP(x) x
	60	+#endif
55	61
56	62	#ifndef EX_RETVAL
57	63	#define EX_RETVAL(x) x
@@ -210,17 +216,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 152 unmodified lines
210	216	sub %o2, %o4, %o2
211	217	alignaddr %o1, %g0, %g1
212	218	add %o1, %o4, %o1
213		− EX_LD(LOAD(ldd, %g1 + 0x00, %f0))
214		−1: EX_LD(LOAD(ldd, %g1 + 0x08, %f2))
	219	+ EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0))
	220	+1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2))
215	221	subcc %o4, 0x40, %o4
216		− EX_LD(LOAD(ldd, %g1 + 0x10, %f4))
217		− EX_LD(LOAD(ldd, %g1 + 0x18, %f6))
218		− EX_LD(LOAD(ldd, %g1 + 0x20, %f8))
219		− EX_LD(LOAD(ldd, %g1 + 0x28, %f10))
220		− EX_LD(LOAD(ldd, %g1 + 0x30, %f12))
221		− EX_LD(LOAD(ldd, %g1 + 0x38, %f14))
	222	+ EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4))
	223	+ EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6))
	224	+ EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8))
	225	+ EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10))
	226	+ EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12))
	227	+ EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14))
222	228	faligndata %f0, %f2, %f16
223		− EX_LD(LOAD(ldd, %g1 + 0x40, %f0))
	229	+ EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0))
224	230	faligndata %f2, %f4, %f18
225	231	add %g1, 0x40, %g1
226	232	faligndata %f4, %f6, %f20
@@ -229,14 +235,14 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 2 unmodified lines
229	235	faligndata %f10, %f12, %f26
230	236	faligndata %f12, %f14, %f28
231	237	faligndata %f14, %f0, %f30
232		− EX_ST(STORE(std, %f16, %o0 + 0x00))
233		− EX_ST(STORE(std, %f18, %o0 + 0x08))
234		− EX_ST(STORE(std, %f20, %o0 + 0x10))
235		− EX_ST(STORE(std, %f22, %o0 + 0x18))
236		− EX_ST(STORE(std, %f24, %o0 + 0x20))
237		− EX_ST(STORE(std, %f26, %o0 + 0x28))
238		− EX_ST(STORE(std, %f28, %o0 + 0x30))
239		− EX_ST(STORE(std, %f30, %o0 + 0x38))
	238	+ EX_ST_FP(STORE(std, %f16, %o0 + 0x00))
	239	+ EX_ST_FP(STORE(std, %f18, %o0 + 0x08))
	240	+ EX_ST_FP(STORE(std, %f20, %o0 + 0x10))
	241	+ EX_ST_FP(STORE(std, %f22, %o0 + 0x18))
	242	+ EX_ST_FP(STORE(std, %f24, %o0 + 0x20))
	243	+ EX_ST_FP(STORE(std, %f26, %o0 + 0x28))
	244	+ EX_ST_FP(STORE(std, %f28, %o0 + 0x30))
	245	+ EX_ST_FP(STORE(std, %f30, %o0 + 0x38))
240	246	add %o0, 0x40, %o0
241	247	bne,pt %icc, 1b
242	248	LOAD(prefetch, %g1 + 0x200, #n_reads_strong)

arch/sparc/lib/U1copy_from_user.S+8−0

@@ -11,6 +11,14 @@ · 10 unmodified lines
11	11	.text; \
12	12	.align 4;
13	13
	14	+#define EX_LD_FP(x) \
	15	+98: x; \
	16	+ .section __ex_table,"a";\
	17	+ .align 4; \
	18	+ .word 98b, __retl_one_fp;\
	19	+ .text; \
	20	+ .align 4;
	21	+
14	22	#define FUNC_NAME ___copy_from_user
15	23	#define LOAD(type,addr,dest) type##a [addr] %asi, dest
16	24	#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_AIUS, dest

arch/sparc/lib/U1copy_to_user.S+8−0

@@ -11,6 +11,14 @@ · 10 unmodified lines
11	11	.text; \
12	12	.align 4;
13	13
	14	+#define EX_ST_FP(x) \
	15	+98: x; \
	16	+ .section __ex_table,"a";\
	17	+ .align 4; \
	18	+ .word 98b, __retl_one_fp;\
	19	+ .text; \
	20	+ .align 4;
	21	+
14	22	#define FUNC_NAME ___copy_to_user
15	23	#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS
16	24	#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS

arch/sparc/lib/U1memcpy.S+27−21

@@ -25,10 +25,16 @@ · 24 unmodified lines
25	25	#ifndef EX_LD
26	26	#define EX_LD(x) x
27	27	#endif
	28	+#ifndef EX_LD_FP
	29	+#define EX_LD_FP(x) x
	30	+#endif
28	31
29	32	#ifndef EX_ST
30	33	#define EX_ST(x) x
31	34	#endif
	35	+#ifndef EX_ST_FP
	36	+#define EX_ST_FP(x) x
	37	+#endif
32	38
33	39	#ifndef EX_RETVAL
34	40	#define EX_RETVAL(x) x
@@ -73,8 +79,8 @@ · 38 unmodified lines
73	79	faligndata %f8, %f9, %f62;
74	80
75	81	#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \
76		− EX_LD(LOAD_BLK(%src, %fdest)); \
77		− EX_ST(STORE_BLK(%fsrc, %dest)); \
	82	+ EX_LD_FP(LOAD_BLK(%src, %fdest)); \
	83	+ EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
78	84	add %src, 0x40, %src; \
79	85	subcc %len, 0x40, %len; \
80	86	be,pn %xcc, jmptgt; \
@@ -89,12 +95,12 @@ · 8 unmodified lines
89	95
90	96	#define DO_SYNC membar #Sync;
91	97	#define STORE_SYNC(dest, fsrc) \
92		− EX_ST(STORE_BLK(%fsrc, %dest)); \
	98	+ EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
93	99	add %dest, 0x40, %dest; \
94	100	DO_SYNC
95	101
96	102	#define STORE_JUMP(dest, fsrc, target) \
97		− EX_ST(STORE_BLK(%fsrc, %dest)); \
	103	+ EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
98	104	add %dest, 0x40, %dest; \
99	105	ba,pt %xcc, target; \
100	106	nop;
@@ -103,7 +109,7 @@ · 2 unmodified lines
103	109	subcc %left, 8, %left;\
104	110	bl,pn %xcc, 95f; \
105	111	faligndata %f0, %f1, %f48; \
106		− EX_ST(STORE(std, %f48, %dest)); \
	112	+ EX_ST_FP(STORE(std, %f48, %dest)); \
107	113	add %dest, 8, %dest;
108	114
109	115	#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
@@ -160,8 +166,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 50 unmodified lines
160	166	and %g2, 0x38, %g2
161	167
162	168	1: subcc %g1, 0x1, %g1
163		− EX_LD(LOAD(ldub, %o1 + 0x00, %o3))
164		− EX_ST(STORE(stb, %o3, %o1 + %GLOBAL_SPARE))
	169	+ EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3))
	170	+ EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE))
165	171	bgu,pt %XCC, 1b
166	172	add %o1, 0x1, %o1
167	173
@@ -172,20 +178,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 4 unmodified lines
172	178	be,pt %icc, 3f
173	179	alignaddr %o1, %g0, %o1
174	180
175		− EX_LD(LOAD(ldd, %o1, %f4))
176		−1: EX_LD(LOAD(ldd, %o1 + 0x8, %f6))
	181	+ EX_LD_FP(LOAD(ldd, %o1, %f4))
	182	+1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6))
177	183	add %o1, 0x8, %o1
178	184	subcc %g2, 0x8, %g2
179	185	faligndata %f4, %f6, %f0
180		− EX_ST(STORE(std, %f0, %o0))
	186	+ EX_ST_FP(STORE(std, %f0, %o0))
181	187	be,pn %icc, 3f
182	188	add %o0, 0x8, %o0
183	189
184		− EX_LD(LOAD(ldd, %o1 + 0x8, %f4))
	190	+ EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4))
185	191	add %o1, 0x8, %o1
186	192	subcc %g2, 0x8, %g2
187	193	faligndata %f6, %f4, %f0
188		− EX_ST(STORE(std, %f0, %o0))
	194	+ EX_ST_FP(STORE(std, %f0, %o0))
189	195	bne,pt %icc, 1b
190	196	add %o0, 0x8, %o0
191	197
@@ -208,13 +214,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 16 unmodified lines
208	214	add %g1, %GLOBAL_SPARE, %g1
209	215	subcc %o2, %g3, %o2
210	216
211		− EX_LD(LOAD_BLK(%o1, %f0))
	217	+ EX_LD_FP(LOAD_BLK(%o1, %f0))
212	218	add %o1, 0x40, %o1
213	219	add %g1, %g3, %g1
214		− EX_LD(LOAD_BLK(%o1, %f16))
	220	+ EX_LD_FP(LOAD_BLK(%o1, %f16))
215	221	add %o1, 0x40, %o1
216	222	sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
217		− EX_LD(LOAD_BLK(%o1, %f32))
	223	+ EX_LD_FP(LOAD_BLK(%o1, %f32))
218	224	add %o1, 0x40, %o1
219	225
220	226	/* There are 8 instances of the unrolled loop,
@@ -426,28 +432,28 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 205 unmodified lines
426	432	62: FINISH_VISCHUNK(o0, f44, f46, g3)
427	433	63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3)
428	434
429		−93: EX_LD(LOAD(ldd, %o1, %f2))
	435	+93: EX_LD_FP(LOAD(ldd, %o1, %f2))
430	436	add %o1, 8, %o1
431	437	subcc %g3, 8, %g3
432	438	faligndata %f0, %f2, %f8
433		− EX_ST(STORE(std, %f8, %o0))
	439	+ EX_ST_FP(STORE(std, %f8, %o0))
434	440	bl,pn %xcc, 95f
435	441	add %o0, 8, %o0
436		− EX_LD(LOAD(ldd, %o1, %f0))
	442	+ EX_LD_FP(LOAD(ldd, %o1, %f0))
437	443	add %o1, 8, %o1
438	444	subcc %g3, 8, %g3
439	445	faligndata %f2, %f0, %f8
440		− EX_ST(STORE(std, %f8, %o0))
	446	+ EX_ST_FP(STORE(std, %f8, %o0))
441	447	bge,pt %xcc, 93b
442	448	add %o0, 8, %o0
443	449
444	450	95: brz,pt %o2, 2f
445	451	mov %g1, %o1
446	452
447		−1: EX_LD(LOAD(ldub, %o1, %o3))
	453	+1: EX_LD_FP(LOAD(ldub, %o1, %o3))
448	454	add %o1, 1, %o1
449	455	subcc %o2, 1, %o2
450		− EX_ST(STORE(stb, %o3, %o0))
	456	+ EX_ST_FP(STORE(stb, %o3, %o0))
451	457	bne,pt %xcc, 1b
452	458	add %o0, 1, %o0
453	459

arch/sparc/lib/U3copy_from_user.S+8−0

@@ -11,6 +11,14 @@ · 10 unmodified lines
11	11	.text; \
12	12	.align 4;
13	13
	14	+#define EX_LD_FP(x) \
	15	+98: x; \
	16	+ .section __ex_table,"a";\
	17	+ .align 4; \
	18	+ .word 98b, __retl_one_fp;\
	19	+ .text; \
	20	+ .align 4;
	21	+
14	22	#define FUNC_NAME U3copy_from_user
15	23	#define LOAD(type,addr,dest) type##a [addr] %asi, dest
16	24	#define EX_RETVAL(x) 0

arch/sparc/lib/U3copy_to_user.S+8−0

@@ -11,6 +11,14 @@ · 10 unmodified lines
11	11	.text; \
12	12	.align 4;
13	13
	14	+#define EX_ST_FP(x) \
	15	+98: x; \
	16	+ .section __ex_table,"a";\
	17	+ .align 4; \
	18	+ .word 98b, __retl_one_fp;\
	19	+ .text; \
	20	+ .align 4;
	21	+
14	22	#define FUNC_NAME U3copy_to_user
15	23	#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS
16	24	#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS

arch/sparc/lib/U3memcpy.S+46−40

@@ -24,10 +24,16 @@ · 23 unmodified lines
24	24	#ifndef EX_LD
25	25	#define EX_LD(x) x
26	26	#endif
	27	+#ifndef EX_LD_FP
	28	+#define EX_LD_FP(x) x
	29	+#endif
27	30
28	31	#ifndef EX_ST
29	32	#define EX_ST(x) x
30	33	#endif
	34	+#ifndef EX_ST_FP
	35	+#define EX_ST_FP(x) x
	36	+#endif
31	37
32	38	#ifndef EX_RETVAL
33	39	#define EX_RETVAL(x) x
@@ -120,8 +126,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 86 unmodified lines
120	126	and %g2, 0x38, %g2
121	127
122	128	1: subcc %g1, 0x1, %g1
123		− EX_LD(LOAD(ldub, %o1 + 0x00, %o3))
124		− EX_ST(STORE(stb, %o3, %o1 + GLOBAL_SPARE))
	129	+ EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3))
	130	+ EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE))
125	131	bgu,pt %XCC, 1b
126	132	add %o1, 0x1, %o1
127	133
@@ -132,20 +138,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 4 unmodified lines
132	138	be,pt %icc, 3f
133	139	alignaddr %o1, %g0, %o1
134	140
135		− EX_LD(LOAD(ldd, %o1, %f4))
136		−1: EX_LD(LOAD(ldd, %o1 + 0x8, %f6))
	141	+ EX_LD_FP(LOAD(ldd, %o1, %f4))
	142	+1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6))
137	143	add %o1, 0x8, %o1
138	144	subcc %g2, 0x8, %g2
139	145	faligndata %f4, %f6, %f0
140		− EX_ST(STORE(std, %f0, %o0))
	146	+ EX_ST_FP(STORE(std, %f0, %o0))
141	147	be,pn %icc, 3f
142	148	add %o0, 0x8, %o0
143	149
144		− EX_LD(LOAD(ldd, %o1 + 0x8, %f4))
	150	+ EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4))
145	151	add %o1, 0x8, %o1
146	152	subcc %g2, 0x8, %g2
147	153	faligndata %f6, %f4, %f2
148		− EX_ST(STORE(std, %f2, %o0))
	154	+ EX_ST_FP(STORE(std, %f2, %o0))
149	155	bne,pt %icc, 1b
150	156	add %o0, 0x8, %o0
151	157
@@ -155,25 +161,25 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 3 unmodified lines
155	161	LOAD(prefetch, %o1 + 0x080, #one_read)
156	162	LOAD(prefetch, %o1 + 0x0c0, #one_read)
157	163	LOAD(prefetch, %o1 + 0x100, #one_read)
158		− EX_LD(LOAD(ldd, %o1 + 0x000, %f0))
	164	+ EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0))
159	165	LOAD(prefetch, %o1 + 0x140, #one_read)
160		− EX_LD(LOAD(ldd, %o1 + 0x008, %f2))
	166	+ EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
161	167	LOAD(prefetch, %o1 + 0x180, #one_read)
162		− EX_LD(LOAD(ldd, %o1 + 0x010, %f4))
	168	+ EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
163	169	LOAD(prefetch, %o1 + 0x1c0, #one_read)
164	170	faligndata %f0, %f2, %f16
165		− EX_LD(LOAD(ldd, %o1 + 0x018, %f6))
	171	+ EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
166	172	faligndata %f2, %f4, %f18
167		− EX_LD(LOAD(ldd, %o1 + 0x020, %f8))
	173	+ EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
168	174	faligndata %f4, %f6, %f20
169		− EX_LD(LOAD(ldd, %o1 + 0x028, %f10))
	175	+ EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
170	176	faligndata %f6, %f8, %f22
171	177
172		− EX_LD(LOAD(ldd, %o1 + 0x030, %f12))
	178	+ EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
173	179	faligndata %f8, %f10, %f24
174		− EX_LD(LOAD(ldd, %o1 + 0x038, %f14))
	180	+ EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
175	181	faligndata %f10, %f12, %f26
176		− EX_LD(LOAD(ldd, %o1 + 0x040, %f0))
	182	+ EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
177	183
178	184	subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE
179	185	add %o1, 0x40, %o1
@@ -184,26 +190,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 4 unmodified lines
184	190
185	191	.align 64
186	192	1:
187		− EX_LD(LOAD(ldd, %o1 + 0x008, %f2))
	193	+ EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
188	194	faligndata %f12, %f14, %f28
189		− EX_LD(LOAD(ldd, %o1 + 0x010, %f4))
	195	+ EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
190	196	faligndata %f14, %f0, %f30
191		− EX_ST(STORE_BLK(%f16, %o0))
192		− EX_LD(LOAD(ldd, %o1 + 0x018, %f6))
	197	+ EX_ST_FP(STORE_BLK(%f16, %o0))
	198	+ EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
193	199	faligndata %f0, %f2, %f16
194	200	add %o0, 0x40, %o0
195	201
196		− EX_LD(LOAD(ldd, %o1 + 0x020, %f8))
	202	+ EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
197	203	faligndata %f2, %f4, %f18
198		− EX_LD(LOAD(ldd, %o1 + 0x028, %f10))
	204	+ EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
199	205	faligndata %f4, %f6, %f20
200		− EX_LD(LOAD(ldd, %o1 + 0x030, %f12))
	206	+ EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
201	207	subcc %o3, 0x01, %o3
202	208	faligndata %f6, %f8, %f22
203		− EX_LD(LOAD(ldd, %o1 + 0x038, %f14))
	209	+ EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
204	210
205	211	faligndata %f8, %f10, %f24
206		− EX_LD(LOAD(ldd, %o1 + 0x040, %f0))
	212	+ EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
207	213	LOAD(prefetch, %o1 + 0x1c0, #one_read)
208	214	faligndata %f10, %f12, %f26
209	215	bg,pt %XCC, 1b
@@ -211,29 +217,29 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 1 unmodified line
211	217
212	218	/* Finally we copy the last full 64-byte block. */
213	219	2:
214		− EX_LD(LOAD(ldd, %o1 + 0x008, %f2))
	220	+ EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
215	221	faligndata %f12, %f14, %f28
216		− EX_LD(LOAD(ldd, %o1 + 0x010, %f4))
	222	+ EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
217	223	faligndata %f14, %f0, %f30
218		− EX_ST(STORE_BLK(%f16, %o0))
219		− EX_LD(LOAD(ldd, %o1 + 0x018, %f6))
	224	+ EX_ST_FP(STORE_BLK(%f16, %o0))
	225	+ EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
220	226	faligndata %f0, %f2, %f16
221		− EX_LD(LOAD(ldd, %o1 + 0x020, %f8))
	227	+ EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
222	228	faligndata %f2, %f4, %f18
223		− EX_LD(LOAD(ldd, %o1 + 0x028, %f10))
	229	+ EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
224	230	faligndata %f4, %f6, %f20
225		− EX_LD(LOAD(ldd, %o1 + 0x030, %f12))
	231	+ EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
226	232	faligndata %f6, %f8, %f22
227		− EX_LD(LOAD(ldd, %o1 + 0x038, %f14))
	233	+ EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
228	234	faligndata %f8, %f10, %f24
229	235	cmp %g1, 0
230	236	be,pt %XCC, 1f
231	237	add %o0, 0x40, %o0
232		− EX_LD(LOAD(ldd, %o1 + 0x040, %f0))
	238	+ EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
233	239	1: faligndata %f10, %f12, %f26
234	240	faligndata %f12, %f14, %f28
235	241	faligndata %f14, %f0, %f30
236		− EX_ST(STORE_BLK(%f16, %o0))
	242	+ EX_ST_FP(STORE_BLK(%f16, %o0))
237	243	add %o0, 0x40, %o0
238	244	add %o1, 0x40, %o1
239	245	membar #Sync
@@ -253,20 +259,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ · 13 unmodified lines
253	259
254	260	sub %o2, %g2, %o2
255	261	be,a,pt %XCC, 1f
256		− EX_LD(LOAD(ldd, %o1 + 0x00, %f0))
	262	+ EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0))
257	263
258		−1: EX_LD(LOAD(ldd, %o1 + 0x08, %f2))
	264	+1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2))
259	265	add %o1, 0x8, %o1
260	266	subcc %g2, 0x8, %g2
261	267	faligndata %f0, %f2, %f8
262		− EX_ST(STORE(std, %f8, %o0))
	268	+ EX_ST_FP(STORE(std, %f8, %o0))
263	269	be,pn %XCC, 2f
264	270	add %o0, 0x8, %o0
265		− EX_LD(LOAD(ldd, %o1 + 0x08, %f0))
	271	+ EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0))
266	272	add %o1, 0x8, %o1
267	273	subcc %g2, 0x8, %g2
268	274	faligndata %f2, %f0, %f8
269		− EX_ST(STORE(std, %f8, %o0))
	275	+ EX_ST_FP(STORE(std, %f8, %o0))
270	276	bne,pn %XCC, 1b
271	277	add %o0, 0x8, %o0
272	278