tests: add test for x86_64 xor REX prefix bug in load()

When load() in x86_64-gen.c generates a zero constant for a 64-bit register, it uses: o(0xc031 + REG_VALUE(r) * 0x900); REG_VALUE(r) masks to (r & 7), losing bit 3, and no orex() call emits the REX prefix needed for registers r8-r15. For TREG_R11 (used by gcall_or_jmp for indirect calls), this emits "xor %ebx,%ebx" (31 db) instead of "xor %r11d,%r11d" (45 31 db), clobbering the wrong register. The test compiles an indirect call through a null function pointer ((void(*)(void))0)() via libtcc, then inspects the generated machine code for the incorrect encoding.

Cyan Ogilvie committed 7 months ago

Commit d5ecb521 parent 6547ea42 files · +135 −0Browse files

Unified · Split

2 files

tests/Makefile +7 −0
tests/libtcc_test_xor_rex.c +128 −0

tests/Makefile+7−0

@@ -13,6 +13,7 @@ TESTS = \ · 12 unmodified lines
13	13	hello-run \
14	14	libtest \
15	15	libtest_mt \
	16	+ libtest_xor_rex \
16	17	test3 \
17	18	abitest \
18	19	asm-c-connect-test \
@@ -40,6 +41,9 @@ endif · 21 unmodified lines
40	41	ifeq (,$(filter i386 x86_64,$(ARCH)))
41	42	TESTS := $(filter-out asm-c-connect-test,$(TESTS))
42	43	endif
	44	+ifeq (,$(filter x86_64,$(ARCH)))
	45	+ TESTS := $(filter-out libtest_xor_rex,$(TESTS))
	46	+endif
43	47	ifeq ($(OS),Windows_NT) # for libtcc_test to find libtcc.dll
44	48	PATH := $(CURDIR)/$(TOP)$(if $(findstring ;,$(PATH)),;,:)$(PATH)
45	49	endif
@@ -103,6 +107,9 @@ libtcc_test$(EXESUF): libtcc_test.c · 57 unmodified lines
103	107	libtcc_test_mt$(EXESUF): libtcc_test_mt.c
104	108	$(CC) -o $@ $< $(CFLAGS) $(-LTCC) $(LIBS)
105	109
	110	+libtcc_test_xor_rex$(EXESUF): libtcc_test_xor_rex.c
	111	+ $(CC) -o $@ $< $(CFLAGS) $(-LTCC) $(LIBS)
	112	+
106	113	%-dir:
107	114	@echo ------------ $@ ------------
108	115	$(MAKE) -k -C $*

tests/libtcc_test_xor_rex.c+128−0

@@ -0,0 +1,128 @@
	1	+/*
	2	+ * Test for x86_64 xor REX prefix bug in load() -- x86_64-gen.c
	3	+ *
	4	+ * Bug: when loading a 64-bit zero constant into registers r8-r15,
	5	+ * load() emits:
	6	+ *
	7	+ * o(0xc031 + REG_VALUE(r) * 0x900); // xor r, r
	8	+ *
	9	+ * REG_VALUE(r) masks to (r & 7), losing bit 3, and no orex() call
	10	+ * emits the REX prefix needed for extended registers.
	11	+ *
	12	+ * Result: r=TREG_R11(11) -> REG_VALUE=3 -> emits 31 db (xor ebx,ebx)
	13	+ * Correct: should emit 45 31 db (REX.RB xor r11d,r11d)
	14	+ *
	15	+ * Fix:
	16	+ * orex(0, r, r, 0x31);
	17	+ * o(0xc0 + REG_VALUE(r) * 9);
	18	+ *
	19	+ * Trigger: an indirect call through a compile-time null function pointer,
	20	+ * e.g. ((void(*)(void))0)(), causes gcall_or_jmp() to fall into the
	21	+ * "indirect call" path which does load(TREG_R11, <constant 0>).
	22	+ *
	23	+ * This test compiles such code via the libtcc API, then inspects the
	24	+ * generated machine code for the incorrect encoding.
	25	+ */
	26	+
	27	+#if !defined __x86_64__
	28	+#include <stdio.h>
	29	+int main(void) { printf("SKIP (x86_64 only)\n"); return 0; }
	30	+#else
	31	+
	32	+#include <stdio.h>
	33	+#include "libtcc.h"
	34	+
	35	+static void handle_error(void opaque, const char msg)
	36	+{
	37	+ fprintf(opaque, "%s\n", msg);
	38	+}
	39	+
	40	+/*
	41	+ * Compiled via libtcc. The cast-to-null indirect call forces
	42	+ * gcall_or_jmp() into its "else" branch (no VT_SYM, so the
	43	+ * condition on line ~650 fails), which does:
	44	+ *
	45	+ * r = TREG_R11;
	46	+ * load(r, vtop); // <-- buggy xor lands here
	47	+ * o(0x41); o(0xff); // call/jmp *r
	48	+ * o(0xd0 + REG_VALUE(r)); // r11 -> 0xd3
	49	+ *
	50	+ * We never execute the function (it would crash); we only
	51	+ * inspect the generated bytes.
	52	+ */
	53	+static const char test_code[] =
	54	+ "void test(void) {\n"
	55	+ " ((void(*)(void))0)();\n"
	56	+ "}\n";
	57	+
	58	+int main(int argc, char **argv)
	59	+{
	60	+ TCCState *s;
	61	+ unsigned char *code;
	62	+ int i;
	63	+ int ret = 0;
	64	+
	65	+ s = tcc_new();
	66	+ if (!s) {
	67	+ fprintf(stderr, "tcc_new() failed\n");
	68	+ return 2;
	69	+ }
	70	+ tcc_set_error_func(s, stderr, handle_error);
	71	+
	72	+ for (i = 1; i < argc; ++i) {
	73	+ char *a = argv[i];
	74	+ if (a[0] == '-') {
	75	+ if (a[1] == 'B')
	76	+ tcc_set_lib_path(s, a + 2);
	77	+ else if (a[1] == 'I')
	78	+ tcc_add_include_path(s, a + 2);
	79	+ else if (a[1] == 'L')
	80	+ tcc_add_library_path(s, a + 2);
	81	+ }
	82	+ }
	83	+
	84	+ tcc_set_output_type(s, TCC_OUTPUT_MEMORY);
	85	+ if (tcc_compile_string(s, test_code) == -1)
	86	+ return 2;
	87	+ if (tcc_relocate(s) < 0)
	88	+ return 2;
	89	+
	90	+ code = (unsigned char *)tcc_get_symbol(s, "test");
	91	+ if (!code) {
	92	+ fprintf(stderr, "symbol 'test' not found\n");
	93	+ return 2;
	94	+ }
	95	+
	96	+ /*
	97	+ * Scan for the 'call *%r11' instruction: 41 ff d3
	98	+ * Then inspect the bytes immediately before it.
	99	+ *
	100	+ * Correct: 45 31 db 41 ff d3 (xor %r11d,%r11d ; call *%r11)
	101	+ * Buggy: 31 db 41 ff d3 (xor %ebx,%ebx ; call *%r11)
	102	+ */
	103	+ for (i = 3; i < 128; i++) {
	104	+ if (code[i] == 0x41 && code[i+1] == 0xff && code[i+2] == 0xd3) {
	105	+ if (i >= 3 && code[i-3] == 0x45
	106	+ && code[i-2] == 0x31
	107	+ && code[i-1] == 0xdb) {
	108	+ printf("xor_rex: OK\n");
	109	+ } else if (i >= 2 && code[i-2] == 0x31 && code[i-1] == 0xdb) {
	110	+ printf("xor_rex: FAIL - xor %%ebx,%%ebx (31 db) emitted"
	111	+ " instead of xor %%r11d,%%r11d (45 31 db)\n");
	112	+ ret = 1;
	113	+ } else {
	114	+ printf("xor_rex: FAIL - unexpected bytes before"
	115	+ " call *%%r11: %02x %02x %02x %02x\n",
	116	+ code[i-4], code[i-3], code[i-2], code[i-1]);
	117	+ ret = 1;
	118	+ }
	119	+ goto done;
	120	+ }
	121	+ }
	122	+ printf("xor_rex: SKIP - call *%%r11 not found in generated code\n");
	123	+
	124	+done:
	125	+ tcc_delete(s);
	126	+ return ret;
	127	+}
	128	+#endif

tests: add test for x86_64 xor REX prefix bug in load()

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code