/* * Copyright 2013 Tilera Corporation. All Rights Reserved. * *   This program is free software; you can redistribute it and/or *   modify it under the terms of the GNU General Public License *   as published by the Free Software Foundation, version 2. * *   This program is distributed in the hope that it will be useful, but *   WITHOUT ANY WARRANTY; without even the implied warranty of *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or *   NON INFRINGEMENT.  See the GNU General Public License for *   more details. * * A code-rewriter that handles unaligned exception. */ #include <linux/smp.h>#include <linux/ptrace.h>#include <linux/slab.h>#include <linux/sched/debug.h>#include <linux/sched/task.h>#include <linux/thread_info.h>#include <linux/uaccess.h>#include <linux/mman.h>#include <linux/types.h>#include <linux/err.h>#include <linux/extable.h>#include <linux/compat.h>#include <linux/prctl.h>#include <asm/cacheflush.h>#include <asm/traps.h>#include <linux/uaccess.h>#include <asm/unaligned.h>#include <arch/abi.h>#include <arch/spr_def.h>#include <arch/opcode.h>  /* * This file handles unaligned exception for tile-Gx. The tilepro's unaligned * exception is supported out of single_step.c */ int unaligned_printk; static int __init setup_unaligned_printk(char *str){	long val;	if (kstrtol(str, 0, &val) != 0)		return 0;	unaligned_printk = val;	pr_info("Printk for each unaligned data accesses is %s\n",		unaligned_printk ? "enabled" : "disabled");	return 1;}__setup("unaligned_printk=", setup_unaligned_printk); unsigned int unaligned_fixup_count; #ifdef __tilegx__ /* * Unalign data jit fixup code fragement. Reserved space is 128 bytes. * The 1st 64-bit word saves fault PC address, 2nd word is the fault * instruction bundle followed by 14 JIT bundles. */ struct unaligned_jit_fragment {	unsigned long       pc;	tilegx_bundle_bits  bundle;	tilegx_bundle_bits  insn[14];}; /* * Check if a nop or fnop at bundle's pipeline X0. */ static bool is_bundle_x0_nop(tilegx_bundle_bits bundle){	return (((get_UnaryOpcodeExtension_X0(bundle) ==		  NOP_UNARY_OPCODE_X0) &&		 (get_RRROpcodeExtension_X0(bundle) ==		  UNARY_RRR_0_OPCODE_X0) &&		 (get_Opcode_X0(bundle) ==		  RRR_0_OPCODE_X0)) ||		((get_UnaryOpcodeExtension_X0(bundle) ==		  FNOP_UNARY_OPCODE_X0) &&		 (get_RRROpcodeExtension_X0(bundle) ==		  UNARY_RRR_0_OPCODE_X0) &&		 (get_Opcode_X0(bundle) ==		  RRR_0_OPCODE_X0)));} /* * Check if nop or fnop at bundle's pipeline X1. */ static bool is_bundle_x1_nop(tilegx_bundle_bits bundle){	return (((get_UnaryOpcodeExtension_X1(bundle) ==		  NOP_UNARY_OPCODE_X1) &&		 (get_RRROpcodeExtension_X1(bundle) ==		  UNARY_RRR_0_OPCODE_X1) &&		 (get_Opcode_X1(bundle) ==		  RRR_0_OPCODE_X1)) ||		((get_UnaryOpcodeExtension_X1(bundle) ==		  FNOP_UNARY_OPCODE_X1) &&		 (get_RRROpcodeExtension_X1(bundle) ==		  UNARY_RRR_0_OPCODE_X1) &&		 (get_Opcode_X1(bundle) ==		  RRR_0_OPCODE_X1)));} /* * Check if nop or fnop at bundle's Y0 pipeline. */ static bool is_bundle_y0_nop(tilegx_bundle_bits bundle){	return (((get_UnaryOpcodeExtension_Y0(bundle) ==		  NOP_UNARY_OPCODE_Y0) &&		 (get_RRROpcodeExtension_Y0(bundle) ==		  UNARY_RRR_1_OPCODE_Y0) &&		 (get_Opcode_Y0(bundle) ==		  RRR_1_OPCODE_Y0)) ||		((get_UnaryOpcodeExtension_Y0(bundle) ==		  FNOP_UNARY_OPCODE_Y0) &&		 (get_RRROpcodeExtension_Y0(bundle) ==		  UNARY_RRR_1_OPCODE_Y0) &&		 (get_Opcode_Y0(bundle) ==		  RRR_1_OPCODE_Y0)));} /* * Check if nop or fnop at bundle's pipeline Y1. */ static bool is_bundle_y1_nop(tilegx_bundle_bits bundle){	return (((get_UnaryOpcodeExtension_Y1(bundle) ==		  NOP_UNARY_OPCODE_Y1) &&		 (get_RRROpcodeExtension_Y1(bundle) ==		  UNARY_RRR_1_OPCODE_Y1) &&		 (get_Opcode_Y1(bundle) ==		  RRR_1_OPCODE_Y1)) ||		((get_UnaryOpcodeExtension_Y1(bundle) ==		  FNOP_UNARY_OPCODE_Y1) &&		 (get_RRROpcodeExtension_Y1(bundle) ==		  UNARY_RRR_1_OPCODE_Y1) &&		 (get_Opcode_Y1(bundle) ==		  RRR_1_OPCODE_Y1)));} /* * Test if a bundle's y0 and y1 pipelines are both nop or fnop. */ static bool is_y0_y1_nop(tilegx_bundle_bits bundle){	return is_bundle_y0_nop(bundle) && is_bundle_y1_nop(bundle);} /* * Test if a bundle's x0 and x1 pipelines are both nop or fnop. */ static bool is_x0_x1_nop(tilegx_bundle_bits bundle){	return is_bundle_x0_nop(bundle) && is_bundle_x1_nop(bundle);} /* * Find the destination, source registers of fault unalign access instruction * at X1 or Y2. Also, allocate up to 3 scratch registers clob1, clob2 and * clob3, which are guaranteed different from any register used in the fault * bundle. r_alias is used to return if the other instructions other than the * unalign load/store shares same register with ra, rb and rd. */ static void find_regs(tilegx_bundle_bits bundle, uint64_t *rd, uint64_t *ra,		      uint64_t *rb, uint64_t *clob1, uint64_t *clob2,		      uint64_t *clob3, bool *r_alias){	int i;	uint64_t reg;	uint64_t reg_map = 0, alias_reg_map = 0, map;	bool alias = false; 	/*	 * Parse fault bundle, find potential used registers and mark	 * corresponding bits in reg_map and alias_map. These 2 bit maps	 * are used to find the scratch registers and determine if there	 * is register alias.	 */	if (bundle & TILEGX_BUNDLE_MODE_MASK) {  /* Y Mode Bundle. */ 		reg = get_SrcA_Y2(bundle);		reg_map |= 1ULL << reg;		*ra = reg;		reg = get_SrcBDest_Y2(bundle);		reg_map |= 1ULL << reg; 		if (rd) {			/* Load. */			*rd = reg;			alias_reg_map = (1ULL << *rd) | (1ULL << *ra);		} else {			/* Store. */			*rb = reg;			alias_reg_map = (1ULL << *ra) | (1ULL << *rb);		} 		if (!is_bundle_y1_nop(bundle)) {			reg = get_SrcA_Y1(bundle);			reg_map |= (1ULL << reg);			map = (1ULL << reg); 			reg = get_SrcB_Y1(bundle);			reg_map |= (1ULL << reg);			map |= (1ULL << reg); 			reg = get_Dest_Y1(bundle);			reg_map |= (1ULL << reg);			map |= (1ULL << reg); 			if (map & alias_reg_map)				alias = true;		} 		if (!is_bundle_y0_nop(bundle)) {			reg = get_SrcA_Y0(bundle);			reg_map |= (1ULL << reg);			map = (1ULL << reg); 			reg = get_SrcB_Y0(bundle);			reg_map |= (1ULL << reg);			map |= (1ULL << reg); 			reg = get_Dest_Y0(bundle);			reg_map |= (1ULL << reg);			map |= (1ULL << reg); 			if (map & alias_reg_map)				alias = true;		}	} else	{ /* X Mode Bundle. */ 		reg = get_SrcA_X1(bundle);		reg_map |= (1ULL << reg);		*ra = reg;		if (rd)	{			/* Load. */			reg = get_Dest_X1(bundle);			reg_map |= (1ULL << reg);			*rd = reg;			alias_reg_map = (1ULL << *rd) | (1ULL << *ra);		} else {			/* Store. */			reg = get_SrcB_X1(bundle);			reg_map |= (1ULL << reg);			*rb = reg;			alias_reg_map = (1ULL << *ra) | (1ULL << *rb);		} 		if (!is_bundle_x0_nop(bundle)) {			reg = get_SrcA_X0(bundle);			reg_map |= (1ULL << reg);			map = (1ULL << reg); 			reg = get_SrcB_X0(bundle);			reg_map |= (1ULL << reg);			map |= (1ULL << reg); 			reg = get_Dest_X0(bundle);			reg_map |= (1ULL << reg);			map |= (1ULL << reg); 			if (map & alias_reg_map)				alias = true;		}	} 	/*	 * "alias" indicates if the unalign access registers have collision	 * with others in the same bundle. We jsut simply test all register	 * operands case (RRR), ignored the case with immidate. If a bundle	 * has no register alias, we may do fixup in a simple or fast manner.	 * So if an immidata field happens to hit with a register, we may end	 * up fall back to the generic handling.	 */ 	*r_alias = alias; 	/* Flip bits on reg_map. */	reg_map ^= -1ULL; 	/* Scan reg_map lower 54(TREG_SP) bits to find 3 set bits. */	for (i = 0; i < TREG_SP; i++) {		if (reg_map & (0x1ULL << i)) {			if (*clob1 == -1) {				*clob1 = i;			} else if (*clob2 == -1) {				*clob2 = i;			} else if (*clob3 == -1) {				*clob3 = i;				return;			}		}	}} /* * Sanity check for register ra, rb, rd, clob1/2/3. Return true if any of them * is unexpected. */ static bool check_regs(uint64_t rd, uint64_t ra, uint64_t rb,		       uint64_t clob1, uint64_t clob2,  uint64_t clob3){	bool unexpected = false;	if ((ra >= 56) && (ra != TREG_ZERO))		unexpected = true; 	if ((clob1 >= 56) || (clob2 >= 56) || (clob3 >= 56))		unexpected = true; 	if (rd != -1) {		if ((rd >= 56) && (rd != TREG_ZERO))			unexpected = true;	} else {		if ((rb >= 56) && (rb != TREG_ZERO))			unexpected = true;	}	return unexpected;}  #define  GX_INSN_X0_MASK   ((1ULL << 31) - 1)#define  GX_INSN_X1_MASK   (((1ULL << 31) - 1) << 31)#define  GX_INSN_Y0_MASK   ((0xFULL << 27) | (0xFFFFFULL))#define  GX_INSN_Y1_MASK   (GX_INSN_Y0_MASK << 31)#define  GX_INSN_Y2_MASK   ((0x7FULL << 51) | (0x7FULL << 20)) #ifdef __LITTLE_ENDIAN#define  GX_INSN_BSWAP(_bundle_)    (_bundle_)#else#define  GX_INSN_BSWAP(_bundle_)    swab64(_bundle_)#endif /* __LITTLE_ENDIAN */ /* * __JIT_CODE(.) creates template bundles in .rodata.unalign_data section. * The corresponding static function jix_x#_###(.) generates partial or * whole bundle based on the template and given arguments. */ #define __JIT_CODE(_X_)						\	asm (".pushsection .rodata.unalign_data, \"a\"\n"	\	     _X_"\n"						\	     ".popsection\n") __JIT_CODE("__unalign_jit_x1_mtspr:   {mtspr 0,  r0}");static tilegx_bundle_bits jit_x1_mtspr(int spr, int reg){	extern  tilegx_bundle_bits __unalign_jit_x1_mtspr;	return (GX_INSN_BSWAP(__unalign_jit_x1_mtspr) & GX_INSN_X1_MASK) |		create_MT_Imm14_X1(spr) | create_SrcA_X1(reg);} __JIT_CODE("__unalign_jit_x1_mfspr:   {mfspr r0, 0}");static tilegx_bundle_bits  jit_x1_mfspr(int reg, int spr){	extern  tilegx_bundle_bits __unalign_jit_x1_mfspr;	return (GX_INSN_BSWAP(__unalign_jit_x1_mfspr) & GX_INSN_X1_MASK) |		create_MF_Imm14_X1(spr) | create_Dest_X1(reg);} __JIT_CODE("__unalign_jit_x0_addi:   {addi  r0, r0, 0; iret}");static tilegx_bundle_bits  jit_x0_addi(int rd, int ra, int imm8){	extern  tilegx_bundle_bits __unalign_jit_x0_addi;	return (GX_INSN_BSWAP(__unalign_jit_x0_addi) & GX_INSN_X0_MASK) |		create_Dest_X0(rd) | create_SrcA_X0(ra) |		create_Imm8_X0(imm8);} __JIT_CODE("__unalign_jit_x1_ldna:   {ldna  r0, r0}");static tilegx_bundle_bits  jit_x1_ldna(int rd, int ra){	extern  tilegx_bundle_bits __unalign_jit_x1_ldna;	return (GX_INSN_BSWAP(__unalign_jit_x1_ldna) &  GX_INSN_X1_MASK) |		create_Dest_X1(rd) | create_SrcA_X1(ra);} __JIT_CODE("__unalign_jit_x0_dblalign:   {dblalign r0, r0 ,r0}");static tilegx_bundle_bits  jit_x0_dblalign(int rd, int ra, int rb){	extern  tilegx_bundle_bits __unalign_jit_x0_dblalign;	return (GX_INSN_BSWAP(__unalign_jit_x0_dblalign) & GX_INSN_X0_MASK) |		create_Dest_X0(rd) | create_SrcA_X0(ra) |		create_SrcB_X0(rb);} __JIT_CODE("__unalign_jit_x1_iret:   {iret}");static tilegx_bundle_bits  jit_x1_iret(void){	extern  tilegx_bundle_bits __unalign_jit_x1_iret;	return GX_INSN_BSWAP(__unalign_jit_x1_iret) & GX_INSN_X1_MASK;} __JIT_CODE("__unalign_jit_x01_fnop:   {fnop;fnop}");static tilegx_bundle_bits  jit_x0_fnop(void){	extern  tilegx_bundle_bits __unalign_jit_x01_fnop;	return GX_INSN_BSWAP(__unalign_jit_x01_fnop) & GX_INSN_X0_MASK;} static tilegx_bundle_bits  jit_x1_fnop(void){	extern  tilegx_bundle_bits __unalign_jit_x01_fnop;	return GX_INSN_BSWAP(__unalign_jit_x01_fnop) & GX_INSN_X1_MASK;} __JIT_CODE("__unalign_jit_y2_dummy:   {fnop; fnop; ld zero, sp}");static tilegx_bundle_bits  jit_y2_dummy(void){	extern  tilegx_bundle_bits __unalign_jit_y2_dummy;	return GX_INSN_BSWAP(__unalign_jit_y2_dummy) & GX_INSN_Y2_MASK;} static tilegx_bundle_bits  jit_y1_fnop(void){	extern  tilegx_bundle_bits __unalign_jit_y2_dummy;	return GX_INSN_BSWAP(__unalign_jit_y2_dummy) & GX_INSN_Y1_MASK;} __JIT_CODE("__unalign_jit_x1_st1_add:  {st1_add r1, r0, 0}");static tilegx_bundle_bits  jit_x1_st1_add(int ra, int rb, int imm8){	extern  tilegx_bundle_bits __unalign_jit_x1_st1_add;	return (GX_INSN_BSWAP(__unalign_jit_x1_st1_add) &		(~create_SrcA_X1(-1)) &		GX_INSN_X1_MASK) | create_SrcA_X1(ra) |		create_SrcB_X1(rb) | create_Dest_Imm8_X1(imm8);} __JIT_CODE("__unalign_jit_x1_st:  {crc32_8 r1, r0, r0; st  r0, r0}");static tilegx_bundle_bits  jit_x1_st(int ra, int rb){	extern  tilegx_bundle_bits __unalign_jit_x1_st;	return (GX_INSN_BSWAP(__unalign_jit_x1_st) & GX_INSN_X1_MASK) |		create_SrcA_X1(ra) | create_SrcB_X1(rb);} __JIT_CODE("__unalign_jit_x1_st_add:  {st_add  r1, r0, 0}");static tilegx_bundle_bits  jit_x1_st_add(int ra, int rb, int imm8){	extern  tilegx_bundle_bits __unalign_jit_x1_st_add;	return (GX_INSN_BSWAP(__unalign_jit_x1_st_add) &		(~create_SrcA_X1(-1)) &		GX_INSN_X1_MASK) | create_SrcA_X1(ra) |		create_SrcB_X1(rb) | create_Dest_Imm8_X1(imm8);} __JIT_CODE("__unalign_jit_x1_ld:  {crc32_8 r1, r0, r0; ld  r0, r0}");static tilegx_bundle_bits  jit_x1_ld(int rd, int ra){	extern  tilegx_bundle_bits __unalign_jit_x1_ld;	return (GX_INSN_BSWAP(__unalign_jit_x1_ld) & GX_INSN_X1_MASK) |		create_Dest_X1(rd) | create_SrcA_X1(ra);} __JIT_CODE("__unalign_jit_x1_ld_add:  {ld_add  r1, r0, 0}");static tilegx_bundle_bits  jit_x1_ld_add(int rd, int ra, int imm8){	extern  tilegx_bundle_bits __unalign_jit_x1_ld_add;	return (GX_INSN_BSWAP(__unalign_jit_x1_ld_add) &		(~create_Dest_X1(-1)) &		GX_INSN_X1_MASK) | create_Dest_X1(rd) |		create_SrcA_X1(ra) | create_Imm8_X1(imm8);} __JIT_CODE("__unalign_jit_x0_bfexts:  {bfexts r0, r0, 0, 0}");static tilegx_bundle_bits  jit_x0_bfexts(int rd, int ra, int bfs, int bfe){	extern  tilegx_bundle_bits __unalign_jit_x0_bfexts;	return (GX_INSN_BSWAP(__unalign_jit_x0_bfexts) &		GX_INSN_X0_MASK) |		create_Dest_X0(rd) | create_SrcA_X0(ra) |		create_BFStart_X0(bfs) | create_BFEnd_X0(bfe);} __JIT_CODE("__unalign_jit_x0_bfextu:  {bfextu r0, r0, 0, 0}");static tilegx_bundle_bits  jit_x0_bfextu(int rd, int ra, int bfs, int bfe){	extern  tilegx_bundle_bits __unalign_jit_x0_bfextu;	return (GX_INSN_BSWAP(__unalign_jit_x0_bfextu) &		GX_INSN_X0_MASK) |		create_Dest_X0(rd) | create_SrcA_X0(ra) |		create_BFStart_X0(bfs) | create_BFEnd_X0(bfe);}

Lines 1–500 of 1597. Load more