// SPDX-License-Identifier: GPL-2.0-only/* * ARC Cache Management * * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com) * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) */ #include <linux/module.h>#include <linux/mm.h>#include <linux/sched.h>#include <linux/cache.h>#include <linux/mmu_context.h>#include <linux/syscalls.h>#include <linux/uaccess.h>#include <linux/pagemap.h>#include <asm/cacheflush.h>#include <asm/cachectl.h>#include <asm/setup.h> #ifdef CONFIG_ISA_ARCV2#define USE_RGN_FLSH	1#endif static int l2_line_sz;static int ioc_exists;int slc_enable = 1, ioc_enable = 1;unsigned long perip_base = ARC_UNCACHED_ADDR_SPACE; /* legacy value for boot */unsigned long perip_end = 0xFFFFFFFF; /* legacy value */ static struct cpuinfo_arc_cache {	unsigned int sz_k, line_len, colors;} ic_info, dc_info, slc_info; void (*_cache_line_loop_ic_fn)(phys_addr_t paddr, unsigned long vaddr,			       unsigned long sz, const int op, const int full_page); void (*__dma_cache_wback_inv)(phys_addr_t start, unsigned long sz);void (*__dma_cache_inv)(phys_addr_t start, unsigned long sz);void (*__dma_cache_wback)(phys_addr_t start, unsigned long sz); static int read_decode_cache_bcr_arcv2(int c, char *buf, int len){	struct cpuinfo_arc_cache *p_slc = &slc_info;	struct bcr_identity ident;	struct bcr_generic sbcr;	struct bcr_clust_cfg cbcr;	struct bcr_volatile vol;	int n = 0; 	READ_BCR(ARC_REG_SLC_BCR, sbcr);	if (sbcr.ver) {		struct bcr_slc_cfg  slc_cfg;		READ_BCR(ARC_REG_SLC_CFG, slc_cfg);		p_slc->sz_k = 128 << slc_cfg.sz;		l2_line_sz = p_slc->line_len = (slc_cfg.lsz == 0) ? 128 : 64;		n += scnprintf(buf + n, len - n,			       "SLC\t\t: %uK, %uB Line%s\n",			       p_slc->sz_k, p_slc->line_len, IS_USED_RUN(slc_enable));	} 	READ_BCR(ARC_REG_CLUSTER_BCR, cbcr);	if (cbcr.c) {		ioc_exists = 1; 		/*		 * As for today we don't support both IOC and ZONE_HIGHMEM enabled		 * simultaneously. This happens because as of today IOC aperture covers		 * only ZONE_NORMAL (low mem) and any dma transactions outside this		 * region won't be HW coherent.		 * If we want to use both IOC and ZONE_HIGHMEM we can use		 * bounce_buffer to handle dma transactions to HIGHMEM.		 * Also it is possible to modify dma_direct cache ops or increase IOC		 * aperture size if we are planning to use HIGHMEM without PAE.		 */		if (IS_ENABLED(CONFIG_HIGHMEM) || is_pae40_enabled())			ioc_enable = 0;	} else {		ioc_enable = 0;	} 	READ_BCR(AUX_IDENTITY, ident); 	/* HS 2.0 didn't have AUX_VOL */	if (ident.family > 0x51) {		READ_BCR(AUX_VOL, vol);		perip_base = vol.start << 28;		/* HS 3.0 has limit and strict-ordering fields */		if (ident.family > 0x52)			perip_end = (vol.limit << 28) - 1;	} 	n += scnprintf(buf + n, len - n, "Peripherals\t: %#lx%s%s\n",		       perip_base,		       IS_AVAIL3(ioc_exists, ioc_enable, ", IO-Coherency (per-device) ")); 	return n;} int arc_cache_mumbojumbo(int c, char *buf, int len){	struct cpuinfo_arc_cache *p_ic = &ic_info, *p_dc = &dc_info;	struct bcr_cache ibcr, dbcr;	int vipt, assoc;	int n = 0; 	READ_BCR(ARC_REG_IC_BCR, ibcr);	if (!ibcr.ver)		goto dc_chk; 	if (is_isa_arcompact() && (ibcr.ver <= 3)) {		BUG_ON(ibcr.config != 3);		assoc = 2;		/* Fixed to 2w set assoc */	} else if (is_isa_arcv2() && (ibcr.ver >= 4)) {		assoc = 1 << ibcr.config;	/* 1,2,4,8 */	} 	p_ic->line_len = 8 << ibcr.line_len;	p_ic->sz_k = 1 << (ibcr.sz - 1);	p_ic->colors = p_ic->sz_k/assoc/TO_KB(PAGE_SIZE); 	n += scnprintf(buf + n, len - n,			"I-Cache\t\t: %uK, %dway/set, %uB Line, VIPT%s%s\n",			p_ic->sz_k, assoc, p_ic->line_len,			p_ic->colors > 1 ? " aliasing" : "",			IS_USED_CFG(CONFIG_ARC_HAS_ICACHE)); dc_chk:	READ_BCR(ARC_REG_DC_BCR, dbcr);	if (!dbcr.ver)		goto slc_chk; 	if (is_isa_arcompact() && (dbcr.ver <= 3)) {		BUG_ON(dbcr.config != 2);		vipt = 1;		assoc = 4;		/* Fixed to 4w set assoc */		p_dc->colors = p_dc->sz_k/assoc/TO_KB(PAGE_SIZE);	} else if (is_isa_arcv2() && (dbcr.ver >= 4)) {		vipt = 0;		assoc = 1 << dbcr.config;	/* 1,2,4,8 */		p_dc->colors = 1;		/* PIPT so can't VIPT alias */	} 	p_dc->line_len = 16 << dbcr.line_len;	p_dc->sz_k = 1 << (dbcr.sz - 1); 	n += scnprintf(buf + n, len - n,			"D-Cache\t\t: %uK, %dway/set, %uB Line, %s%s\n",			p_dc->sz_k, assoc, p_dc->line_len,			vipt ? "VIPT" : "PIPT",			IS_USED_CFG(CONFIG_ARC_HAS_DCACHE)); slc_chk:	if (is_isa_arcv2())		n += read_decode_cache_bcr_arcv2(c, buf + n, len - n); 	return n;} /* * Line Operation on {I,D}-Cache */ #define OP_INV		0x1#define OP_FLUSH	0x2#define OP_FLUSH_N_INV	0x3#define OP_INV_IC	0x4 /* * Cache Flush programming model * * ARC700 MMUv3 I$ and D$ are both VIPT and can potentially alias. * Programming model requires both paddr and vaddr irrespecive of aliasing * considerations: *  - vaddr in {I,D}C_IV?L *  - paddr in {I,D}C_PTAG * * In HS38x (MMUv4), D$ is PIPT, I$ is VIPT and can still alias. * Programming model is different for aliasing vs. non-aliasing I$ *  - D$ / Non-aliasing I$: only paddr in {I,D}C_IV?L *  - Aliasing I$: same as ARC700 above (so MMUv3 routine used for MMUv4 I$) * *  - If PAE40 is enabled, independent of aliasing considerations, the higher *    bits needs to be written into PTAG_HI */ static inlinevoid __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr,			  unsigned long sz, const int op, const int full_page){	unsigned int aux_cmd, aux_tag;	int num_lines; 	if (op == OP_INV_IC) {		aux_cmd = ARC_REG_IC_IVIL;		aux_tag = ARC_REG_IC_PTAG;	} else {		aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL;		aux_tag = ARC_REG_DC_PTAG;	} 	/* Ensure we properly floor/ceil the non-line aligned/sized requests	 * and have @paddr - aligned to cache line and integral @num_lines.	 * This however can be avoided for page sized since:	 *  -@paddr will be cache-line aligned already (being page aligned)	 *  -@sz will be integral multiple of line size (being page sized).	 */	if (!full_page) {		sz += paddr & ~CACHE_LINE_MASK;		paddr &= CACHE_LINE_MASK;		vaddr &= CACHE_LINE_MASK;	}	num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 	/*	 * MMUv3, cache ops require paddr in PTAG reg	 * if V-P const for loop, PTAG can be written once outside loop	 */	if (full_page)		write_aux_reg(aux_tag, paddr); 	/*	 * This is technically for MMU v4, using the MMU v3 programming model	 * Special work for HS38 aliasing I-cache configuration with PAE40	 *   - upper 8 bits of paddr need to be written into PTAG_HI	 *   - (and needs to be written before the lower 32 bits)	 * Note that PTAG_HI is hoisted outside the line loop	 */	if (is_pae40_enabled() && op == OP_INV_IC)		write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 	while (num_lines-- > 0) {		if (!full_page) {			write_aux_reg(aux_tag, paddr);			paddr += L1_CACHE_BYTES;		} 		write_aux_reg(aux_cmd, vaddr);		vaddr += L1_CACHE_BYTES;	}} #ifndef USE_RGN_FLSH /* */static inlinevoid __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr,			  unsigned long sz, const int op, const int full_page){	unsigned int aux_cmd;	int num_lines; 	if (op == OP_INV_IC) {		aux_cmd = ARC_REG_IC_IVIL;	} else {		/* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */		aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL;	} 	/* Ensure we properly floor/ceil the non-line aligned/sized requests	 * and have @paddr - aligned to cache line and integral @num_lines.	 * This however can be avoided for page sized since:	 *  -@paddr will be cache-line aligned already (being page aligned)	 *  -@sz will be integral multiple of line size (being page sized).	 */	if (!full_page) {		sz += paddr & ~CACHE_LINE_MASK;		paddr &= CACHE_LINE_MASK;	} 	num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 	/*	 * For HS38 PAE40 configuration	 *   - upper 8 bits of paddr need to be written into PTAG_HI	 *   - (and needs to be written before the lower 32 bits)	 */	if (is_pae40_enabled()) {		if (op == OP_INV_IC)			/*			 * Non aliasing I-cache in HS38,			 * aliasing I-cache handled in __cache_line_loop_v3()			 */			write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32);		else			write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32);	} 	while (num_lines-- > 0) {		write_aux_reg(aux_cmd, paddr);		paddr += L1_CACHE_BYTES;	}} #else /* * optimized flush operation which takes a region as opposed to iterating per line */static inlinevoid __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr,			  unsigned long sz, const int op, const int full_page){	unsigned int s, e; 	/* Only for Non aliasing I-cache in HS38 */	if (op == OP_INV_IC) {		s = ARC_REG_IC_IVIR;		e = ARC_REG_IC_ENDR;	} else {		s = ARC_REG_DC_STARTR;		e = ARC_REG_DC_ENDR;	} 	if (!full_page) {		/* for any leading gap between @paddr and start of cache line */		sz += paddr & ~CACHE_LINE_MASK;		paddr &= CACHE_LINE_MASK; 		/*		 *  account for any trailing gap to end of cache line		 *  this is equivalent to DIV_ROUND_UP() in line ops above		 */		sz += L1_CACHE_BYTES - 1;	} 	if (is_pae40_enabled()) {		/* TBD: check if crossing 4TB boundary */		if (op == OP_INV_IC)			write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32);		else			write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32);	} 	/* ENDR needs to be set ahead of START */	write_aux_reg(e, paddr + sz);	/* ENDR is exclusive */	write_aux_reg(s, paddr); 	/* caller waits on DC_CTRL.FS */} #endif #ifdef CONFIG_ARC_MMU_V3#define __cache_line_loop	__cache_line_loop_v3#else#define __cache_line_loop	__cache_line_loop_v4#endif #ifdef CONFIG_ARC_HAS_DCACHE /*************************************************************** * Machine specific helpers for Entire D-Cache or Per Line ops */ #ifndef USE_RGN_FLSH/* * this version avoids extra read/write of DC_CTRL for flush or invalid ops * in the non region flush regime (such as for ARCompact) */static inline void __before_dc_op(const int op){	if (op == OP_FLUSH_N_INV) {		/* Dcache provides 2 cmd: FLUSH or INV		 * INV in turn has sub-modes: DISCARD or FLUSH-BEFORE		 * flush-n-inv is achieved by INV cmd but with IM=1		 * So toggle INV sub-mode depending on op request and default		 */		const unsigned int ctl = ARC_REG_DC_CTRL;		write_aux_reg(ctl, read_aux_reg(ctl) | DC_CTRL_INV_MODE_FLUSH);	}} #else static inline void __before_dc_op(const int op){	const unsigned int ctl = ARC_REG_DC_CTRL;	unsigned int val = read_aux_reg(ctl); 	if (op == OP_FLUSH_N_INV) {		val |= DC_CTRL_INV_MODE_FLUSH;	} 	if (op != OP_INV_IC) {		/*		 * Flush / Invalidate is provided by DC_CTRL.RNG_OP 0 or 1		 * combined Flush-n-invalidate uses DC_CTRL.IM = 1 set above		 */		val &= ~DC_CTRL_RGN_OP_MSK;		if (op & OP_INV)			val |= DC_CTRL_RGN_OP_INV;	}	write_aux_reg(ctl, val);} #endif  static inline void __after_dc_op(const int op){	if (op & OP_FLUSH) {		const unsigned int ctl = ARC_REG_DC_CTRL;		unsigned int reg; 		/* flush / flush-n-inv both wait */		while ((reg = read_aux_reg(ctl)) & DC_CTRL_FLUSH_STATUS)			; 		/* Switch back to default Invalidate mode */		if (op == OP_FLUSH_N_INV)			write_aux_reg(ctl, reg & ~DC_CTRL_INV_MODE_FLUSH);	}} /* * Operation on Entire D-Cache * @op = {OP_INV, OP_FLUSH, OP_FLUSH_N_INV} * Note that constant propagation ensures all the checks are gone * in generated code */static inline void __dc_entire_op(const int op){	int aux; 	__before_dc_op(op); 	if (op & OP_INV)	/* Inv or flush-n-inv use same cmd reg */		aux = ARC_REG_DC_IVDC;	else		aux = ARC_REG_DC_FLSH; 	write_aux_reg(aux, 0x1); 	__after_dc_op(op);} static inline void __dc_disable(void){	const int r = ARC_REG_DC_CTRL; 	__dc_entire_op(OP_FLUSH_N_INV);	write_aux_reg(r, read_aux_reg(r) | DC_CTRL_DIS);} static void __dc_enable(void){	const int r = ARC_REG_DC_CTRL; 	write_aux_reg(r, read_aux_reg(r) & ~DC_CTRL_DIS);} /* For kernel mappings cache operation: index is same as paddr */#define __dc_line_op_k(p, sz, op)	__dc_line_op(p, p, sz, op) /* * D-Cache Line ops: Per Line INV (discard or wback+discard) or FLUSH (wback) */static inline void __dc_line_op(phys_addr_t paddr, unsigned long vaddr,				unsigned long sz, const int op){	const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE;	unsigned long flags; 	local_irq_save(flags); 	__before_dc_op(op); 	__cache_line_loop(paddr, vaddr, sz, op, full_page); 	__after_dc_op(op); 	local_irq_restore(flags);} #else #define __dc_entire_op(op)#define __dc_disable()#define __dc_enable()#define __dc_line_op(paddr, vaddr, sz, op)#define __dc_line_op_k(paddr, sz, op) #endif /* CONFIG_ARC_HAS_DCACHE */ #ifdef CONFIG_ARC_HAS_ICACHE static inline void __ic_entire_inv(void){	write_aux_reg(ARC_REG_IC_IVIC, 1);	read_aux_reg(ARC_REG_IC_CTRL);	/* blocks */} static inline void__ic_line_inv_vaddr_local(phys_addr_t paddr, unsigned long vaddr,			  unsigned long sz){	const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE;	unsigned long flags;

Lines 1–500 of 1094. Load more