// SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved. * Copyright (C) 2018-2024 Linaro Ltd. */ #include <linux/bits.h>#include <linux/bug.h>#include <linux/completion.h>#include <linux/interrupt.h>#include <linux/mutex.h>#include <linux/netdevice.h>#include <linux/platform_device.h>#include <linux/types.h> #include "gsi.h"#include "gsi_private.h"#include "gsi_reg.h"#include "gsi_trans.h"#include "ipa_data.h"#include "ipa_gsi.h"#include "ipa_version.h"#include "reg.h" /** * DOC: The IPA Generic Software Interface * * The generic software interface (GSI) is an integral component of the IPA, * providing a well-defined communication layer between the AP subsystem * and the IPA core.  The modem uses the GSI layer as well. * *	--------	     --------- *	|      |	     |	     | *	|  AP  +<---.	.----+ Modem | *	|      +--. |	| .->+	     | *	|      |  | |	| |  |	     | *	--------  | |	| |  --------- *		  v |	v | *		--+-+---+-+-- *		|    GSI    | *		|-----------| *		|	    | *		|    IPA    | *		|	    | *		------------- * * In the above diagram, the AP and Modem represent "execution environments" * (EEs), which are independent operating environments that use the IPA for * data transfer. * * Each EE uses a set of unidirectional GSI "channels," which allow transfer * of data to or from the IPA.  A channel is implemented as a ring buffer, * with a DRAM-resident array of "transfer elements" (TREs) available to * describe transfers to or from other EEs through the IPA.  A transfer * element can also contain an immediate command, requesting the IPA perform * actions other than data transfer. * * Each TRE refers to a block of data--also located in DRAM.  After writing * one or more TREs to a channel, the writer (either the IPA or an EE) writes * a doorbell register to inform the receiving side how many elements have * been written. * * Each channel has a GSI "event ring" associated with it.  An event ring * is implemented very much like a channel ring, but is always directed from * the IPA to an EE.  The IPA notifies an EE (such as the AP) about channel * events by adding an entry to the event ring associated with the channel. * The GSI then writes its doorbell for the event ring, causing the target * EE to be interrupted.  Each entry in an event ring contains a pointer * to the channel TRE whose completion the event represents. * * Each TRE in a channel ring has a set of flags.  One flag indicates whether * the completion of the transfer operation generates an entry (and possibly * an interrupt) in the channel's event ring.  Other flags allow transfer * elements to be chained together, forming a single logical transaction. * TRE flags are used to control whether and when interrupts are generated * to signal completion of channel transfers. * * Elements in channel and event rings are completed (or consumed) strictly * in order.  Completion of one entry implies the completion of all preceding * entries.  A single completion interrupt can therefore communicate the * completion of many transfers. * * Note that all GSI registers are little-endian, which is the assumed * endianness of I/O space accesses.  The accessor functions perform byte * swapping if needed (i.e., for a big endian CPU). */ /* Delay period for interrupt moderation (in 32KHz IPA internal timer ticks) */#define GSI_EVT_RING_INT_MODT		(32 * 1) /* 1ms under 32KHz clock */ #define GSI_CMD_TIMEOUT			50	/* milliseconds */ #define GSI_CHANNEL_STOP_RETRIES	10#define GSI_CHANNEL_MODEM_HALT_RETRIES	10#define GSI_CHANNEL_MODEM_FLOW_RETRIES	5	/* disable flow control only */ #define GSI_MHI_EVENT_ID_START		10	/* 1st reserved event id */#define GSI_MHI_EVENT_ID_END		16	/* Last reserved event id */ #define GSI_ISR_MAX_ITER		50	/* Detect interrupt storms */ /* An entry in an event ring */struct gsi_event {	__le64 xfer_ptr;	__le16 len;	u8 reserved1;	u8 code;	__le16 reserved2;	u8 type;	u8 chid;}; /** gsi_channel_scratch_gpi - GPI protocol scratch register * @max_outstanding_tre: *	Defines the maximum number of TREs allowed in a single transaction *	on a channel (in bytes).  This determines the amount of prefetch *	performed by the hardware.  We configure this to equal the size of *	the TLV FIFO for the channel. * @outstanding_threshold: *	Defines the threshold (in bytes) determining when the sequencer *	should update the channel doorbell.  We configure this to equal *	the size of two TREs. */struct gsi_channel_scratch_gpi {	u64 reserved1;	u16 reserved2;	u16 max_outstanding_tre;	u16 reserved3;	u16 outstanding_threshold;}; /** gsi_channel_scratch - channel scratch configuration area * * The exact interpretation of this register is protocol-specific. * We only use GPI channels; see struct gsi_channel_scratch_gpi, above. */union gsi_channel_scratch {	struct gsi_channel_scratch_gpi gpi;	struct {		u32 word1;		u32 word2;		u32 word3;		u32 word4;	} data;}; /* Check things that can be validated at build time. */static void gsi_validate_build(void){	/* This is used as a divisor */	BUILD_BUG_ON(!GSI_RING_ELEMENT_SIZE); 	/* Code assumes the size of channel and event ring element are	 * the same (and fixed).  Make sure the size of an event ring	 * element is what's expected.	 */	BUILD_BUG_ON(sizeof(struct gsi_event) != GSI_RING_ELEMENT_SIZE); 	/* Hardware requires a 2^n ring size.  We ensure the number of	 * elements in an event ring is a power of 2 elsewhere; this	 * ensure the elements themselves meet the requirement.	 */	BUILD_BUG_ON(!is_power_of_2(GSI_RING_ELEMENT_SIZE));} /* Return the channel id associated with a given channel */static u32 gsi_channel_id(struct gsi_channel *channel){	return channel - &channel->gsi->channel[0];} /* An initialized channel has a non-null GSI pointer */static bool gsi_channel_initialized(struct gsi_channel *channel){	return !!channel->gsi;} /* Encode the channel protocol for the CH_C_CNTXT_0 register */static u32 ch_c_cntxt_0_type_encode(enum ipa_version version,				    const struct reg *reg,				    enum gsi_channel_type type){	u32 val; 	val = reg_encode(reg, CHTYPE_PROTOCOL, type);	if (version < IPA_VERSION_4_5 || version >= IPA_VERSION_5_0)		return val; 	type >>= hweight32(reg_fmask(reg, CHTYPE_PROTOCOL)); 	return val | reg_encode(reg, CHTYPE_PROTOCOL_MSB, type);} /* Update the GSI IRQ type register with the cached value */static void gsi_irq_type_update(struct gsi *gsi, u32 val){	const struct reg *reg = gsi_reg(gsi, CNTXT_TYPE_IRQ_MSK); 	gsi->type_enabled_bitmap = val;	iowrite32(val, gsi->virt + reg_offset(reg));} static void gsi_irq_type_enable(struct gsi *gsi, enum gsi_irq_type_id type_id){	gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | type_id);} static void gsi_irq_type_disable(struct gsi *gsi, enum gsi_irq_type_id type_id){	gsi_irq_type_update(gsi, gsi->type_enabled_bitmap & ~type_id);} /* Event ring commands are performed one at a time.  Their completion * is signaled by the event ring control GSI interrupt type, which is * only enabled when we issue an event ring command.  Only the event * ring being operated on has this interrupt enabled. */static void gsi_irq_ev_ctrl_enable(struct gsi *gsi, u32 evt_ring_id){	u32 val = BIT(evt_ring_id);	const struct reg *reg; 	/* There's a small chance that a previous command completed	 * after the interrupt was disabled, so make sure we have no	 * pending interrupts before we enable them.	 */	reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ_CLR);	iowrite32(~0, gsi->virt + reg_offset(reg)); 	reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ_MSK);	iowrite32(val, gsi->virt + reg_offset(reg));	gsi_irq_type_enable(gsi, GSI_EV_CTRL);} /* Disable event ring control interrupts */static void gsi_irq_ev_ctrl_disable(struct gsi *gsi){	const struct reg *reg; 	gsi_irq_type_disable(gsi, GSI_EV_CTRL); 	reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ_MSK);	iowrite32(0, gsi->virt + reg_offset(reg));} /* Channel commands are performed one at a time.  Their completion is * signaled by the channel control GSI interrupt type, which is only * enabled when we issue a channel command.  Only the channel being * operated on has this interrupt enabled. */static void gsi_irq_ch_ctrl_enable(struct gsi *gsi, u32 channel_id){	u32 val = BIT(channel_id);	const struct reg *reg; 	/* There's a small chance that a previous command completed	 * after the interrupt was disabled, so make sure we have no	 * pending interrupts before we enable them.	 */	reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ_CLR);	iowrite32(~0, gsi->virt + reg_offset(reg)); 	reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ_MSK);	iowrite32(val, gsi->virt + reg_offset(reg)); 	gsi_irq_type_enable(gsi, GSI_CH_CTRL);} /* Disable channel control interrupts */static void gsi_irq_ch_ctrl_disable(struct gsi *gsi){	const struct reg *reg; 	gsi_irq_type_disable(gsi, GSI_CH_CTRL); 	reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ_MSK);	iowrite32(0, gsi->virt + reg_offset(reg));} static void gsi_irq_ieob_enable_one(struct gsi *gsi, u32 evt_ring_id){	bool enable_ieob = !gsi->ieob_enabled_bitmap;	const struct reg *reg;	u32 val; 	gsi->ieob_enabled_bitmap |= BIT(evt_ring_id); 	reg = gsi_reg(gsi, CNTXT_SRC_IEOB_IRQ_MSK);	val = gsi->ieob_enabled_bitmap;	iowrite32(val, gsi->virt + reg_offset(reg)); 	/* Enable the interrupt type if this is the first channel enabled */	if (enable_ieob)		gsi_irq_type_enable(gsi, GSI_IEOB);} static void gsi_irq_ieob_disable(struct gsi *gsi, u32 event_mask){	const struct reg *reg;	u32 val; 	gsi->ieob_enabled_bitmap &= ~event_mask; 	/* Disable the interrupt type if this was the last enabled channel */	if (!gsi->ieob_enabled_bitmap)		gsi_irq_type_disable(gsi, GSI_IEOB); 	reg = gsi_reg(gsi, CNTXT_SRC_IEOB_IRQ_MSK);	val = gsi->ieob_enabled_bitmap;	iowrite32(val, gsi->virt + reg_offset(reg));} static void gsi_irq_ieob_disable_one(struct gsi *gsi, u32 evt_ring_id){	gsi_irq_ieob_disable(gsi, BIT(evt_ring_id));} /* Enable all GSI_interrupt types */static void gsi_irq_enable(struct gsi *gsi){	const struct reg *reg;	u32 val; 	/* Global interrupts include hardware error reports.  Enable	 * that so we can at least report the error should it occur.	 */	reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_EN);	iowrite32(ERROR_INT, gsi->virt + reg_offset(reg)); 	gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | GSI_GLOB_EE); 	/* General GSI interrupts are reported to all EEs; if they occur	 * they are unrecoverable (without reset).  A breakpoint interrupt	 * also exists, but we don't support that.  We want to be notified	 * of errors so we can report them, even if they can't be handled.	 */	reg = gsi_reg(gsi, CNTXT_GSI_IRQ_EN);	val = BUS_ERROR;	val |= CMD_FIFO_OVRFLOW;	val |= MCS_STACK_OVRFLOW;	iowrite32(val, gsi->virt + reg_offset(reg)); 	gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | GSI_GENERAL);} /* Disable all GSI interrupt types */static void gsi_irq_disable(struct gsi *gsi){	const struct reg *reg; 	gsi_irq_type_update(gsi, 0); 	/* Clear the type-specific interrupt masks set by gsi_irq_enable() */	reg = gsi_reg(gsi, CNTXT_GSI_IRQ_EN);	iowrite32(0, gsi->virt + reg_offset(reg)); 	reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_EN);	iowrite32(0, gsi->virt + reg_offset(reg));} /* Return the virtual address associated with a ring index */void *gsi_ring_virt(struct gsi_ring *ring, u32 index){	/* Note: index *must* be used modulo the ring count here */	return ring->virt + (index % ring->count) * GSI_RING_ELEMENT_SIZE;} /* Return the 32-bit DMA address associated with a ring index */static u32 gsi_ring_addr(struct gsi_ring *ring, u32 index){	return lower_32_bits(ring->addr) + index * GSI_RING_ELEMENT_SIZE;} /* Return the ring index of a 32-bit ring offset */static u32 gsi_ring_index(struct gsi_ring *ring, u32 offset){	return (offset - gsi_ring_addr(ring, 0)) / GSI_RING_ELEMENT_SIZE;} /* Issue a GSI command by writing a value to a register, then wait for * completion to be signaled.  Returns true if the command completes * or false if it times out. */static bool gsi_command(struct gsi *gsi, u32 reg, u32 val){	unsigned long timeout = msecs_to_jiffies(GSI_CMD_TIMEOUT);	struct completion *completion = &gsi->completion; 	reinit_completion(completion); 	iowrite32(val, gsi->virt + reg); 	return !!wait_for_completion_timeout(completion, timeout);} /* Return the hardware's notion of the current state of an event ring */static enum gsi_evt_ring_stategsi_evt_ring_state(struct gsi *gsi, u32 evt_ring_id){	const struct reg *reg = gsi_reg(gsi, EV_CH_E_CNTXT_0);	u32 val; 	val = ioread32(gsi->virt + reg_n_offset(reg, evt_ring_id)); 	return reg_decode(reg, EV_CHSTATE, val);} /* Issue an event ring command and wait for it to complete */static void gsi_evt_ring_command(struct gsi *gsi, u32 evt_ring_id,				 enum gsi_evt_cmd_opcode opcode){	struct device *dev = gsi->dev;	const struct reg *reg;	bool timeout;	u32 val; 	/* Enable the completion interrupt for the command */	gsi_irq_ev_ctrl_enable(gsi, evt_ring_id); 	reg = gsi_reg(gsi, EV_CH_CMD);	val = reg_encode(reg, EV_CHID, evt_ring_id);	val |= reg_encode(reg, EV_OPCODE, opcode); 	timeout = !gsi_command(gsi, reg_offset(reg), val); 	gsi_irq_ev_ctrl_disable(gsi); 	if (!timeout)		return; 	dev_err(dev, "GSI command %u for event ring %u timed out, state %u\n",		opcode, evt_ring_id, gsi_evt_ring_state(gsi, evt_ring_id));} /* Allocate an event ring in NOT_ALLOCATED state */static int gsi_evt_ring_alloc_command(struct gsi *gsi, u32 evt_ring_id){	enum gsi_evt_ring_state state; 	/* Get initial event ring state */	state = gsi_evt_ring_state(gsi, evt_ring_id);	if (state != GSI_EVT_RING_STATE_NOT_ALLOCATED) {		dev_err(gsi->dev, "event ring %u bad state %u before alloc\n",			evt_ring_id, state);		return -EINVAL;	} 	gsi_evt_ring_command(gsi, evt_ring_id, GSI_EVT_ALLOCATE); 	/* If successful the event ring state will have changed */	state = gsi_evt_ring_state(gsi, evt_ring_id);	if (state == GSI_EVT_RING_STATE_ALLOCATED)		return 0; 	dev_err(gsi->dev, "event ring %u bad state %u after alloc\n",		evt_ring_id, state); 	return -EIO;} /* Reset a GSI event ring in ALLOCATED or ERROR state. */static void gsi_evt_ring_reset_command(struct gsi *gsi, u32 evt_ring_id){	enum gsi_evt_ring_state state; 	state = gsi_evt_ring_state(gsi, evt_ring_id);	if (state != GSI_EVT_RING_STATE_ALLOCATED &&	    state != GSI_EVT_RING_STATE_ERROR) {		dev_err(gsi->dev, "event ring %u bad state %u before reset\n",			evt_ring_id, state);		return;	} 	gsi_evt_ring_command(gsi, evt_ring_id, GSI_EVT_RESET); 	/* If successful the event ring state will have changed */	state = gsi_evt_ring_state(gsi, evt_ring_id);	if (state == GSI_EVT_RING_STATE_ALLOCATED)		return; 	dev_err(gsi->dev, "event ring %u bad state %u after reset\n",		evt_ring_id, state);} /* Issue a hardware de-allocation request for an allocated event ring */static void gsi_evt_ring_de_alloc_command(struct gsi *gsi, u32 evt_ring_id){	enum gsi_evt_ring_state state; 	state = gsi_evt_ring_state(gsi, evt_ring_id);	if (state != GSI_EVT_RING_STATE_ALLOCATED) {		dev_err(gsi->dev, "event ring %u state %u before dealloc\n",			evt_ring_id, state);		return;	} 	gsi_evt_ring_command(gsi, evt_ring_id, GSI_EVT_DE_ALLOC); 	/* If successful the event ring state will have changed */	state = gsi_evt_ring_state(gsi, evt_ring_id);

Lines 1–500 of 2434. Load more