habanalabs: add h/w queues module

This patch adds the H/W queues module and the code to initialize Goya's various compute and DMA engines and their queues. Goya has 5 DMA channels, 8 TPC engines and a single MME engine. For each channel/engine, there is a H/W queue logic which is used to pass commands from the user to the H/W. That logic is called QMAN. There are two types of QMANs: external and internal. The DMA QMANs are considered external while the TPC and MME QMANs are considered internal. For each external queue there is a completion queue, which is located on the Host memory. The differences between external and internal QMANs are: 1. The location of the queue's memory. External QMANs are located on the Host memory while internal QMANs are located on the on-chip memory. 2. The external QMAN write an entry to a completion queue and sends an MSI-X interrupt upon completion of a command buffer that was given to it. The internal QMAN doesn't do that.

Oded Gabbay committed 7 years ago

Commit 9494a8d1 parent 839c48013 files · +2915 −116Browse files

Unified · Split

13 files

drivers/misc/habanalabs/Makefile +1 −1
drivers/misc/habanalabs/device.c +73 −2
drivers/misc/habanalabs/goya/goya.c +1415 −112
drivers/misc/habanalabs/goya/goyaP.h +7 −0
drivers/misc/habanalabs/habanalabs.h +173 −1
drivers/misc/habanalabs/habanalabs_drv.c +5 −0
drivers/misc/habanalabs/hw_queue.c +400 −0
drivers/misc/habanalabs/include/armcp_if.h +292 −0
drivers/misc/habanalabs/include/goya/goya_async_events.h +186 −0
drivers/misc/habanalabs/include/goya/goya_packets.h +129 −0
drivers/misc/habanalabs/include/qman_if.h +56 −0
drivers/misc/habanalabs/irq.c +149 −0
include/uapi/misc/habanalabs.h +29 −0

drivers/misc/habanalabs/Makefile+1−1

@@ -5,7 +5,7 @@ · 4 unmodified lines
5	5	obj-m := habanalabs.o
6	6
7	7	habanalabs-y := habanalabs_drv.o device.o context.o asid.o habanalabs_ioctl.o \
8		− command_buffer.o
	8	+ command_buffer.o hw_queue.o irq.o
9	9
10	10	include $(src)/goya/Makefile
11	11	habanalabs-y += $(HL_GOYA_FILES)

drivers/misc/habanalabs/device.c+73−2

@@ -174,13 +174,23 @@ static int device_early_init(struct hl_device *hdev) · 173 unmodified lines
174	174	if (rc)
175	175	goto early_fini;
176	176
	177	+ hdev->cq_wq = alloc_workqueue("hl-free-jobs", WQ_UNBOUND, 0);
	178	+ if (hdev->cq_wq == NULL) {
	179	+ dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
	180	+ rc = -ENOMEM;
	181	+ goto asid_fini;
	182	+ }
	183	+
177	184	hl_cb_mgr_init(&hdev->kernel_cb_mgr);
178	185
179	186	mutex_init(&hdev->fd_open_cnt_lock);
	187	+ mutex_init(&hdev->send_cpu_message_lock);
180	188	atomic_set(&hdev->fd_open_cnt, 0);
181	189
182	190	return 0;
183	191
	192	+asid_fini:
	193	+ hl_asid_fini(hdev);
184	194	early_fini:
185	195	if (hdev->asic_funcs->early_fini)
186	196	hdev->asic_funcs->early_fini(hdev);
@@ -196,9 +206,12 @@ early_fini: · 9 unmodified lines
196	206	*/
197	207	static void device_early_fini(struct hl_device *hdev)
198	208	{
	209	+ mutex_destroy(&hdev->send_cpu_message_lock);
199	210
200	211	hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
201	212
	213	+ destroy_workqueue(hdev->cq_wq);
	214	+
202	215	hl_asid_fini(hdev);
203	216
204	217	if (hdev->asic_funcs->early_fini)
@@ -277,7 +290,7 @@ int hl_device_resume(struct hl_device *hdev) · 72 unmodified lines
277	290	*/
278	291	int hl_device_init(struct hl_device hdev, struct class hclass)
279	292	{
280		− int rc;
	293	+ int i, rc, cq_ready_cnt;
281	294
282	295	/* Create device */
283	296	rc = device_setup_cdev(hdev, hclass, hdev->id, &hl_ops);
@@ -298,11 +311,48 @@ int hl_device_init(struct hl_device hdev, struct class hclass) · 14 unmodified lines
298	311	if (rc)
299	312	goto early_fini;
300	313
	314	+ /*
	315	+ * Initialize the H/W queues. Must be done before hw_init, because
	316	+ * there the addresses of the kernel queue are being written to the
	317	+ * registers of the device
	318	+ */
	319	+ rc = hl_hw_queues_create(hdev);
	320	+ if (rc) {
	321	+ dev_err(hdev->dev, "failed to initialize kernel queues\n");
	322	+ goto sw_fini;
	323	+ }
	324	+
	325	+ /*
	326	+ * Initialize the completion queues. Must be done before hw_init,
	327	+ * because there the addresses of the completion queues are being
	328	+ * passed as arguments to request_irq
	329	+ */
	330	+ hdev->completion_queue =
	331	+ kcalloc(hdev->asic_prop.completion_queues_count,
	332	+ sizeof(*hdev->completion_queue), GFP_KERNEL);
	333	+
	334	+ if (!hdev->completion_queue) {
	335	+ dev_err(hdev->dev, "failed to allocate completion queues\n");
	336	+ rc = -ENOMEM;
	337	+ goto hw_queues_destroy;
	338	+ }
	339	+
	340	+ for (i = 0, cq_ready_cnt = 0;
	341	+ i < hdev->asic_prop.completion_queues_count;
	342	+ i++, cq_ready_cnt++) {
	343	+ rc = hl_cq_init(hdev, &hdev->completion_queue[i], i);
	344	+ if (rc) {
	345	+ dev_err(hdev->dev,
	346	+ "failed to initialize completion queue\n");
	347	+ goto cq_fini;
	348	+ }
	349	+ }
	350	+
301	351	/* Allocate the kernel context */
302	352	hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
303	353	if (!hdev->kernel_ctx) {
304	354	rc = -ENOMEM;
305		− goto sw_fini;
	355	+ goto cq_fini;
306	356	}
307	357
308	358	hdev->user_ctx = NULL;
@@ -328,6 +378,14 @@ int hl_device_init(struct hl_device hdev, struct class hclass) · 19 unmodified lines
328	378
329	379	hdev->disabled = false;
330	380
	381	+ /* Check that the communication with the device is working */
	382	+ rc = hdev->asic_funcs->test_queues(hdev);
	383	+ if (rc) {
	384	+ dev_err(hdev->dev, "Failed to detect if device is alive\n");
	385	+ rc = 0;
	386	+ goto out_disabled;
	387	+ }
	388	+
331	389	dev_notice(hdev->dev,
332	390	"Successfully added device to habanalabs driver\n");
333	391
@@ -339,6 +397,12 @@ release_ctx: · 5 unmodified lines
339	397	"kernel ctx is still alive on initialization failure\n");
340	398	free_ctx:
341	399	kfree(hdev->kernel_ctx);
	400	+cq_fini:
	401	+ for (i = 0 ; i < cq_ready_cnt ; i++)
	402	+ hl_cq_fini(hdev, &hdev->completion_queue[i]);
	403	+ kfree(hdev->completion_queue);
	404	+hw_queues_destroy:
	405	+ hl_hw_queues_destroy(hdev);
342	406	sw_fini:
343	407	hdev->asic_funcs->sw_fini(hdev);
344	408	early_fini:
@@ -368,6 +432,7 @@ out_disabled: · 23 unmodified lines
368	432	*/
369	433	void hl_device_fini(struct hl_device *hdev)
370	434	{
	435	+ int i;
371	436	dev_info(hdev->dev, "Removing device\n");
372	437
373	438	/* Mark device as disabled */
@@ -382,6 +447,12 @@ void hl_device_fini(struct hl_device *hdev) · 8 unmodified lines
382	447	/* Reset the H/W. It will be in idle state after this returns */
383	448	hdev->asic_funcs->hw_fini(hdev, true);
384	449
	450	+ for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
	451	+ hl_cq_fini(hdev, &hdev->completion_queue[i]);
	452	+ kfree(hdev->completion_queue);
	453	+
	454	+ hl_hw_queues_destroy(hdev);
	455	+
385	456	/* Call ASIC S/W finalize function */
386	457	hdev->asic_funcs->sw_fini(hdev);
387	458

drivers/misc/habanalabs/goya/goya.c+1415−112

@@ -90,6 +90,26 @@ · 89 unmodified lines
90	90	static void goya_get_fixed_properties(struct hl_device *hdev)
91	91	{
92	92	struct asic_fixed_properties *prop = &hdev->asic_prop;
	93	+ int i;
	94	+
	95	+ for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) {
	96	+ prop->hw_queues_props[i].type = QUEUE_TYPE_EXT;
	97	+ prop->hw_queues_props[i].kmd_only = 0;
	98	+ }
	99	+
	100	+ for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES ; i++) {
	101	+ prop->hw_queues_props[i].type = QUEUE_TYPE_CPU;
	102	+ prop->hw_queues_props[i].kmd_only = 1;
	103	+ }
	104	+
	105	+ for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES +
	106	+ NUMBER_OF_INT_HW_QUEUES; i++) {
	107	+ prop->hw_queues_props[i].type = QUEUE_TYPE_INT;
	108	+ prop->hw_queues_props[i].kmd_only = 0;
	109	+ }
	110	+
	111	+ for (; i < HL_MAX_QUEUES; i++)
	112	+ prop->hw_queues_props[i].type = QUEUE_TYPE_NA;
93	113
94	114	prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;
95	115
@@ -118,6 +138,18 @@ static void goya_get_fixed_properties(struct hl_device *hdev) · 22 unmodified lines
118	138	prop->high_pll = PLL_HIGH_DEFAULT;
119	139	}
120	140
	141	+int goya_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
	142	+{
	143	+ struct armcp_packet pkt;
	144	+
	145	+ memset(&pkt, 0, sizeof(pkt));
	146	+
	147	+ pkt.ctl = opcode << ARMCP_PKT_CTL_OPCODE_SHIFT;
	148	+
	149	+ return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
	150	+ sizeof(pkt), HL_DEVICE_TIMEOUT_USEC, NULL);
	151	+}
	152	+
121	153	/*
122	154	* goya_pci_bars_map - Map PCI BARS of Goya device
123	155	*
@@ -510,6 +542,8 @@ static int goya_sw_init(struct hl_device *hdev) · 386 unmodified lines
510	542	if (!goya)
511	543	return -ENOMEM;
512	544
	545	+ goya->test_cpu_queue = goya_test_cpu_queue;
	546	+
513	547	/* according to goya_init_iatu */
514	548	goya->ddr_bar_cur_addr = DRAM_PHYS_BASE;
515	549	hdev->asic_specific = goya;
@@ -596,6 +630,311 @@ int goya_sw_fini(struct hl_device *hdev) · 80 unmodified lines
596	630	return 0;
597	631	}
598	632
	633	+static void goya_init_dma_qman(struct hl_device *hdev, int dma_id,
	634	+ dma_addr_t bus_address)
	635	+{
	636	+ struct goya_device *goya = hdev->asic_specific;
	637	+ u32 mtr_base_lo, mtr_base_hi;
	638	+ u32 so_base_lo, so_base_hi;
	639	+ u32 gic_base_lo, gic_base_hi;
	640	+ u32 reg_off = dma_id * (mmDMA_QM_1_PQ_PI - mmDMA_QM_0_PQ_PI);
	641	+
	642	+ mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
	643	+ mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
	644	+ so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
	645	+ so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
	646	+
	647	+ gic_base_lo =
	648	+ lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
	649	+ gic_base_hi =
	650	+ upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
	651	+
	652	+ WREG32(mmDMA_QM_0_PQ_BASE_LO + reg_off, lower_32_bits(bus_address));
	653	+ WREG32(mmDMA_QM_0_PQ_BASE_HI + reg_off, upper_32_bits(bus_address));
	654	+
	655	+ WREG32(mmDMA_QM_0_PQ_SIZE + reg_off, ilog2(HL_QUEUE_LENGTH));
	656	+ WREG32(mmDMA_QM_0_PQ_PI + reg_off, 0);
	657	+ WREG32(mmDMA_QM_0_PQ_CI + reg_off, 0);
	658	+
	659	+ WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
	660	+ WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
	661	+ WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
	662	+ WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
	663	+ WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
	664	+ WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
	665	+ WREG32(mmDMA_QM_0_GLBL_ERR_WDATA + reg_off,
	666	+ GOYA_ASYNC_EVENT_ID_DMA0_QM + dma_id);
	667	+
	668	+ /* PQ has buffer of 2 cache lines, while CQ has 8 lines */
	669	+ WREG32(mmDMA_QM_0_PQ_CFG1 + reg_off, 0x00020002);
	670	+ WREG32(mmDMA_QM_0_CQ_CFG1 + reg_off, 0x00080008);
	671	+
	672	+ if (dma_id == 0)
	673	+ WREG32(mmDMA_QM_0_GLBL_PROT + reg_off, QMAN_DMA_FULLY_TRUSTED);
	674	+ else
	675	+ if (goya->hw_cap_initialized & HW_CAP_MMU)
	676	+ WREG32(mmDMA_QM_0_GLBL_PROT + reg_off,
	677	+ QMAN_DMA_PARTLY_TRUSTED);
	678	+ else
	679	+ WREG32(mmDMA_QM_0_GLBL_PROT + reg_off,
	680	+ QMAN_DMA_FULLY_TRUSTED);
	681	+
	682	+ WREG32(mmDMA_QM_0_GLBL_ERR_CFG + reg_off, QMAN_DMA_ERR_MSG_EN);
	683	+ WREG32(mmDMA_QM_0_GLBL_CFG0 + reg_off, QMAN_DMA_ENABLE);
	684	+}
	685	+
	686	+static void goya_init_dma_ch(struct hl_device *hdev, int dma_id)
	687	+{
	688	+ u32 gic_base_lo, gic_base_hi;
	689	+ u64 sob_addr;
	690	+ u32 reg_off = dma_id * (mmDMA_CH_1_CFG1 - mmDMA_CH_0_CFG1);
	691	+
	692	+ gic_base_lo =
	693	+ lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
	694	+ gic_base_hi =
	695	+ upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
	696	+
	697	+ WREG32(mmDMA_CH_0_ERRMSG_ADDR_LO + reg_off, gic_base_lo);
	698	+ WREG32(mmDMA_CH_0_ERRMSG_ADDR_HI + reg_off, gic_base_hi);
	699	+ WREG32(mmDMA_CH_0_ERRMSG_WDATA + reg_off,
	700	+ GOYA_ASYNC_EVENT_ID_DMA0_CH + dma_id);
	701	+
	702	+ if (dma_id) {
	703	+ sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1000 +
	704	+ (dma_id - 1) * 4;
	705	+ WREG32(mmDMA_CH_0_WR_COMP_ADDR_LO + reg_off,
	706	+ lower_32_bits(sob_addr));
	707	+ WREG32(mmDMA_CH_0_WR_COMP_ADDR_HI + reg_off,
	708	+ upper_32_bits(sob_addr));
	709	+ WREG32(mmDMA_CH_0_WR_COMP_WDATA + reg_off, 0x80000001);
	710	+ }
	711	+}
	712	+
	713	+/*
	714	+ * goya_init_dma_qmans - Initialize QMAN DMA registers
	715	+ *
	716	+ * @hdev: pointer to hl_device structure
	717	+ *
	718	+ * Initialize the H/W registers of the QMAN DMA channels
	719	+ *
	720	+ */
	721	+static void goya_init_dma_qmans(struct hl_device *hdev)
	722	+{
	723	+ struct goya_device *goya = hdev->asic_specific;
	724	+ struct hl_hw_queue *q;
	725	+ dma_addr_t bus_address;
	726	+ int i;
	727	+
	728	+ if (goya->hw_cap_initialized & HW_CAP_DMA)
	729	+ return;
	730	+
	731	+ q = &hdev->kernel_queues[0];
	732	+
	733	+ for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++, q++) {
	734	+ bus_address = q->bus_address +
	735	+ hdev->asic_prop.host_phys_base_address;
	736	+
	737	+ goya_init_dma_qman(hdev, i, bus_address);
	738	+ goya_init_dma_ch(hdev, i);
	739	+ }
	740	+
	741	+ goya->hw_cap_initialized \|= HW_CAP_DMA;
	742	+}
	743	+
	744	+/*
	745	+ * goya_disable_external_queues - Disable external queues
	746	+ *
	747	+ * @hdev: pointer to hl_device structure
	748	+ *
	749	+ */
	750	+static void goya_disable_external_queues(struct hl_device *hdev)
	751	+{
	752	+ WREG32(mmDMA_QM_0_GLBL_CFG0, 0);
	753	+ WREG32(mmDMA_QM_1_GLBL_CFG0, 0);
	754	+ WREG32(mmDMA_QM_2_GLBL_CFG0, 0);
	755	+ WREG32(mmDMA_QM_3_GLBL_CFG0, 0);
	756	+ WREG32(mmDMA_QM_4_GLBL_CFG0, 0);
	757	+}
	758	+
	759	+static int goya_stop_queue(struct hl_device *hdev, u32 cfg_reg,
	760	+ u32 cp_sts_reg, u32 glbl_sts0_reg)
	761	+{
	762	+ int rc;
	763	+ u32 status;
	764	+
	765	+ /* use the values of TPC0 as they are all the same*/
	766	+
	767	+ WREG32(cfg_reg, 1 << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
	768	+
	769	+ status = RREG32(cp_sts_reg);
	770	+ if (status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK) {
	771	+ rc = hl_poll_timeout(
	772	+ hdev,
	773	+ cp_sts_reg,
	774	+ status,
	775	+ !(status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK),
	776	+ 1000,
	777	+ QMAN_FENCE_TIMEOUT_USEC);
	778	+
	779	+ /* if QMAN is stuck in fence no need to check for stop */
	780	+ if (rc)
	781	+ return 0;
	782	+ }
	783	+
	784	+ rc = hl_poll_timeout(
	785	+ hdev,
	786	+ glbl_sts0_reg,
	787	+ status,
	788	+ (status & TPC0_QM_GLBL_STS0_CP_IS_STOP_MASK),
	789	+ 1000,
	790	+ QMAN_STOP_TIMEOUT_USEC);
	791	+
	792	+ if (rc) {
	793	+ dev_err(hdev->dev,
	794	+ "Timeout while waiting for QMAN to stop\n");
	795	+ return -EINVAL;
	796	+ }
	797	+
	798	+ return 0;
	799	+}
	800	+
	801	+/*
	802	+ * goya_stop_external_queues - Stop external queues
	803	+ *
	804	+ * @hdev: pointer to hl_device structure
	805	+ *
	806	+ * Returns 0 on success
	807	+ *
	808	+ */
	809	+static int goya_stop_external_queues(struct hl_device *hdev)
	810	+{
	811	+ int rc, retval = 0;
	812	+
	813	+ rc = goya_stop_queue(hdev,
	814	+ mmDMA_QM_0_GLBL_CFG1,
	815	+ mmDMA_QM_0_CP_STS,
	816	+ mmDMA_QM_0_GLBL_STS0);
	817	+
	818	+ if (rc) {
	819	+ dev_err(hdev->dev, "failed to stop DMA QMAN 0\n");
	820	+ retval = -EIO;
	821	+ }
	822	+
	823	+ rc = goya_stop_queue(hdev,
	824	+ mmDMA_QM_1_GLBL_CFG1,
	825	+ mmDMA_QM_1_CP_STS,
	826	+ mmDMA_QM_1_GLBL_STS0);
	827	+
	828	+ if (rc) {
	829	+ dev_err(hdev->dev, "failed to stop DMA QMAN 1\n");
	830	+ retval = -EIO;
	831	+ }
	832	+
	833	+ rc = goya_stop_queue(hdev,
	834	+ mmDMA_QM_2_GLBL_CFG1,
	835	+ mmDMA_QM_2_CP_STS,
	836	+ mmDMA_QM_2_GLBL_STS0);
	837	+
	838	+ if (rc) {
	839	+ dev_err(hdev->dev, "failed to stop DMA QMAN 2\n");
	840	+ retval = -EIO;
	841	+ }
	842	+
	843	+ rc = goya_stop_queue(hdev,
	844	+ mmDMA_QM_3_GLBL_CFG1,
	845	+ mmDMA_QM_3_CP_STS,
	846	+ mmDMA_QM_3_GLBL_STS0);
	847	+
	848	+ if (rc) {
	849	+ dev_err(hdev->dev, "failed to stop DMA QMAN 3\n");
	850	+ retval = -EIO;
	851	+ }
	852	+
	853	+ rc = goya_stop_queue(hdev,
	854	+ mmDMA_QM_4_GLBL_CFG1,
	855	+ mmDMA_QM_4_CP_STS,
	856	+ mmDMA_QM_4_GLBL_STS0);
	857	+
	858	+ if (rc) {
	859	+ dev_err(hdev->dev, "failed to stop DMA QMAN 4\n");
	860	+ retval = -EIO;
	861	+ }
	862	+
	863	+ return retval;
	864	+}
	865	+
	866	+static void goya_resume_external_queues(struct hl_device *hdev)
	867	+{
	868	+ WREG32(mmDMA_QM_0_GLBL_CFG1, 0);
	869	+ WREG32(mmDMA_QM_1_GLBL_CFG1, 0);
	870	+ WREG32(mmDMA_QM_2_GLBL_CFG1, 0);
	871	+ WREG32(mmDMA_QM_3_GLBL_CFG1, 0);
	872	+ WREG32(mmDMA_QM_4_GLBL_CFG1, 0);
	873	+}
	874	+
	875	+/*
	876	+ * goya_init_cpu_queues - Initialize PQ/CQ/EQ of CPU
	877	+ *
	878	+ * @hdev: pointer to hl_device structure
	879	+ *
	880	+ * Returns 0 on success
	881	+ *
	882	+ */
	883	+int goya_init_cpu_queues(struct hl_device *hdev)
	884	+{
	885	+ struct goya_device *goya = hdev->asic_specific;
	886	+ dma_addr_t bus_address;
	887	+ u32 status;
	888	+ struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ];
	889	+ int err;
	890	+
	891	+ if (!hdev->cpu_queues_enable)
	892	+ return 0;
	893	+
	894	+ if (goya->hw_cap_initialized & HW_CAP_CPU_Q)
	895	+ return 0;
	896	+
	897	+ bus_address = cpu_pq->bus_address +
	898	+ hdev->asic_prop.host_phys_base_address;
	899	+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_0, lower_32_bits(bus_address));
	900	+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_1, upper_32_bits(bus_address));
	901	+
	902	+ bus_address = hdev->cpu_accessible_dma_address +
	903	+ hdev->asic_prop.host_phys_base_address;
	904	+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_8, lower_32_bits(bus_address));
	905	+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_9, upper_32_bits(bus_address));
	906	+
	907	+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_5, HL_QUEUE_SIZE_IN_BYTES);
	908	+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_10, CPU_ACCESSIBLE_MEM_SIZE);
	909	+
	910	+ /* Used for EQ CI */
	911	+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_6, 0);
	912	+
	913	+ WREG32(mmCPU_IF_PF_PQ_PI, 0);
	914	+
	915	+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_7, PQ_INIT_STATUS_READY_FOR_CP);
	916	+
	917	+ WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
	918	+ GOYA_ASYNC_EVENT_ID_PI_UPDATE);
	919	+
	920	+ err = hl_poll_timeout(
	921	+ hdev,
	922	+ mmPSOC_GLOBAL_CONF_SCRATCHPAD_7,
	923	+ status,
	924	+ (status == PQ_INIT_STATUS_READY_FOR_HOST),
	925	+ 1000,
	926	+ GOYA_CPU_TIMEOUT_USEC);
	927	+
	928	+ if (err) {
	929	+ dev_err(hdev->dev,
	930	+ "Failed to communicate with ARM CPU (ArmCP timeout)\n");
	931	+ return -EIO;
	932	+ }
	933	+
	934	+ goya->hw_cap_initialized \|= HW_CAP_CPU_Q;
	935	+ return 0;
	936	+}
	937	+
599	938	static void goya_set_pll_refclk(struct hl_device *hdev)
600	939	{
601	940	WREG32(mmCPU_PLL_DIV_SEL_0, 0x0);
@@ -1023,144 +1362,644 @@ static void goya_init_golden_registers(struct hl_device *hdev) · 421 unmodified lines
1023	1362	goya->hw_cap_initialized \|= HW_CAP_GOLDEN;
1024	1363	}
1025	1364
1026		−
1027		−/*
1028		− * goya_push_fw_to_device - Push FW code to device
1029		− *
1030		− * @hdev: pointer to hl_device structure
1031		− *
1032		− * Copy fw code from firmware file to device memory.
1033		− * Returns 0 on success
1034		− *
1035		− */
1036		−static int goya_push_fw_to_device(struct hl_device hdev, const char fw_name,
1037		− void __iomem *dst)
	1365	+static void goya_init_mme_qman(struct hl_device *hdev)
1038	1366	{
1039		− const struct firmware *fw;
1040		− const u64 *fw_data;
1041		− size_t fw_size, i;
1042		− int rc;
	1367	+ u32 mtr_base_lo, mtr_base_hi;
	1368	+ u32 so_base_lo, so_base_hi;
	1369	+ u32 gic_base_lo, gic_base_hi;
	1370	+ u64 qman_base_addr;
1043	1371
1044		− rc = request_firmware(&fw, fw_name, hdev->dev);
	1372	+ mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
	1373	+ mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
	1374	+ so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
	1375	+ so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1045	1376
1046		− if (rc) {
1047		− dev_err(hdev->dev, "Failed to request %s\n", fw_name);
1048		− goto out;
1049		− }
	1377	+ gic_base_lo =
	1378	+ lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
	1379	+ gic_base_hi =
	1380	+ upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1050	1381
1051		− fw_size = fw->size;
1052		− if ((fw_size % 4) != 0) {
1053		− dev_err(hdev->dev, "illegal %s firmware size %zu\n",
1054		− fw_name, fw_size);
1055		− rc = -EINVAL;
1056		− goto out;
1057		− }
	1382	+ qman_base_addr = hdev->asic_prop.sram_base_address +
	1383	+ MME_QMAN_BASE_OFFSET;
1058	1384
1059		− dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
	1385	+ WREG32(mmMME_QM_PQ_BASE_LO, lower_32_bits(qman_base_addr));
	1386	+ WREG32(mmMME_QM_PQ_BASE_HI, upper_32_bits(qman_base_addr));
	1387	+ WREG32(mmMME_QM_PQ_SIZE, ilog2(MME_QMAN_LENGTH));
	1388	+ WREG32(mmMME_QM_PQ_PI, 0);
	1389	+ WREG32(mmMME_QM_PQ_CI, 0);
	1390	+ WREG32(mmMME_QM_CP_LDMA_SRC_BASE_LO_OFFSET, 0x10C0);
	1391	+ WREG32(mmMME_QM_CP_LDMA_SRC_BASE_HI_OFFSET, 0x10C4);
	1392	+ WREG32(mmMME_QM_CP_LDMA_TSIZE_OFFSET, 0x10C8);
	1393	+ WREG32(mmMME_QM_CP_LDMA_COMMIT_OFFSET, 0x10CC);
1060	1394
1061		− fw_data = (const u64 *) fw->data;
	1395	+ WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_LO, mtr_base_lo);
	1396	+ WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_HI, mtr_base_hi);
	1397	+ WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_LO, so_base_lo);
	1398	+ WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_HI, so_base_hi);
1062	1399
1063		− if ((fw->size % 8) != 0)
1064		− fw_size -= 8;
	1400	+ /* QMAN CQ has 8 cache lines */
	1401	+ WREG32(mmMME_QM_CQ_CFG1, 0x00080008);
1065	1402
1066		− for (i = 0 ; i < fw_size ; i += 8, fw_data++, dst += 8) {
1067		− if (!(i & (0x80000 - 1))) {
1068		− dev_dbg(hdev->dev,
1069		− "copied so far %zu out of %zu for %s firmware",
1070		− i, fw_size, fw_name);
1071		− usleep_range(20, 100);
1072		− }
	1403	+ WREG32(mmMME_QM_GLBL_ERR_ADDR_LO, gic_base_lo);
	1404	+ WREG32(mmMME_QM_GLBL_ERR_ADDR_HI, gic_base_hi);
1073	1405
1074		− writeq(*fw_data, dst);
1075		− }
	1406	+ WREG32(mmMME_QM_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_QM);
1076	1407
1077		− if ((fw->size % 8) != 0)
1078		− writel((const u32 ) fw_data, dst);
	1408	+ WREG32(mmMME_QM_GLBL_ERR_CFG, QMAN_MME_ERR_MSG_EN);
1079	1409
1080		−out:
1081		− release_firmware(fw);
1082		− return rc;
	1410	+ WREG32(mmMME_QM_GLBL_PROT, QMAN_MME_ERR_PROT);
	1411	+
	1412	+ WREG32(mmMME_QM_GLBL_CFG0, QMAN_MME_ENABLE);
1083	1413	}
1084	1414
1085		−static int goya_pldm_init_cpu(struct hl_device *hdev)
	1415	+static void goya_init_mme_cmdq(struct hl_device *hdev)
1086	1416	{
1087		− char fw_name[200];
1088		− void __iomem *dst;
1089		− u32 val, unit_rst_val;
1090		− int rc;
	1417	+ u32 mtr_base_lo, mtr_base_hi;
	1418	+ u32 so_base_lo, so_base_hi;
	1419	+ u32 gic_base_lo, gic_base_hi;
	1420	+ u64 qman_base_addr;
1091	1421
1092		− /* Must initialize SRAM scrambler before pushing u-boot to SRAM */
1093		− goya_init_golden_registers(hdev);
	1422	+ mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
	1423	+ mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
	1424	+ so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
	1425	+ so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1094	1426
1095		− /* Put ARM cores into reset */
1096		− WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL, CPU_RESET_ASSERT);
1097		− val = RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);
	1427	+ gic_base_lo =
	1428	+ lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
	1429	+ gic_base_hi =
	1430	+ upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1098	1431
1099		− /* Reset the CA53 MACRO */
1100		− unit_rst_val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
1101		− WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, CA53_RESET);
1102		− val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
1103		− WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, unit_rst_val);
1104		− val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
	1432	+ qman_base_addr = hdev->asic_prop.sram_base_address +
	1433	+ MME_QMAN_BASE_OFFSET;
1105	1434
1106		− snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-u-boot.bin");
1107		− dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + UBOOT_FW_OFFSET;
1108		− rc = goya_push_fw_to_device(hdev, fw_name, dst);
1109		− if (rc)
1110		− return rc;
	1435	+ WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_LO, mtr_base_lo);
	1436	+ WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_HI, mtr_base_hi);
	1437	+ WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_LO, so_base_lo);
	1438	+ WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_HI, so_base_hi);
1111	1439
1112		− snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-fit.itb");
1113		− dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;
1114		− rc = goya_push_fw_to_device(hdev, fw_name, dst);
1115		− if (rc)
1116		− return rc;
	1440	+ /* CMDQ CQ has 20 cache lines */
	1441	+ WREG32(mmMME_CMDQ_CQ_CFG1, 0x00140014);
1117	1442
1118		− WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_FIT_RDY);
1119		− WREG32(mmPSOC_GLOBAL_CONF_WARM_REBOOT, CPU_BOOT_STATUS_NA);
	1443	+ WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_LO, gic_base_lo);
	1444	+ WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_HI, gic_base_hi);
1120	1445
1121		− WREG32(mmCPU_CA53_CFG_RST_ADDR_LSB_0,
1122		− lower_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
1123		− WREG32(mmCPU_CA53_CFG_RST_ADDR_MSB_0,
1124		− upper_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
	1446	+ WREG32(mmMME_CMDQ_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_CMDQ);
1125	1447
1126		− /* Release ARM core 0 from reset */
1127		− WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL,
1128		− CPU_RESET_CORE0_DEASSERT);
1129		− val = RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);
	1448	+ WREG32(mmMME_CMDQ_GLBL_ERR_CFG, CMDQ_MME_ERR_MSG_EN);
1130	1449
1131		− return 0;
	1450	+ WREG32(mmMME_CMDQ_GLBL_PROT, CMDQ_MME_ERR_PROT);
	1451	+
	1452	+ WREG32(mmMME_CMDQ_GLBL_CFG0, CMDQ_MME_ENABLE);
1132	1453	}
1133	1454
1134		−/*
1135		− * FW component passes an offset from SRAM_BASE_ADDR in SCRATCHPAD_xx.
1136		− * The version string should be located by that offset.
1137		− */
1138		−static void goya_read_device_fw_version(struct hl_device *hdev,
1139		− enum goya_fw_component fwc)
	1455	+static void goya_init_mme_qmans(struct hl_device *hdev)
1140	1456	{
1141		− const char *name;
1142		− u32 ver_off;
1143		− char *dest;
	1457	+ struct goya_device *goya = hdev->asic_specific;
	1458	+ u32 so_base_lo, so_base_hi;
1144	1459
1145		− switch (fwc) {
1146		− case FW_COMP_UBOOT:
1147		− ver_off = RREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_29);
1148		− dest = hdev->asic_prop.uboot_ver;
1149		− name = "U-Boot";
1150		− break;
1151		− case FW_COMP_PREBOOT:
1152		− ver_off = RREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_28);
1153		− dest = hdev->asic_prop.preboot_ver;
1154		− name = "Preboot";
1155		− break;
1156		− default:
1157		− dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
	1460	+ if (goya->hw_cap_initialized & HW_CAP_MME)
1158	1461	return;
1159		− }
1160	1462
1161		− ver_off &= ~((u32)SRAM_BASE_ADDR);
	1463	+ so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
	1464	+ so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1162	1465
1163		− if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) {
	1466	+ WREG32(mmMME_SM_BASE_ADDRESS_LOW, so_base_lo);
	1467	+ WREG32(mmMME_SM_BASE_ADDRESS_HIGH, so_base_hi);
	1468	+
	1469	+ goya_init_mme_qman(hdev);
	1470	+ goya_init_mme_cmdq(hdev);
	1471	+
	1472	+ goya->hw_cap_initialized \|= HW_CAP_MME;
	1473	+}
	1474	+
	1475	+static void goya_init_tpc_qman(struct hl_device *hdev, u32 base_off, int tpc_id)
	1476	+{
	1477	+ u32 mtr_base_lo, mtr_base_hi;
	1478	+ u32 so_base_lo, so_base_hi;
	1479	+ u32 gic_base_lo, gic_base_hi;
	1480	+ u64 qman_base_addr;
	1481	+ u32 reg_off = tpc_id * (mmTPC1_QM_PQ_PI - mmTPC0_QM_PQ_PI);
	1482	+
	1483	+ mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
	1484	+ mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
	1485	+ so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
	1486	+ so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
	1487	+
	1488	+ gic_base_lo =
	1489	+ lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
	1490	+ gic_base_hi =
	1491	+ upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
	1492	+
	1493	+ qman_base_addr = hdev->asic_prop.sram_base_address + base_off;
	1494	+
	1495	+ WREG32(mmTPC0_QM_PQ_BASE_LO + reg_off, lower_32_bits(qman_base_addr));
	1496	+ WREG32(mmTPC0_QM_PQ_BASE_HI + reg_off, upper_32_bits(qman_base_addr));
	1497	+ WREG32(mmTPC0_QM_PQ_SIZE + reg_off, ilog2(TPC_QMAN_LENGTH));
	1498	+ WREG32(mmTPC0_QM_PQ_PI + reg_off, 0);
	1499	+ WREG32(mmTPC0_QM_PQ_CI + reg_off, 0);
	1500	+ WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET + reg_off, 0x10C0);
	1501	+ WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_HI_OFFSET + reg_off, 0x10C4);
	1502	+ WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET + reg_off, 0x10C8);
	1503	+ WREG32(mmTPC0_QM_CP_LDMA_COMMIT_OFFSET + reg_off, 0x10CC);
	1504	+
	1505	+ WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
	1506	+ WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
	1507	+ WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
	1508	+ WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
	1509	+
	1510	+ WREG32(mmTPC0_QM_CQ_CFG1 + reg_off, 0x00080008);
	1511	+
	1512	+ WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
	1513	+ WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
	1514	+
	1515	+ WREG32(mmTPC0_QM_GLBL_ERR_WDATA + reg_off,
	1516	+ GOYA_ASYNC_EVENT_ID_TPC0_QM + tpc_id);
	1517	+
	1518	+ WREG32(mmTPC0_QM_GLBL_ERR_CFG + reg_off, QMAN_TPC_ERR_MSG_EN);
	1519	+
	1520	+ WREG32(mmTPC0_QM_GLBL_PROT + reg_off, QMAN_TPC_ERR_PROT);
	1521	+
	1522	+ WREG32(mmTPC0_QM_GLBL_CFG0 + reg_off, QMAN_TPC_ENABLE);
	1523	+}
	1524	+
	1525	+static void goya_init_tpc_cmdq(struct hl_device *hdev, int tpc_id)
	1526	+{
	1527	+ u32 mtr_base_lo, mtr_base_hi;
	1528	+ u32 so_base_lo, so_base_hi;
	1529	+ u32 gic_base_lo, gic_base_hi;
	1530	+ u32 reg_off = tpc_id * (mmTPC1_CMDQ_CQ_CFG1 - mmTPC0_CMDQ_CQ_CFG1);
	1531	+
	1532	+ mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
	1533	+ mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
	1534	+ so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
	1535	+ so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
	1536	+
	1537	+ gic_base_lo =
	1538	+ lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
	1539	+ gic_base_hi =
	1540	+ upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
	1541	+
	1542	+ WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
	1543	+ WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
	1544	+ WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
	1545	+ WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
	1546	+
	1547	+ WREG32(mmTPC0_CMDQ_CQ_CFG1 + reg_off, 0x00140014);
	1548	+
	1549	+ WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
	1550	+ WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
	1551	+
	1552	+ WREG32(mmTPC0_CMDQ_GLBL_ERR_WDATA + reg_off,
	1553	+ GOYA_ASYNC_EVENT_ID_TPC0_CMDQ + tpc_id);
	1554	+
	1555	+ WREG32(mmTPC0_CMDQ_GLBL_ERR_CFG + reg_off, CMDQ_TPC_ERR_MSG_EN);
	1556	+
	1557	+ WREG32(mmTPC0_CMDQ_GLBL_PROT + reg_off, CMDQ_TPC_ERR_PROT);
	1558	+
	1559	+ WREG32(mmTPC0_CMDQ_GLBL_CFG0 + reg_off, CMDQ_TPC_ENABLE);
	1560	+}
	1561	+
	1562	+static void goya_init_tpc_qmans(struct hl_device *hdev)
	1563	+{
	1564	+ struct goya_device *goya = hdev->asic_specific;
	1565	+ u32 so_base_lo, so_base_hi;
	1566	+ u32 cfg_off = mmTPC1_CFG_SM_BASE_ADDRESS_LOW -
	1567	+ mmTPC0_CFG_SM_BASE_ADDRESS_LOW;
	1568	+ int i;
	1569	+
	1570	+ if (goya->hw_cap_initialized & HW_CAP_TPC)
	1571	+ return;
	1572	+
	1573	+ so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
	1574	+ so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
	1575	+
	1576	+ for (i = 0 ; i < TPC_MAX_NUM ; i++) {
	1577	+ WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_LOW + i * cfg_off,
	1578	+ so_base_lo);
	1579	+ WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_HIGH + i * cfg_off,
	1580	+ so_base_hi);
	1581	+ }
	1582	+
	1583	+ goya_init_tpc_qman(hdev, TPC0_QMAN_BASE_OFFSET, 0);
	1584	+ goya_init_tpc_qman(hdev, TPC1_QMAN_BASE_OFFSET, 1);
	1585	+ goya_init_tpc_qman(hdev, TPC2_QMAN_BASE_OFFSET, 2);
	1586	+ goya_init_tpc_qman(hdev, TPC3_QMAN_BASE_OFFSET, 3);
	1587	+ goya_init_tpc_qman(hdev, TPC4_QMAN_BASE_OFFSET, 4);
	1588	+ goya_init_tpc_qman(hdev, TPC5_QMAN_BASE_OFFSET, 5);
	1589	+ goya_init_tpc_qman(hdev, TPC6_QMAN_BASE_OFFSET, 6);
	1590	+ goya_init_tpc_qman(hdev, TPC7_QMAN_BASE_OFFSET, 7);
	1591	+
	1592	+ for (i = 0 ; i < TPC_MAX_NUM ; i++)
	1593	+ goya_init_tpc_cmdq(hdev, i);
	1594	+
	1595	+ goya->hw_cap_initialized \|= HW_CAP_TPC;
	1596	+}
	1597	+
	1598	+/*
	1599	+ * goya_disable_internal_queues - Disable internal queues
	1600	+ *
	1601	+ * @hdev: pointer to hl_device structure
	1602	+ *
	1603	+ */
	1604	+static void goya_disable_internal_queues(struct hl_device *hdev)
	1605	+{
	1606	+ WREG32(mmMME_QM_GLBL_CFG0, 0);
	1607	+ WREG32(mmMME_CMDQ_GLBL_CFG0, 0);
	1608	+
	1609	+ WREG32(mmTPC0_QM_GLBL_CFG0, 0);
	1610	+ WREG32(mmTPC0_CMDQ_GLBL_CFG0, 0);
	1611	+
	1612	+ WREG32(mmTPC1_QM_GLBL_CFG0, 0);
	1613	+ WREG32(mmTPC1_CMDQ_GLBL_CFG0, 0);
	1614	+
	1615	+ WREG32(mmTPC2_QM_GLBL_CFG0, 0);
	1616	+ WREG32(mmTPC2_CMDQ_GLBL_CFG0, 0);
	1617	+
	1618	+ WREG32(mmTPC3_QM_GLBL_CFG0, 0);
	1619	+ WREG32(mmTPC3_CMDQ_GLBL_CFG0, 0);
	1620	+
	1621	+ WREG32(mmTPC4_QM_GLBL_CFG0, 0);
	1622	+ WREG32(mmTPC4_CMDQ_GLBL_CFG0, 0);
	1623	+
	1624	+ WREG32(mmTPC5_QM_GLBL_CFG0, 0);
	1625	+ WREG32(mmTPC5_CMDQ_GLBL_CFG0, 0);
	1626	+
	1627	+ WREG32(mmTPC6_QM_GLBL_CFG0, 0);
	1628	+ WREG32(mmTPC6_CMDQ_GLBL_CFG0, 0);
	1629	+
	1630	+ WREG32(mmTPC7_QM_GLBL_CFG0, 0);
	1631	+ WREG32(mmTPC7_CMDQ_GLBL_CFG0, 0);
	1632	+}
	1633	+
	1634	+/*
	1635	+ * goya_stop_internal_queues - Stop internal queues
	1636	+ *
	1637	+ * @hdev: pointer to hl_device structure
	1638	+ *
	1639	+ * Returns 0 on success
	1640	+ *
	1641	+ */
	1642	+static int goya_stop_internal_queues(struct hl_device *hdev)
	1643	+{
	1644	+ int rc, retval = 0;
	1645	+
	1646	+ /*
	1647	+ * Each queue (QMAN) is a separate H/W logic. That means that each
	1648	+ * QMAN can be stopped independently and failure to stop one does NOT
	1649	+ * mandate we should not try to stop other QMANs
	1650	+ */
	1651	+
	1652	+ rc = goya_stop_queue(hdev,
	1653	+ mmMME_QM_GLBL_CFG1,
	1654	+ mmMME_QM_CP_STS,
	1655	+ mmMME_QM_GLBL_STS0);
	1656	+
	1657	+ if (rc) {
	1658	+ dev_err(hdev->dev, "failed to stop MME QMAN\n");
	1659	+ retval = -EIO;
	1660	+ }
	1661	+
	1662	+ rc = goya_stop_queue(hdev,
	1663	+ mmMME_CMDQ_GLBL_CFG1,
	1664	+ mmMME_CMDQ_CP_STS,
	1665	+ mmMME_CMDQ_GLBL_STS0);
	1666	+
	1667	+ if (rc) {
	1668	+ dev_err(hdev->dev, "failed to stop MME CMDQ\n");
	1669	+ retval = -EIO;
	1670	+ }
	1671	+
	1672	+ rc = goya_stop_queue(hdev,
	1673	+ mmTPC0_QM_GLBL_CFG1,
	1674	+ mmTPC0_QM_CP_STS,
	1675	+ mmTPC0_QM_GLBL_STS0);
	1676	+
	1677	+ if (rc) {
	1678	+ dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n");
	1679	+ retval = -EIO;
	1680	+ }
	1681	+
	1682	+ rc = goya_stop_queue(hdev,
	1683	+ mmTPC0_CMDQ_GLBL_CFG1,
	1684	+ mmTPC0_CMDQ_CP_STS,
	1685	+ mmTPC0_CMDQ_GLBL_STS0);
	1686	+
	1687	+ if (rc) {
	1688	+ dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n");
	1689	+ retval = -EIO;
	1690	+ }
	1691	+
	1692	+ rc = goya_stop_queue(hdev,
	1693	+ mmTPC1_QM_GLBL_CFG1,
	1694	+ mmTPC1_QM_CP_STS,
	1695	+ mmTPC1_QM_GLBL_STS0);
	1696	+
	1697	+ if (rc) {
	1698	+ dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n");
	1699	+ retval = -EIO;
	1700	+ }
	1701	+
	1702	+ rc = goya_stop_queue(hdev,
	1703	+ mmTPC1_CMDQ_GLBL_CFG1,
	1704	+ mmTPC1_CMDQ_CP_STS,
	1705	+ mmTPC1_CMDQ_GLBL_STS0);
	1706	+
	1707	+ if (rc) {
	1708	+ dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n");
	1709	+ retval = -EIO;
	1710	+ }
	1711	+
	1712	+ rc = goya_stop_queue(hdev,
	1713	+ mmTPC2_QM_GLBL_CFG1,
	1714	+ mmTPC2_QM_CP_STS,
	1715	+ mmTPC2_QM_GLBL_STS0);
	1716	+
	1717	+ if (rc) {
	1718	+ dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n");
	1719	+ retval = -EIO;
	1720	+ }
	1721	+
	1722	+ rc = goya_stop_queue(hdev,
	1723	+ mmTPC2_CMDQ_GLBL_CFG1,
	1724	+ mmTPC2_CMDQ_CP_STS,
	1725	+ mmTPC2_CMDQ_GLBL_STS0);
	1726	+
	1727	+ if (rc) {
	1728	+ dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n");
	1729	+ retval = -EIO;
	1730	+ }
	1731	+
	1732	+ rc = goya_stop_queue(hdev,
	1733	+ mmTPC3_QM_GLBL_CFG1,
	1734	+ mmTPC3_QM_CP_STS,
	1735	+ mmTPC3_QM_GLBL_STS0);
	1736	+
	1737	+ if (rc) {
	1738	+ dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n");
	1739	+ retval = -EIO;
	1740	+ }
	1741	+
	1742	+ rc = goya_stop_queue(hdev,
	1743	+ mmTPC3_CMDQ_GLBL_CFG1,
	1744	+ mmTPC3_CMDQ_CP_STS,
	1745	+ mmTPC3_CMDQ_GLBL_STS0);
	1746	+
	1747	+ if (rc) {
	1748	+ dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n");
	1749	+ retval = -EIO;
	1750	+ }
	1751	+
	1752	+ rc = goya_stop_queue(hdev,
	1753	+ mmTPC4_QM_GLBL_CFG1,
	1754	+ mmTPC4_QM_CP_STS,
	1755	+ mmTPC4_QM_GLBL_STS0);
	1756	+
	1757	+ if (rc) {
	1758	+ dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n");
	1759	+ retval = -EIO;
	1760	+ }
	1761	+
	1762	+ rc = goya_stop_queue(hdev,
	1763	+ mmTPC4_CMDQ_GLBL_CFG1,
	1764	+ mmTPC4_CMDQ_CP_STS,
	1765	+ mmTPC4_CMDQ_GLBL_STS0);
	1766	+
	1767	+ if (rc) {
	1768	+ dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n");
	1769	+ retval = -EIO;
	1770	+ }
	1771	+
	1772	+ rc = goya_stop_queue(hdev,
	1773	+ mmTPC5_QM_GLBL_CFG1,
	1774	+ mmTPC5_QM_CP_STS,
	1775	+ mmTPC5_QM_GLBL_STS0);
	1776	+
	1777	+ if (rc) {
	1778	+ dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n");
	1779	+ retval = -EIO;
	1780	+ }
	1781	+
	1782	+ rc = goya_stop_queue(hdev,
	1783	+ mmTPC5_CMDQ_GLBL_CFG1,
	1784	+ mmTPC5_CMDQ_CP_STS,
	1785	+ mmTPC5_CMDQ_GLBL_STS0);
	1786	+
	1787	+ if (rc) {
	1788	+ dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n");
	1789	+ retval = -EIO;
	1790	+ }
	1791	+
	1792	+ rc = goya_stop_queue(hdev,
	1793	+ mmTPC6_QM_GLBL_CFG1,
	1794	+ mmTPC6_QM_CP_STS,
	1795	+ mmTPC6_QM_GLBL_STS0);
	1796	+
	1797	+ if (rc) {
	1798	+ dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n");
	1799	+ retval = -EIO;
	1800	+ }
	1801	+
	1802	+ rc = goya_stop_queue(hdev,
	1803	+ mmTPC6_CMDQ_GLBL_CFG1,
	1804	+ mmTPC6_CMDQ_CP_STS,
	1805	+ mmTPC6_CMDQ_GLBL_STS0);
	1806	+
	1807	+ if (rc) {
	1808	+ dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n");
	1809	+ retval = -EIO;
	1810	+ }
	1811	+
	1812	+ rc = goya_stop_queue(hdev,
	1813	+ mmTPC7_QM_GLBL_CFG1,
	1814	+ mmTPC7_QM_CP_STS,
	1815	+ mmTPC7_QM_GLBL_STS0);
	1816	+
	1817	+ if (rc) {
	1818	+ dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n");
	1819	+ retval = -EIO;
	1820	+ }
	1821	+
	1822	+ rc = goya_stop_queue(hdev,
	1823	+ mmTPC7_CMDQ_GLBL_CFG1,
	1824	+ mmTPC7_CMDQ_CP_STS,
	1825	+ mmTPC7_CMDQ_GLBL_STS0);
	1826	+
	1827	+ if (rc) {
	1828	+ dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n");
	1829	+ retval = -EIO;
	1830	+ }
	1831	+
	1832	+ return retval;
	1833	+}
	1834	+
	1835	+static void goya_resume_internal_queues(struct hl_device *hdev)
	1836	+{
	1837	+ WREG32(mmMME_QM_GLBL_CFG1, 0);
	1838	+ WREG32(mmMME_CMDQ_GLBL_CFG1, 0);
	1839	+
	1840	+ WREG32(mmTPC0_QM_GLBL_CFG1, 0);
	1841	+ WREG32(mmTPC0_CMDQ_GLBL_CFG1, 0);
	1842	+
	1843	+ WREG32(mmTPC1_QM_GLBL_CFG1, 0);
	1844	+ WREG32(mmTPC1_CMDQ_GLBL_CFG1, 0);
	1845	+
	1846	+ WREG32(mmTPC2_QM_GLBL_CFG1, 0);
	1847	+ WREG32(mmTPC2_CMDQ_GLBL_CFG1, 0);
	1848	+
	1849	+ WREG32(mmTPC3_QM_GLBL_CFG1, 0);
	1850	+ WREG32(mmTPC3_CMDQ_GLBL_CFG1, 0);
	1851	+
	1852	+ WREG32(mmTPC4_QM_GLBL_CFG1, 0);
	1853	+ WREG32(mmTPC4_CMDQ_GLBL_CFG1, 0);
	1854	+
	1855	+ WREG32(mmTPC5_QM_GLBL_CFG1, 0);
	1856	+ WREG32(mmTPC5_CMDQ_GLBL_CFG1, 0);
	1857	+
	1858	+ WREG32(mmTPC6_QM_GLBL_CFG1, 0);
	1859	+ WREG32(mmTPC6_CMDQ_GLBL_CFG1, 0);
	1860	+
	1861	+ WREG32(mmTPC7_QM_GLBL_CFG1, 0);
	1862	+ WREG32(mmTPC7_CMDQ_GLBL_CFG1, 0);
	1863	+}
	1864	+
	1865	+
	1866	+/*
	1867	+ * goya_push_fw_to_device - Push FW code to device
	1868	+ *
	1869	+ * @hdev: pointer to hl_device structure
	1870	+ *
	1871	+ * Copy fw code from firmware file to device memory.
	1872	+ * Returns 0 on success
	1873	+ *
	1874	+ */
	1875	+static int goya_push_fw_to_device(struct hl_device hdev, const char fw_name,
	1876	+ void __iomem *dst)
	1877	+{
	1878	+ const struct firmware *fw;
	1879	+ const u64 *fw_data;
	1880	+ size_t fw_size, i;
	1881	+ int rc;
	1882	+
	1883	+ rc = request_firmware(&fw, fw_name, hdev->dev);
	1884	+
	1885	+ if (rc) {
	1886	+ dev_err(hdev->dev, "Failed to request %s\n", fw_name);
	1887	+ goto out;
	1888	+ }
	1889	+
	1890	+ fw_size = fw->size;
	1891	+ if ((fw_size % 4) != 0) {
	1892	+ dev_err(hdev->dev, "illegal %s firmware size %zu\n",
	1893	+ fw_name, fw_size);
	1894	+ rc = -EINVAL;
	1895	+ goto out;
	1896	+ }
	1897	+
	1898	+ dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
	1899	+
	1900	+ fw_data = (const u64 *) fw->data;
	1901	+
	1902	+ if ((fw->size % 8) != 0)
	1903	+ fw_size -= 8;
	1904	+
	1905	+ for (i = 0 ; i < fw_size ; i += 8, fw_data++, dst += 8) {
	1906	+ if (!(i & (0x80000 - 1))) {
	1907	+ dev_dbg(hdev->dev,
	1908	+ "copied so far %zu out of %zu for %s firmware",
	1909	+ i, fw_size, fw_name);
	1910	+ usleep_range(20, 100);
	1911	+ }
	1912	+
	1913	+ writeq(*fw_data, dst);
	1914	+ }
	1915	+
	1916	+ if ((fw->size % 8) != 0)
	1917	+ writel((const u32 ) fw_data, dst);
	1918	+
	1919	+out:
	1920	+ release_firmware(fw);
	1921	+ return rc;
	1922	+}
	1923	+
	1924	+static int goya_pldm_init_cpu(struct hl_device *hdev)
	1925	+{
	1926	+ char fw_name[200];
	1927	+ void __iomem *dst;
	1928	+ u32 val, unit_rst_val;
	1929	+ int rc;
	1930	+
	1931	+ /* Must initialize SRAM scrambler before pushing u-boot to SRAM */
	1932	+ goya_init_golden_registers(hdev);
	1933	+
	1934	+ /* Put ARM cores into reset */
	1935	+ WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL, CPU_RESET_ASSERT);
	1936	+ val = RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);
	1937	+
	1938	+ /* Reset the CA53 MACRO */
	1939	+ unit_rst_val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
	1940	+ WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, CA53_RESET);
	1941	+ val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
	1942	+ WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, unit_rst_val);
	1943	+ val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
	1944	+
	1945	+ snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-u-boot.bin");
	1946	+ dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + UBOOT_FW_OFFSET;
	1947	+ rc = goya_push_fw_to_device(hdev, fw_name, dst);
	1948	+ if (rc)
	1949	+ return rc;
	1950	+
	1951	+ snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-fit.itb");
	1952	+ dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;
	1953	+ rc = goya_push_fw_to_device(hdev, fw_name, dst);
	1954	+ if (rc)
	1955	+ return rc;
	1956	+
	1957	+ WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_FIT_RDY);
	1958	+ WREG32(mmPSOC_GLOBAL_CONF_WARM_REBOOT, CPU_BOOT_STATUS_NA);
	1959	+
	1960	+ WREG32(mmCPU_CA53_CFG_RST_ADDR_LSB_0,
	1961	+ lower_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
	1962	+ WREG32(mmCPU_CA53_CFG_RST_ADDR_MSB_0,
	1963	+ upper_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
	1964	+
	1965	+ /* Release ARM core 0 from reset */
	1966	+ WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL,
	1967	+ CPU_RESET_CORE0_DEASSERT);
	1968	+ val = RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);
	1969	+
	1970	+ return 0;
	1971	+}
	1972	+
	1973	+/*
	1974	+ * FW component passes an offset from SRAM_BASE_ADDR in SCRATCHPAD_xx.
	1975	+ * The version string should be located by that offset.
	1976	+ */
	1977	+static void goya_read_device_fw_version(struct hl_device *hdev,
	1978	+ enum goya_fw_component fwc)
	1979	+{
	1980	+ const char *name;
	1981	+ u32 ver_off;
	1982	+ char *dest;
	1983	+
	1984	+ switch (fwc) {
	1985	+ case FW_COMP_UBOOT:
	1986	+ ver_off = RREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_29);
	1987	+ dest = hdev->asic_prop.uboot_ver;
	1988	+ name = "U-Boot";
	1989	+ break;
	1990	+ case FW_COMP_PREBOOT:
	1991	+ ver_off = RREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_28);
	1992	+ dest = hdev->asic_prop.preboot_ver;
	1993	+ name = "Preboot";
	1994	+ break;
	1995	+ default:
	1996	+ dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
	1997	+ return;
	1998	+ }
	1999	+
	2000	+ ver_off &= ~((u32)SRAM_BASE_ADDR);
	2001	+
	2002	+ if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) {
1164	2003	memcpy_fromio(dest, hdev->pcie_bar[SRAM_CFG_BAR_ID] + ver_off,
1165	2004	VERSION_MAX_LEN);
1166	2005	} else {
@@ -1344,6 +2183,19 @@ static int goya_hw_init(struct hl_device *hdev) · 177 unmodified lines
1344	2183
1345	2184	goya_init_security(hdev);
1346	2185
	2186	+ goya_init_dma_qmans(hdev);
	2187	+
	2188	+ goya_init_mme_qmans(hdev);
	2189	+
	2190	+ goya_init_tpc_qmans(hdev);
	2191	+
	2192	+ rc = goya_init_cpu_queues(hdev);
	2193	+ if (rc) {
	2194	+ dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n",
	2195	+ rc);
	2196	+ goto disable_queues;
	2197	+ }
	2198	+
1347	2199	/* CPU initialization is finished, we can now move to 48 bit DMA mask */
1348	2200	rc = pci_set_dma_mask(hdev->pdev, DMA_BIT_MASK(48));
1349	2201	if (rc) {
@@ -1352,7 +2204,7 @@ static int goya_hw_init(struct hl_device *hdev) · 2 unmodified lines
1352	2204	if (rc) {
1353	2205	dev_err(hdev->dev,
1354	2206	"Unable to set pci dma mask to 32 bits\n");
1355		− return rc;
	2207	+ goto disable_pci_access;
1356	2208	}
1357	2209	}
1358	2210
@@ -1364,7 +2216,7 @@ static int goya_hw_init(struct hl_device *hdev) · 5 unmodified lines
1364	2216	if (rc) {
1365	2217	dev_err(hdev->dev,
1366	2218	"Unable to set pci consistent dma mask to 32 bits\n");
1367		− return rc;
	2219	+ goto disable_pci_access;
1368	2220	}
1369	2221	}
1370	2222
@@ -1372,6 +2224,14 @@ static int goya_hw_init(struct hl_device *hdev) · 1 unmodified line
1372	2224	val = RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
1373	2225
1374	2226	return 0;
	2227	+
	2228	+disable_pci_access:
	2229	+ goya_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
	2230	+disable_queues:
	2231	+ goya_disable_internal_queues(hdev);
	2232	+ goya_disable_external_queues(hdev);
	2233	+
	2234	+ return rc;
1375	2235	}
1376	2236
1377	2237	/*
@@ -1460,12 +2320,40 @@ static void goya_hw_fini(struct hl_device *hdev, bool hard_reset) · 82 unmodified lines
1460	2320
1461	2321	int goya_suspend(struct hl_device *hdev)
1462	2322	{
1463		− return 0;
	2323	+ int rc;
	2324	+
	2325	+ rc = goya_stop_internal_queues(hdev);
	2326	+
	2327	+ if (rc) {
	2328	+ dev_err(hdev->dev, "failed to stop internal queues\n");
	2329	+ return rc;
	2330	+ }
	2331	+
	2332	+ rc = goya_stop_external_queues(hdev);
	2333	+
	2334	+ if (rc) {
	2335	+ dev_err(hdev->dev, "failed to stop external queues\n");
	2336	+ return rc;
	2337	+ }
	2338	+
	2339	+ rc = goya_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
	2340	+ if (rc)
	2341	+ dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
	2342	+
	2343	+ return rc;
1464	2344	}
1465	2345
1466	2346	int goya_resume(struct hl_device *hdev)
1467	2347	{
1468		− return 0;
	2348	+ int rc;
	2349	+
	2350	+ goya_resume_external_queues(hdev);
	2351	+ goya_resume_internal_queues(hdev);
	2352	+
	2353	+ rc = goya_send_pci_access_msg(hdev, ARMCP_PACKET_ENABLE_PCI_ACCESS);
	2354	+ if (rc)
	2355	+ dev_err(hdev->dev, "Failed to enable PCI access from CPU\n");
	2356	+ return rc;
1469	2357	}
1470	2358
1471	2359	int goya_mmap(struct hl_fpriv hpriv, struct vm_area_struct vma)
@@ -1489,6 +2377,101 @@ int goya_cb_mmap(struct hl_device hdev, struct vm_area_struct vma, · 17 unmodified lines
1489	2377	return rc;
1490	2378	}
1491	2379
	2380	+void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
	2381	+{
	2382	+ u32 db_reg_offset, db_value;
	2383	+ bool invalid_queue = false;
	2384	+
	2385	+ switch (hw_queue_id) {
	2386	+ case GOYA_QUEUE_ID_DMA_0:
	2387	+ db_reg_offset = mmDMA_QM_0_PQ_PI;
	2388	+ break;
	2389	+
	2390	+ case GOYA_QUEUE_ID_DMA_1:
	2391	+ db_reg_offset = mmDMA_QM_1_PQ_PI;
	2392	+ break;
	2393	+
	2394	+ case GOYA_QUEUE_ID_DMA_2:
	2395	+ db_reg_offset = mmDMA_QM_2_PQ_PI;
	2396	+ break;
	2397	+
	2398	+ case GOYA_QUEUE_ID_DMA_3:
	2399	+ db_reg_offset = mmDMA_QM_3_PQ_PI;
	2400	+ break;
	2401	+
	2402	+ case GOYA_QUEUE_ID_DMA_4:
	2403	+ db_reg_offset = mmDMA_QM_4_PQ_PI;
	2404	+ break;
	2405	+
	2406	+ case GOYA_QUEUE_ID_CPU_PQ:
	2407	+ if (hdev->cpu_queues_enable)
	2408	+ db_reg_offset = mmCPU_IF_PF_PQ_PI;
	2409	+ else
	2410	+ invalid_queue = true;
	2411	+ break;
	2412	+
	2413	+ case GOYA_QUEUE_ID_MME:
	2414	+ db_reg_offset = mmMME_QM_PQ_PI;
	2415	+ break;
	2416	+
	2417	+ case GOYA_QUEUE_ID_TPC0:
	2418	+ db_reg_offset = mmTPC0_QM_PQ_PI;
	2419	+ break;
	2420	+
	2421	+ case GOYA_QUEUE_ID_TPC1:
	2422	+ db_reg_offset = mmTPC1_QM_PQ_PI;
	2423	+ break;
	2424	+
	2425	+ case GOYA_QUEUE_ID_TPC2:
	2426	+ db_reg_offset = mmTPC2_QM_PQ_PI;
	2427	+ break;
	2428	+
	2429	+ case GOYA_QUEUE_ID_TPC3:
	2430	+ db_reg_offset = mmTPC3_QM_PQ_PI;
	2431	+ break;
	2432	+
	2433	+ case GOYA_QUEUE_ID_TPC4:
	2434	+ db_reg_offset = mmTPC4_QM_PQ_PI;
	2435	+ break;
	2436	+
	2437	+ case GOYA_QUEUE_ID_TPC5:
	2438	+ db_reg_offset = mmTPC5_QM_PQ_PI;
	2439	+ break;
	2440	+
	2441	+ case GOYA_QUEUE_ID_TPC6:
	2442	+ db_reg_offset = mmTPC6_QM_PQ_PI;
	2443	+ break;
	2444	+
	2445	+ case GOYA_QUEUE_ID_TPC7:
	2446	+ db_reg_offset = mmTPC7_QM_PQ_PI;
	2447	+ break;
	2448	+
	2449	+ default:
	2450	+ invalid_queue = true;
	2451	+ }
	2452	+
	2453	+ if (invalid_queue) {
	2454	+ /* Should never get here */
	2455	+ dev_err(hdev->dev, "h/w queue %d is invalid. Can't set pi\n",
	2456	+ hw_queue_id);
	2457	+ return;
	2458	+ }
	2459	+
	2460	+ db_value = pi;
	2461	+
	2462	+ /* ring the doorbell */
	2463	+ WREG32(db_reg_offset, db_value);
	2464	+
	2465	+ if (hw_queue_id == GOYA_QUEUE_ID_CPU_PQ)
	2466	+ WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
	2467	+ GOYA_ASYNC_EVENT_ID_PI_UPDATE);
	2468	+}
	2469	+
	2470	+void goya_flush_pq_write(struct hl_device hdev, u64 pq, u64 exp_val)
	2471	+{
	2472	+ /* Not needed in Goya */
	2473	+}
	2474	+
1492	2475	void goya_dma_alloc_coherent(struct hl_device hdev, size_t size,
1493	2476	dma_addr_t *dma_handle, gfp_t flags)
1494	2477	{
@@ -1501,6 +2484,315 @@ void goya_dma_free_coherent(struct hl_device hdev, size_t size, void cpu_addr, · 6 unmodified lines
1501	2484	dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle);
1502	2485	}
1503	2486
	2487	+void goya_get_int_queue_base(struct hl_device hdev, u32 queue_id,
	2488	+ dma_addr_t dma_handle, u16 queue_len)
	2489	+{
	2490	+ void *base;
	2491	+ u32 offset;
	2492	+
	2493	+ *dma_handle = hdev->asic_prop.sram_base_address;
	2494	+
	2495	+ base = hdev->pcie_bar[SRAM_CFG_BAR_ID];
	2496	+
	2497	+ switch (queue_id) {
	2498	+ case GOYA_QUEUE_ID_MME:
	2499	+ offset = MME_QMAN_BASE_OFFSET;
	2500	+ *queue_len = MME_QMAN_LENGTH;
	2501	+ break;
	2502	+ case GOYA_QUEUE_ID_TPC0:
	2503	+ offset = TPC0_QMAN_BASE_OFFSET;
	2504	+ *queue_len = TPC_QMAN_LENGTH;
	2505	+ break;
	2506	+ case GOYA_QUEUE_ID_TPC1:
	2507	+ offset = TPC1_QMAN_BASE_OFFSET;
	2508	+ *queue_len = TPC_QMAN_LENGTH;
	2509	+ break;
	2510	+ case GOYA_QUEUE_ID_TPC2:
	2511	+ offset = TPC2_QMAN_BASE_OFFSET;
	2512	+ *queue_len = TPC_QMAN_LENGTH;
	2513	+ break;
	2514	+ case GOYA_QUEUE_ID_TPC3:
	2515	+ offset = TPC3_QMAN_BASE_OFFSET;
	2516	+ *queue_len = TPC_QMAN_LENGTH;
	2517	+ break;
	2518	+ case GOYA_QUEUE_ID_TPC4:
	2519	+ offset = TPC4_QMAN_BASE_OFFSET;
	2520	+ *queue_len = TPC_QMAN_LENGTH;
	2521	+ break;
	2522	+ case GOYA_QUEUE_ID_TPC5:
	2523	+ offset = TPC5_QMAN_BASE_OFFSET;
	2524	+ *queue_len = TPC_QMAN_LENGTH;
	2525	+ break;
	2526	+ case GOYA_QUEUE_ID_TPC6:
	2527	+ offset = TPC6_QMAN_BASE_OFFSET;
	2528	+ *queue_len = TPC_QMAN_LENGTH;
	2529	+ break;
	2530	+ case GOYA_QUEUE_ID_TPC7:
	2531	+ offset = TPC7_QMAN_BASE_OFFSET;
	2532	+ *queue_len = TPC_QMAN_LENGTH;
	2533	+ break;
	2534	+ default:
	2535	+ dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id);
	2536	+ return NULL;
	2537	+ }
	2538	+
	2539	+ base += offset;
	2540	+ *dma_handle += offset;
	2541	+
	2542	+ return base;
	2543	+}
	2544	+
	2545	+int goya_send_cpu_message(struct hl_device hdev, u32 msg, u16 len,
	2546	+ u32 timeout, long *result)
	2547	+{
	2548	+ struct goya_device *goya = hdev->asic_specific;
	2549	+ struct armcp_packet *pkt;
	2550	+ dma_addr_t pkt_dma_addr;
	2551	+ u32 tmp;
	2552	+ int rc = 0;
	2553	+
	2554	+ if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) {
	2555	+ if (result)
	2556	+ *result = 0;
	2557	+ return 0;
	2558	+ }
	2559	+
	2560	+ if (len > CPU_CB_SIZE) {
	2561	+ dev_err(hdev->dev, "Invalid CPU message size of %d bytes\n",
	2562	+ len);
	2563	+ return -ENOMEM;
	2564	+ }
	2565	+
	2566	+ pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len,
	2567	+ &pkt_dma_addr);
	2568	+ if (!pkt) {
	2569	+ dev_err(hdev->dev,
	2570	+ "Failed to allocate DMA memory for packet to CPU\n");
	2571	+ return -ENOMEM;
	2572	+ }
	2573	+
	2574	+ memcpy(pkt, msg, len);
	2575	+
	2576	+ mutex_lock(&hdev->send_cpu_message_lock);
	2577	+
	2578	+ if (hdev->disabled)
	2579	+ goto out;
	2580	+
	2581	+ rc = hl_hw_queue_send_cb_no_cmpl(hdev, GOYA_QUEUE_ID_CPU_PQ, len,
	2582	+ pkt_dma_addr);
	2583	+ if (rc) {
	2584	+ dev_err(hdev->dev, "Failed to send CB on CPU PQ (%d)\n", rc);
	2585	+ goto out;
	2586	+ }
	2587	+
	2588	+ rc = hl_poll_timeout_memory(hdev, (u64) (uintptr_t) &pkt->fence,
	2589	+ timeout, &tmp);
	2590	+
	2591	+ hl_hw_queue_inc_ci_kernel(hdev, GOYA_QUEUE_ID_CPU_PQ);
	2592	+
	2593	+ if (rc == -ETIMEDOUT) {
	2594	+ dev_err(hdev->dev,
	2595	+ "Timeout while waiting for CPU packet fence\n");
	2596	+ goto out;
	2597	+ }
	2598	+
	2599	+ if (tmp == ARMCP_PACKET_FENCE_VAL) {
	2600	+ rc = (pkt->ctl & ARMCP_PKT_CTL_RC_MASK) >>
	2601	+ ARMCP_PKT_CTL_RC_SHIFT;
	2602	+ if (rc) {
	2603	+ dev_err(hdev->dev,
	2604	+ "F/W ERROR %d for CPU packet %d\n",
	2605	+ rc, (pkt->ctl & ARMCP_PKT_CTL_OPCODE_MASK)
	2606	+ >> ARMCP_PKT_CTL_OPCODE_SHIFT);
	2607	+ rc = -EINVAL;
	2608	+ } else if (result) {
	2609	+ *result = pkt->result;
	2610	+ }
	2611	+ } else {
	2612	+ dev_err(hdev->dev, "CPU packet wrong fence value\n");
	2613	+ rc = -EINVAL;
	2614	+ }
	2615	+
	2616	+out:
	2617	+ mutex_unlock(&hdev->send_cpu_message_lock);
	2618	+
	2619	+ hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, len, pkt);
	2620	+
	2621	+ return rc;
	2622	+}
	2623	+
	2624	+int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id)
	2625	+{
	2626	+ struct packet_msg_prot *fence_pkt;
	2627	+ dma_addr_t pkt_dma_addr;
	2628	+ u32 fence_val, tmp;
	2629	+ dma_addr_t fence_dma_addr;
	2630	+ u32 *fence_ptr;
	2631	+ int rc;
	2632	+
	2633	+ fence_val = GOYA_QMAN0_FENCE_VAL;
	2634	+
	2635	+ fence_ptr = hdev->asic_funcs->dma_pool_zalloc(hdev, 4, GFP_KERNEL,
	2636	+ &fence_dma_addr);
	2637	+ if (!fence_ptr) {
	2638	+ dev_err(hdev->dev,
	2639	+ "Failed to allocate memory for queue testing\n");
	2640	+ return -ENOMEM;
	2641	+ }
	2642	+
	2643	+ *fence_ptr = 0;
	2644	+
	2645	+ fence_pkt = hdev->asic_funcs->dma_pool_zalloc(hdev,
	2646	+ sizeof(struct packet_msg_prot),
	2647	+ GFP_KERNEL, &pkt_dma_addr);
	2648	+ if (!fence_pkt) {
	2649	+ dev_err(hdev->dev,
	2650	+ "Failed to allocate packet for queue testing\n");
	2651	+ rc = -ENOMEM;
	2652	+ goto free_fence_ptr;
	2653	+ }
	2654	+
	2655	+ fence_pkt->ctl = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) \|
	2656	+ (1 << GOYA_PKT_CTL_EB_SHIFT) \|
	2657	+ (1 << GOYA_PKT_CTL_MB_SHIFT);
	2658	+ fence_pkt->value = fence_val;
	2659	+ fence_pkt->addr = fence_dma_addr +
	2660	+ hdev->asic_prop.host_phys_base_address;
	2661	+
	2662	+ rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id,
	2663	+ sizeof(struct packet_msg_prot),
	2664	+ pkt_dma_addr);
	2665	+ if (rc) {
	2666	+ dev_err(hdev->dev,
	2667	+ "Failed to send fence packet\n");
	2668	+ goto free_pkt;
	2669	+ }
	2670	+
	2671	+ rc = hl_poll_timeout_memory(hdev, (u64) (uintptr_t) fence_ptr,
	2672	+ GOYA_TEST_QUEUE_WAIT_USEC, &tmp);
	2673	+
	2674	+ hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
	2675	+
	2676	+ if ((!rc) && (tmp == fence_val)) {
	2677	+ dev_info(hdev->dev,
	2678	+ "queue test on H/W queue %d succeeded\n",
	2679	+ hw_queue_id);
	2680	+ } else {
	2681	+ dev_err(hdev->dev,
	2682	+ "H/W queue %d test failed (scratch(0x%08llX) == 0x%08X)\n",
	2683	+ hw_queue_id, (unsigned long long) fence_dma_addr, tmp);
	2684	+ rc = -EINVAL;
	2685	+ }
	2686	+
	2687	+free_pkt:
	2688	+ hdev->asic_funcs->dma_pool_free(hdev, (void *) fence_pkt,
	2689	+ pkt_dma_addr);
	2690	+free_fence_ptr:
	2691	+ hdev->asic_funcs->dma_pool_free(hdev, (void *) fence_ptr,
	2692	+ fence_dma_addr);
	2693	+ return rc;
	2694	+}
	2695	+
	2696	+int goya_test_cpu_queue(struct hl_device *hdev)
	2697	+{
	2698	+ struct armcp_packet test_pkt;
	2699	+ long result;
	2700	+ int rc;
	2701	+
	2702	+ /* cpu_queues_enable flag is always checked in send cpu message */
	2703	+
	2704	+ memset(&test_pkt, 0, sizeof(test_pkt));
	2705	+
	2706	+ test_pkt.ctl = ARMCP_PACKET_TEST << ARMCP_PKT_CTL_OPCODE_SHIFT;
	2707	+ test_pkt.value = ARMCP_PACKET_FENCE_VAL;
	2708	+
	2709	+ rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
	2710	+ sizeof(test_pkt), HL_DEVICE_TIMEOUT_USEC, &result);
	2711	+
	2712	+ if (!rc)
	2713	+ dev_info(hdev->dev, "queue test on CPU queue succeeded\n");
	2714	+ else
	2715	+ dev_err(hdev->dev, "CPU queue test failed (0x%08lX)\n", result);
	2716	+
	2717	+ return rc;
	2718	+}
	2719	+
	2720	+static int goya_test_queues(struct hl_device *hdev)
	2721	+{
	2722	+ struct goya_device *goya = hdev->asic_specific;
	2723	+ int i, rc, ret_val = 0;
	2724	+
	2725	+ for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) {
	2726	+ rc = goya_test_queue(hdev, i);
	2727	+ if (rc)
	2728	+ ret_val = -EINVAL;
	2729	+ }
	2730	+
	2731	+ if (hdev->cpu_queues_enable) {
	2732	+ rc = goya->test_cpu_queue(hdev);
	2733	+ if (rc)
	2734	+ ret_val = -EINVAL;
	2735	+ }
	2736	+
	2737	+ return ret_val;
	2738	+}
	2739	+
	2740	+void goya_dma_pool_zalloc(struct hl_device hdev, size_t size, gfp_t mem_flags,
	2741	+ dma_addr_t *dma_handle)
	2742	+{
	2743	+ if (size > GOYA_DMA_POOL_BLK_SIZE)
	2744	+ return NULL;
	2745	+
	2746	+ return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
	2747	+}
	2748	+
	2749	+void goya_dma_pool_free(struct hl_device hdev, void vaddr,
	2750	+ dma_addr_t dma_addr)
	2751	+{
	2752	+ dma_pool_free(hdev->dma_pool, vaddr, dma_addr);
	2753	+}
	2754	+
	2755	+void goya_cpu_accessible_dma_pool_alloc(struct hl_device hdev, size_t size,
	2756	+ dma_addr_t *dma_handle)
	2757	+{
	2758	+ u64 kernel_addr;
	2759	+
	2760	+ /* roundup to CPU_PKT_SIZE */
	2761	+ size = (size + (CPU_PKT_SIZE - 1)) & CPU_PKT_MASK;
	2762	+
	2763	+ kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
	2764	+
	2765	+ *dma_handle = hdev->cpu_accessible_dma_address +
	2766	+ (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
	2767	+
	2768	+ return (void *) (uintptr_t) kernel_addr;
	2769	+}
	2770	+
	2771	+void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
	2772	+ void *vaddr)
	2773	+{
	2774	+ /* roundup to CPU_PKT_SIZE */
	2775	+ size = (size + (CPU_PKT_SIZE - 1)) & CPU_PKT_MASK;
	2776	+
	2777	+ gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
	2778	+ size);
	2779	+}
	2780	+
	2781	+
	2782	+static void goya_hw_queues_lock(struct hl_device *hdev)
	2783	+{
	2784	+ struct goya_device *goya = hdev->asic_specific;
	2785	+
	2786	+ spin_lock(&goya->hw_queues_lock);
	2787	+}
	2788	+
	2789	+static void goya_hw_queues_unlock(struct hl_device *hdev)
	2790	+{
	2791	+ struct goya_device *goya = hdev->asic_specific;
	2792	+
	2793	+ spin_unlock(&goya->hw_queues_lock);
	2794	+}
	2795	+
1504	2796	static const struct hl_asic_funcs goya_funcs = {
1505	2797	.early_init = goya_early_init,
1506	2798	.early_fini = goya_early_fini,
@@ -1512,8 +2804,19 @@ static const struct hl_asic_funcs goya_funcs = { · 5 unmodified lines
1512	2804	.resume = goya_resume,
1513	2805	.mmap = goya_mmap,
1514	2806	.cb_mmap = goya_cb_mmap,
	2807	+ .ring_doorbell = goya_ring_doorbell,
	2808	+ .flush_pq_write = goya_flush_pq_write,
1515	2809	.dma_alloc_coherent = goya_dma_alloc_coherent,
1516	2810	.dma_free_coherent = goya_dma_free_coherent,
	2811	+ .get_int_queue_base = goya_get_int_queue_base,
	2812	+ .test_queues = goya_test_queues,
	2813	+ .dma_pool_zalloc = goya_dma_pool_zalloc,
	2814	+ .dma_pool_free = goya_dma_pool_free,
	2815	+ .cpu_accessible_dma_pool_alloc = goya_cpu_accessible_dma_pool_alloc,
	2816	+ .cpu_accessible_dma_pool_free = goya_cpu_accessible_dma_pool_free,
	2817	+ .hw_queues_lock = goya_hw_queues_lock,
	2818	+ .hw_queues_unlock = goya_hw_queues_unlock,
	2819	+ .send_cpu_message = goya_send_cpu_message
1517	2820	};
1518	2821
1519	2822	/*

drivers/misc/habanalabs/goya/goyaP.h+7−0

@@ -11,7 +11,9 @@ · 10 unmodified lines
11	11	#include <uapi/misc/habanalabs.h>
12	12	#include "habanalabs.h"
13	13	#include "include/hl_boot_if.h"
	14	+#include "include/goya/goya_packets.h"
14	15	#include "include/goya/goya.h"
	16	+#include "include/goya/goya_async_events.h"
15	17	#include "include/goya/goya_fw_if.h"
16	18
17	19	#define NUMBER_OF_CMPLT_QUEUES 5
@@ -145,12 +147,17 @@ enum goya_fw_component { · 127 unmodified lines
145	147	};
146	148
147	149	struct goya_device {
	150	+ int (test_cpu_queue)(struct hl_device hdev);
	151	+
148	152	/* TODO: remove hw_queues_lock after moving to scheduler code */
149	153	spinlock_t hw_queues_lock;
150	154	u64 ddr_bar_cur_addr;
151	155	u32 hw_cap_initialized;
152	156	};
153	157
	158	+int goya_test_cpu_queue(struct hl_device *hdev);
	159	+int goya_send_cpu_message(struct hl_device hdev, u32 msg, u16 len,
	160	+ u32 timeout, long *result);
154	161	void goya_init_security(struct hl_device *hdev);
155	162
156	163	#endif /* GOYAP_H_ */

drivers/misc/habanalabs/habanalabs.h+173−1

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	#define HABANALABSP_H_
10	10
11	11	#include "include/armcp_if.h"
	12	+#include "include/qman_if.h"
12	13
13	14	#define pr_fmt(fmt) "habanalabs: " fmt
14	15
@@ -26,9 +27,36 @@ · 11 unmodified lines
26	27	struct hl_device;
27	28	struct hl_fpriv;
28	29
	30	+/**
	31	+ * enum hl_queue_type - Supported QUEUE types.
	32	+ * @QUEUE_TYPE_NA: queue is not available.
	33	+ * @QUEUE_TYPE_EXT: external queue which is a DMA channel that may access the
	34	+ * host.
	35	+ * @QUEUE_TYPE_INT: internal queue that performs DMA inside the device's
	36	+ * memories and/or operates the compute engines.
	37	+ * @QUEUE_TYPE_CPU: S/W queue for communication with the device's CPU.
	38	+ */
	39	+enum hl_queue_type {
	40	+ QUEUE_TYPE_NA,
	41	+ QUEUE_TYPE_EXT,
	42	+ QUEUE_TYPE_INT,
	43	+ QUEUE_TYPE_CPU
	44	+};
	45	+
	46	+/**
	47	+ * struct hw_queue_properties - queue information.
	48	+ * @type: queue type.
	49	+ * @kmd_only: true if only KMD is allowed to send a job to this queue, false
	50	+ * otherwise.
	51	+ */
	52	+struct hw_queue_properties {
	53	+ enum hl_queue_type type;
	54	+ u8 kmd_only;
	55	+};
29	56
30	57	/**
31	58	* struct asic_fixed_properties - ASIC specific immutable properties.
	59	+ * @hw_queues_props: H/W queues properties.
32	60	* @uboot_ver: F/W U-boot version.
33	61	* @preboot_ver: F/W Preboot version.
34	62	* @sram_base_address: SRAM physical start address.
@@ -59,6 +87,7 @@ struct hl_fpriv; · 24 unmodified lines
59	87	* @tpc_enabled_mask: which TPCs are enabled.
60	88	*/
61	89	struct asic_fixed_properties {
	90	+ struct hw_queue_properties hw_queues_props[HL_MAX_QUEUES];
62	91	char uboot_ver[VERSION_MAX_LEN];
63	92	char preboot_ver[VERSION_MAX_LEN];
64	93	u64 sram_base_address;
@@ -132,7 +161,89 @@ struct hl_cb { · 67 unmodified lines
132	161	};
133	162
134	163
	164	+/*
	165	+ * QUEUES
	166	+ */
	167	+
	168	+struct hl_cs_job;
	169	+
	170	+/*
	171	+ * Currently, there are two limitations on the maximum length of a queue:
	172	+ *
	173	+ * 1. The memory footprint of the queue. The current allocated space for the
	174	+ * queue is PAGE_SIZE. Because each entry in the queue is HL_BD_SIZE,
	175	+ * the maximum length of the queue can be PAGE_SIZE / HL_BD_SIZE,
	176	+ * which currently is 4096/16 = 256 entries.
	177	+ *
	178	+ * To increase that, we need either to decrease the size of the
	179	+ * BD (difficult), or allocate more than a single page (easier).
	180	+ *
	181	+ * 2. Because the size of the JOB handle field in the BD CTL / completion queue
	182	+ * is 10-bit, we can have up to 1024 open jobs per hardware queue.
	183	+ * Therefore, each queue can hold up to 1024 entries.
	184	+ *
	185	+ * HL_QUEUE_LENGTH is in units of struct hl_bd.
	186	+ * HL_QUEUE_LENGTH * sizeof(struct hl_bd) should be <= HL_PAGE_SIZE
	187	+ */
	188	+
	189	+#define HL_PAGE_SIZE 4096 /* minimum page size */
	190	+/* Must be power of 2 (HL_PAGE_SIZE / HL_BD_SIZE) */
135	191	#define HL_QUEUE_LENGTH 256
	192	+#define HL_QUEUE_SIZE_IN_BYTES (HL_QUEUE_LENGTH * HL_BD_SIZE)
	193	+
	194	+/*
	195	+ * HL_CQ_LENGTH is in units of struct hl_cq_entry.
	196	+ * HL_CQ_LENGTH should be <= HL_PAGE_SIZE
	197	+ */
	198	+#define HL_CQ_LENGTH HL_QUEUE_LENGTH
	199	+#define HL_CQ_SIZE_IN_BYTES (HL_CQ_LENGTH * HL_CQ_ENTRY_SIZE)
	200	+
	201	+
	202	+
	203	+/**
	204	+ * struct hl_hw_queue - describes a H/W transport queue.
	205	+ * @shadow_queue: pointer to a shadow queue that holds pointers to jobs.
	206	+ * @queue_type: type of queue.
	207	+ * @kernel_address: holds the queue's kernel virtual address.
	208	+ * @bus_address: holds the queue's DMA address.
	209	+ * @pi: holds the queue's pi value.
	210	+ * @ci: holds the queue's ci value, AS CALCULATED BY THE DRIVER (not real ci).
	211	+ * @hw_queue_id: the id of the H/W queue.
	212	+ * @int_queue_len: length of internal queue (number of entries).
	213	+ * @valid: is the queue valid (we have array of 32 queues, not all of them
	214	+ * exists).
	215	+ */
	216	+struct hl_hw_queue {
	217	+ struct hl_cs_job **shadow_queue;
	218	+ enum hl_queue_type queue_type;
	219	+ u64 kernel_address;
	220	+ dma_addr_t bus_address;
	221	+ u32 pi;
	222	+ u32 ci;
	223	+ u32 hw_queue_id;
	224	+ u16 int_queue_len;
	225	+ u8 valid;
	226	+};
	227	+
	228	+/**
	229	+ * struct hl_cq - describes a completion queue
	230	+ * @hdev: pointer to the device structure
	231	+ * @kernel_address: holds the queue's kernel virtual address
	232	+ * @bus_address: holds the queue's DMA address
	233	+ * @hw_queue_id: the id of the matching H/W queue
	234	+ * @ci: ci inside the queue
	235	+ * @pi: pi inside the queue
	236	+ * @free_slots_cnt: counter of free slots in queue
	237	+ */
	238	+struct hl_cq {
	239	+ struct hl_device *hdev;
	240	+ u64 kernel_address;
	241	+ dma_addr_t bus_address;
	242	+ u32 hw_queue_id;
	243	+ u32 ci;
	244	+ u32 pi;
	245	+ atomic_t free_slots_cnt;
	246	+};
136	247
137	248
138	249	/*
@@ -164,6 +275,8 @@ enum hl_asic_type { · 25 unmodified lines
164	275	* @resume: handles IP specific H/W or SW changes for resume.
165	276	* @mmap: mmap function, does nothing.
166	277	* @cb_mmap: maps a CB.
	278	+ * @ring_doorbell: increment PI on a given QMAN.
	279	+ * @flush_pq_write: flush PQ entry write if necessary, WARN if flushing failed.
167	280	* @dma_alloc_coherent: Allocate coherent DMA memory by calling
168	281	* dma_alloc_coherent(). This is ASIC function because its
169	282	* implementation is not trivial when the driver is loaded
@@ -172,6 +285,16 @@ enum hl_asic_type { · 2 unmodified lines
172	285	* This is ASIC function because its implementation is not
173	286	* trivial when the driver is loaded in simulation mode
174	287	* (not upstreamed).
	288	+ * @get_int_queue_base: get the internal queue base address.
	289	+ * @test_queues: run simple test on all queues for sanity check.
	290	+ * @dma_pool_zalloc: small DMA allocation of coherent memory from DMA pool.
	291	+ * size of allocation is HL_DMA_POOL_BLK_SIZE.
	292	+ * @dma_pool_free: free small DMA allocation from pool.
	293	+ * @cpu_accessible_dma_pool_alloc: allocate CPU PQ packet from DMA pool.
	294	+ * @cpu_accessible_dma_pool_free: free CPU PQ packet from DMA pool.
	295	+ * @hw_queues_lock: acquire H/W queues lock.
	296	+ * @hw_queues_unlock: release H/W queues lock.
	297	+ * @send_cpu_message: send buffer to ArmCP.
175	298	*/
176	299	struct hl_asic_funcs {
177	300	int (early_init)(struct hl_device hdev);
@@ -185,10 +308,27 @@ struct hl_asic_funcs { · 7 unmodified lines
185	308	int (mmap)(struct hl_fpriv hpriv, struct vm_area_struct *vma);
186	309	int (cb_mmap)(struct hl_device hdev, struct vm_area_struct *vma,
187	310	u64 kaddress, phys_addr_t paddress, u32 size);
	311	+ void (ring_doorbell)(struct hl_device hdev, u32 hw_queue_id, u32 pi);
	312	+ void (flush_pq_write)(struct hl_device hdev, u64 *pq, u64 exp_val);
188	313	void* (dma_alloc_coherent)(struct hl_device hdev, size_t size,
189	314	dma_addr_t *dma_handle, gfp_t flag);
190	315	void (dma_free_coherent)(struct hl_device hdev, size_t size,
191	316	void *cpu_addr, dma_addr_t dma_handle);
	317	+ void* (get_int_queue_base)(struct hl_device hdev, u32 queue_id,
	318	+ dma_addr_t dma_handle, u16 queue_len);
	319	+ int (test_queues)(struct hl_device hdev);
	320	+ void* (dma_pool_zalloc)(struct hl_device hdev, size_t size,
	321	+ gfp_t mem_flags, dma_addr_t *dma_handle);
	322	+ void (dma_pool_free)(struct hl_device hdev, void *vaddr,
	323	+ dma_addr_t dma_addr);
	324	+ void* (cpu_accessible_dma_pool_alloc)(struct hl_device hdev,
	325	+ size_t size, dma_addr_t *dma_handle);
	326	+ void (cpu_accessible_dma_pool_free)(struct hl_device hdev,
	327	+ size_t size, void *vaddr);
	328	+ void (hw_queues_lock)(struct hl_device hdev);
	329	+ void (hw_queues_unlock)(struct hl_device hdev);
	330	+ int (send_cpu_message)(struct hl_device hdev, u32 *msg,
	331	+ u16 len, u32 timeout, long *result);
192	332	};
193	333
194	334
@@ -224,6 +364,17 @@ struct hl_ctx_mgr { · 29 unmodified lines
224	364	};
225	365
226	366
	367	+
	368	+
	369	+/**
	370	+ * struct hl_cs_job - command submission job.
	371	+ * @finish_work: workqueue object to run when job is completed.
	372	+ * @id: the id of this job inside a CS.
	373	+ */
	374	+struct hl_cs_job {
	375	+ struct work_struct finish_work;
	376	+ u32 id;
	377	+};
227	378	/*
228	379	* FILE PRIVATE STRUCTURE
229	380	*/
@@ -298,7 +449,11 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val); · 68 unmodified lines
298	449	* @dev: realted kernel basic device structure.
299	450	* @asic_name: ASIC specific nmae.
300	451	* @asic_type: ASIC specific type.
	452	+ * @completion_queue: array of hl_cq.
	453	+ * @cq_wq: work queue of completion queues for executing work in process context
	454	+ * @eq_wq: work queue of event queue for executing work in process context.
301	455	* @kernel_ctx: KMD context structure.
	456	+ * @kernel_queues: array of hl_hw_queue.
302	457	* @kernel_cb_mgr: command buffer manager for creating/destroying/handling CGs.
303	458	* @dma_pool: DMA pool for small allocations.
304	459	* @cpu_accessible_dma_mem: KMD <-> ArmCP shared memory CPU address.
@@ -312,6 +467,7 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val); · 7 unmodified lines
312	467	* only a single process at a time. In addition, we need a
313	468	* lock here so we can flush user processes which are opening
314	469	* the device while we are trying to hard reset it
	470	+ * @send_cpu_message_lock: enforces only one message in KMD <-> ArmCP queue.
315	471	* @asic_prop: ASIC specific immutable properties.
316	472	* @asic_funcs: ASIC specific functions.
317	473	* @asic_specific: ASIC specific information to use only from ASIC files.
@@ -331,7 +487,10 @@ struct hl_device { · 13 unmodified lines
331	487	struct device *dev;
332	488	char asic_name[16];
333	489	enum hl_asic_type asic_type;
	490	+ struct hl_cq *completion_queue;
	491	+ struct workqueue_struct *cq_wq;
334	492	struct hl_ctx *kernel_ctx;
	493	+ struct hl_hw_queue *kernel_queues;
335	494	struct hl_cb_mgr kernel_cb_mgr;
336	495	struct dma_pool *dma_pool;
337	496	void *cpu_accessible_dma_mem;
@@ -341,6 +500,7 @@ struct hl_device { · 3 unmodified lines
341	500	struct mutex asid_mutex;
342	501	/* TODO: remove fd_open_cnt_lock for multiple process support */
343	502	struct mutex fd_open_cnt_lock;
	503	+ struct mutex send_cpu_message_lock;
344	504	struct asic_fixed_properties asic_prop;
345	505	const struct hl_asic_funcs *asic_funcs;
346	506	void *asic_specific;
@@ -358,6 +518,7 @@ struct hl_device { · 11 unmodified lines
358	518	/* Parameters for bring-up */
359	519	u8 cpu_enable;
360	520	u8 reset_pcilink;
	521	+ u8 cpu_queues_enable;
361	522	u8 fw_loading;
362	523	u8 pldm;
363	524	};
@@ -400,7 +561,18 @@ int hl_poll_timeout_memory(struct hl_device *hdev, u64 addr, u32 timeout_us, · 36 unmodified lines
400	561	u32 *val);
401	562	int hl_poll_timeout_device_memory(struct hl_device hdev, void __iomem addr,
402	563	u32 timeout_us, u32 *val);
403		−
	564	+int hl_hw_queues_create(struct hl_device *hdev);
	565	+void hl_hw_queues_destroy(struct hl_device *hdev);
	566	+int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
	567	+ u32 cb_size, u64 cb_ptr);
	568	+u32 hl_hw_queue_add_ptr(u32 ptr, u16 val);
	569	+void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id);
	570	+
	571	+#define hl_queue_inc_ptr(p) hl_hw_queue_add_ptr(p, 1)
	572	+#define hl_pi_2_offset(pi) ((pi) & (HL_QUEUE_LENGTH - 1))
	573	+
	574	+int hl_cq_init(struct hl_device hdev, struct hl_cq q, u32 hw_queue_id);
	575	+void hl_cq_fini(struct hl_device hdev, struct hl_cq q);
404	576	int hl_asid_init(struct hl_device *hdev);
405	577	void hl_asid_fini(struct hl_device *hdev);
406	578	unsigned long hl_asid_alloc(struct hl_device *hdev);

drivers/misc/habanalabs/habanalabs_drv.c+5−0

@@ -169,6 +169,7 @@ int create_hdev(struct hl_device *dev, struct pci_dev pdev, · 168 unmodified lines
169	169	/* Parameters for bring-up - set them to defaults */
170	170	hdev->cpu_enable = 1;
171	171	hdev->reset_pcilink = 0;
	172	+ hdev->cpu_queues_enable = 1;
172	173	hdev->fw_loading = 1;
173	174	hdev->pldm = 0;
174	175
@@ -176,6 +177,10 @@ int create_hdev(struct hl_device *dev, struct pci_dev pdev, · 1 unmodified line
176	177	if (!hdev->cpu_enable)
177	178	hdev->fw_loading = 0;
178	179
	180	+ /* If we don't load FW, no need to initialize CPU queues */
	181	+ if (!hdev->fw_loading)
	182	+ hdev->cpu_queues_enable = 0;
	183	+
179	184	hdev->disabled = true;
180	185	hdev->pdev = pdev; /* can be NULL in case of simulator device */
181	186

drivers/misc/habanalabs/hw_queue.c+400−0

@@ -0,0 +1,400 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+/*
	4	+ * Copyright 2016-2019 HabanaLabs, Ltd.
	5	+ * All Rights Reserved.
	6	+ */
	7	+
	8	+#include "habanalabs.h"
	9	+
	10	+#include <linux/slab.h>
	11	+
	12	+/*
	13	+ * hl_queue_add_ptr - add to pi or ci and checks if it wraps around
	14	+ *
	15	+ * @ptr: the current pi/ci value
	16	+ * @val: the amount to add
	17	+ *
	18	+ * Add val to ptr. It can go until twice the queue length.
	19	+ */
	20	+inline u32 hl_hw_queue_add_ptr(u32 ptr, u16 val)
	21	+{
	22	+ ptr += val;
	23	+ ptr &= ((HL_QUEUE_LENGTH << 1) - 1);
	24	+ return ptr;
	25	+}
	26	+
	27	+static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len)
	28	+{
	29	+ int delta = (q->pi - q->ci);
	30	+
	31	+ if (delta >= 0)
	32	+ return (queue_len - delta);
	33	+ else
	34	+ return (abs(delta) - queue_len);
	35	+}
	36	+
	37	+/*
	38	+ * ext_queue_submit_bd - Submit a buffer descriptor to an external queue
	39	+ *
	40	+ * @hdev: pointer to habanalabs device structure
	41	+ * @q: pointer to habanalabs queue structure
	42	+ * @ctl: BD's control word
	43	+ * @len: BD's length
	44	+ * @ptr: BD's pointer
	45	+ *
	46	+ * This function assumes there is enough space on the queue to submit a new
	47	+ * BD to it. It initializes the next BD and calls the device specific
	48	+ * function to set the pi (and doorbell)
	49	+ *
	50	+ * This function must be called when the scheduler mutex is taken
	51	+ *
	52	+ */
	53	+static void ext_queue_submit_bd(struct hl_device hdev, struct hl_hw_queue q,
	54	+ u32 ctl, u32 len, u64 ptr)
	55	+{
	56	+ struct hl_bd *bd;
	57	+
	58	+ bd = (struct hl_bd *) (uintptr_t) q->kernel_address;
	59	+ bd += hl_pi_2_offset(q->pi);
	60	+ bd->ctl = ctl;
	61	+ bd->len = len;
	62	+ bd->ptr = ptr + hdev->asic_prop.host_phys_base_address;
	63	+
	64	+ q->pi = hl_queue_inc_ptr(q->pi);
	65	+ hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);
	66	+}
	67	+
	68	+/*
	69	+ * ext_queue_sanity_checks - perform some sanity checks on external queue
	70	+ *
	71	+ * @hdev : pointer to hl_device structure
	72	+ * @q : pointer to hl_hw_queue structure
	73	+ * @num_of_entries : how many entries to check for space
	74	+ * @reserve_cq_entry : whether to reserve an entry in the cq
	75	+ *
	76	+ * H/W queues spinlock should be taken before calling this function
	77	+ *
	78	+ * Perform the following:
	79	+ * - Make sure we have enough space in the h/w queue
	80	+ * - Make sure we have enough space in the completion queue
	81	+ * - Reserve space in the completion queue (needs to be reversed if there
	82	+ * is a failure down the road before the actual submission of work). Only
	83	+ * do this action if reserve_cq_entry is true
	84	+ *
	85	+ */
	86	+static int ext_queue_sanity_checks(struct hl_device *hdev,
	87	+ struct hl_hw_queue *q, int num_of_entries,
	88	+ bool reserve_cq_entry)
	89	+{
	90	+ atomic_t *free_slots =
	91	+ &hdev->completion_queue[q->hw_queue_id].free_slots_cnt;
	92	+ int free_slots_cnt;
	93	+
	94	+ /* Check we have enough space in the queue */
	95	+ free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);
	96	+
	97	+ if (free_slots_cnt < num_of_entries) {
	98	+ dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",
	99	+ q->hw_queue_id, num_of_entries);
	100	+ return -EAGAIN;
	101	+ }
	102	+
	103	+ if (reserve_cq_entry) {
	104	+ /*
	105	+ * Check we have enough space in the completion queue
	106	+ * Add -1 to counter (decrement) unless counter was already 0
	107	+ * In that case, CQ is full so we can't submit a new CB because
	108	+ * we won't get ack on its completion
	109	+ * atomic_add_unless will return 0 if counter was already 0
	110	+ */
	111	+ if (atomic_add_negative(num_of_entries * -1, free_slots)) {
	112	+ dev_dbg(hdev->dev, "No space for %d on CQ %d\n",
	113	+ num_of_entries, q->hw_queue_id);
	114	+ atomic_add(num_of_entries, free_slots);
	115	+ return -EAGAIN;
	116	+ }
	117	+ }
	118	+
	119	+ return 0;
	120	+}
	121	+
	122	+/*
	123	+ * hl_hw_queue_send_cb_no_cmpl - send a single CB (not a JOB) without completion
	124	+ *
	125	+ * @hdev: pointer to hl_device structure
	126	+ * @hw_queue_id: Queue's type
	127	+ * @cb_size: size of CB
	128	+ * @cb_ptr: pointer to CB location
	129	+ *
	130	+ * This function sends a single CB, that must NOT generate a completion entry
	131	+ *
	132	+ */
	133	+int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
	134	+ u32 cb_size, u64 cb_ptr)
	135	+{
	136	+ struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];
	137	+ int rc;
	138	+
	139	+ /*
	140	+ * The CPU queue is a synchronous queue with an effective depth of
	141	+ * a single entry (although it is allocated with room for multiple
	142	+ * entries). Therefore, there is a different lock, called
	143	+ * send_cpu_message_lock, that serializes accesses to the CPU queue.
	144	+ * As a result, we don't need to lock the access to the entire H/W
	145	+ * queues module when submitting a JOB to the CPU queue
	146	+ */
	147	+ if (q->queue_type != QUEUE_TYPE_CPU)
	148	+ hdev->asic_funcs->hw_queues_lock(hdev);
	149	+
	150	+ if (hdev->disabled) {
	151	+ rc = -EPERM;
	152	+ goto out;
	153	+ }
	154	+
	155	+ rc = ext_queue_sanity_checks(hdev, q, 1, false);
	156	+ if (rc)
	157	+ goto out;
	158	+
	159	+ ext_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);
	160	+
	161	+out:
	162	+ if (q->queue_type != QUEUE_TYPE_CPU)
	163	+ hdev->asic_funcs->hw_queues_unlock(hdev);
	164	+
	165	+ return rc;
	166	+}
	167	+
	168	+/*
	169	+ * hl_hw_queue_inc_ci_kernel - increment ci for kernel's queue
	170	+ *
	171	+ * @hdev: pointer to hl_device structure
	172	+ * @hw_queue_id: which queue to increment its ci
	173	+ */
	174	+void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id)
	175	+{
	176	+ struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];
	177	+
	178	+ q->ci = hl_queue_inc_ptr(q->ci);
	179	+}
	180	+
	181	+static int ext_and_cpu_hw_queue_init(struct hl_device *hdev,
	182	+ struct hl_hw_queue *q)
	183	+{
	184	+ void *p;
	185	+ int rc;
	186	+
	187	+ p = hdev->asic_funcs->dma_alloc_coherent(hdev,
	188	+ HL_QUEUE_SIZE_IN_BYTES,
	189	+ &q->bus_address, GFP_KERNEL \| __GFP_ZERO);
	190	+ if (!p)
	191	+ return -ENOMEM;
	192	+
	193	+ q->kernel_address = (u64) (uintptr_t) p;
	194	+
	195	+ q->shadow_queue = kmalloc_array(HL_QUEUE_LENGTH,
	196	+ sizeof(*q->shadow_queue),
	197	+ GFP_KERNEL);
	198	+ if (!q->shadow_queue) {
	199	+ dev_err(hdev->dev,
	200	+ "Failed to allocate shadow queue for H/W queue %d\n",
	201	+ q->hw_queue_id);
	202	+ rc = -ENOMEM;
	203	+ goto free_queue;
	204	+ }
	205	+
	206	+ /* Make sure read/write pointers are initialized to start of queue */
	207	+ q->ci = 0;
	208	+ q->pi = 0;
	209	+
	210	+ return 0;
	211	+
	212	+free_queue:
	213	+ hdev->asic_funcs->dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES,
	214	+ (void *) (uintptr_t) q->kernel_address, q->bus_address);
	215	+
	216	+ return rc;
	217	+}
	218	+
	219	+static int int_hw_queue_init(struct hl_device hdev, struct hl_hw_queue q)
	220	+{
	221	+ void *p;
	222	+
	223	+ p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id,
	224	+ &q->bus_address, &q->int_queue_len);
	225	+ if (!p) {
	226	+ dev_err(hdev->dev,
	227	+ "Failed to get base address for internal queue %d\n",
	228	+ q->hw_queue_id);
	229	+ return -EFAULT;
	230	+ }
	231	+
	232	+ q->kernel_address = (u64) (uintptr_t) p;
	233	+ q->pi = 0;
	234	+ q->ci = 0;
	235	+
	236	+ return 0;
	237	+}
	238	+
	239	+static int cpu_hw_queue_init(struct hl_device hdev, struct hl_hw_queue q)
	240	+{
	241	+ return ext_and_cpu_hw_queue_init(hdev, q);
	242	+}
	243	+
	244	+static int ext_hw_queue_init(struct hl_device hdev, struct hl_hw_queue q)
	245	+{
	246	+ return ext_and_cpu_hw_queue_init(hdev, q);
	247	+}
	248	+
	249	+/*
	250	+ * hw_queue_init - main initialization function for H/W queue object
	251	+ *
	252	+ * @hdev: pointer to hl_device device structure
	253	+ * @q: pointer to hl_hw_queue queue structure
	254	+ * @hw_queue_id: The id of the H/W queue
	255	+ *
	256	+ * Allocate dma-able memory for the queue and initialize fields
	257	+ * Returns 0 on success
	258	+ */
	259	+static int hw_queue_init(struct hl_device hdev, struct hl_hw_queue q,
	260	+ u32 hw_queue_id)
	261	+{
	262	+ int rc;
	263	+
	264	+ BUILD_BUG_ON(HL_QUEUE_SIZE_IN_BYTES > HL_PAGE_SIZE);
	265	+
	266	+ q->hw_queue_id = hw_queue_id;
	267	+
	268	+ switch (q->queue_type) {
	269	+ case QUEUE_TYPE_EXT:
	270	+ rc = ext_hw_queue_init(hdev, q);
	271	+ break;
	272	+
	273	+ case QUEUE_TYPE_INT:
	274	+ rc = int_hw_queue_init(hdev, q);
	275	+ break;
	276	+
	277	+ case QUEUE_TYPE_CPU:
	278	+ rc = cpu_hw_queue_init(hdev, q);
	279	+ break;
	280	+
	281	+ case QUEUE_TYPE_NA:
	282	+ q->valid = 0;
	283	+ return 0;
	284	+
	285	+ default:
	286	+ dev_crit(hdev->dev, "wrong queue type %d during init\n",
	287	+ q->queue_type);
	288	+ rc = -EINVAL;
	289	+ break;
	290	+ }
	291	+
	292	+ if (rc)
	293	+ return rc;
	294	+
	295	+ q->valid = 1;
	296	+
	297	+ return 0;
	298	+}
	299	+
	300	+/*
	301	+ * hw_queue_fini - destroy queue
	302	+ *
	303	+ * @hdev: pointer to hl_device device structure
	304	+ * @q: pointer to hl_hw_queue queue structure
	305	+ *
	306	+ * Free the queue memory
	307	+ */
	308	+static void hw_queue_fini(struct hl_device hdev, struct hl_hw_queue q)
	309	+{
	310	+ if (!q->valid)
	311	+ return;
	312	+
	313	+ /*
	314	+ * If we arrived here, there are no jobs waiting on this queue
	315	+ * so we can safely remove it.
	316	+ * This is because this function can only called when:
	317	+ * 1. Either a context is deleted, which only can occur if all its
	318	+ * jobs were finished
	319	+ * 2. A context wasn't able to be created due to failure or timeout,
	320	+ * which means there are no jobs on the queue yet
	321	+ *
	322	+ * The only exception are the queues of the kernel context, but
	323	+ * if they are being destroyed, it means that the entire module is
	324	+ * being removed. If the module is removed, it means there is no open
	325	+ * user context. It also means that if a job was submitted by
	326	+ * the kernel driver (e.g. context creation), the job itself was
	327	+ * released by the kernel driver when a timeout occurred on its
	328	+ * Completion. Thus, we don't need to release it again.
	329	+ */
	330	+
	331	+ if (q->queue_type == QUEUE_TYPE_INT)
	332	+ return;
	333	+
	334	+ kfree(q->shadow_queue);
	335	+
	336	+ hdev->asic_funcs->dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES,
	337	+ (void *) (uintptr_t) q->kernel_address, q->bus_address);
	338	+}
	339	+
	340	+int hl_hw_queues_create(struct hl_device *hdev)
	341	+{
	342	+ struct asic_fixed_properties *asic = &hdev->asic_prop;
	343	+ struct hl_hw_queue *q;
	344	+ int i, rc, q_ready_cnt;
	345	+
	346	+ hdev->kernel_queues = kcalloc(HL_MAX_QUEUES,
	347	+ sizeof(*hdev->kernel_queues), GFP_KERNEL);
	348	+
	349	+ if (!hdev->kernel_queues) {
	350	+ dev_err(hdev->dev, "Not enough memory for H/W queues\n");
	351	+ return -ENOMEM;
	352	+ }
	353	+
	354	+ /* Initialize the H/W queues */
	355	+ for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues;
	356	+ i < HL_MAX_QUEUES ; i++, q_ready_cnt++, q++) {
	357	+
	358	+ q->queue_type = asic->hw_queues_props[i].type;
	359	+ rc = hw_queue_init(hdev, q, i);
	360	+ if (rc) {
	361	+ dev_err(hdev->dev,
	362	+ "failed to initialize queue %d\n", i);
	363	+ goto release_queues;
	364	+ }
	365	+ }
	366	+
	367	+ return 0;
	368	+
	369	+release_queues:
	370	+ for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++)
	371	+ hw_queue_fini(hdev, q);
	372	+
	373	+ kfree(hdev->kernel_queues);
	374	+
	375	+ return rc;
	376	+}
	377	+
	378	+void hl_hw_queues_destroy(struct hl_device *hdev)
	379	+{
	380	+ struct hl_hw_queue *q;
	381	+ int i;
	382	+
	383	+ for (i = 0, q = hdev->kernel_queues ; i < HL_MAX_QUEUES ; i++, q++)
	384	+ hw_queue_fini(hdev, q);
	385	+
	386	+ kfree(hdev->kernel_queues);
	387	+}
	388	+
	389	+void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset)
	390	+{
	391	+ struct hl_hw_queue *q;
	392	+ int i;
	393	+
	394	+ for (i = 0, q = hdev->kernel_queues ; i < HL_MAX_QUEUES ; i++, q++) {
	395	+ if ((!q->valid) \|\|
	396	+ ((!hard_reset) && (q->queue_type == QUEUE_TYPE_CPU)))
	397	+ continue;
	398	+ q->pi = q->ci = 0;
	399	+ }
	400	+}

drivers/misc/habanalabs/include/armcp_if.h+292−0

@@ -10,10 +10,302 @@ · 9 unmodified lines
10	10
11	11	#include <linux/types.h>
12	12
	13	+enum pq_init_status {
	14	+ PQ_INIT_STATUS_NA = 0,
	15	+ PQ_INIT_STATUS_READY_FOR_CP,
	16	+ PQ_INIT_STATUS_READY_FOR_HOST
	17	+};
	18	+
	19	+/*
	20	+ * ArmCP Primary Queue Packets
	21	+ *
	22	+ * During normal operation, KMD needs to send various messages to ArmCP,
	23	+ * usually either to SET some value into a H/W periphery or to GET the current
	24	+ * value of some H/W periphery. For example, SET the frequency of MME/TPC and
	25	+ * GET the value of the thermal sensor.
	26	+ *
	27	+ * These messages can be initiated either by the User application or by KMD
	28	+ * itself, e.g. power management code. In either case, the communication from
	29	+ * KMD to ArmCP will always be in synchronous mode, meaning that KMD will
	30	+ * send a single message and poll until the message was acknowledged and the
	31	+ * results are ready (if results are needed).
	32	+ *
	33	+ * This means that only a single message can be sent at a time and KMD must
	34	+ * wait for its result before sending the next message. Having said that,
	35	+ * because these are control messages which are sent in a relatively low
	36	+ * frequency, this limitation seems acceptable. It's important to note that
	37	+ * in case of multiple devices, messages to different devices can be sent
	38	+ * at the same time.
	39	+ *
	40	+ * The message, inputs/outputs (if relevant) and fence object will be located
	41	+ * on the device DDR at an address that will be determined by KMD. During
	42	+ * device initialization phase, KMD will pass to ArmCP that address. Most of
	43	+ * the message types will contain inputs/outputs inside the message itself.
	44	+ * The common part of each message will contain the opcode of the message (its
	45	+ * type) and a field representing a fence object.
	46	+ *
	47	+ * When KMD wishes to send a message to ArmCP, it will write the message
	48	+ * contents to the device DDR, clear the fence object and then write the
	49	+ * value 484 to the mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR register to issue
	50	+ * the 484 interrupt-id to the ARM core.
	51	+ *
	52	+ * Upon receiving the 484 interrupt-id, ArmCP will read the message from the
	53	+ * DDR. In case the message is a SET operation, ArmCP will first perform the
	54	+ * operation and then write to the fence object on the device DDR. In case the
	55	+ * message is a GET operation, ArmCP will first fill the results section on the
	56	+ * device DDR and then write to the fence object. If an error occurred, ArmCP
	57	+ * will fill the rc field with the right error code.
	58	+ *
	59	+ * In the meantime, KMD will poll on the fence object. Once KMD sees that the
	60	+ * fence object is signaled, it will read the results from the device DDR
	61	+ * (if relevant) and resume the code execution in KMD.
	62	+ *
	63	+ * To use QMAN packets, the opcode must be the QMAN opcode, shifted by 8
	64	+ * so the value being put by the KMD matches the value read by ArmCP
	65	+ *
	66	+ * Non-QMAN packets should be limited to values 1 through (2^8 - 1)
	67	+ *
	68	+ * Detailed description:
	69	+ *
	70	+ * ARMCP_PACKET_DISABLE_PCI_ACCESS -
	71	+ * After receiving this packet the embedded CPU must NOT issue PCI
	72	+ * transactions (read/write) towards the Host CPU. This also include
	73	+ * sending MSI-X interrupts.
	74	+ * This packet is usually sent before the device is moved to D3Hot state.
	75	+ *
	76	+ * ARMCP_PACKET_ENABLE_PCI_ACCESS -
	77	+ * After receiving this packet the embedded CPU is allowed to issue PCI
	78	+ * transactions towards the Host CPU, including sending MSI-X interrupts.
	79	+ * This packet is usually send after the device is moved to D0 state.
	80	+ *
	81	+ * ARMCP_PACKET_TEMPERATURE_GET -
	82	+ * Fetch the current temperature / Max / Max Hyst / Critical /
	83	+ * Critical Hyst of a specified thermal sensor. The packet's
	84	+ * arguments specify the desired sensor and the field to get.
	85	+ *
	86	+ * ARMCP_PACKET_VOLTAGE_GET -
	87	+ * Fetch the voltage / Max / Min of a specified sensor. The packet's
	88	+ * arguments specify the sensor and type.
	89	+ *
	90	+ * ARMCP_PACKET_CURRENT_GET -
	91	+ * Fetch the current / Max / Min of a specified sensor. The packet's
	92	+ * arguments specify the sensor and type.
	93	+ *
	94	+ * ARMCP_PACKET_FAN_SPEED_GET -
	95	+ * Fetch the speed / Max / Min of a specified fan. The packet's
	96	+ * arguments specify the sensor and type.
	97	+ *
	98	+ * ARMCP_PACKET_PWM_GET -
	99	+ * Fetch the pwm value / mode of a specified pwm. The packet's
	100	+ * arguments specify the sensor and type.
	101	+ *
	102	+ * ARMCP_PACKET_PWM_SET -
	103	+ * Set the pwm value / mode of a specified pwm. The packet's
	104	+ * arguments specify the sensor, type and value.
	105	+ *
	106	+ * ARMCP_PACKET_FREQUENCY_SET -
	107	+ * Set the frequency of a specified PLL. The packet's arguments specify
	108	+ * the PLL and the desired frequency. The actual frequency in the device
	109	+ * might differ from the requested frequency.
	110	+ *
	111	+ * ARMCP_PACKET_FREQUENCY_GET -
	112	+ * Fetch the frequency of a specified PLL. The packet's arguments specify
	113	+ * the PLL.
	114	+ *
	115	+ * ARMCP_PACKET_LED_SET -
	116	+ * Set the state of a specified led. The packet's arguments
	117	+ * specify the led and the desired state.
	118	+ *
	119	+ * ARMCP_PACKET_I2C_WR -
	120	+ * Write 32-bit value to I2C device. The packet's arguments specify the
	121	+ * I2C bus, address and value.
	122	+ *
	123	+ * ARMCP_PACKET_I2C_RD -
	124	+ * Read 32-bit value from I2C device. The packet's arguments specify the
	125	+ * I2C bus and address.
	126	+ *
	127	+ * ARMCP_PACKET_INFO_GET -
	128	+ * Fetch information from the device as specified in the packet's
	129	+ * structure. KMD passes the max size it allows the ArmCP to write to
	130	+ * the structure, to prevent data corruption in case of mismatched
	131	+ * KMD/FW versions.
	132	+ *
	133	+ * ARMCP_PACKET_FLASH_PROGRAM_REMOVED - this packet was removed
	134	+ *
	135	+ * ARMCP_PACKET_UNMASK_RAZWI_IRQ -
	136	+ * Unmask the given IRQ. The IRQ number is specified in the value field.
	137	+ * The packet is sent after receiving an interrupt and printing its
	138	+ * relevant information.
	139	+ *
	140	+ * ARMCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY -
	141	+ * Unmask the given IRQs. The IRQs numbers are specified in an array right
	142	+ * after the armcp_packet structure, where its first element is the array
	143	+ * length. The packet is sent after a soft reset was done in order to
	144	+ * handle any interrupts that were sent during the reset process.
	145	+ *
	146	+ * ARMCP_PACKET_TEST -
	147	+ * Test packet for ArmCP connectivity. The CPU will put the fence value
	148	+ * in the result field.
	149	+ *
	150	+ * ARMCP_PACKET_FREQUENCY_CURR_GET -
	151	+ * Fetch the current frequency of a specified PLL. The packet's arguments
	152	+ * specify the PLL.
	153	+ *
	154	+ * ARMCP_PACKET_MAX_POWER_GET -
	155	+ * Fetch the maximal power of the device.
	156	+ *
	157	+ * ARMCP_PACKET_MAX_POWER_SET -
	158	+ * Set the maximal power of the device. The packet's arguments specify
	159	+ * the power.
	160	+ *
	161	+ * ARMCP_PACKET_EEPROM_DATA_GET -
	162	+ * Get EEPROM data from the ArmCP kernel. The buffer is specified in the
	163	+ * addr field. The CPU will put the returned data size in the result
	164	+ * field. In addition, KMD passes the max size it allows the ArmCP to
	165	+ * write to the structure, to prevent data corruption in case of
	166	+ * mismatched KMD/FW versions.
	167	+ *
	168	+ */
	169	+
	170	+enum armcp_packet_id {
	171	+ ARMCP_PACKET_DISABLE_PCI_ACCESS = 1, /* internal */
	172	+ ARMCP_PACKET_ENABLE_PCI_ACCESS, /* internal */
	173	+ ARMCP_PACKET_TEMPERATURE_GET, /* sysfs */
	174	+ ARMCP_PACKET_VOLTAGE_GET, /* sysfs */
	175	+ ARMCP_PACKET_CURRENT_GET, /* sysfs */
	176	+ ARMCP_PACKET_FAN_SPEED_GET, /* sysfs */
	177	+ ARMCP_PACKET_PWM_GET, /* sysfs */
	178	+ ARMCP_PACKET_PWM_SET, /* sysfs */
	179	+ ARMCP_PACKET_FREQUENCY_SET, /* sysfs */
	180	+ ARMCP_PACKET_FREQUENCY_GET, /* sysfs */
	181	+ ARMCP_PACKET_LED_SET, /* debugfs */
	182	+ ARMCP_PACKET_I2C_WR, /* debugfs */
	183	+ ARMCP_PACKET_I2C_RD, /* debugfs */
	184	+ ARMCP_PACKET_INFO_GET, /* IOCTL */
	185	+ ARMCP_PACKET_FLASH_PROGRAM_REMOVED,
	186	+ ARMCP_PACKET_UNMASK_RAZWI_IRQ, /* internal */
	187	+ ARMCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY, /* internal */
	188	+ ARMCP_PACKET_TEST, /* internal */
	189	+ ARMCP_PACKET_FREQUENCY_CURR_GET, /* sysfs */
	190	+ ARMCP_PACKET_MAX_POWER_GET, /* sysfs */
	191	+ ARMCP_PACKET_MAX_POWER_SET, /* sysfs */
	192	+ ARMCP_PACKET_EEPROM_DATA_GET, /* sysfs */
	193	+};
	194	+
	195	+#define ARMCP_PACKET_FENCE_VAL 0xFE8CE7A5
	196	+
	197	+#define ARMCP_PKT_CTL_RC_SHIFT 12
	198	+#define ARMCP_PKT_CTL_RC_MASK 0x0000F000
	199	+
	200	+#define ARMCP_PKT_CTL_OPCODE_SHIFT 16
	201	+#define ARMCP_PKT_CTL_OPCODE_MASK 0x1FFF0000
	202	+
	203	+struct armcp_packet {
	204	+ union {
	205	+ __le64 value; /* For SET packets */
	206	+ __le64 result; /* For GET packets */
	207	+ __le64 addr; /* For PQ */
	208	+ };
	209	+
	210	+ __le32 ctl;
	211	+
	212	+ __le32 fence; /* Signal to KMD that message is completed */
	213	+
	214	+ union {
	215	+ struct {/* For temperature/current/voltage/fan/pwm get/set */
	216	+ __le16 sensor_index;
	217	+ __le16 type;
	218	+ };
	219	+
	220	+ struct { /* For I2C read/write */
	221	+ __u8 i2c_bus;
	222	+ __u8 i2c_addr;
	223	+ __u8 i2c_reg;
	224	+ __u8 pad; /* unused */
	225	+ };
	226	+
	227	+ /* For frequency get/set */
	228	+ __le32 pll_index;
	229	+
	230	+ /* For led set */
	231	+ __le32 led_index;
	232	+
	233	+ /* For get Armcp info/EEPROM data */
	234	+ __le32 data_max_size;
	235	+ };
	236	+};
	237	+
	238	+struct armcp_unmask_irq_arr_packet {
	239	+ struct armcp_packet armcp_pkt;
	240	+ __le32 length;
	241	+ __le32 irqs[0];
	242	+};
	243	+
	244	+enum armcp_packet_rc {
	245	+ armcp_packet_success,
	246	+ armcp_packet_invalid,
	247	+ armcp_packet_fault
	248	+};
	249	+
	250	+enum armcp_temp_type {
	251	+ armcp_temp_input,
	252	+ armcp_temp_max = 6,
	253	+ armcp_temp_max_hyst,
	254	+ armcp_temp_crit,
	255	+ armcp_temp_crit_hyst
	256	+};
	257	+
	258	+enum armcp_in_attributes {
	259	+ armcp_in_input,
	260	+ armcp_in_min,
	261	+ armcp_in_max
	262	+};
	263	+
	264	+enum armcp_curr_attributes {
	265	+ armcp_curr_input,
	266	+ armcp_curr_min,
	267	+ armcp_curr_max
	268	+};
	269	+
	270	+enum armcp_fan_attributes {
	271	+ armcp_fan_input,
	272	+ armcp_fan_min = 2,
	273	+ armcp_fan_max
	274	+};
	275	+
	276	+enum armcp_pwm_attributes {
	277	+ armcp_pwm_input,
	278	+ armcp_pwm_enable
	279	+};
	280	+
	281	+/* Event Queue Packets */
	282	+
	283	+struct eq_generic_event {
	284	+ __le64 data[7];
	285	+};
	286	+
13	287	/*
14	288	* ArmCP info
15	289	*/
16	290
17	291	#define VERSION_MAX_LEN 128
	292	+#define ARMCP_MAX_SENSORS 128
	293	+
	294	+struct armcp_sensor {
	295	+ __le32 type;
	296	+ __le32 flags;
	297	+};
	298	+
	299	+struct armcp_info {
	300	+ struct armcp_sensor sensors[ARMCP_MAX_SENSORS];
	301	+ __u8 kernel_version[VERSION_MAX_LEN];
	302	+ __le32 reserved[3];
	303	+ __le32 cpld_version;
	304	+ __le32 infineon_version;
	305	+ __u8 fuse_version[VERSION_MAX_LEN];
	306	+ __u8 thermal_version[VERSION_MAX_LEN];
	307	+ __u8 armcp_version[VERSION_MAX_LEN];
	308	+ __le64 dram_size;
	309	+};
18	310
19	311	#endif /* ARMCP_IF_H */

drivers/misc/habanalabs/include/goya/goya_async_events.h+186−0

@@ -0,0 +1,186 @@
	1	+/* SPDX-License-Identifier: GPL-2.0
	2	+ *
	3	+ * Copyright 2018 HabanaLabs, Ltd.
	4	+ * All Rights Reserved.
	5	+ *
	6	+ */
	7	+
	8	+#ifndef __GOYA_ASYNC_EVENTS_H_
	9	+#define __GOYA_ASYNC_EVENTS_H_
	10	+
	11	+enum goya_async_event_id {
	12	+ GOYA_ASYNC_EVENT_ID_PCIE_IF = 33,
	13	+ GOYA_ASYNC_EVENT_ID_TPC0_ECC = 36,
	14	+ GOYA_ASYNC_EVENT_ID_TPC1_ECC = 39,
	15	+ GOYA_ASYNC_EVENT_ID_TPC2_ECC = 42,
	16	+ GOYA_ASYNC_EVENT_ID_TPC3_ECC = 45,
	17	+ GOYA_ASYNC_EVENT_ID_TPC4_ECC = 48,
	18	+ GOYA_ASYNC_EVENT_ID_TPC5_ECC = 51,
	19	+ GOYA_ASYNC_EVENT_ID_TPC6_ECC = 54,
	20	+ GOYA_ASYNC_EVENT_ID_TPC7_ECC = 57,
	21	+ GOYA_ASYNC_EVENT_ID_MME_ECC = 60,
	22	+ GOYA_ASYNC_EVENT_ID_MME_ECC_EXT = 61,
	23	+ GOYA_ASYNC_EVENT_ID_MMU_ECC = 63,
	24	+ GOYA_ASYNC_EVENT_ID_DMA_MACRO = 64,
	25	+ GOYA_ASYNC_EVENT_ID_DMA_ECC = 66,
	26	+ GOYA_ASYNC_EVENT_ID_CPU_IF_ECC = 75,
	27	+ GOYA_ASYNC_EVENT_ID_PSOC_MEM = 78,
	28	+ GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT = 79,
	29	+ GOYA_ASYNC_EVENT_ID_SRAM0 = 81,
	30	+ GOYA_ASYNC_EVENT_ID_SRAM1 = 82,
	31	+ GOYA_ASYNC_EVENT_ID_SRAM2 = 83,
	32	+ GOYA_ASYNC_EVENT_ID_SRAM3 = 84,
	33	+ GOYA_ASYNC_EVENT_ID_SRAM4 = 85,
	34	+ GOYA_ASYNC_EVENT_ID_SRAM5 = 86,
	35	+ GOYA_ASYNC_EVENT_ID_SRAM6 = 87,
	36	+ GOYA_ASYNC_EVENT_ID_SRAM7 = 88,
	37	+ GOYA_ASYNC_EVENT_ID_SRAM8 = 89,
	38	+ GOYA_ASYNC_EVENT_ID_SRAM9 = 90,
	39	+ GOYA_ASYNC_EVENT_ID_SRAM10 = 91,
	40	+ GOYA_ASYNC_EVENT_ID_SRAM11 = 92,
	41	+ GOYA_ASYNC_EVENT_ID_SRAM12 = 93,
	42	+ GOYA_ASYNC_EVENT_ID_SRAM13 = 94,
	43	+ GOYA_ASYNC_EVENT_ID_SRAM14 = 95,
	44	+ GOYA_ASYNC_EVENT_ID_SRAM15 = 96,
	45	+ GOYA_ASYNC_EVENT_ID_SRAM16 = 97,
	46	+ GOYA_ASYNC_EVENT_ID_SRAM17 = 98,
	47	+ GOYA_ASYNC_EVENT_ID_SRAM18 = 99,
	48	+ GOYA_ASYNC_EVENT_ID_SRAM19 = 100,
	49	+ GOYA_ASYNC_EVENT_ID_SRAM20 = 101,
	50	+ GOYA_ASYNC_EVENT_ID_SRAM21 = 102,
	51	+ GOYA_ASYNC_EVENT_ID_SRAM22 = 103,
	52	+ GOYA_ASYNC_EVENT_ID_SRAM23 = 104,
	53	+ GOYA_ASYNC_EVENT_ID_SRAM24 = 105,
	54	+ GOYA_ASYNC_EVENT_ID_SRAM25 = 106,
	55	+ GOYA_ASYNC_EVENT_ID_SRAM26 = 107,
	56	+ GOYA_ASYNC_EVENT_ID_SRAM27 = 108,
	57	+ GOYA_ASYNC_EVENT_ID_SRAM28 = 109,
	58	+ GOYA_ASYNC_EVENT_ID_SRAM29 = 110,
	59	+ GOYA_ASYNC_EVENT_ID_GIC500 = 112,
	60	+ GOYA_ASYNC_EVENT_ID_PCIE_DEC = 115,
	61	+ GOYA_ASYNC_EVENT_ID_TPC0_DEC = 117,
	62	+ GOYA_ASYNC_EVENT_ID_TPC1_DEC = 120,
	63	+ GOYA_ASYNC_EVENT_ID_TPC2_DEC = 123,
	64	+ GOYA_ASYNC_EVENT_ID_TPC3_DEC = 126,
	65	+ GOYA_ASYNC_EVENT_ID_TPC4_DEC = 129,
	66	+ GOYA_ASYNC_EVENT_ID_TPC5_DEC = 132,
	67	+ GOYA_ASYNC_EVENT_ID_TPC6_DEC = 135,
	68	+ GOYA_ASYNC_EVENT_ID_TPC7_DEC = 138,
	69	+ GOYA_ASYNC_EVENT_ID_AXI_ECC = 139,
	70	+ GOYA_ASYNC_EVENT_ID_L2_RAM_ECC = 140,
	71	+ GOYA_ASYNC_EVENT_ID_MME_WACS = 141,
	72	+ GOYA_ASYNC_EVENT_ID_MME_WACSD = 142,
	73	+ GOYA_ASYNC_EVENT_ID_PLL0 = 143,
	74	+ GOYA_ASYNC_EVENT_ID_PLL1 = 144,
	75	+ GOYA_ASYNC_EVENT_ID_PLL3 = 146,
	76	+ GOYA_ASYNC_EVENT_ID_PLL4 = 147,
	77	+ GOYA_ASYNC_EVENT_ID_PLL5 = 148,
	78	+ GOYA_ASYNC_EVENT_ID_PLL6 = 149,
	79	+ GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER = 155,
	80	+ GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC = 159,
	81	+ GOYA_ASYNC_EVENT_ID_PSOC = 160,
	82	+ GOYA_ASYNC_EVENT_ID_PCIE_FLR = 171,
	83	+ GOYA_ASYNC_EVENT_ID_PCIE_HOT_RESET = 172,
	84	+ GOYA_ASYNC_EVENT_ID_PCIE_QID0_ENG0 = 174,
	85	+ GOYA_ASYNC_EVENT_ID_PCIE_QID0_ENG1 = 175,
	86	+ GOYA_ASYNC_EVENT_ID_PCIE_QID0_ENG2 = 176,
	87	+ GOYA_ASYNC_EVENT_ID_PCIE_QID0_ENG3 = 177,
	88	+ GOYA_ASYNC_EVENT_ID_PCIE_QID1_ENG0 = 178,
	89	+ GOYA_ASYNC_EVENT_ID_PCIE_QID1_ENG1 = 179,
	90	+ GOYA_ASYNC_EVENT_ID_PCIE_QID1_ENG2 = 180,
	91	+ GOYA_ASYNC_EVENT_ID_PCIE_QID1_ENG3 = 181,
	92	+ GOYA_ASYNC_EVENT_ID_PCIE_APB = 182,
	93	+ GOYA_ASYNC_EVENT_ID_PCIE_QDB = 183,
	94	+ GOYA_ASYNC_EVENT_ID_PCIE_BM_D_P_WR = 184,
	95	+ GOYA_ASYNC_EVENT_ID_PCIE_BM_D_RD = 185,
	96	+ GOYA_ASYNC_EVENT_ID_PCIE_BM_U_P_WR = 186,
	97	+ GOYA_ASYNC_EVENT_ID_PCIE_BM_U_RD = 187,
	98	+ GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU = 190,
	99	+ GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR = 191,
	100	+ GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU = 200,
	101	+ GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR = 201,
	102	+ GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU = 210,
	103	+ GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR = 211,
	104	+ GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU = 220,
	105	+ GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR = 221,
	106	+ GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU = 230,
	107	+ GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR = 231,
	108	+ GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU = 240,
	109	+ GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR = 241,
	110	+ GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU = 250,
	111	+ GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR = 251,
	112	+ GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU = 260,
	113	+ GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR = 261,
	114	+ GOYA_ASYNC_EVENT_ID_MMU_SBA_SPMU0 = 270,
	115	+ GOYA_ASYNC_EVENT_ID_MMU_SBA_SPMU1 = 271,
	116	+ GOYA_ASYNC_EVENT_ID_MME_WACS_UP = 272,
	117	+ GOYA_ASYNC_EVENT_ID_MME_WACS_DOWN = 273,
	118	+ GOYA_ASYNC_EVENT_ID_MMU_PAGE_FAULT = 280,
	119	+ GOYA_ASYNC_EVENT_ID_MMU_WR_PERM = 281,
	120	+ GOYA_ASYNC_EVENT_ID_MMU_DBG_BM = 282,
	121	+ GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 = 290,
	122	+ GOYA_ASYNC_EVENT_ID_DMA_BM_CH1 = 291,
	123	+ GOYA_ASYNC_EVENT_ID_DMA_BM_CH2 = 292,
	124	+ GOYA_ASYNC_EVENT_ID_DMA_BM_CH3 = 293,
	125	+ GOYA_ASYNC_EVENT_ID_DMA_BM_CH4 = 294,
	126	+ GOYA_ASYNC_EVENT_ID_DDR0_PHY_DFI = 300,
	127	+ GOYA_ASYNC_EVENT_ID_DDR0_ECC_SCRUB = 301,
	128	+ GOYA_ASYNC_EVENT_ID_DDR0_DB_ECC = 302,
	129	+ GOYA_ASYNC_EVENT_ID_DDR0_SB_ECC = 303,
	130	+ GOYA_ASYNC_EVENT_ID_DDR0_SB_ECC_MC = 304,
	131	+ GOYA_ASYNC_EVENT_ID_DDR0_AXI_RD = 305,
	132	+ GOYA_ASYNC_EVENT_ID_DDR0_AXI_WR = 306,
	133	+ GOYA_ASYNC_EVENT_ID_DDR1_PHY_DFI = 310,
	134	+ GOYA_ASYNC_EVENT_ID_DDR1_ECC_SCRUB = 311,
	135	+ GOYA_ASYNC_EVENT_ID_DDR1_DB_ECC = 312,
	136	+ GOYA_ASYNC_EVENT_ID_DDR1_SB_ECC = 313,
	137	+ GOYA_ASYNC_EVENT_ID_DDR1_SB_ECC_MC = 314,
	138	+ GOYA_ASYNC_EVENT_ID_DDR1_AXI_RD = 315,
	139	+ GOYA_ASYNC_EVENT_ID_DDR1_AXI_WR = 316,
	140	+ GOYA_ASYNC_EVENT_ID_CPU_BMON = 320,
	141	+ GOYA_ASYNC_EVENT_ID_TS_EAST = 322,
	142	+ GOYA_ASYNC_EVENT_ID_TS_WEST = 323,
	143	+ GOYA_ASYNC_EVENT_ID_TS_NORTH = 324,
	144	+ GOYA_ASYNC_EVENT_ID_PSOC_GPIO_U16_0 = 330,
	145	+ GOYA_ASYNC_EVENT_ID_PSOC_GPIO_U16_1 = 331,
	146	+ GOYA_ASYNC_EVENT_ID_PSOC_GPIO_U16_2 = 332,
	147	+ GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET = 356,
	148	+ GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT = 361,
	149	+ GOYA_ASYNC_EVENT_ID_TPC0_CMDQ = 430,
	150	+ GOYA_ASYNC_EVENT_ID_TPC1_CMDQ = 431,
	151	+ GOYA_ASYNC_EVENT_ID_TPC2_CMDQ = 432,
	152	+ GOYA_ASYNC_EVENT_ID_TPC3_CMDQ = 433,
	153	+ GOYA_ASYNC_EVENT_ID_TPC4_CMDQ = 434,
	154	+ GOYA_ASYNC_EVENT_ID_TPC5_CMDQ = 435,
	155	+ GOYA_ASYNC_EVENT_ID_TPC6_CMDQ = 436,
	156	+ GOYA_ASYNC_EVENT_ID_TPC7_CMDQ = 437,
	157	+ GOYA_ASYNC_EVENT_ID_TPC0_QM = 438,
	158	+ GOYA_ASYNC_EVENT_ID_TPC1_QM = 439,
	159	+ GOYA_ASYNC_EVENT_ID_TPC2_QM = 440,
	160	+ GOYA_ASYNC_EVENT_ID_TPC3_QM = 441,
	161	+ GOYA_ASYNC_EVENT_ID_TPC4_QM = 442,
	162	+ GOYA_ASYNC_EVENT_ID_TPC5_QM = 443,
	163	+ GOYA_ASYNC_EVENT_ID_TPC6_QM = 444,
	164	+ GOYA_ASYNC_EVENT_ID_TPC7_QM = 445,
	165	+ GOYA_ASYNC_EVENT_ID_MME_QM = 447,
	166	+ GOYA_ASYNC_EVENT_ID_MME_CMDQ = 448,
	167	+ GOYA_ASYNC_EVENT_ID_DMA0_QM = 449,
	168	+ GOYA_ASYNC_EVENT_ID_DMA1_QM = 450,
	169	+ GOYA_ASYNC_EVENT_ID_DMA2_QM = 451,
	170	+ GOYA_ASYNC_EVENT_ID_DMA3_QM = 452,
	171	+ GOYA_ASYNC_EVENT_ID_DMA4_QM = 453,
	172	+ GOYA_ASYNC_EVENT_ID_DMA_ON_HBW = 454,
	173	+ GOYA_ASYNC_EVENT_ID_DMA0_CH = 455,
	174	+ GOYA_ASYNC_EVENT_ID_DMA1_CH = 456,
	175	+ GOYA_ASYNC_EVENT_ID_DMA2_CH = 457,
	176	+ GOYA_ASYNC_EVENT_ID_DMA3_CH = 458,
	177	+ GOYA_ASYNC_EVENT_ID_DMA4_CH = 459,
	178	+ GOYA_ASYNC_EVENT_ID_PI_UPDATE = 484,
	179	+ GOYA_ASYNC_EVENT_ID_HALT_MACHINE = 485,
	180	+ GOYA_ASYNC_EVENT_ID_INTS_REGISTER = 486,
	181	+ GOYA_ASYNC_EVENT_ID_SOFT_RESET = 487,
	182	+ GOYA_ASYNC_EVENT_ID_LAST_VALID_ID = 1023,
	183	+ GOYA_ASYNC_EVENT_ID_SIZE
	184	+};
	185	+
	186	+#endif /* __GOYA_ASYNC_EVENTS_H_ */

drivers/misc/habanalabs/include/goya/goya_packets.h+129−0

@@ -0,0 +1,129 @@
	1	+/* SPDX-License-Identifier: GPL-2.0
	2	+ *
	3	+ * Copyright 2017-2018 HabanaLabs, Ltd.
	4	+ * All Rights Reserved.
	5	+ *
	6	+ */
	7	+
	8	+#ifndef GOYA_PACKETS_H
	9	+#define GOYA_PACKETS_H
	10	+
	11	+#include <linux/types.h>
	12	+
	13	+#define PACKET_HEADER_PACKET_ID_SHIFT 56
	14	+#define PACKET_HEADER_PACKET_ID_MASK 0x1F00000000000000ull
	15	+
	16	+enum packet_id {
	17	+ PACKET_WREG_32 = 0x1,
	18	+ PACKET_WREG_BULK = 0x2,
	19	+ PACKET_MSG_LONG = 0x3,
	20	+ PACKET_MSG_SHORT = 0x4,
	21	+ PACKET_CP_DMA = 0x5,
	22	+ PACKET_MSG_PROT = 0x7,
	23	+ PACKET_FENCE = 0x8,
	24	+ PACKET_LIN_DMA = 0x9,
	25	+ PACKET_NOP = 0xA,
	26	+ PACKET_STOP = 0xB,
	27	+ MAX_PACKET_ID = (PACKET_HEADER_PACKET_ID_MASK >>
	28	+ PACKET_HEADER_PACKET_ID_SHIFT) + 1
	29	+};
	30	+
	31	+enum goya_dma_direction {
	32	+ DMA_HOST_TO_DRAM,
	33	+ DMA_HOST_TO_SRAM,
	34	+ DMA_DRAM_TO_SRAM,
	35	+ DMA_SRAM_TO_DRAM,
	36	+ DMA_SRAM_TO_HOST,
	37	+ DMA_DRAM_TO_HOST,
	38	+ DMA_DRAM_TO_DRAM,
	39	+ DMA_SRAM_TO_SRAM,
	40	+ DMA_ENUM_MAX
	41	+};
	42	+
	43	+#define GOYA_PKT_CTL_OPCODE_SHIFT 24
	44	+#define GOYA_PKT_CTL_OPCODE_MASK 0x1F000000
	45	+
	46	+#define GOYA_PKT_CTL_EB_SHIFT 29
	47	+#define GOYA_PKT_CTL_EB_MASK 0x20000000
	48	+
	49	+#define GOYA_PKT_CTL_RB_SHIFT 30
	50	+#define GOYA_PKT_CTL_RB_MASK 0x40000000
	51	+
	52	+#define GOYA_PKT_CTL_MB_SHIFT 31
	53	+#define GOYA_PKT_CTL_MB_MASK 0x80000000
	54	+
	55	+struct packet_nop {
	56	+ __le32 reserved;
	57	+ __le32 ctl;
	58	+};
	59	+
	60	+struct packet_stop {
	61	+ __le32 reserved;
	62	+ __le32 ctl;
	63	+};
	64	+
	65	+#define GOYA_PKT_WREG32_CTL_REG_OFFSET_SHIFT 0
	66	+#define GOYA_PKT_WREG32_CTL_REG_OFFSET_MASK 0x0000FFFF
	67	+
	68	+struct packet_wreg32 {
	69	+ __le32 value;
	70	+ __le32 ctl;
	71	+};
	72	+
	73	+struct packet_wreg_bulk {
	74	+ __le32 size64;
	75	+ __le32 ctl;
	76	+ __le64 values[0]; /* data starts here */
	77	+};
	78	+
	79	+struct packet_msg_long {
	80	+ __le32 value;
	81	+ __le32 ctl;
	82	+ __le64 addr;
	83	+};
	84	+
	85	+struct packet_msg_short {
	86	+ __le32 value;
	87	+ __le32 ctl;
	88	+};
	89	+
	90	+struct packet_msg_prot {
	91	+ __le32 value;
	92	+ __le32 ctl;
	93	+ __le64 addr;
	94	+};
	95	+
	96	+struct packet_fence {
	97	+ __le32 cfg;
	98	+ __le32 ctl;
	99	+};
	100	+
	101	+#define GOYA_PKT_LIN_DMA_CTL_WO_SHIFT 0
	102	+#define GOYA_PKT_LIN_DMA_CTL_WO_MASK 0x00000001
	103	+
	104	+#define GOYA_PKT_LIN_DMA_CTL_RDCOMP_SHIFT 1
	105	+#define GOYA_PKT_LIN_DMA_CTL_RDCOMP_MASK 0x00000002
	106	+
	107	+#define GOYA_PKT_LIN_DMA_CTL_WRCOMP_SHIFT 2
	108	+#define GOYA_PKT_LIN_DMA_CTL_WRCOMP_MASK 0x00000004
	109	+
	110	+#define GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT 6
	111	+#define GOYA_PKT_LIN_DMA_CTL_MEMSET_MASK 0x00000040
	112	+
	113	+#define GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT 20
	114	+#define GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK 0x00700000
	115	+
	116	+struct packet_lin_dma {
	117	+ __le32 tsize;
	118	+ __le32 ctl;
	119	+ __le64 src_addr;
	120	+ __le64 dst_addr;
	121	+};
	122	+
	123	+struct packet_cp_dma {
	124	+ __le32 tsize;
	125	+ __le32 ctl;
	126	+ __le64 src_addr;
	127	+};
	128	+
	129	+#endif /* GOYA_PACKETS_H */

drivers/misc/habanalabs/include/qman_if.h+56−0

@@ -0,0 +1,56 @@
	1	+/* SPDX-License-Identifier: GPL-2.0
	2	+ *
	3	+ * Copyright 2016-2018 HabanaLabs, Ltd.
	4	+ * All Rights Reserved.
	5	+ *
	6	+ */
	7	+
	8	+#ifndef QMAN_IF_H
	9	+#define QMAN_IF_H
	10	+
	11	+#include <linux/types.h>
	12	+
	13	+/*
	14	+ * PRIMARY QUEUE
	15	+ */
	16	+
	17	+struct hl_bd {
	18	+ __le64 ptr;
	19	+ __le32 len;
	20	+ __le32 ctl;
	21	+};
	22	+
	23	+#define HL_BD_SIZE sizeof(struct hl_bd)
	24	+
	25	+/*
	26	+ * BD_CTL_REPEAT_VALID tells the CP whether the repeat field in the BD CTL is
	27	+ * valid. 1 means the repeat field is valid, 0 means not-valid,
	28	+ * i.e. repeat == 1
	29	+ */
	30	+#define BD_CTL_REPEAT_VALID_SHIFT 24
	31	+#define BD_CTL_REPEAT_VALID_MASK 0x01000000
	32	+
	33	+#define BD_CTL_SHADOW_INDEX_SHIFT 0
	34	+#define BD_CTL_SHADOW_INDEX_MASK 0x00000FFF
	35	+
	36	+/*
	37	+ * COMPLETION QUEUE
	38	+ */
	39	+
	40	+struct hl_cq_entry {
	41	+ __le32 data;
	42	+};
	43	+
	44	+#define HL_CQ_ENTRY_SIZE sizeof(struct hl_cq_entry)
	45	+
	46	+#define CQ_ENTRY_READY_SHIFT 31
	47	+#define CQ_ENTRY_READY_MASK 0x80000000
	48	+
	49	+#define CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT 30
	50	+#define CQ_ENTRY_SHADOW_INDEX_VALID_MASK 0x40000000
	51	+
	52	+#define CQ_ENTRY_SHADOW_INDEX_SHIFT BD_CTL_SHADOW_INDEX_SHIFT
	53	+#define CQ_ENTRY_SHADOW_INDEX_MASK BD_CTL_SHADOW_INDEX_MASK
	54	+
	55	+
	56	+#endif /* QMAN_IF_H */

drivers/misc/habanalabs/irq.c+149−0

@@ -0,0 +1,149 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+/*
	4	+ * Copyright 2016-2019 HabanaLabs, Ltd.
	5	+ * All Rights Reserved.
	6	+ */
	7	+
	8	+#include "habanalabs.h"
	9	+
	10	+#include <linux/irqreturn.h>
	11	+
	12	+/*
	13	+ * hl_cq_inc_ptr - increment ci or pi of cq
	14	+ *
	15	+ * @ptr: the current ci or pi value of the completion queue
	16	+ *
	17	+ * Increment ptr by 1. If it reaches the number of completion queue
	18	+ * entries, set it to 0
	19	+ */
	20	+inline u32 hl_cq_inc_ptr(u32 ptr)
	21	+{
	22	+ ptr++;
	23	+ if (unlikely(ptr == HL_CQ_LENGTH))
	24	+ ptr = 0;
	25	+ return ptr;
	26	+}
	27	+
	28	+/*
	29	+ * hl_irq_handler_cq - irq handler for completion queue
	30	+ *
	31	+ * @irq: irq number
	32	+ * @arg: pointer to completion queue structure
	33	+ *
	34	+ */
	35	+irqreturn_t hl_irq_handler_cq(int irq, void *arg)
	36	+{
	37	+ struct hl_cq *cq = arg;
	38	+ struct hl_device *hdev = cq->hdev;
	39	+ struct hl_hw_queue *queue;
	40	+ struct hl_cs_job *job;
	41	+ bool shadow_index_valid;
	42	+ u16 shadow_index;
	43	+ u32 *cq_entry;
	44	+ u32 *cq_base;
	45	+
	46	+ if (hdev->disabled) {
	47	+ dev_dbg(hdev->dev,
	48	+ "Device disabled but received IRQ %d for CQ %d\n",
	49	+ irq, cq->hw_queue_id);
	50	+ return IRQ_HANDLED;
	51	+ }
	52	+
	53	+ cq_base = (u32 *) (uintptr_t) cq->kernel_address;
	54	+
	55	+ while (1) {
	56	+ bool entry_ready = ((cq_base[cq->ci] & CQ_ENTRY_READY_MASK)
	57	+ >> CQ_ENTRY_READY_SHIFT);
	58	+
	59	+ if (!entry_ready)
	60	+ break;
	61	+
	62	+ cq_entry = (u32 *) &cq_base[cq->ci];
	63	+
	64	+ /*
	65	+ * Make sure we read CQ entry contents after we've
	66	+ * checked the ownership bit.
	67	+ */
	68	+ dma_rmb();
	69	+
	70	+ shadow_index_valid =
	71	+ ((*cq_entry & CQ_ENTRY_SHADOW_INDEX_VALID_MASK)
	72	+ >> CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT);
	73	+
	74	+ shadow_index = (u16)
	75	+ ((*cq_entry & CQ_ENTRY_SHADOW_INDEX_MASK)
	76	+ >> CQ_ENTRY_SHADOW_INDEX_SHIFT);
	77	+
	78	+ queue = &hdev->kernel_queues[cq->hw_queue_id];
	79	+
	80	+ if ((shadow_index_valid) && (!hdev->disabled)) {
	81	+ job = queue->shadow_queue[hl_pi_2_offset(shadow_index)];
	82	+ queue_work(hdev->cq_wq, &job->finish_work);
	83	+ }
	84	+
	85	+ /*
	86	+ * Update ci of the context's queue. There is no
	87	+ * need to protect it with spinlock because this update is
	88	+ * done only inside IRQ and there is a different IRQ per
	89	+ * queue
	90	+ */
	91	+ queue->ci = hl_queue_inc_ptr(queue->ci);
	92	+
	93	+ /* Clear CQ entry ready bit */
	94	+ cq_base[cq->ci] &= ~CQ_ENTRY_READY_MASK;
	95	+
	96	+ cq->ci = hl_cq_inc_ptr(cq->ci);
	97	+
	98	+ /* Increment free slots */
	99	+ atomic_inc(&cq->free_slots_cnt);
	100	+ }
	101	+
	102	+ return IRQ_HANDLED;
	103	+}
	104	+
	105	+/*
	106	+ * hl_cq_init - main initialization function for an cq object
	107	+ *
	108	+ * @hdev: pointer to device structure
	109	+ * @q: pointer to cq structure
	110	+ * @hw_queue_id: The H/W queue ID this completion queue belongs to
	111	+ *
	112	+ * Allocate dma-able memory for the completion queue and initialize fields
	113	+ * Returns 0 on success
	114	+ */
	115	+int hl_cq_init(struct hl_device hdev, struct hl_cq q, u32 hw_queue_id)
	116	+{
	117	+ void *p;
	118	+
	119	+ BUILD_BUG_ON(HL_CQ_SIZE_IN_BYTES > HL_PAGE_SIZE);
	120	+
	121	+ p = hdev->asic_funcs->dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
	122	+ &q->bus_address, GFP_KERNEL \| __GFP_ZERO);
	123	+ if (!p)
	124	+ return -ENOMEM;
	125	+
	126	+ q->hdev = hdev;
	127	+ q->kernel_address = (u64) (uintptr_t) p;
	128	+ q->hw_queue_id = hw_queue_id;
	129	+ q->ci = 0;
	130	+ q->pi = 0;
	131	+
	132	+ atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
	133	+
	134	+ return 0;
	135	+}
	136	+
	137	+/*
	138	+ * hl_cq_fini - destroy completion queue
	139	+ *
	140	+ * @hdev: pointer to device structure
	141	+ * @q: pointer to cq structure
	142	+ *
	143	+ * Free the completion queue memory
	144	+ */
	145	+void hl_cq_fini(struct hl_device hdev, struct hl_cq q)
	146	+{
	147	+ hdev->asic_funcs->dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
	148	+ (void *) (uintptr_t) q->kernel_address, q->bus_address);
	149	+}

include/uapi/misc/habanalabs.h+29−0

@@ -17,6 +17,35 @@ · 16 unmodified lines
17	17	*/
18	18	#define GOYA_KMD_SRAM_RESERVED_SIZE_FROM_START 0x8000 /* 32KB */
19	19
	20	+/*
	21	+ * Queue Numbering
	22	+ *
	23	+ * The external queues (DMA channels + CPU) MUST be before the internal queues
	24	+ * and each group (DMA channels + CPU and internal) must be contiguous inside
	25	+ * itself but there can be a gap between the two groups (although not
	26	+ * recommended)
	27	+ */
	28	+
	29	+enum goya_queue_id {
	30	+ GOYA_QUEUE_ID_DMA_0 = 0,
	31	+ GOYA_QUEUE_ID_DMA_1,
	32	+ GOYA_QUEUE_ID_DMA_2,
	33	+ GOYA_QUEUE_ID_DMA_3,
	34	+ GOYA_QUEUE_ID_DMA_4,
	35	+ GOYA_QUEUE_ID_CPU_PQ,
	36	+ GOYA_QUEUE_ID_MME,
	37	+ GOYA_QUEUE_ID_TPC0,
	38	+ GOYA_QUEUE_ID_TPC1,
	39	+ GOYA_QUEUE_ID_TPC2,
	40	+ GOYA_QUEUE_ID_TPC3,
	41	+ GOYA_QUEUE_ID_TPC4,
	42	+ GOYA_QUEUE_ID_TPC5,
	43	+ GOYA_QUEUE_ID_TPC6,
	44	+ GOYA_QUEUE_ID_TPC7,
	45	+ GOYA_QUEUE_ID_SIZE
	46	+};
	47	+
	48	+
20	49	/* Opcode to create a new command buffer */
21	50	#define HL_CB_OP_CREATE 0
22	51	/* Opcode to destroy previously created command buffer */