habanalabs: add h/w queues module

This patch adds the H/W queues module and the code to initialize Goya's various compute and DMA engines and their queues. Goya has 5 DMA channels, 8 TPC engines and a single MME engine. For each channel/engine, there is a H/W queue logic which is used to pass commands from the user to the H/W. That logic is called QMAN. There are two types of QMANs: external and internal. The DMA QMANs are considered external while the TPC and MME QMANs are considered internal. For each external queue there is a completion queue, which is located on the Host memory. The differences between external and internal QMANs are: 1. The location of the queue's memory. External QMANs are located on the Host memory while internal QMANs are located on the on-chip memory. 2. The external QMAN write an entry to a completion queue and sends an MSI-X interrupt upon completion of a command buffer that was given to it. The internal QMAN doesn't do that.

Oded Gabbay committed 7 years ago

Commit 9494a8d1 parent 839c48013 files · +2915 −116Browse files

Unified · Split

13 files

drivers/misc/habanalabs/Makefile +1 −1
drivers/misc/habanalabs/device.c +73 −2
drivers/misc/habanalabs/goya/goya.c +1415 −112
drivers/misc/habanalabs/goya/goyaP.h +7 −0
drivers/misc/habanalabs/habanalabs.h +173 −1
drivers/misc/habanalabs/habanalabs_drv.c +5 −0
drivers/misc/habanalabs/hw_queue.c +400 −0
drivers/misc/habanalabs/include/armcp_if.h +292 −0
drivers/misc/habanalabs/include/goya/goya_async_events.h +186 −0
drivers/misc/habanalabs/include/goya/goya_packets.h +129 −0
drivers/misc/habanalabs/include/qman_if.h +56 −0
drivers/misc/habanalabs/irq.c +149 −0
include/uapi/misc/habanalabs.h +29 −0

drivers/misc/habanalabs/Makefile+1−1

@@ -5,7 +5,7 @@ · 4 unmodified lines
5	obj-m := habanalabs.o	5	obj-m := habanalabs.o
6		6
7	habanalabs-y := habanalabs_drv.o device.o context.o asid.o habanalabs_ioctl.o \	7	habanalabs-y := habanalabs_drv.o device.o context.o asid.o habanalabs_ioctl.o \
8	command_buffer.o	8	command_buffer.o hw_queue.o irq.o
9		9
10	include $(src)/goya/Makefile	10	include $(src)/goya/Makefile
11	habanalabs-y += $(HL_GOYA_FILES)	11	habanalabs-y += $(HL_GOYA_FILES)

drivers/misc/habanalabs/device.c+73−2

@@ -174,13 +174,23 @@ static int device_early_init(struct hl_device *hdev) · 173 unmodified lines
174	if (rc)	174	if (rc)
175	goto early_fini;	175	goto early_fini;
176		176
		177	hdev->cq_wq = alloc_workqueue("hl-free-jobs", WQ_UNBOUND, 0);
		178	if (hdev->cq_wq == NULL) {
		179	dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
		180	rc = -ENOMEM;
		181	goto asid_fini;
		182	}
		183
177	hl_cb_mgr_init(&hdev->kernel_cb_mgr);	184	hl_cb_mgr_init(&hdev->kernel_cb_mgr);
178		185
179	mutex_init(&hdev->fd_open_cnt_lock);	186	mutex_init(&hdev->fd_open_cnt_lock);
		187	mutex_init(&hdev->send_cpu_message_lock);
180	atomic_set(&hdev->fd_open_cnt, 0);	188	atomic_set(&hdev->fd_open_cnt, 0);
181		189
182	return 0;	190	return 0;
183		191
		192	asid_fini:
		193	hl_asid_fini(hdev);
184	early_fini:	194	early_fini:
185	if (hdev->asic_funcs->early_fini)	195	if (hdev->asic_funcs->early_fini)
186	hdev->asic_funcs->early_fini(hdev);	196	hdev->asic_funcs->early_fini(hdev);
@@ -196,9 +206,12 @@ early_fini: · 9 unmodified lines
196	*/	206	*/
197	static void device_early_fini(struct hl_device *hdev)	207	static void device_early_fini(struct hl_device *hdev)
198	{	208	{
		209	mutex_destroy(&hdev->send_cpu_message_lock);
199		210
200	hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);	211	hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
201		212
		213	destroy_workqueue(hdev->cq_wq);
		214
202	hl_asid_fini(hdev);	215	hl_asid_fini(hdev);
203		216
204	if (hdev->asic_funcs->early_fini)	217	if (hdev->asic_funcs->early_fini)
@@ -277,7 +290,7 @@ int hl_device_resume(struct hl_device *hdev) · 72 unmodified lines
277	*/	290	*/
278	int hl_device_init(struct hl_device hdev, struct class hclass)	291	int hl_device_init(struct hl_device hdev, struct class hclass)
279	{	292	{
280	int rc;	293	int i, rc, cq_ready_cnt;
281		294
282	/* Create device */	295	/* Create device */
283	rc = device_setup_cdev(hdev, hclass, hdev->id, &hl_ops);	296	rc = device_setup_cdev(hdev, hclass, hdev->id, &hl_ops);
@@ -298,11 +311,48 @@ int hl_device_init(struct hl_device hdev, struct class hclass) · 14 unmodified lines
298	if (rc)	311	if (rc)
299	goto early_fini;	312	goto early_fini;
300		313
		314	/*
		315	* Initialize the H/W queues. Must be done before hw_init, because
		316	* there the addresses of the kernel queue are being written to the
		317	* registers of the device
		318	*/
		319	rc = hl_hw_queues_create(hdev);
		320	if (rc) {
		321	dev_err(hdev->dev, "failed to initialize kernel queues\n");
		322	goto sw_fini;
		323	}
		324
		325	/*
		326	* Initialize the completion queues. Must be done before hw_init,
		327	* because there the addresses of the completion queues are being
		328	* passed as arguments to request_irq
		329	*/
		330	hdev->completion_queue =
		331	kcalloc(hdev->asic_prop.completion_queues_count,
		332	sizeof(*hdev->completion_queue), GFP_KERNEL);
		333
		334	if (!hdev->completion_queue) {
		335	dev_err(hdev->dev, "failed to allocate completion queues\n");
		336	rc = -ENOMEM;
		337	goto hw_queues_destroy;
		338	}
		339
		340	for (i = 0, cq_ready_cnt = 0;
		341	i < hdev->asic_prop.completion_queues_count;
		342	i++, cq_ready_cnt++) {
		343	rc = hl_cq_init(hdev, &hdev->completion_queue[i], i);
		344	if (rc) {
		345	dev_err(hdev->dev,
		346	"failed to initialize completion queue\n");
		347	goto cq_fini;
		348	}
		349	}
		350
301	/* Allocate the kernel context */	351	/* Allocate the kernel context */
302	hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);	352	hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
303	if (!hdev->kernel_ctx) {	353	if (!hdev->kernel_ctx) {
304	rc = -ENOMEM;	354	rc = -ENOMEM;
305	goto sw_fini;	355	goto cq_fini;
306	}	356	}
307		357
308	hdev->user_ctx = NULL;	358	hdev->user_ctx = NULL;
@@ -328,6 +378,14 @@ int hl_device_init(struct hl_device hdev, struct class hclass) · 19 unmodified lines
328		378
329	hdev->disabled = false;	379	hdev->disabled = false;
330		380
		381	/* Check that the communication with the device is working */
		382	rc = hdev->asic_funcs->test_queues(hdev);
		383	if (rc) {
		384	dev_err(hdev->dev, "Failed to detect if device is alive\n");
		385	rc = 0;
		386	goto out_disabled;
		387	}
		388
331	dev_notice(hdev->dev,	389	dev_notice(hdev->dev,
332	"Successfully added device to habanalabs driver\n");	390	"Successfully added device to habanalabs driver\n");
333		391
@@ -339,6 +397,12 @@ release_ctx: · 5 unmodified lines
339	"kernel ctx is still alive on initialization failure\n");	397	"kernel ctx is still alive on initialization failure\n");
340	free_ctx:	398	free_ctx:
341	kfree(hdev->kernel_ctx);	399	kfree(hdev->kernel_ctx);
		400	cq_fini:
		401	for (i = 0 ; i < cq_ready_cnt ; i++)
		402	hl_cq_fini(hdev, &hdev->completion_queue[i]);
		403	kfree(hdev->completion_queue);
		404	hw_queues_destroy:
		405	hl_hw_queues_destroy(hdev);
342	sw_fini:	406	sw_fini:
343	hdev->asic_funcs->sw_fini(hdev);	407	hdev->asic_funcs->sw_fini(hdev);
344	early_fini:	408	early_fini:
@@ -368,6 +432,7 @@ out_disabled: · 23 unmodified lines
368	*/	432	*/
369	void hl_device_fini(struct hl_device *hdev)	433	void hl_device_fini(struct hl_device *hdev)
370	{	434	{
		435	int i;
371	dev_info(hdev->dev, "Removing device\n");	436	dev_info(hdev->dev, "Removing device\n");
372		437
373	/* Mark device as disabled */	438	/* Mark device as disabled */
@@ -382,6 +447,12 @@ void hl_device_fini(struct hl_device *hdev) · 8 unmodified lines
382	/* Reset the H/W. It will be in idle state after this returns */	447	/* Reset the H/W. It will be in idle state after this returns */
383	hdev->asic_funcs->hw_fini(hdev, true);	448	hdev->asic_funcs->hw_fini(hdev, true);
384		449
		450	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
		451	hl_cq_fini(hdev, &hdev->completion_queue[i]);
		452	kfree(hdev->completion_queue);
		453
		454	hl_hw_queues_destroy(hdev);
		455
385	/* Call ASIC S/W finalize function */	456	/* Call ASIC S/W finalize function */
386	hdev->asic_funcs->sw_fini(hdev);	457	hdev->asic_funcs->sw_fini(hdev);
387		458

drivers/misc/habanalabs/goya/goya.c+1415−112

@@ -90,6 +90,26 @@ · 89 unmodified lines
90	static void goya_get_fixed_properties(struct hl_device *hdev)	90	static void goya_get_fixed_properties(struct hl_device *hdev)
91	{	91	{
92	struct asic_fixed_properties *prop = &hdev->asic_prop;	92	struct asic_fixed_properties *prop = &hdev->asic_prop;
		93	int i;
		94
		95	for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) {
		96	prop->hw_queues_props[i].type = QUEUE_TYPE_EXT;
		97	prop->hw_queues_props[i].kmd_only = 0;
		98	}
		99
		100	for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES ; i++) {
		101	prop->hw_queues_props[i].type = QUEUE_TYPE_CPU;
		102	prop->hw_queues_props[i].kmd_only = 1;
		103	}
		104
		105	for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES +
		106	NUMBER_OF_INT_HW_QUEUES; i++) {
		107	prop->hw_queues_props[i].type = QUEUE_TYPE_INT;
		108	prop->hw_queues_props[i].kmd_only = 0;
		109	}
		110
		111	for (; i < HL_MAX_QUEUES; i++)
		112	prop->hw_queues_props[i].type = QUEUE_TYPE_NA;
93		113
94	prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;	114	prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;
95		115
@@ -118,6 +138,18 @@ static void goya_get_fixed_properties(struct hl_device *hdev) · 22 unmodified lines
118	prop->high_pll = PLL_HIGH_DEFAULT;	138	prop->high_pll = PLL_HIGH_DEFAULT;
119	}	139	}
120		140
		141	int goya_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
		142	{
		143	struct armcp_packet pkt;
		144
		145	memset(&pkt, 0, sizeof(pkt));
		146
		147	pkt.ctl = opcode << ARMCP_PKT_CTL_OPCODE_SHIFT;
		148
		149	return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
		150	sizeof(pkt), HL_DEVICE_TIMEOUT_USEC, NULL);
		151	}
		152
121	/*	153	/*
122	* goya_pci_bars_map - Map PCI BARS of Goya device	154	* goya_pci_bars_map - Map PCI BARS of Goya device
123	*	155	*
@@ -510,6 +542,8 @@ static int goya_sw_init(struct hl_device *hdev) · 386 unmodified lines
510	if (!goya)	542	if (!goya)
511	return -ENOMEM;	543	return -ENOMEM;
512		544
		545	goya->test_cpu_queue = goya_test_cpu_queue;
		546
513	/* according to goya_init_iatu */	547	/* according to goya_init_iatu */
514	goya->ddr_bar_cur_addr = DRAM_PHYS_BASE;	548	goya->ddr_bar_cur_addr = DRAM_PHYS_BASE;
515	hdev->asic_specific = goya;	549	hdev->asic_specific = goya;
@@ -596,6 +630,311 @@ int goya_sw_fini(struct hl_device *hdev) · 80 unmodified lines
596	return 0;	630	return 0;
597	}	631	}
598		632
		633	static void goya_init_dma_qman(struct hl_device *hdev, int dma_id,
		634	dma_addr_t bus_address)
		635	{
		636	struct goya_device *goya = hdev->asic_specific;
		637	u32 mtr_base_lo, mtr_base_hi;
		638	u32 so_base_lo, so_base_hi;
		639	u32 gic_base_lo, gic_base_hi;
		640	u32 reg_off = dma_id * (mmDMA_QM_1_PQ_PI - mmDMA_QM_0_PQ_PI);
		641
		642	mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
		643	mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
		644	so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
		645	so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
		646
		647	gic_base_lo =
		648	lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
		649	gic_base_hi =
		650	upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
		651
		652	WREG32(mmDMA_QM_0_PQ_BASE_LO + reg_off, lower_32_bits(bus_address));
		653	WREG32(mmDMA_QM_0_PQ_BASE_HI + reg_off, upper_32_bits(bus_address));
		654
		655	WREG32(mmDMA_QM_0_PQ_SIZE + reg_off, ilog2(HL_QUEUE_LENGTH));
		656	WREG32(mmDMA_QM_0_PQ_PI + reg_off, 0);
		657	WREG32(mmDMA_QM_0_PQ_CI + reg_off, 0);
		658
		659	WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
		660	WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
		661	WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
		662	WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
		663	WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
		664	WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
		665	WREG32(mmDMA_QM_0_GLBL_ERR_WDATA + reg_off,
		666	GOYA_ASYNC_EVENT_ID_DMA0_QM + dma_id);
		667
		668	/* PQ has buffer of 2 cache lines, while CQ has 8 lines */
		669	WREG32(mmDMA_QM_0_PQ_CFG1 + reg_off, 0x00020002);
		670	WREG32(mmDMA_QM_0_CQ_CFG1 + reg_off, 0x00080008);
		671
		672	if (dma_id == 0)
		673	WREG32(mmDMA_QM_0_GLBL_PROT + reg_off, QMAN_DMA_FULLY_TRUSTED);
		674	else
		675	if (goya->hw_cap_initialized & HW_CAP_MMU)
		676	WREG32(mmDMA_QM_0_GLBL_PROT + reg_off,
		677	QMAN_DMA_PARTLY_TRUSTED);
		678	else
		679	WREG32(mmDMA_QM_0_GLBL_PROT + reg_off,
		680	QMAN_DMA_FULLY_TRUSTED);
		681
		682	WREG32(mmDMA_QM_0_GLBL_ERR_CFG + reg_off, QMAN_DMA_ERR_MSG_EN);
		683	WREG32(mmDMA_QM_0_GLBL_CFG0 + reg_off, QMAN_DMA_ENABLE);
		684	}
		685
		686	static void goya_init_dma_ch(struct hl_device *hdev, int dma_id)
		687	{
		688	u32 gic_base_lo, gic_base_hi;
		689	u64 sob_addr;
		690	u32 reg_off = dma_id * (mmDMA_CH_1_CFG1 - mmDMA_CH_0_CFG1);
		691
		692	gic_base_lo =
		693	lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
		694	gic_base_hi =
		695	upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
		696
		697	WREG32(mmDMA_CH_0_ERRMSG_ADDR_LO + reg_off, gic_base_lo);
		698	WREG32(mmDMA_CH_0_ERRMSG_ADDR_HI + reg_off, gic_base_hi);
		699	WREG32(mmDMA_CH_0_ERRMSG_WDATA + reg_off,
		700	GOYA_ASYNC_EVENT_ID_DMA0_CH + dma_id);
		701
		702	if (dma_id) {
		703	sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1000 +
		704	(dma_id - 1) * 4;
		705	WREG32(mmDMA_CH_0_WR_COMP_ADDR_LO + reg_off,
		706	lower_32_bits(sob_addr));
		707	WREG32(mmDMA_CH_0_WR_COMP_ADDR_HI + reg_off,
		708	upper_32_bits(sob_addr));
		709	WREG32(mmDMA_CH_0_WR_COMP_WDATA + reg_off, 0x80000001);
		710	}
		711	}
		712
		713	/*
		714	* goya_init_dma_qmans - Initialize QMAN DMA registers
		715	*
		716	* @hdev: pointer to hl_device structure
		717	*
		718	* Initialize the H/W registers of the QMAN DMA channels
		719	*
		720	*/
		721	static void goya_init_dma_qmans(struct hl_device *hdev)
		722	{
		723	struct goya_device *goya = hdev->asic_specific;
		724	struct hl_hw_queue *q;
		725	dma_addr_t bus_address;
		726	int i;
		727
		728	if (goya->hw_cap_initialized & HW_CAP_DMA)
		729	return;
		730
		731	q = &hdev->kernel_queues[0];
		732
		733	for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++, q++) {
		734	bus_address = q->bus_address +
		735	hdev->asic_prop.host_phys_base_address;
		736
		737	goya_init_dma_qman(hdev, i, bus_address);
		738	goya_init_dma_ch(hdev, i);
		739	}
		740
		741	goya->hw_cap_initialized \|= HW_CAP_DMA;
		742	}
		743
		744	/*
		745	* goya_disable_external_queues - Disable external queues
		746	*
		747	* @hdev: pointer to hl_device structure
		748	*
		749	*/
		750	static void goya_disable_external_queues(struct hl_device *hdev)
		751	{
		752	WREG32(mmDMA_QM_0_GLBL_CFG0, 0);
		753	WREG32(mmDMA_QM_1_GLBL_CFG0, 0);
		754	WREG32(mmDMA_QM_2_GLBL_CFG0, 0);
		755	WREG32(mmDMA_QM_3_GLBL_CFG0, 0);
		756	WREG32(mmDMA_QM_4_GLBL_CFG0, 0);
		757	}
		758
		759	static int goya_stop_queue(struct hl_device *hdev, u32 cfg_reg,
		760	u32 cp_sts_reg, u32 glbl_sts0_reg)
		761	{
		762	int rc;
		763	u32 status;
		764
		765	/* use the values of TPC0 as they are all the same*/
		766
		767	WREG32(cfg_reg, 1 << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
		768
		769	status = RREG32(cp_sts_reg);
		770	if (status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK) {
		771	rc = hl_poll_timeout(
		772	hdev,
		773	cp_sts_reg,
		774	status,
		775	!(status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK),
		776	1000,
		777	QMAN_FENCE_TIMEOUT_USEC);
		778
		779	/* if QMAN is stuck in fence no need to check for stop */
		780	if (rc)
		781	return 0;
		782	}
		783
		784	rc = hl_poll_timeout(
		785	hdev,
		786	glbl_sts0_reg,
		787	status,
		788	(status & TPC0_QM_GLBL_STS0_CP_IS_STOP_MASK),
		789	1000,
		790	QMAN_STOP_TIMEOUT_USEC);
		791
		792	if (rc) {
		793	dev_err(hdev->dev,
		794	"Timeout while waiting for QMAN to stop\n");
		795	return -EINVAL;
		796	}
		797
		798	return 0;
		799	}
		800
		801	/*
		802	* goya_stop_external_queues - Stop external queues
		803	*
		804	* @hdev: pointer to hl_device structure
		805	*
		806	* Returns 0 on success
		807	*
		808	*/
		809	static int goya_stop_external_queues(struct hl_device *hdev)
		810	{
		811	int rc, retval = 0;
		812
		813	rc = goya_stop_queue(hdev,
		814	mmDMA_QM_0_GLBL_CFG1,
		815	mmDMA_QM_0_CP_STS,
		816	mmDMA_QM_0_GLBL_STS0);
		817
		818	if (rc) {
		819	dev_err(hdev->dev, "failed to stop DMA QMAN 0\n");
		820	retval = -EIO;
		821	}
		822
		823	rc = goya_stop_queue(hdev,
		824	mmDMA_QM_1_GLBL_CFG1,
		825	mmDMA_QM_1_CP_STS,
		826	mmDMA_QM_1_GLBL_STS0);
		827
		828	if (rc) {
		829	dev_err(hdev->dev, "failed to stop DMA QMAN 1\n");
		830	retval = -EIO;
		831	}
		832
		833	rc = goya_stop_queue(hdev,
		834	mmDMA_QM_2_GLBL_CFG1,
		835	mmDMA_QM_2_CP_STS,
		836	mmDMA_QM_2_GLBL_STS0);
		837
		838	if (rc) {
		839	dev_err(hdev->dev, "failed to stop DMA QMAN 2\n");
		840	retval = -EIO;
		841	}
		842
		843	rc = goya_stop_queue(hdev,
		844	mmDMA_QM_3_GLBL_CFG1,
		845	mmDMA_QM_3_CP_STS,
		846	mmDMA_QM_3_GLBL_STS0);
		847
		848	if (rc) {
		849	dev_err(hdev->dev, "failed to stop DMA QMAN 3\n");
		850	retval = -EIO;
		851	}
		852
		853	rc = goya_stop_queue(hdev,
		854	mmDMA_QM_4_GLBL_CFG1,
		855	mmDMA_QM_4_CP_STS,
		856	mmDMA_QM_4_GLBL_STS0);
		857
		858	if (rc) {
		859	dev_err(hdev->dev, "failed to stop DMA QMAN 4\n");
		860	retval = -EIO;
		861	}
		862
		863	return retval;
		864	}
		865
		866	static void goya_resume_external_queues(struct hl_device *hdev)
		867	{
		868	WREG32(mmDMA_QM_0_GLBL_CFG1, 0);
		869	WREG32(mmDMA_QM_1_GLBL_CFG1, 0);
		870	WREG32(mmDMA_QM_2_GLBL_CFG1, 0);
		871	WREG32(mmDMA_QM_3_GLBL_CFG1, 0);
		872	WREG32(mmDMA_QM_4_GLBL_CFG1, 0);
		873	}
		874
		875	/*
		876	* goya_init_cpu_queues - Initialize PQ/CQ/EQ of CPU
		877	*
		878	* @hdev: pointer to hl_device structure
		879	*
		880	* Returns 0 on success
		881	*
		882	*/
		883	int goya_init_cpu_queues(struct hl_device *hdev)
		884	{
		885	struct goya_device *goya = hdev->asic_specific;
		886	dma_addr_t bus_address;
		887	u32 status;
		888	struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ];
		889	int err;
		890
		891	if (!hdev->cpu_queues_enable)
		892	return 0;
		893
		894	if (goya->hw_cap_initialized & HW_CAP_CPU_Q)
		895	return 0;
		896
		897	bus_address = cpu_pq->bus_address +
		898	hdev->asic_prop.host_phys_base_address;
		899	WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_0, lower_32_bits(bus_address));
		900	WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_1, upper_32_bits(bus_address));
		901
		902	bus_address = hdev->cpu_accessible_dma_address +
		903	hdev->asic_prop.host_phys_base_address;
		904	WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_8, lower_32_bits(bus_address));
		905	WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_9, upper_32_bits(bus_address));
		906
		907	WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_5, HL_QUEUE_SIZE_IN_BYTES);
		908	WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_10, CPU_ACCESSIBLE_MEM_SIZE);
		909
		910	/* Used for EQ CI */
		911	WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_6, 0);
		912
		913	WREG32(mmCPU_IF_PF_PQ_PI, 0);
		914
		915	WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_7, PQ_INIT_STATUS_READY_FOR_CP);
		916
		917	WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
		918	GOYA_ASYNC_EVENT_ID_PI_UPDATE);
		919
		920	err = hl_poll_timeout(
		921	hdev,
		922	mmPSOC_GLOBAL_CONF_SCRATCHPAD_7,
		923	status,
		924	(status == PQ_INIT_STATUS_READY_FOR_HOST),
		925	1000,
		926	GOYA_CPU_TIMEOUT_USEC);
		927
		928	if (err) {
		929	dev_err(hdev->dev,
		930	"Failed to communicate with ARM CPU (ArmCP timeout)\n");
		931	return -EIO;
		932	}
		933
		934	goya->hw_cap_initialized \|= HW_CAP_CPU_Q;
		935	return 0;
		936	}
		937
599	static void goya_set_pll_refclk(struct hl_device *hdev)	938	static void goya_set_pll_refclk(struct hl_device *hdev)
600	{	939	{
601	WREG32(mmCPU_PLL_DIV_SEL_0, 0x0);	940	WREG32(mmCPU_PLL_DIV_SEL_0, 0x0);
@@ -1023,144 +1362,644 @@ static void goya_init_golden_registers(struct hl_device *hdev) · 421 unmodified lines
1023	goya->hw_cap_initialized \|= HW_CAP_GOLDEN;	1362	goya->hw_cap_initialized \|= HW_CAP_GOLDEN;
1024	}	1363	}
1025		1364
1026		1365	static void goya_init_mme_qman(struct hl_device *hdev)
1027	/*
1028	* goya_push_fw_to_device - Push FW code to device
1029	*
1030	* @hdev: pointer to hl_device structure
1031	*
1032	* Copy fw code from firmware file to device memory.
1033	* Returns 0 on success
1034	*
1035	*/
1036	static int goya_push_fw_to_device(struct hl_device hdev, const char fw_name,
1037	void __iomem *dst)
1038	{	1366	{
1039	const struct firmware *fw;	1367	u32 mtr_base_lo, mtr_base_hi;
1040	const u64 *fw_data;	1368	u32 so_base_lo, so_base_hi;
1041	size_t fw_size, i;	1369	u32 gic_base_lo, gic_base_hi;
1042	int rc;	1370	u64 qman_base_addr;
1043		1371
1044	rc = request_firmware(&fw, fw_name, hdev->dev);	1372	mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
		1373	mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
		1374	so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
		1375	so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1045		1376
1046	if (rc) {	1377	gic_base_lo =
1047	dev_err(hdev->dev, "Failed to request %s\n", fw_name);	1378	lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1048	goto out;	1379	gic_base_hi =
1049	}	1380	upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1050		1381
1051	fw_size = fw->size;	1382	qman_base_addr = hdev->asic_prop.sram_base_address +
1052	if ((fw_size % 4) != 0) {	1383	MME_QMAN_BASE_OFFSET;
1053	dev_err(hdev->dev, "illegal %s firmware size %zu\n",
1054	fw_name, fw_size);
1055	rc = -EINVAL;
1056	goto out;
1057	}
1058		1384
1059	dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);	1385	WREG32(mmMME_QM_PQ_BASE_LO, lower_32_bits(qman_base_addr));
		1386	WREG32(mmMME_QM_PQ_BASE_HI, upper_32_bits(qman_base_addr));
		1387	WREG32(mmMME_QM_PQ_SIZE, ilog2(MME_QMAN_LENGTH));
		1388	WREG32(mmMME_QM_PQ_PI, 0);
		1389	WREG32(mmMME_QM_PQ_CI, 0);
		1390	WREG32(mmMME_QM_CP_LDMA_SRC_BASE_LO_OFFSET, 0x10C0);
		1391	WREG32(mmMME_QM_CP_LDMA_SRC_BASE_HI_OFFSET, 0x10C4);
		1392	WREG32(mmMME_QM_CP_LDMA_TSIZE_OFFSET, 0x10C8);
		1393	WREG32(mmMME_QM_CP_LDMA_COMMIT_OFFSET, 0x10CC);
1060		1394
1061	fw_data = (const u64 *) fw->data;	1395	WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_LO, mtr_base_lo);
		1396	WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_HI, mtr_base_hi);
		1397	WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_LO, so_base_lo);
		1398	WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_HI, so_base_hi);
1062		1399
1063	if ((fw->size % 8) != 0)	1400	/* QMAN CQ has 8 cache lines */
1064	fw_size -= 8;	1401	WREG32(mmMME_QM_CQ_CFG1, 0x00080008);
1065		1402
1066	for (i = 0 ; i < fw_size ; i += 8, fw_data++, dst += 8) {	1403	WREG32(mmMME_QM_GLBL_ERR_ADDR_LO, gic_base_lo);
1067	if (!(i & (0x80000 - 1))) {	1404	WREG32(mmMME_QM_GLBL_ERR_ADDR_HI, gic_base_hi);
1068	dev_dbg(hdev->dev,
1069	"copied so far %zu out of %zu for %s firmware",
1070	i, fw_size, fw_name);
1071	usleep_range(20, 100);
1072	}
1073		1405
1074	writeq(*fw_data, dst);	1406	WREG32(mmMME_QM_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_QM);
1075	}
1076		1407
1077	if ((fw->size % 8) != 0)	1408	WREG32(mmMME_QM_GLBL_ERR_CFG, QMAN_MME_ERR_MSG_EN);
1078	writel((const u32 ) fw_data, dst);
1079		1409
1080	out:	1410	WREG32(mmMME_QM_GLBL_PROT, QMAN_MME_ERR_PROT);
1081	release_firmware(fw);	1411
1082	return rc;	1412	WREG32(mmMME_QM_GLBL_CFG0, QMAN_MME_ENABLE);
1083	}	1413	}
1084		1414
1085	static int goya_pldm_init_cpu(struct hl_device *hdev)	1415	static void goya_init_mme_cmdq(struct hl_device *hdev)
1086	{	1416	{
1087	char fw_name[200];	1417	u32 mtr_base_lo, mtr_base_hi;
1088	void __iomem *dst;	1418	u32 so_base_lo, so_base_hi;
1089	u32 val, unit_rst_val;	1419	u32 gic_base_lo, gic_base_hi;
1090	int rc;	1420	u64 qman_base_addr;
1091		1421
1092	/* Must initialize SRAM scrambler before pushing u-boot to SRAM */	1422	mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1093	goya_init_golden_registers(hdev);	1423	mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
		1424	so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
		1425	so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1094		1426
1095	/* Put ARM cores into reset */	1427	gic_base_lo =
1096	WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL, CPU_RESET_ASSERT);	1428	lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1097	val = RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);	1429	gic_base_hi =
		1430	upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1098		1431
1099	/* Reset the CA53 MACRO */	1432	qman_base_addr = hdev->asic_prop.sram_base_address +
1100	unit_rst_val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);	1433	MME_QMAN_BASE_OFFSET;
1101	WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, CA53_RESET);
1102	val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
1103	WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, unit_rst_val);
1104	val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
1105		1434
1106	snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-u-boot.bin");	1435	WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_LO, mtr_base_lo);
1107	dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + UBOOT_FW_OFFSET;	1436	WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_HI, mtr_base_hi);
1108	rc = goya_push_fw_to_device(hdev, fw_name, dst);	1437	WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_LO, so_base_lo);
1109	if (rc)	1438	WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_HI, so_base_hi);
1110	return rc;
1111		1439
1112	snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-fit.itb");	1440	/* CMDQ CQ has 20 cache lines */
1113	dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;	1441	WREG32(mmMME_CMDQ_CQ_CFG1, 0x00140014);
1114	rc = goya_push_fw_to_device(hdev, fw_name, dst);
1115	if (rc)
1116	return rc;
1117		1442
1118	WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_FIT_RDY);	1443	WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_LO, gic_base_lo);
1119	WREG32(mmPSOC_GLOBAL_CONF_WARM_REBOOT, CPU_BOOT_STATUS_NA);	1444	WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_HI, gic_base_hi);
1120		1445
1121	WREG32(mmCPU_CA53_CFG_RST_ADDR_LSB_0,	1446	WREG32(mmMME_CMDQ_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_CMDQ);
1122	lower_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
1123	WREG32(mmCPU_CA53_CFG_RST_ADDR_MSB_0,
1124	upper_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
1125		1447
1126	/* Release ARM core 0 from reset */	1448	WREG32(mmMME_CMDQ_GLBL_ERR_CFG, CMDQ_MME_ERR_MSG_EN);
1127	WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL,
1128	CPU_RESET_CORE0_DEASSERT);
1129	val = RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);
1130		1449
1131	return 0;	1450	WREG32(mmMME_CMDQ_GLBL_PROT, CMDQ_MME_ERR_PROT);
		1451
		1452	WREG32(mmMME_CMDQ_GLBL_CFG0, CMDQ_MME_ENABLE);
1132	}	1453	}
1133		1454
1134	/*	1455	static void goya_init_mme_qmans(struct hl_device *hdev)
1135	* FW component passes an offset from SRAM_BASE_ADDR in SCRATCHPAD_xx.
1136	* The version string should be located by that offset.
1137	*/
1138	static void goya_read_device_fw_version(struct hl_device *hdev,
1139	enum goya_fw_component fwc)
1140	{	1456	{
1141	const char *name;	1457	struct goya_device *goya = hdev->asic_specific;
1142	u32 ver_off;	1458	u32 so_base_lo, so_base_hi;
1143	char *dest;
1144		1459
1145	switch (fwc) {	1460	if (goya->hw_cap_initialized & HW_CAP_MME)
1146	case FW_COMP_UBOOT:
1147	ver_off = RREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_29);
1148	dest = hdev->asic_prop.uboot_ver;
1149	name = "U-Boot";
1150	break;
1151	case FW_COMP_PREBOOT:
1152	ver_off = RREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_28);
1153	dest = hdev->asic_prop.preboot_ver;
1154	name = "Preboot";
1155	break;
1156	default:
1157	dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1158	return;	1461	return;
1159	}
1160		1462
1161	ver_off &= ~((u32)SRAM_BASE_ADDR);	1463	so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
		1464	so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1162		1465
1163	if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) {	1466	WREG32(mmMME_SM_BASE_ADDRESS_LOW, so_base_lo);
		1467	WREG32(mmMME_SM_BASE_ADDRESS_HIGH, so_base_hi);
		1468
		1469	goya_init_mme_qman(hdev);
		1470	goya_init_mme_cmdq(hdev);
		1471
		1472	goya->hw_cap_initialized \|= HW_CAP_MME;
		1473	}
		1474
		1475	static void goya_init_tpc_qman(struct hl_device *hdev, u32 base_off, int tpc_id)
		1476	{
		1477	u32 mtr_base_lo, mtr_base_hi;
		1478	u32 so_base_lo, so_base_hi;
		1479	u32 gic_base_lo, gic_base_hi;
		1480	u64 qman_base_addr;
		1481	u32 reg_off = tpc_id * (mmTPC1_QM_PQ_PI - mmTPC0_QM_PQ_PI);
		1482
		1483	mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
		1484	mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
		1485	so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
		1486	so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
		1487
		1488	gic_base_lo =
		1489	lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
		1490	gic_base_hi =
		1491	upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
		1492
		1493	qman_base_addr = hdev->asic_prop.sram_base_address + base_off;
		1494
		1495	WREG32(mmTPC0_QM_PQ_BASE_LO + reg_off, lower_32_bits(qman_base_addr));
		1496	WREG32(mmTPC0_QM_PQ_BASE_HI + reg_off, upper_32_bits(qman_base_addr));
		1497	WREG32(mmTPC0_QM_PQ_SIZE + reg_off, ilog2(TPC_QMAN_LENGTH));
		1498	WREG32(mmTPC0_QM_PQ_PI + reg_off, 0);
		1499	WREG32(mmTPC0_QM_PQ_CI + reg_off, 0);
		1500	WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET + reg_off, 0x10C0);
		1501	WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_HI_OFFSET + reg_off, 0x10C4);
		1502	WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET + reg_off, 0x10C8);
		1503	WREG32(mmTPC0_QM_CP_LDMA_COMMIT_OFFSET + reg_off, 0x10CC);
		1504
		1505	WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
		1506	WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
		1507	WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
		1508	WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
		1509
		1510	WREG32(mmTPC0_QM_CQ_CFG1 + reg_off, 0x00080008);
		1511
		1512	WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
		1513	WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
		1514
		1515	WREG32(mmTPC0_QM_GLBL_ERR_WDATA + reg_off,
		1516	GOYA_ASYNC_EVENT_ID_TPC0_QM + tpc_id);
		1517
		1518	WREG32(mmTPC0_QM_GLBL_ERR_CFG + reg_off, QMAN_TPC_ERR_MSG_EN);
		1519
		1520	WREG32(mmTPC0_QM_GLBL_PROT + reg_off, QMAN_TPC_ERR_PROT);
		1521
		1522	WREG32(mmTPC0_QM_GLBL_CFG0 + reg_off, QMAN_TPC_ENABLE);
		1523	}
		1524
		1525	static void goya_init_tpc_cmdq(struct hl_device *hdev, int tpc_id)
		1526	{
		1527	u32 mtr_base_lo, mtr_base_hi;
		1528	u32 so_base_lo, so_base_hi;
		1529	u32 gic_base_lo, gic_base_hi;
		1530	u32 reg_off = tpc_id * (mmTPC1_CMDQ_CQ_CFG1 - mmTPC0_CMDQ_CQ_CFG1);
		1531
		1532	mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
		1533	mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
		1534	so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
		1535	so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
		1536
		1537	gic_base_lo =
		1538	lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
		1539	gic_base_hi =
		1540	upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
		1541
		1542	WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
		1543	WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
		1544	WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
		1545	WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
		1546
		1547	WREG32(mmTPC0_CMDQ_CQ_CFG1 + reg_off, 0x00140014);
		1548
		1549	WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
		1550	WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
		1551
		1552	WREG32(mmTPC0_CMDQ_GLBL_ERR_WDATA + reg_off,
		1553	GOYA_ASYNC_EVENT_ID_TPC0_CMDQ + tpc_id);
		1554
		1555	WREG32(mmTPC0_CMDQ_GLBL_ERR_CFG + reg_off, CMDQ_TPC_ERR_MSG_EN);
		1556
		1557	WREG32(mmTPC0_CMDQ_GLBL_PROT + reg_off, CMDQ_TPC_ERR_PROT);
		1558
		1559	WREG32(mmTPC0_CMDQ_GLBL_CFG0 + reg_off, CMDQ_TPC_ENABLE);
		1560	}
		1561
		1562	static void goya_init_tpc_qmans(struct hl_device *hdev)
		1563	{
		1564	struct goya_device *goya = hdev->asic_specific;
		1565	u32 so_base_lo, so_base_hi;
		1566	u32 cfg_off = mmTPC1_CFG_SM_BASE_ADDRESS_LOW -
		1567	mmTPC0_CFG_SM_BASE_ADDRESS_LOW;
		1568	int i;
		1569
		1570	if (goya->hw_cap_initialized & HW_CAP_TPC)
		1571	return;
		1572
		1573	so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
		1574	so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
		1575
		1576	for (i = 0 ; i < TPC_MAX_NUM ; i++) {
		1577	WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_LOW + i * cfg_off,
		1578	so_base_lo);
		1579	WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_HIGH + i * cfg_off,
		1580	so_base_hi);
		1581	}
		1582
		1583	goya_init_tpc_qman(hdev, TPC0_QMAN_BASE_OFFSET, 0);
		1584	goya_init_tpc_qman(hdev, TPC1_QMAN_BASE_OFFSET, 1);
		1585	goya_init_tpc_qman(hdev, TPC2_QMAN_BASE_OFFSET, 2);
		1586	goya_init_tpc_qman(hdev, TPC3_QMAN_BASE_OFFSET, 3);
		1587	goya_init_tpc_qman(hdev, TPC4_QMAN_BASE_OFFSET, 4);
		1588	goya_init_tpc_qman(hdev, TPC5_QMAN_BASE_OFFSET, 5);
		1589	goya_init_tpc_qman(hdev, TPC6_QMAN_BASE_OFFSET, 6);
		1590	goya_init_tpc_qman(hdev, TPC7_QMAN_BASE_OFFSET, 7);
		1591
		1592	for (i = 0 ; i < TPC_MAX_NUM ; i++)
		1593	goya_init_tpc_cmdq(hdev, i);
		1594
		1595	goya->hw_cap_initialized \|= HW_CAP_TPC;
		1596	}
		1597
		1598	/*
		1599	* goya_disable_internal_queues - Disable internal queues
		1600	*
		1601	* @hdev: pointer to hl_device structure
		1602	*
		1603	*/
		1604	static void goya_disable_internal_queues(struct hl_device *hdev)
		1605	{
		1606	WREG32(mmMME_QM_GLBL_CFG0, 0);
		1607	WREG32(mmMME_CMDQ_GLBL_CFG0, 0);
		1608
		1609	WREG32(mmTPC0_QM_GLBL_CFG0, 0);
		1610	WREG32(mmTPC0_CMDQ_GLBL_CFG0, 0);
		1611
		1612	WREG32(mmTPC1_QM_GLBL_CFG0, 0);
		1613	WREG32(mmTPC1_CMDQ_GLBL_CFG0, 0);
		1614
		1615	WREG32(mmTPC2_QM_GLBL_CFG0, 0);
		1616	WREG32(mmTPC2_CMDQ_GLBL_CFG0, 0);
		1617
		1618	WREG32(mmTPC3_QM_GLBL_CFG0, 0);
		1619	WREG32(mmTPC3_CMDQ_GLBL_CFG0, 0);
		1620
		1621	WREG32(mmTPC4_QM_GLBL_CFG0, 0);
		1622	WREG32(mmTPC4_CMDQ_GLBL_CFG0, 0);
		1623
		1624	WREG32(mmTPC5_QM_GLBL_CFG0, 0);
		1625	WREG32(mmTPC5_CMDQ_GLBL_CFG0, 0);
		1626
		1627	WREG32(mmTPC6_QM_GLBL_CFG0, 0);
		1628	WREG32(mmTPC6_CMDQ_GLBL_CFG0, 0);
		1629
		1630	WREG32(mmTPC7_QM_GLBL_CFG0, 0);
		1631	WREG32(mmTPC7_CMDQ_GLBL_CFG0, 0);
		1632	}
		1633
		1634	/*
		1635	* goya_stop_internal_queues - Stop internal queues
		1636	*
		1637	* @hdev: pointer to hl_device structure
		1638	*
		1639	* Returns 0 on success
		1640	*
		1641	*/
		1642	static int goya_stop_internal_queues(struct hl_device *hdev)
		1643	{
		1644	int rc, retval = 0;
		1645
		1646	/*
		1647	* Each queue (QMAN) is a separate H/W logic. That means that each
		1648	* QMAN can be stopped independently and failure to stop one does NOT
		1649	* mandate we should not try to stop other QMANs
		1650	*/
		1651
		1652	rc = goya_stop_queue(hdev,
		1653	mmMME_QM_GLBL_CFG1,
		1654	mmMME_QM_CP_STS,
		1655	mmMME_QM_GLBL_STS0);
		1656
		1657	if (rc) {
		1658	dev_err(hdev->dev, "failed to stop MME QMAN\n");
		1659	retval = -EIO;
		1660	}
		1661
		1662	rc = goya_stop_queue(hdev,
		1663	mmMME_CMDQ_GLBL_CFG1,
		1664	mmMME_CMDQ_CP_STS,
		1665	mmMME_CMDQ_GLBL_STS0);
		1666
		1667	if (rc) {
		1668	dev_err(hdev->dev, "failed to stop MME CMDQ\n");
		1669	retval = -EIO;
		1670	}
		1671
		1672	rc = goya_stop_queue(hdev,
		1673	mmTPC0_QM_GLBL_CFG1,
		1674	mmTPC0_QM_CP_STS,
		1675	mmTPC0_QM_GLBL_STS0);
		1676
		1677	if (rc) {
		1678	dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n");
		1679	retval = -EIO;
		1680	}
		1681
		1682	rc = goya_stop_queue(hdev,
		1683	mmTPC0_CMDQ_GLBL_CFG1,
		1684	mmTPC0_CMDQ_CP_STS,
		1685	mmTPC0_CMDQ_GLBL_STS0);
		1686
		1687	if (rc) {
		1688	dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n");
		1689	retval = -EIO;
		1690	}
		1691
		1692	rc = goya_stop_queue(hdev,
		1693	mmTPC1_QM_GLBL_CFG1,
		1694	mmTPC1_QM_CP_STS,
		1695	mmTPC1_QM_GLBL_STS0);
		1696
		1697	if (rc) {
		1698	dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n");
		1699	retval = -EIO;
		1700	}
		1701
		1702	rc = goya_stop_queue(hdev,
		1703	mmTPC1_CMDQ_GLBL_CFG1,
		1704	mmTPC1_CMDQ_CP_STS,
		1705	mmTPC1_CMDQ_GLBL_STS0);
		1706
		1707	if (rc) {
		1708	dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n");
		1709	retval = -EIO;
		1710	}
		1711
		1712	rc = goya_stop_queue(hdev,
		1713	mmTPC2_QM_GLBL_CFG1,
		1714	mmTPC2_QM_CP_STS,
		1715	mmTPC2_QM_GLBL_STS0);
		1716
		1717	if (rc) {
		1718	dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n");
		1719	retval = -EIO;
		1720	}
		1721
		1722	rc = goya_stop_queue(hdev,
		1723	mmTPC2_CMDQ_GLBL_CFG1,
		1724	mmTPC2_CMDQ_CP_STS,
		1725	mmTPC2_CMDQ_GLBL_STS0);
		1726
		1727	if (rc) {
		1728	dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n");
		1729	retval = -EIO;
		1730	}
		1731
		1732	rc = goya_stop_queue(hdev,
		1733	mmTPC3_QM_GLBL_CFG1,
		1734	mmTPC3_QM_CP_STS,
		1735	mmTPC3_QM_GLBL_STS0);
		1736
		1737	if (rc) {
		1738	dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n");
		1739	retval = -EIO;
		1740	}
		1741
		1742	rc = goya_stop_queue(hdev,
		1743	mmTPC3_CMDQ_GLBL_CFG1,
		1744	mmTPC3_CMDQ_CP_STS,
		1745	mmTPC3_CMDQ_GLBL_STS0);
		1746
		1747	if (rc) {
		1748	dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n");
		1749	retval = -EIO;
		1750	}
		1751
		1752	rc = goya_stop_queue(hdev,
		1753	mmTPC4_QM_GLBL_CFG1,
		1754	mmTPC4_QM_CP_STS,
		1755	mmTPC4_QM_GLBL_STS0);
		1756
		1757	if (rc) {
		1758	dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n");
		1759	retval = -EIO;
		1760	}
		1761
		1762	rc = goya_stop_queue(hdev,
		1763	mmTPC4_CMDQ_GLBL_CFG1,
		1764	mmTPC4_CMDQ_CP_STS,
		1765	mmTPC4_CMDQ_GLBL_STS0);
		1766
		1767	if (rc) {
		1768	dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n");
		1769	retval = -EIO;
		1770	}
		1771
		1772	rc = goya_stop_queue(hdev,
		1773	mmTPC5_QM_GLBL_CFG1,
		1774	mmTPC5_QM_CP_STS,
		1775	mmTPC5_QM_GLBL_STS0);
		1776
		1777	if (rc) {
		1778	dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n");
		1779	retval = -EIO;
		1780	}
		1781
		1782	rc = goya_stop_queue(hdev,
		1783	mmTPC5_CMDQ_GLBL_CFG1,
		1784	mmTPC5_CMDQ_CP_STS,
		1785	mmTPC5_CMDQ_GLBL_STS0);
		1786
		1787	if (rc) {
		1788	dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n");
		1789	retval = -EIO;
		1790	}
		1791
		1792	rc = goya_stop_queue(hdev,
		1793	mmTPC6_QM_GLBL_CFG1,
		1794	mmTPC6_QM_CP_STS,
		1795	mmTPC6_QM_GLBL_STS0);
		1796
		1797	if (rc) {
		1798	dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n");
		1799	retval = -EIO;
		1800	}
		1801
		1802	rc = goya_stop_queue(hdev,
		1803	mmTPC6_CMDQ_GLBL_CFG1,
		1804	mmTPC6_CMDQ_CP_STS,
		1805	mmTPC6_CMDQ_GLBL_STS0);
		1806
		1807	if (rc) {
		1808	dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n");
		1809	retval = -EIO;
		1810	}
		1811
		1812	rc = goya_stop_queue(hdev,
		1813	mmTPC7_QM_GLBL_CFG1,
		1814	mmTPC7_QM_CP_STS,
		1815	mmTPC7_QM_GLBL_STS0);
		1816
		1817	if (rc) {
		1818	dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n");
		1819	retval = -EIO;
		1820	}
		1821
		1822	rc = goya_stop_queue(hdev,
		1823	mmTPC7_CMDQ_GLBL_CFG1,
		1824	mmTPC7_CMDQ_CP_STS,
		1825	mmTPC7_CMDQ_GLBL_STS0);
		1826
		1827	if (rc) {
		1828	dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n");
		1829	retval = -EIO;
		1830	}
		1831
		1832	return retval;
		1833	}
		1834
		1835	static void goya_resume_internal_queues(struct hl_device *hdev)
		1836	{
		1837	WREG32(mmMME_QM_GLBL_CFG1, 0);
		1838	WREG32(mmMME_CMDQ_GLBL_CFG1, 0);
		1839
		1840	WREG32(mmTPC0_QM_GLBL_CFG1, 0);
		1841	WREG32(mmTPC0_CMDQ_GLBL_CFG1, 0);
		1842
		1843	WREG32(mmTPC1_QM_GLBL_CFG1, 0);
		1844	WREG32(mmTPC1_CMDQ_GLBL_CFG1, 0);
		1845
		1846	WREG32(mmTPC2_QM_GLBL_CFG1, 0);
		1847	WREG32(mmTPC2_CMDQ_GLBL_CFG1, 0);
		1848
		1849	WREG32(mmTPC3_QM_GLBL_CFG1, 0);
		1850	WREG32(mmTPC3_CMDQ_GLBL_CFG1, 0);
		1851
		1852	WREG32(mmTPC4_QM_GLBL_CFG1, 0);
		1853	WREG32(mmTPC4_CMDQ_GLBL_CFG1, 0);
		1854
		1855	WREG32(mmTPC5_QM_GLBL_CFG1, 0);
		1856	WREG32(mmTPC5_CMDQ_GLBL_CFG1, 0);
		1857
		1858	WREG32(mmTPC6_QM_GLBL_CFG1, 0);
		1859	WREG32(mmTPC6_CMDQ_GLBL_CFG1, 0);
		1860
		1861	WREG32(mmTPC7_QM_GLBL_CFG1, 0);
		1862	WREG32(mmTPC7_CMDQ_GLBL_CFG1, 0);
		1863	}
		1864
		1865
		1866	/*
		1867	* goya_push_fw_to_device - Push FW code to device
		1868	*
		1869	* @hdev: pointer to hl_device structure
		1870	*
		1871	* Copy fw code from firmware file to device memory.
		1872	* Returns 0 on success
		1873	*
		1874	*/
		1875	static int goya_push_fw_to_device(struct hl_device hdev, const char fw_name,
		1876	void __iomem *dst)
		1877	{
		1878	const struct firmware *fw;
		1879	const u64 *fw_data;
		1880	size_t fw_size, i;
		1881	int rc;
		1882
		1883	rc = request_firmware(&fw, fw_name, hdev->dev);
		1884
		1885	if (rc) {
		1886	dev_err(hdev->dev, "Failed to request %s\n", fw_name);
		1887	goto out;
		1888	}
		1889
		1890	fw_size = fw->size;
		1891	if ((fw_size % 4) != 0) {
		1892	dev_err(hdev->dev, "illegal %s firmware size %zu\n",
		1893	fw_name, fw_size);
		1894	rc = -EINVAL;
		1895	goto out;
		1896	}
		1897
		1898	dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
		1899
		1900	fw_data = (const u64 *) fw->data;
		1901
		1902	if ((fw->size % 8) != 0)
		1903	fw_size -= 8;
		1904
		1905	for (i = 0 ; i < fw_size ; i += 8, fw_data++, dst += 8) {
		1906	if (!(i & (0x80000 - 1))) {
		1907	dev_dbg(hdev->dev,
		1908	"copied so far %zu out of %zu for %s firmware",
		1909	i, fw_size, fw_name);
		1910	usleep_range(20, 100);
		1911	}
		1912
		1913	writeq(*fw_data, dst);
		1914	}
		1915
		1916	if ((fw->size % 8) != 0)
		1917	writel((const u32 ) fw_data, dst);
		1918
		1919	out:
		1920	release_firmware(fw);
		1921	return rc;
		1922	}
		1923
		1924	static int goya_pldm_init_cpu(struct hl_device *hdev)
		1925	{
		1926	char fw_name[200];
		1927	void __iomem *dst;
		1928	u32 val, unit_rst_val;
		1929	int rc;
		1930
		1931	/* Must initialize SRAM scrambler before pushing u-boot to SRAM */
		1932	goya_init_golden_registers(hdev);
		1933
		1934	/* Put ARM cores into reset */
		1935	WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL, CPU_RESET_ASSERT);
		1936	val = RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);
		1937
		1938	/* Reset the CA53 MACRO */
		1939	unit_rst_val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
		1940	WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, CA53_RESET);
		1941	val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
		1942	WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, unit_rst_val);
		1943	val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
		1944
		1945	snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-u-boot.bin");
		1946	dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + UBOOT_FW_OFFSET;
		1947	rc = goya_push_fw_to_device(hdev, fw_name, dst);
		1948	if (rc)
		1949	return rc;
		1950
		1951	snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-fit.itb");
		1952	dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;
		1953	rc = goya_push_fw_to_device(hdev, fw_name, dst);
		1954	if (rc)
		1955	return rc;
		1956
		1957	WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_FIT_RDY);
		1958	WREG32(mmPSOC_GLOBAL_CONF_WARM_REBOOT, CPU_BOOT_STATUS_NA);
		1959
		1960	WREG32(mmCPU_CA53_CFG_RST_ADDR_LSB_0,
		1961	lower_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
		1962	WREG32(mmCPU_CA53_CFG_RST_ADDR_MSB_0,
		1963	upper_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
		1964
		1965	/* Release ARM core 0 from reset */
		1966	WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL,
		1967	CPU_RESET_CORE0_DEASSERT);
		1968	val = RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);
		1969
		1970	return 0;
		1971	}
		1972
		1973	/*
		1974	* FW component passes an offset from SRAM_BASE_ADDR in SCRATCHPAD_xx.
		1975	* The version string should be located by that offset.
		1976	*/
		1977	static void goya_read_device_fw_version(struct hl_device *hdev,
		1978	enum goya_fw_component fwc)
		1979	{
		1980	const char *name;
		1981	u32 ver_off;
		1982	char *dest;
		1983
		1984	switch (fwc) {
		1985	case FW_COMP_UBOOT:
		1986	ver_off = RREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_29);
		1987	dest = hdev->asic_prop.uboot_ver;
		1988	name = "U-Boot";
		1989	break;
		1990	case FW_COMP_PREBOOT:
		1991	ver_off = RREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_28);
		1992	dest = hdev->asic_prop.preboot_ver;
		1993	name = "Preboot";
		1994	break;
		1995	default:
		1996	dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
		1997	return;
		1998	}
		1999
		2000	ver_off &= ~((u32)SRAM_BASE_ADDR);
		2001
		2002	if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) {
1164	memcpy_fromio(dest, hdev->pcie_bar[SRAM_CFG_BAR_ID] + ver_off,	2003	memcpy_fromio(dest, hdev->pcie_bar[SRAM_CFG_BAR_ID] + ver_off,
1165	VERSION_MAX_LEN);	2004	VERSION_MAX_LEN);
1166	} else {	2005	} else {
@@ -1344,6 +2183,19 @@ static int goya_hw_init(struct hl_device *hdev) · 177 unmodified lines
1344		2183
1345	goya_init_security(hdev);	2184	goya_init_security(hdev);
1346		2185
		2186	goya_init_dma_qmans(hdev);
		2187
		2188	goya_init_mme_qmans(hdev);
		2189
		2190	goya_init_tpc_qmans(hdev);
		2191
		2192	rc = goya_init_cpu_queues(hdev);
		2193	if (rc) {
		2194	dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n",
		2195	rc);
		2196	goto disable_queues;
		2197	}
		2198
1347	/* CPU initialization is finished, we can now move to 48 bit DMA mask */	2199	/* CPU initialization is finished, we can now move to 48 bit DMA mask */
1348	rc = pci_set_dma_mask(hdev->pdev, DMA_BIT_MASK(48));	2200	rc = pci_set_dma_mask(hdev->pdev, DMA_BIT_MASK(48));
1349	if (rc) {	2201	if (rc) {
@@ -1352,7 +2204,7 @@ static int goya_hw_init(struct hl_device *hdev) · 2 unmodified lines
1352	if (rc) {	2204	if (rc) {
1353	dev_err(hdev->dev,	2205	dev_err(hdev->dev,
1354	"Unable to set pci dma mask to 32 bits\n");	2206	"Unable to set pci dma mask to 32 bits\n");
1355	return rc;	2207	goto disable_pci_access;
1356	}	2208	}
1357	}	2209	}
1358		2210
@@ -1364,7 +2216,7 @@ static int goya_hw_init(struct hl_device *hdev) · 5 unmodified lines
1364	if (rc) {	2216	if (rc) {
1365	dev_err(hdev->dev,	2217	dev_err(hdev->dev,
1366	"Unable to set pci consistent dma mask to 32 bits\n");	2218	"Unable to set pci consistent dma mask to 32 bits\n");
1367	return rc;	2219	goto disable_pci_access;
1368	}	2220	}
1369	}	2221	}
1370		2222
@@ -1372,6 +2224,14 @@ static int goya_hw_init(struct hl_device *hdev) · 1 unmodified line
1372	val = RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);	2224	val = RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
1373		2225
1374	return 0;	2226	return 0;
		2227
		2228	disable_pci_access:
		2229	goya_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
		2230	disable_queues:
		2231	goya_disable_internal_queues(hdev);
		2232	goya_disable_external_queues(hdev);
		2233
		2234	return rc;
1375	}	2235	}
1376		2236
1377	/*	2237	/*
@@ -1460,12 +2320,40 @@ static void goya_hw_fini(struct hl_device *hdev, bool hard_reset) · 82 unmodified lines
1460		2320
1461	int goya_suspend(struct hl_device *hdev)	2321	int goya_suspend(struct hl_device *hdev)
1462	{	2322	{
1463	return 0;	2323	int rc;
		2324
		2325	rc = goya_stop_internal_queues(hdev);
		2326
		2327	if (rc) {
		2328	dev_err(hdev->dev, "failed to stop internal queues\n");
		2329	return rc;
		2330	}
		2331
		2332	rc = goya_stop_external_queues(hdev);
		2333
		2334	if (rc) {
		2335	dev_err(hdev->dev, "failed to stop external queues\n");
		2336	return rc;
		2337	}
		2338
		2339	rc = goya_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
		2340	if (rc)
		2341	dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
		2342
		2343	return rc;
1464	}	2344	}
1465		2345
1466	int goya_resume(struct hl_device *hdev)	2346	int goya_resume(struct hl_device *hdev)
1467	{	2347	{
1468	return 0;	2348	int rc;
		2349
		2350	goya_resume_external_queues(hdev);
		2351	goya_resume_internal_queues(hdev);
		2352
		2353	rc = goya_send_pci_access_msg(hdev, ARMCP_PACKET_ENABLE_PCI_ACCESS);
		2354	if (rc)
		2355	dev_err(hdev->dev, "Failed to enable PCI access from CPU\n");
		2356	return rc;
1469	}	2357	}
1470		2358
1471	int goya_mmap(struct hl_fpriv hpriv, struct vm_area_struct vma)	2359	int goya_mmap(struct hl_fpriv hpriv, struct vm_area_struct vma)
@@ -1489,6 +2377,101 @@ int goya_cb_mmap(struct hl_device hdev, struct vm_area_struct vma, · 17 unmodified lines
1489	return rc;	2377	return rc;
1490	}	2378	}
1491		2379
		2380	void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
		2381	{
		2382	u32 db_reg_offset, db_value;
		2383	bool invalid_queue = false;
		2384
		2385	switch (hw_queue_id) {
		2386	case GOYA_QUEUE_ID_DMA_0:
		2387	db_reg_offset = mmDMA_QM_0_PQ_PI;
		2388	break;
		2389
		2390	case GOYA_QUEUE_ID_DMA_1:
		2391	db_reg_offset = mmDMA_QM_1_PQ_PI;
		2392	break;
		2393
		2394	case GOYA_QUEUE_ID_DMA_2:
		2395	db_reg_offset = mmDMA_QM_2_PQ_PI;
		2396	break;
		2397
		2398	case GOYA_QUEUE_ID_DMA_3:
		2399	db_reg_offset = mmDMA_QM_3_PQ_PI;
		2400	break;
		2401
		2402	case GOYA_QUEUE_ID_DMA_4:
		2403	db_reg_offset = mmDMA_QM_4_PQ_PI;
		2404	break;
		2405
		2406	case GOYA_QUEUE_ID_CPU_PQ:
		2407	if (hdev->cpu_queues_enable)
		2408	db_reg_offset = mmCPU_IF_PF_PQ_PI;
		2409	else
		2410	invalid_queue = true;
		2411	break;
		2412
		2413	case GOYA_QUEUE_ID_MME:
		2414	db_reg_offset = mmMME_QM_PQ_PI;
		2415	break;
		2416
		2417	case GOYA_QUEUE_ID_TPC0:
		2418	db_reg_offset = mmTPC0_QM_PQ_PI;
		2419	break;
		2420
		2421	case GOYA_QUEUE_ID_TPC1:
		2422	db_reg_offset = mmTPC1_QM_PQ_PI;
		2423	break;
		2424
		2425	case GOYA_QUEUE_ID_TPC2:
		2426	db_reg_offset = mmTPC2_QM_PQ_PI;
		2427	break;
		2428
		2429	case GOYA_QUEUE_ID_TPC3:
		2430	db_reg_offset = mmTPC3_QM_PQ_PI;
		2431	break;
		2432
		2433	case GOYA_QUEUE_ID_TPC4:
		2434	db_reg_offset = mmTPC4_QM_PQ_PI;
		2435	break;
		2436
		2437	case GOYA_QUEUE_ID_TPC5:
		2438	db_reg_offset = mmTPC5_QM_PQ_PI;
		2439	break;
		2440
		2441	case GOYA_QUEUE_ID_TPC6:
		2442	db_reg_offset = mmTPC6_QM_PQ_PI;
		2443	break;
		2444
		2445	case GOYA_QUEUE_ID_TPC7:
		2446	db_reg_offset = mmTPC7_QM_PQ_PI;
		2447	break;
		2448
		2449	default:
		2450	invalid_queue = true;
		2451	}
		2452
		2453	if (invalid_queue) {
		2454	/* Should never get here */
		2455	dev_err(hdev->dev, "h/w queue %d is invalid. Can't set pi\n",
		2456	hw_queue_id);
		2457	return;
		2458	}
		2459
		2460	db_value = pi;
		2461
		2462	/* ring the doorbell */
		2463	WREG32(db_reg_offset, db_value);
		2464
		2465	if (hw_queue_id == GOYA_QUEUE_ID_CPU_PQ)
		2466	WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
		2467	GOYA_ASYNC_EVENT_ID_PI_UPDATE);
		2468	}
		2469
		2470	void goya_flush_pq_write(struct hl_device hdev, u64 pq, u64 exp_val)
		2471	{
		2472	/* Not needed in Goya */
		2473	}
		2474
1492	void goya_dma_alloc_coherent(struct hl_device hdev, size_t size,	2475	void goya_dma_alloc_coherent(struct hl_device hdev, size_t size,
1493	dma_addr_t *dma_handle, gfp_t flags)	2476	dma_addr_t *dma_handle, gfp_t flags)
1494	{	2477	{
@@ -1501,6 +2484,315 @@ void goya_dma_free_coherent(struct hl_device hdev, size_t size, void cpu_addr, · 6 unmodified lines
1501	dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle);	2484	dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle);
1502	}	2485	}
1503		2486
		2487	void goya_get_int_queue_base(struct hl_device hdev, u32 queue_id,
		2488	dma_addr_t dma_handle, u16 queue_len)
		2489	{
		2490	void *base;
		2491	u32 offset;
		2492
		2493	*dma_handle = hdev->asic_prop.sram_base_address;
		2494
		2495	base = hdev->pcie_bar[SRAM_CFG_BAR_ID];
		2496
		2497	switch (queue_id) {
		2498	case GOYA_QUEUE_ID_MME:
		2499	offset = MME_QMAN_BASE_OFFSET;
		2500	*queue_len = MME_QMAN_LENGTH;
		2501	break;
		2502	case GOYA_QUEUE_ID_TPC0:
		2503	offset = TPC0_QMAN_BASE_OFFSET;
		2504	*queue_len = TPC_QMAN_LENGTH;
		2505	break;
		2506	case GOYA_QUEUE_ID_TPC1:
		2507	offset = TPC1_QMAN_BASE_OFFSET;
		2508	*queue_len = TPC_QMAN_LENGTH;
		2509	break;
		2510	case GOYA_QUEUE_ID_TPC2:
		2511	offset = TPC2_QMAN_BASE_OFFSET;
		2512	*queue_len = TPC_QMAN_LENGTH;
		2513	break;
		2514	case GOYA_QUEUE_ID_TPC3:
		2515	offset = TPC3_QMAN_BASE_OFFSET;
		2516	*queue_len = TPC_QMAN_LENGTH;
		2517	break;
		2518	case GOYA_QUEUE_ID_TPC4:
		2519	offset = TPC4_QMAN_BASE_OFFSET;
		2520	*queue_len = TPC_QMAN_LENGTH;
		2521	break;
		2522	case GOYA_QUEUE_ID_TPC5:
		2523	offset = TPC5_QMAN_BASE_OFFSET;
		2524	*queue_len = TPC_QMAN_LENGTH;
		2525	break;
		2526	case GOYA_QUEUE_ID_TPC6:
		2527	offset = TPC6_QMAN_BASE_OFFSET;
		2528	*queue_len = TPC_QMAN_LENGTH;
		2529	break;
		2530	case GOYA_QUEUE_ID_TPC7:
		2531	offset = TPC7_QMAN_BASE_OFFSET;
		2532	*queue_len = TPC_QMAN_LENGTH;
		2533	break;
		2534	default:
		2535	dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id);
		2536	return NULL;
		2537	}
		2538
		2539	base += offset;
		2540	*dma_handle += offset;
		2541
		2542	return base;
		2543	}
		2544
		2545	int goya_send_cpu_message(struct hl_device hdev, u32 msg, u16 len,
		2546	u32 timeout, long *result)
		2547	{
		2548	struct goya_device *goya = hdev->asic_specific;
		2549	struct armcp_packet *pkt;
		2550	dma_addr_t pkt_dma_addr;
		2551	u32 tmp;
		2552	int rc = 0;
		2553
		2554	if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) {
		2555	if (result)
		2556	*result = 0;
		2557	return 0;
		2558	}
		2559
		2560	if (len > CPU_CB_SIZE) {
		2561	dev_err(hdev->dev, "Invalid CPU message size of %d bytes\n",
		2562	len);
		2563	return -ENOMEM;
		2564	}
		2565
		2566	pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len,
		2567	&pkt_dma_addr);
		2568	if (!pkt) {
		2569	dev_err(hdev->dev,
		2570	"Failed to allocate DMA memory for packet to CPU\n");
		2571	return -ENOMEM;
		2572	}
		2573
		2574	memcpy(pkt, msg, len);
		2575
		2576	mutex_lock(&hdev->send_cpu_message_lock);
		2577
		2578	if (hdev->disabled)
		2579	goto out;
		2580
		2581	rc = hl_hw_queue_send_cb_no_cmpl(hdev, GOYA_QUEUE_ID_CPU_PQ, len,
		2582	pkt_dma_addr);
		2583	if (rc) {
		2584	dev_err(hdev->dev, "Failed to send CB on CPU PQ (%d)\n", rc);
		2585	goto out;
		2586	}
		2587
		2588	rc = hl_poll_timeout_memory(hdev, (u64) (uintptr_t) &pkt->fence,
		2589	timeout, &tmp);
		2590
		2591	hl_hw_queue_inc_ci_kernel(hdev, GOYA_QUEUE_ID_CPU_PQ);
		2592
		2593	if (rc == -ETIMEDOUT) {
		2594	dev_err(hdev->dev,
		2595	"Timeout while waiting for CPU packet fence\n");
		2596	goto out;
		2597	}
		2598
		2599	if (tmp == ARMCP_PACKET_FENCE_VAL) {
		2600	rc = (pkt->ctl & ARMCP_PKT_CTL_RC_MASK) >>
		2601	ARMCP_PKT_CTL_RC_SHIFT;
		2602	if (rc) {
		2603	dev_err(hdev->dev,
		2604	"F/W ERROR %d for CPU packet %d\n",
		2605	rc, (pkt->ctl & ARMCP_PKT_CTL_OPCODE_MASK)
		2606	>> ARMCP_PKT_CTL_OPCODE_SHIFT);
		2607	rc = -EINVAL;
		2608	} else if (result) {
		2609	*result = pkt->result;
		2610	}
		2611	} else {
		2612	dev_err(hdev->dev, "CPU packet wrong fence value\n");
		2613	rc = -EINVAL;
		2614	}
		2615
		2616	out:
		2617	mutex_unlock(&hdev->send_cpu_message_lock);
		2618
		2619	hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, len, pkt);
		2620
		2621	return rc;
		2622	}
		2623
		2624	int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id)
		2625	{
		2626	struct packet_msg_prot *fence_pkt;
		2627	dma_addr_t pkt_dma_addr;
		2628	u32 fence_val, tmp;
		2629	dma_addr_t fence_dma_addr;
		2630	u32 *fence_ptr;
		2631	int rc;
		2632
		2633	fence_val = GOYA_QMAN0_FENCE_VAL;
		2634
		2635	fence_ptr = hdev->asic_funcs->dma_pool_zalloc(hdev, 4, GFP_KERNEL,
		2636	&fence_dma_addr);
		2637	if (!fence_ptr) {
		2638	dev_err(hdev->dev,
		2639	"Failed to allocate memory for queue testing\n");
		2640	return -ENOMEM;
		2641	}
		2642
		2643	*fence_ptr = 0;
		2644
		2645	fence_pkt = hdev->asic_funcs->dma_pool_zalloc(hdev,
		2646	sizeof(struct packet_msg_prot),
		2647	GFP_KERNEL, &pkt_dma_addr);
		2648	if (!fence_pkt) {
		2649	dev_err(hdev->dev,
		2650	"Failed to allocate packet for queue testing\n");
		2651	rc = -ENOMEM;
		2652	goto free_fence_ptr;
		2653	}
		2654
		2655	fence_pkt->ctl = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) \|
		2656	(1 << GOYA_PKT_CTL_EB_SHIFT) \|
		2657	(1 << GOYA_PKT_CTL_MB_SHIFT);
		2658	fence_pkt->value = fence_val;
		2659	fence_pkt->addr = fence_dma_addr +
		2660	hdev->asic_prop.host_phys_base_address;
		2661
		2662	rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id,
		2663	sizeof(struct packet_msg_prot),
		2664	pkt_dma_addr);
		2665	if (rc) {
		2666	dev_err(hdev->dev,
		2667	"Failed to send fence packet\n");
		2668	goto free_pkt;
		2669	}
		2670
		2671	rc = hl_poll_timeout_memory(hdev, (u64) (uintptr_t) fence_ptr,
		2672	GOYA_TEST_QUEUE_WAIT_USEC, &tmp);
		2673
		2674	hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
		2675
		2676	if ((!rc) && (tmp == fence_val)) {
		2677	dev_info(hdev->dev,
		2678	"queue test on H/W queue %d succeeded\n",
		2679	hw_queue_id);
		2680	} else {
		2681	dev_err(hdev->dev,
		2682	"H/W queue %d test failed (scratch(0x%08llX) == 0x%08X)\n",
		2683	hw_queue_id, (unsigned long long) fence_dma_addr, tmp);
		2684	rc = -EINVAL;
		2685	}
		2686
		2687	free_pkt:
		2688	hdev->asic_funcs->dma_pool_free(hdev, (void *) fence_pkt,
		2689	pkt_dma_addr);
		2690	free_fence_ptr:
		2691	hdev->asic_funcs->dma_pool_free(hdev, (void *) fence_ptr,
		2692	fence_dma_addr);
		2693	return rc;
		2694	}
		2695
		2696	int goya_test_cpu_queue(struct hl_device *hdev)
		2697	{
		2698	struct armcp_packet test_pkt;
		2699	long result;
		2700	int rc;
		2701
		2702	/* cpu_queues_enable flag is always checked in send cpu message */
		2703
		2704	memset(&test_pkt, 0, sizeof(test_pkt));
		2705
		2706	test_pkt.ctl = ARMCP_PACKET_TEST << ARMCP_PKT_CTL_OPCODE_SHIFT;
		2707	test_pkt.value = ARMCP_PACKET_FENCE_VAL;
		2708
		2709	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
		2710	sizeof(test_pkt), HL_DEVICE_TIMEOUT_USEC, &result);
		2711
		2712	if (!rc)
		2713	dev_info(hdev->dev, "queue test on CPU queue succeeded\n");
		2714	else
		2715	dev_err(hdev->dev, "CPU queue test failed (0x%08lX)\n", result);
		2716
		2717	return rc;
		2718	}
		2719
		2720	static int goya_test_queues(struct hl_device *hdev)
		2721	{
		2722	struct goya_device *goya = hdev->asic_specific;
		2723	int i, rc, ret_val = 0;
		2724
		2725	for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) {
		2726	rc = goya_test_queue(hdev, i);
		2727	if (rc)
		2728	ret_val = -EINVAL;
		2729	}
		2730
		2731	if (hdev->cpu_queues_enable) {
		2732	rc = goya->test_cpu_queue(hdev);
		2733	if (rc)
		2734	ret_val = -EINVAL;
		2735	}
		2736
		2737	return ret_val;
		2738	}
		2739
		2740	void goya_dma_pool_zalloc(struct hl_device hdev, size_t size, gfp_t mem_flags,
		2741	dma_addr_t *dma_handle)
		2742	{
		2743	if (size > GOYA_DMA_POOL_BLK_SIZE)
		2744	return NULL;
		2745
		2746	return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
		2747	}
		2748
		2749	void goya_dma_pool_free(struct hl_device hdev, void vaddr,
		2750	dma_addr_t dma_addr)
		2751	{
		2752	dma_pool_free(hdev->dma_pool, vaddr, dma_addr);
		2753	}
		2754
		2755	void goya_cpu_accessible_dma_pool_alloc(struct hl_device hdev, size_t size,
		2756	dma_addr_t *dma_handle)
		2757	{
		2758	u64 kernel_addr;
		2759
		2760	/* roundup to CPU_PKT_SIZE */
		2761	size = (size + (CPU_PKT_SIZE - 1)) & CPU_PKT_MASK;
		2762
		2763	kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
		2764
		2765	*dma_handle = hdev->cpu_accessible_dma_address +
		2766	(kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
		2767
		2768	return (void *) (uintptr_t) kernel_addr;
		2769	}
		2770
		2771	void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
		2772	void *vaddr)
		2773	{
		2774	/* roundup to CPU_PKT_SIZE */
		2775	size = (size + (CPU_PKT_SIZE - 1)) & CPU_PKT_MASK;
		2776
		2777	gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
		2778	size);
		2779	}
		2780
		2781
		2782	static void goya_hw_queues_lock(struct hl_device *hdev)
		2783	{
		2784	struct goya_device *goya = hdev->asic_specific;
		2785
		2786	spin_lock(&goya->hw_queues_lock);
		2787	}
		2788
		2789	static void goya_hw_queues_unlock(struct hl_device *hdev)
		2790	{
		2791	struct goya_device *goya = hdev->asic_specific;
		2792
		2793	spin_unlock(&goya->hw_queues_lock);
		2794	}
		2795
1504	static const struct hl_asic_funcs goya_funcs = {	2796	static const struct hl_asic_funcs goya_funcs = {
1505	.early_init = goya_early_init,	2797	.early_init = goya_early_init,
1506	.early_fini = goya_early_fini,	2798	.early_fini = goya_early_fini,
@@ -1512,8 +2804,19 @@ static const struct hl_asic_funcs goya_funcs = { · 5 unmodified lines
1512	.resume = goya_resume,	2804	.resume = goya_resume,
1513	.mmap = goya_mmap,	2805	.mmap = goya_mmap,
1514	.cb_mmap = goya_cb_mmap,	2806	.cb_mmap = goya_cb_mmap,
		2807	.ring_doorbell = goya_ring_doorbell,
		2808	.flush_pq_write = goya_flush_pq_write,
1515	.dma_alloc_coherent = goya_dma_alloc_coherent,	2809	.dma_alloc_coherent = goya_dma_alloc_coherent,
1516	.dma_free_coherent = goya_dma_free_coherent,	2810	.dma_free_coherent = goya_dma_free_coherent,
		2811	.get_int_queue_base = goya_get_int_queue_base,
		2812	.test_queues = goya_test_queues,
		2813	.dma_pool_zalloc = goya_dma_pool_zalloc,
		2814	.dma_pool_free = goya_dma_pool_free,
		2815	.cpu_accessible_dma_pool_alloc = goya_cpu_accessible_dma_pool_alloc,
		2816	.cpu_accessible_dma_pool_free = goya_cpu_accessible_dma_pool_free,
		2817	.hw_queues_lock = goya_hw_queues_lock,
		2818	.hw_queues_unlock = goya_hw_queues_unlock,
		2819	.send_cpu_message = goya_send_cpu_message
1517	};	2820	};
1518		2821
1519	/*	2822	/*

drivers/misc/habanalabs/goya/goyaP.h+7−0

 #include <uapi/misc/habanalabs.h>
 #include "habanalabs.h"
 #include "include/hl_boot_if.h"
+#include "include/goya/goya_packets.h"
 #include "include/goya/goya.h"
+#include "include/goya/goya_async_events.h"
 #include "include/goya/goya_fw_if.h"
 #define NUMBER_OF_CMPLT_QUEUES		5
 };
 struct goya_device {
+	int (*test_cpu_queue)(struct hl_device *hdev);
 	/* TODO: remove hw_queues_lock after moving to scheduler code */
 	spinlock_t	hw_queues_lock;
 	u64		ddr_bar_cur_addr;
 	u32		hw_cap_initialized;
 };
+int goya_test_cpu_queue(struct hl_device *hdev);
+int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len,
+				u32 timeout, long *result);
 void goya_init_security(struct hl_device *hdev);
 #endif /* GOYAP_H_ */

drivers/misc/habanalabs/habanalabs.h+173−1

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	#define HABANALABSP_H_	9	#define HABANALABSP_H_
10		10
11	#include "include/armcp_if.h"	11	#include "include/armcp_if.h"
		12	#include "include/qman_if.h"
12		13
13	#define pr_fmt(fmt) "habanalabs: " fmt	14	#define pr_fmt(fmt) "habanalabs: " fmt
14		15
@@ -26,9 +27,36 @@ · 11 unmodified lines
26	struct hl_device;	27	struct hl_device;
27	struct hl_fpriv;	28	struct hl_fpriv;
28		29
		30	/**
		31	* enum hl_queue_type - Supported QUEUE types.
		32	* @QUEUE_TYPE_NA: queue is not available.
		33	* @QUEUE_TYPE_EXT: external queue which is a DMA channel that may access the
		34	* host.
		35	* @QUEUE_TYPE_INT: internal queue that performs DMA inside the device's
		36	* memories and/or operates the compute engines.
		37	* @QUEUE_TYPE_CPU: S/W queue for communication with the device's CPU.
		38	*/
		39	enum hl_queue_type {
		40	QUEUE_TYPE_NA,
		41	QUEUE_TYPE_EXT,
		42	QUEUE_TYPE_INT,
		43	QUEUE_TYPE_CPU
		44	};
		45
		46	/**
		47	* struct hw_queue_properties - queue information.
		48	* @type: queue type.
		49	* @kmd_only: true if only KMD is allowed to send a job to this queue, false
		50	* otherwise.
		51	*/
		52	struct hw_queue_properties {
		53	enum hl_queue_type type;
		54	u8 kmd_only;
		55	};
29		56
30	/**	57	/**
31	* struct asic_fixed_properties - ASIC specific immutable properties.	58	* struct asic_fixed_properties - ASIC specific immutable properties.
		59	* @hw_queues_props: H/W queues properties.
32	* @uboot_ver: F/W U-boot version.	60	* @uboot_ver: F/W U-boot version.
33	* @preboot_ver: F/W Preboot version.	61	* @preboot_ver: F/W Preboot version.
34	* @sram_base_address: SRAM physical start address.	62	* @sram_base_address: SRAM physical start address.
@@ -59,6 +87,7 @@ struct hl_fpriv; · 24 unmodified lines
59	* @tpc_enabled_mask: which TPCs are enabled.	87	* @tpc_enabled_mask: which TPCs are enabled.
60	*/	88	*/
61	struct asic_fixed_properties {	89	struct asic_fixed_properties {
		90	struct hw_queue_properties hw_queues_props[HL_MAX_QUEUES];
62	char uboot_ver[VERSION_MAX_LEN];	91	char uboot_ver[VERSION_MAX_LEN];
63	char preboot_ver[VERSION_MAX_LEN];	92	char preboot_ver[VERSION_MAX_LEN];
64	u64 sram_base_address;	93	u64 sram_base_address;
@@ -132,7 +161,89 @@ struct hl_cb { · 67 unmodified lines
132	};	161	};
133		162
134		163
		164	/*
		165	* QUEUES
		166	*/
		167
		168	struct hl_cs_job;
		169
		170	/*
		171	* Currently, there are two limitations on the maximum length of a queue:
		172	*
		173	* 1. The memory footprint of the queue. The current allocated space for the
		174	* queue is PAGE_SIZE. Because each entry in the queue is HL_BD_SIZE,
		175	* the maximum length of the queue can be PAGE_SIZE / HL_BD_SIZE,
		176	* which currently is 4096/16 = 256 entries.
		177	*
		178	* To increase that, we need either to decrease the size of the
		179	* BD (difficult), or allocate more than a single page (easier).
		180	*
		181	* 2. Because the size of the JOB handle field in the BD CTL / completion queue
		182	* is 10-bit, we can have up to 1024 open jobs per hardware queue.
		183	* Therefore, each queue can hold up to 1024 entries.
		184	*
		185	* HL_QUEUE_LENGTH is in units of struct hl_bd.
		186	* HL_QUEUE_LENGTH * sizeof(struct hl_bd) should be <= HL_PAGE_SIZE
		187	*/
		188
		189	#define HL_PAGE_SIZE 4096 /* minimum page size */
		190	/* Must be power of 2 (HL_PAGE_SIZE / HL_BD_SIZE) */
135	#define HL_QUEUE_LENGTH 256	191	#define HL_QUEUE_LENGTH 256
		192	#define HL_QUEUE_SIZE_IN_BYTES (HL_QUEUE_LENGTH * HL_BD_SIZE)
		193
		194	/*
		195	* HL_CQ_LENGTH is in units of struct hl_cq_entry.
		196	* HL_CQ_LENGTH should be <= HL_PAGE_SIZE
		197	*/
		198	#define HL_CQ_LENGTH HL_QUEUE_LENGTH
		199	#define HL_CQ_SIZE_IN_BYTES (HL_CQ_LENGTH * HL_CQ_ENTRY_SIZE)
		200
		201
		202
		203	/**
		204	* struct hl_hw_queue - describes a H/W transport queue.
		205	* @shadow_queue: pointer to a shadow queue that holds pointers to jobs.
		206	* @queue_type: type of queue.
		207	* @kernel_address: holds the queue's kernel virtual address.
		208	* @bus_address: holds the queue's DMA address.
		209	* @pi: holds the queue's pi value.
		210	* @ci: holds the queue's ci value, AS CALCULATED BY THE DRIVER (not real ci).
		211	* @hw_queue_id: the id of the H/W queue.
		212	* @int_queue_len: length of internal queue (number of entries).
		213	* @valid: is the queue valid (we have array of 32 queues, not all of them
		214	* exists).
		215	*/
		216	struct hl_hw_queue {
		217	struct hl_cs_job **shadow_queue;
		218	enum hl_queue_type queue_type;
		219	u64 kernel_address;
		220	dma_addr_t bus_address;
		221	u32 pi;
		222	u32 ci;
		223	u32 hw_queue_id;
		224	u16 int_queue_len;
		225	u8 valid;
		226	};
		227
		228	/**
		229	* struct hl_cq - describes a completion queue
		230	* @hdev: pointer to the device structure
		231	* @kernel_address: holds the queue's kernel virtual address
		232	* @bus_address: holds the queue's DMA address
		233	* @hw_queue_id: the id of the matching H/W queue
		234	* @ci: ci inside the queue
		235	* @pi: pi inside the queue
		236	* @free_slots_cnt: counter of free slots in queue
		237	*/
		238	struct hl_cq {
		239	struct hl_device *hdev;
		240	u64 kernel_address;
		241	dma_addr_t bus_address;
		242	u32 hw_queue_id;
		243	u32 ci;
		244	u32 pi;
		245	atomic_t free_slots_cnt;
		246	};
136		247
137		248
138	/*	249	/*
@@ -164,6 +275,8 @@ enum hl_asic_type { · 25 unmodified lines
164	* @resume: handles IP specific H/W or SW changes for resume.	275	* @resume: handles IP specific H/W or SW changes for resume.
165	* @mmap: mmap function, does nothing.	276	* @mmap: mmap function, does nothing.
166	* @cb_mmap: maps a CB.	277	* @cb_mmap: maps a CB.
		278	* @ring_doorbell: increment PI on a given QMAN.
		279	* @flush_pq_write: flush PQ entry write if necessary, WARN if flushing failed.
167	* @dma_alloc_coherent: Allocate coherent DMA memory by calling	280	* @dma_alloc_coherent: Allocate coherent DMA memory by calling
168	* dma_alloc_coherent(). This is ASIC function because its	281	* dma_alloc_coherent(). This is ASIC function because its
169	* implementation is not trivial when the driver is loaded	282	* implementation is not trivial when the driver is loaded
@@ -172,6 +285,16 @@ enum hl_asic_type { · 2 unmodified lines
172	* This is ASIC function because its implementation is not	285	* This is ASIC function because its implementation is not
173	* trivial when the driver is loaded in simulation mode	286	* trivial when the driver is loaded in simulation mode
174	* (not upstreamed).	287	* (not upstreamed).
		288	* @get_int_queue_base: get the internal queue base address.
		289	* @test_queues: run simple test on all queues for sanity check.
		290	* @dma_pool_zalloc: small DMA allocation of coherent memory from DMA pool.
		291	* size of allocation is HL_DMA_POOL_BLK_SIZE.
		292	* @dma_pool_free: free small DMA allocation from pool.
		293	* @cpu_accessible_dma_pool_alloc: allocate CPU PQ packet from DMA pool.
		294	* @cpu_accessible_dma_pool_free: free CPU PQ packet from DMA pool.
		295	* @hw_queues_lock: acquire H/W queues lock.
		296	* @hw_queues_unlock: release H/W queues lock.
		297	* @send_cpu_message: send buffer to ArmCP.
175	*/	298	*/
176	struct hl_asic_funcs {	299	struct hl_asic_funcs {
177	int (early_init)(struct hl_device hdev);	300	int (early_init)(struct hl_device hdev);
@@ -185,10 +308,27 @@ struct hl_asic_funcs { · 7 unmodified lines
185	int (mmap)(struct hl_fpriv hpriv, struct vm_area_struct *vma);	308	int (mmap)(struct hl_fpriv hpriv, struct vm_area_struct *vma);
186	int (cb_mmap)(struct hl_device hdev, struct vm_area_struct *vma,	309	int (cb_mmap)(struct hl_device hdev, struct vm_area_struct *vma,
187	u64 kaddress, phys_addr_t paddress, u32 size);	310	u64 kaddress, phys_addr_t paddress, u32 size);
		311	void (ring_doorbell)(struct hl_device hdev, u32 hw_queue_id, u32 pi);
		312	void (flush_pq_write)(struct hl_device hdev, u64 *pq, u64 exp_val);
188	void* (dma_alloc_coherent)(struct hl_device hdev, size_t size,	313	void* (dma_alloc_coherent)(struct hl_device hdev, size_t size,
189	dma_addr_t *dma_handle, gfp_t flag);	314	dma_addr_t *dma_handle, gfp_t flag);
190	void (dma_free_coherent)(struct hl_device hdev, size_t size,	315	void (dma_free_coherent)(struct hl_device hdev, size_t size,
191	void *cpu_addr, dma_addr_t dma_handle);	316	void *cpu_addr, dma_addr_t dma_handle);
		317	void* (get_int_queue_base)(struct hl_device hdev, u32 queue_id,
		318	dma_addr_t dma_handle, u16 queue_len);
		319	int (test_queues)(struct hl_device hdev);
		320	void* (dma_pool_zalloc)(struct hl_device hdev, size_t size,
		321	gfp_t mem_flags, dma_addr_t *dma_handle);
		322	void (dma_pool_free)(struct hl_device hdev, void *vaddr,
		323	dma_addr_t dma_addr);
		324	void* (cpu_accessible_dma_pool_alloc)(struct hl_device hdev,
		325	size_t size, dma_addr_t *dma_handle);
		326	void (cpu_accessible_dma_pool_free)(struct hl_device hdev,
		327	size_t size, void *vaddr);
		328	void (hw_queues_lock)(struct hl_device hdev);
		329	void (hw_queues_unlock)(struct hl_device hdev);
		330	int (send_cpu_message)(struct hl_device hdev, u32 *msg,
		331	u16 len, u32 timeout, long *result);
192	};	332	};
193		333
194		334
@@ -224,6 +364,17 @@ struct hl_ctx_mgr { · 29 unmodified lines
224	};	364	};
225		365
226		366
		367
		368
		369	/**
		370	* struct hl_cs_job - command submission job.
		371	* @finish_work: workqueue object to run when job is completed.
		372	* @id: the id of this job inside a CS.
		373	*/
		374	struct hl_cs_job {
		375	struct work_struct finish_work;
		376	u32 id;
		377	};
227	/*	378	/*
228	* FILE PRIVATE STRUCTURE	379	* FILE PRIVATE STRUCTURE
229	*/	380	*/
@@ -298,7 +449,11 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val); · 68 unmodified lines
298	* @dev: realted kernel basic device structure.	449	* @dev: realted kernel basic device structure.
299	* @asic_name: ASIC specific nmae.	450	* @asic_name: ASIC specific nmae.
300	* @asic_type: ASIC specific type.	451	* @asic_type: ASIC specific type.
		452	* @completion_queue: array of hl_cq.
		453	* @cq_wq: work queue of completion queues for executing work in process context
		454	* @eq_wq: work queue of event queue for executing work in process context.
301	* @kernel_ctx: KMD context structure.	455	* @kernel_ctx: KMD context structure.
		456	* @kernel_queues: array of hl_hw_queue.
302	* @kernel_cb_mgr: command buffer manager for creating/destroying/handling CGs.	457	* @kernel_cb_mgr: command buffer manager for creating/destroying/handling CGs.
303	* @dma_pool: DMA pool for small allocations.	458	* @dma_pool: DMA pool for small allocations.
304	* @cpu_accessible_dma_mem: KMD <-> ArmCP shared memory CPU address.	459	* @cpu_accessible_dma_mem: KMD <-> ArmCP shared memory CPU address.
@@ -312,6 +467,7 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val); · 7 unmodified lines
312	* only a single process at a time. In addition, we need a	467	* only a single process at a time. In addition, we need a
313	* lock here so we can flush user processes which are opening	468	* lock here so we can flush user processes which are opening
314	* the device while we are trying to hard reset it	469	* the device while we are trying to hard reset it
		470	* @send_cpu_message_lock: enforces only one message in KMD <-> ArmCP queue.
315	* @asic_prop: ASIC specific immutable properties.	471	* @asic_prop: ASIC specific immutable properties.
316	* @asic_funcs: ASIC specific functions.	472	* @asic_funcs: ASIC specific functions.
317	* @asic_specific: ASIC specific information to use only from ASIC files.	473	* @asic_specific: ASIC specific information to use only from ASIC files.
@@ -331,7 +487,10 @@ struct hl_device { · 13 unmodified lines
331	struct device *dev;	487	struct device *dev;
332	char asic_name[16];	488	char asic_name[16];
333	enum hl_asic_type asic_type;	489	enum hl_asic_type asic_type;
		490	struct hl_cq *completion_queue;
		491	struct workqueue_struct *cq_wq;
334	struct hl_ctx *kernel_ctx;	492	struct hl_ctx *kernel_ctx;
		493	struct hl_hw_queue *kernel_queues;
335	struct hl_cb_mgr kernel_cb_mgr;	494	struct hl_cb_mgr kernel_cb_mgr;
336	struct dma_pool *dma_pool;	495	struct dma_pool *dma_pool;
337	void *cpu_accessible_dma_mem;	496	void *cpu_accessible_dma_mem;
@@ -341,6 +500,7 @@ struct hl_device { · 3 unmodified lines
341	struct mutex asid_mutex;	500	struct mutex asid_mutex;
342	/* TODO: remove fd_open_cnt_lock for multiple process support */	501	/* TODO: remove fd_open_cnt_lock for multiple process support */
343	struct mutex fd_open_cnt_lock;	502	struct mutex fd_open_cnt_lock;
		503	struct mutex send_cpu_message_lock;
344	struct asic_fixed_properties asic_prop;	504	struct asic_fixed_properties asic_prop;
345	const struct hl_asic_funcs *asic_funcs;	505	const struct hl_asic_funcs *asic_funcs;
346	void *asic_specific;	506	void *asic_specific;
@@ -358,6 +518,7 @@ struct hl_device { · 11 unmodified lines
358	/* Parameters for bring-up */	518	/* Parameters for bring-up */
359	u8 cpu_enable;	519	u8 cpu_enable;
360	u8 reset_pcilink;	520	u8 reset_pcilink;
		521	u8 cpu_queues_enable;
361	u8 fw_loading;	522	u8 fw_loading;
362	u8 pldm;	523	u8 pldm;
363	};	524	};
@@ -400,7 +561,18 @@ int hl_poll_timeout_memory(struct hl_device *hdev, u64 addr, u32 timeout_us, · 36 unmodified lines
400	u32 *val);	561	u32 *val);
401	int hl_poll_timeout_device_memory(struct hl_device hdev, void __iomem addr,	562	int hl_poll_timeout_device_memory(struct hl_device hdev, void __iomem addr,
402	u32 timeout_us, u32 *val);	563	u32 timeout_us, u32 *val);
403		564	int hl_hw_queues_create(struct hl_device *hdev);
		565	void hl_hw_queues_destroy(struct hl_device *hdev);
		566	int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
		567	u32 cb_size, u64 cb_ptr);
		568	u32 hl_hw_queue_add_ptr(u32 ptr, u16 val);
		569	void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id);
		570
		571	#define hl_queue_inc_ptr(p) hl_hw_queue_add_ptr(p, 1)
		572	#define hl_pi_2_offset(pi) ((pi) & (HL_QUEUE_LENGTH - 1))
		573
		574	int hl_cq_init(struct hl_device hdev, struct hl_cq q, u32 hw_queue_id);
		575	void hl_cq_fini(struct hl_device hdev, struct hl_cq q);
404	int hl_asid_init(struct hl_device *hdev);	576	int hl_asid_init(struct hl_device *hdev);
405	void hl_asid_fini(struct hl_device *hdev);	577	void hl_asid_fini(struct hl_device *hdev);
406	unsigned long hl_asid_alloc(struct hl_device *hdev);	578	unsigned long hl_asid_alloc(struct hl_device *hdev);

drivers/misc/habanalabs/habanalabs_drv.c+5−0

 	/* Parameters for bring-up - set them to defaults */
 	hdev->cpu_enable = 1;
 	hdev->reset_pcilink = 0;
+	hdev->cpu_queues_enable = 1;
 	hdev->fw_loading = 1;
 	hdev->pldm = 0;
 	if (!hdev->cpu_enable)
 		hdev->fw_loading = 0;
+	/* If we don't load FW, no need to initialize CPU queues */
+	if (!hdev->fw_loading)
+		hdev->cpu_queues_enable = 0;
 	hdev->disabled = true;
 	hdev->pdev = pdev; /* can be NULL in case of simulator device */

drivers/misc/habanalabs/hw_queue.c+400−0

@@ -0,0 +1,400 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	/*
	4	* Copyright 2016-2019 HabanaLabs, Ltd.
	5	* All Rights Reserved.
	6	*/
	7
	8	#include "habanalabs.h"
	9
	10	#include <linux/slab.h>
	11
	12	/*
	13	* hl_queue_add_ptr - add to pi or ci and checks if it wraps around
	14	*
	15	* @ptr: the current pi/ci value
	16	* @val: the amount to add
	17	*
	18	* Add val to ptr. It can go until twice the queue length.
	19	*/
	20	inline u32 hl_hw_queue_add_ptr(u32 ptr, u16 val)
	21	{
	22	ptr += val;
	23	ptr &= ((HL_QUEUE_LENGTH << 1) - 1);
	24	return ptr;
	25	}
	26
	27	static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len)
	28	{
	29	int delta = (q->pi - q->ci);
	30
	31	if (delta >= 0)
	32	return (queue_len - delta);
	33	else
	34	return (abs(delta) - queue_len);
	35	}
	36
	37	/*
	38	* ext_queue_submit_bd - Submit a buffer descriptor to an external queue
	39	*
	40	* @hdev: pointer to habanalabs device structure
	41	* @q: pointer to habanalabs queue structure
	42	* @ctl: BD's control word
	43	* @len: BD's length
	44	* @ptr: BD's pointer
	45	*
	46	* This function assumes there is enough space on the queue to submit a new
	47	* BD to it. It initializes the next BD and calls the device specific
	48	* function to set the pi (and doorbell)
	49	*
	50	* This function must be called when the scheduler mutex is taken
	51	*
	52	*/
	53	static void ext_queue_submit_bd(struct hl_device hdev, struct hl_hw_queue q,
	54	u32 ctl, u32 len, u64 ptr)
	55	{
	56	struct hl_bd *bd;
	57
	58	bd = (struct hl_bd *) (uintptr_t) q->kernel_address;
	59	bd += hl_pi_2_offset(q->pi);
	60	bd->ctl = ctl;
	61	bd->len = len;
	62	bd->ptr = ptr + hdev->asic_prop.host_phys_base_address;
	63
	64	q->pi = hl_queue_inc_ptr(q->pi);
	65	hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);
	66	}
	67
	68	/*
	69	* ext_queue_sanity_checks - perform some sanity checks on external queue
	70	*
	71	* @hdev : pointer to hl_device structure
	72	* @q : pointer to hl_hw_queue structure
	73	* @num_of_entries : how many entries to check for space
	74	* @reserve_cq_entry : whether to reserve an entry in the cq
	75	*
	76	* H/W queues spinlock should be taken before calling this function
	77	*
	78	* Perform the following:
	79	* - Make sure we have enough space in the h/w queue
	80	* - Make sure we have enough space in the completion queue
	81	* - Reserve space in the completion queue (needs to be reversed if there
	82	* is a failure down the road before the actual submission of work). Only
	83	* do this action if reserve_cq_entry is true
	84	*
	85	*/
	86	static int ext_queue_sanity_checks(struct hl_device *hdev,
	87	struct hl_hw_queue *q, int num_of_entries,
	88	bool reserve_cq_entry)
	89	{
	90	atomic_t *free_slots =
	91	&hdev->completion_queue[q->hw_queue_id].free_slots_cnt;
	92	int free_slots_cnt;
	93
	94	/* Check we have enough space in the queue */
	95	free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);
	96
	97	if (free_slots_cnt < num_of_entries) {
	98	dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",
	99	q->hw_queue_id, num_of_entries);
	100	return -EAGAIN;
	101	}
	102
	103	if (reserve_cq_entry) {
	104	/*
	105	* Check we have enough space in the completion queue
	106	* Add -1 to counter (decrement) unless counter was already 0
	107	* In that case, CQ is full so we can't submit a new CB because
	108	* we won't get ack on its completion
	109	* atomic_add_unless will return 0 if counter was already 0
	110	*/
	111	if (atomic_add_negative(num_of_entries * -1, free_slots)) {
	112	dev_dbg(hdev->dev, "No space for %d on CQ %d\n",
	113	num_of_entries, q->hw_queue_id);
	114	atomic_add(num_of_entries, free_slots);
	115	return -EAGAIN;
	116	}
	117	}
	118
	119	return 0;
	120	}
	121
	122	/*
	123	* hl_hw_queue_send_cb_no_cmpl - send a single CB (not a JOB) without completion
	124	*
	125	* @hdev: pointer to hl_device structure
	126	* @hw_queue_id: Queue's type
	127	* @cb_size: size of CB
	128	* @cb_ptr: pointer to CB location
	129	*
	130	* This function sends a single CB, that must NOT generate a completion entry
	131	*
	132	*/
	133	int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
	134	u32 cb_size, u64 cb_ptr)
	135	{
	136	struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];
	137	int rc;
	138
	139	/*
	140	* The CPU queue is a synchronous queue with an effective depth of
	141	* a single entry (although it is allocated with room for multiple
	142	* entries). Therefore, there is a different lock, called
	143	* send_cpu_message_lock, that serializes accesses to the CPU queue.
	144	* As a result, we don't need to lock the access to the entire H/W
	145	* queues module when submitting a JOB to the CPU queue
	146	*/
	147	if (q->queue_type != QUEUE_TYPE_CPU)
	148	hdev->asic_funcs->hw_queues_lock(hdev);
	149
	150	if (hdev->disabled) {
	151	rc = -EPERM;
	152	goto out;
	153	}
	154
	155	rc = ext_queue_sanity_checks(hdev, q, 1, false);
	156	if (rc)
	157	goto out;
	158
	159	ext_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);
	160
	161	out:
	162	if (q->queue_type != QUEUE_TYPE_CPU)
	163	hdev->asic_funcs->hw_queues_unlock(hdev);
	164
	165	return rc;
	166	}
	167
	168	/*
	169	* hl_hw_queue_inc_ci_kernel - increment ci for kernel's queue
	170	*
	171	* @hdev: pointer to hl_device structure
	172	* @hw_queue_id: which queue to increment its ci
	173	*/
	174	void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id)
	175	{
	176	struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];
	177
	178	q->ci = hl_queue_inc_ptr(q->ci);
	179	}
	180
	181	static int ext_and_cpu_hw_queue_init(struct hl_device *hdev,
	182	struct hl_hw_queue *q)
	183	{
	184	void *p;
	185	int rc;
	186
	187	p = hdev->asic_funcs->dma_alloc_coherent(hdev,
	188	HL_QUEUE_SIZE_IN_BYTES,
	189	&q->bus_address, GFP_KERNEL \| __GFP_ZERO);
	190	if (!p)
	191	return -ENOMEM;
	192
	193	q->kernel_address = (u64) (uintptr_t) p;
	194
	195	q->shadow_queue = kmalloc_array(HL_QUEUE_LENGTH,
	196	sizeof(*q->shadow_queue),
	197	GFP_KERNEL);
	198	if (!q->shadow_queue) {
	199	dev_err(hdev->dev,
	200	"Failed to allocate shadow queue for H/W queue %d\n",
	201	q->hw_queue_id);
	202	rc = -ENOMEM;
	203	goto free_queue;
	204	}
	205
	206	/* Make sure read/write pointers are initialized to start of queue */
	207	q->ci = 0;
	208	q->pi = 0;
	209
	210	return 0;
	211
	212	free_queue:
	213	hdev->asic_funcs->dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES,
	214	(void *) (uintptr_t) q->kernel_address, q->bus_address);
	215
	216	return rc;
	217	}
	218
	219	static int int_hw_queue_init(struct hl_device hdev, struct hl_hw_queue q)
	220	{
	221	void *p;
	222
	223	p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id,
	224	&q->bus_address, &q->int_queue_len);
	225	if (!p) {
	226	dev_err(hdev->dev,
	227	"Failed to get base address for internal queue %d\n",
	228	q->hw_queue_id);
	229	return -EFAULT;
	230	}
	231
	232	q->kernel_address = (u64) (uintptr_t) p;
	233	q->pi = 0;
	234	q->ci = 0;
	235
	236	return 0;
	237	}
	238
	239	static int cpu_hw_queue_init(struct hl_device hdev, struct hl_hw_queue q)
	240	{
	241	return ext_and_cpu_hw_queue_init(hdev, q);
	242	}
	243
	244	static int ext_hw_queue_init(struct hl_device hdev, struct hl_hw_queue q)
	245	{
	246	return ext_and_cpu_hw_queue_init(hdev, q);
	247	}
	248
	249	/*
	250	* hw_queue_init - main initialization function for H/W queue object
	251	*
	252	* @hdev: pointer to hl_device device structure
	253	* @q: pointer to hl_hw_queue queue structure
	254	* @hw_queue_id: The id of the H/W queue
	255	*
	256	* Allocate dma-able memory for the queue and initialize fields
	257	* Returns 0 on success
	258	*/
	259	static int hw_queue_init(struct hl_device hdev, struct hl_hw_queue q,
	260	u32 hw_queue_id)
	261	{
	262	int rc;
	263
	264	BUILD_BUG_ON(HL_QUEUE_SIZE_IN_BYTES > HL_PAGE_SIZE);
	265
	266	q->hw_queue_id = hw_queue_id;
	267
	268	switch (q->queue_type) {
	269	case QUEUE_TYPE_EXT:
	270	rc = ext_hw_queue_init(hdev, q);
	271	break;
	272
	273	case QUEUE_TYPE_INT:
	274	rc = int_hw_queue_init(hdev, q);
	275	break;
	276
	277	case QUEUE_TYPE_CPU:
	278	rc = cpu_hw_queue_init(hdev, q);
	279	break;
	280
	281	case QUEUE_TYPE_NA:
	282	q->valid = 0;
	283	return 0;
	284
	285	default:
	286	dev_crit(hdev->dev, "wrong queue type %d during init\n",
	287	q->queue_type);
	288	rc = -EINVAL;
	289	break;
	290	}
	291
	292	if (rc)
	293	return rc;
	294
	295	q->valid = 1;
	296
	297	return 0;
	298	}
	299
	300	/*
	301	* hw_queue_fini - destroy queue
	302	*
	303	* @hdev: pointer to hl_device device structure
	304	* @q: pointer to hl_hw_queue queue structure
	305	*
	306	* Free the queue memory
	307	*/
	308	static void hw_queue_fini(struct hl_device hdev, struct hl_hw_queue q)
	309	{
	310	if (!q->valid)
	311	return;
	312
	313	/*
	314	* If we arrived here, there are no jobs waiting on this queue
	315	* so we can safely remove it.
	316	* This is because this function can only called when:
	317	* 1. Either a context is deleted, which only can occur if all its
	318	* jobs were finished
	319	* 2. A context wasn't able to be created due to failure or timeout,
	320	* which means there are no jobs on the queue yet
	321	*
	322	* The only exception are the queues of the kernel context, but
	323	* if they are being destroyed, it means that the entire module is
	324	* being removed. If the module is removed, it means there is no open
	325	* user context. It also means that if a job was submitted by
	326	* the kernel driver (e.g. context creation), the job itself was
	327	* released by the kernel driver when a timeout occurred on its
	328	* Completion. Thus, we don't need to release it again.
	329	*/
	330
	331	if (q->queue_type == QUEUE_TYPE_INT)
	332	return;
	333
	334	kfree(q->shadow_queue);
	335
	336	hdev->asic_funcs->dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES,
	337	(void *) (uintptr_t) q->kernel_address, q->bus_address);
	338	}
	339
	340	int hl_hw_queues_create(struct hl_device *hdev)
	341	{
	342	struct asic_fixed_properties *asic = &hdev->asic_prop;
	343	struct hl_hw_queue *q;
	344	int i, rc, q_ready_cnt;
	345
	346	hdev->kernel_queues = kcalloc(HL_MAX_QUEUES,
	347	sizeof(*hdev->kernel_queues), GFP_KERNEL);
	348
	349	if (!hdev->kernel_queues) {
	350	dev_err(hdev->dev, "Not enough memory for H/W queues\n");
	351	return -ENOMEM;
	352	}
	353
	354	/* Initialize the H/W queues */
	355	for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues;
	356	i < HL_MAX_QUEUES ; i++, q_ready_cnt++, q++) {
	357
	358	q->queue_type = asic->hw_queues_props[i].type;
	359	rc = hw_queue_init(hdev, q, i);
	360	if (rc) {
	361	dev_err(hdev->dev,
	362	"failed to initialize queue %d\n", i);
	363	goto release_queues;
	364	}
	365	}
	366
	367	return 0;
	368
	369	release_queues:
	370	for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++)
	371	hw_queue_fini(hdev, q);
	372
	373	kfree(hdev->kernel_queues);
	374
	375	return rc;
	376	}
	377
	378	void hl_hw_queues_destroy(struct hl_device *hdev)
	379	{
	380	struct hl_hw_queue *q;
	381	int i;
	382
	383	for (i = 0, q = hdev->kernel_queues ; i < HL_MAX_QUEUES ; i++, q++)
	384	hw_queue_fini(hdev, q);
	385
	386	kfree(hdev->kernel_queues);
	387	}
	388
	389	void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset)
	390	{
	391	struct hl_hw_queue *q;
	392	int i;
	393
	394	for (i = 0, q = hdev->kernel_queues ; i < HL_MAX_QUEUES ; i++, q++) {
	395	if ((!q->valid) \|\|
	396	((!hard_reset) && (q->queue_type == QUEUE_TYPE_CPU)))
	397	continue;
	398	q->pi = q->ci = 0;
	399	}
	400	}

drivers/misc/habanalabs/include/armcp_if.h+292−0

@@ -10,10 +10,302 @@ · 9 unmodified lines
10		10
11	#include <linux/types.h>	11	#include <linux/types.h>
12		12
		13	enum pq_init_status {
		14	PQ_INIT_STATUS_NA = 0,
		15	PQ_INIT_STATUS_READY_FOR_CP,
		16	PQ_INIT_STATUS_READY_FOR_HOST
		17	};
		18
		19	/*
		20	* ArmCP Primary Queue Packets
		21	*
		22	* During normal operation, KMD needs to send various messages to ArmCP,
		23	* usually either to SET some value into a H/W periphery or to GET the current
		24	* value of some H/W periphery. For example, SET the frequency of MME/TPC and
		25	* GET the value of the thermal sensor.
		26	*
		27	* These messages can be initiated either by the User application or by KMD
		28	* itself, e.g. power management code. In either case, the communication from
		29	* KMD to ArmCP will always be in synchronous mode, meaning that KMD will
		30	* send a single message and poll until the message was acknowledged and the
		31	* results are ready (if results are needed).
		32	*
		33	* This means that only a single message can be sent at a time and KMD must
		34	* wait for its result before sending the next message. Having said that,
		35	* because these are control messages which are sent in a relatively low
		36	* frequency, this limitation seems acceptable. It's important to note that
		37	* in case of multiple devices, messages to different devices can be sent
		38	* at the same time.
		39	*
		40	* The message, inputs/outputs (if relevant) and fence object will be located
		41	* on the device DDR at an address that will be determined by KMD. During
		42	* device initialization phase, KMD will pass to ArmCP that address. Most of
		43	* the message types will contain inputs/outputs inside the message itself.
		44	* The common part of each message will contain the opcode of the message (its
		45	* type) and a field representing a fence object.
		46	*
		47	* When KMD wishes to send a message to ArmCP, it will write the message
		48	* contents to the device DDR, clear the fence object and then write the
		49	* value 484 to the mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR register to issue
		50	* the 484 interrupt-id to the ARM core.
		51	*
		52	* Upon receiving the 484 interrupt-id, ArmCP will read the message from the
		53	* DDR. In case the message is a SET operation, ArmCP will first perform the
		54	* operation and then write to the fence object on the device DDR. In case the
		55	* message is a GET operation, ArmCP will first fill the results section on the
		56	* device DDR and then write to the fence object. If an error occurred, ArmCP
		57	* will fill the rc field with the right error code.
		58	*
		59	* In the meantime, KMD will poll on the fence object. Once KMD sees that the
		60	* fence object is signaled, it will read the results from the device DDR
		61	* (if relevant) and resume the code execution in KMD.
		62	*
		63	* To use QMAN packets, the opcode must be the QMAN opcode, shifted by 8
		64	* so the value being put by the KMD matches the value read by ArmCP
		65	*
		66	* Non-QMAN packets should be limited to values 1 through (2^8 - 1)
		67	*
		68	* Detailed description:
		69	*
		70	* ARMCP_PACKET_DISABLE_PCI_ACCESS -
		71	* After receiving this packet the embedded CPU must NOT issue PCI
		72	* transactions (read/write) towards the Host CPU. This also include
		73	* sending MSI-X interrupts.
		74	* This packet is usually sent before the device is moved to D3Hot state.
		75	*
		76	* ARMCP_PACKET_ENABLE_PCI_ACCESS -
		77	* After receiving this packet the embedded CPU is allowed to issue PCI
		78	* transactions towards the Host CPU, including sending MSI-X interrupts.
		79	* This packet is usually send after the device is moved to D0 state.
		80	*
		81	* ARMCP_PACKET_TEMPERATURE_GET -
		82	* Fetch the current temperature / Max / Max Hyst / Critical /
		83	* Critical Hyst of a specified thermal sensor. The packet's
		84	* arguments specify the desired sensor and the field to get.
		85	*
		86	* ARMCP_PACKET_VOLTAGE_GET -
		87	* Fetch the voltage / Max / Min of a specified sensor. The packet's
		88	* arguments specify the sensor and type.
		89	*
		90	* ARMCP_PACKET_CURRENT_GET -
		91	* Fetch the current / Max / Min of a specified sensor. The packet's
		92	* arguments specify the sensor and type.
		93	*
		94	* ARMCP_PACKET_FAN_SPEED_GET -
		95	* Fetch the speed / Max / Min of a specified fan. The packet's
		96	* arguments specify the sensor and type.
		97	*
		98	* ARMCP_PACKET_PWM_GET -
		99	* Fetch the pwm value / mode of a specified pwm. The packet's
		100	* arguments specify the sensor and type.
		101	*
		102	* ARMCP_PACKET_PWM_SET -
		103	* Set the pwm value / mode of a specified pwm. The packet's
		104	* arguments specify the sensor, type and value.
		105	*
		106	* ARMCP_PACKET_FREQUENCY_SET -
		107	* Set the frequency of a specified PLL. The packet's arguments specify
		108	* the PLL and the desired frequency. The actual frequency in the device
		109	* might differ from the requested frequency.
		110	*
		111	* ARMCP_PACKET_FREQUENCY_GET -
		112	* Fetch the frequency of a specified PLL. The packet's arguments specify
		113	* the PLL.
		114	*
		115	* ARMCP_PACKET_LED_SET -
		116	* Set the state of a specified led. The packet's arguments
		117	* specify the led and the desired state.
		118	*
		119	* ARMCP_PACKET_I2C_WR -
		120	* Write 32-bit value to I2C device. The packet's arguments specify the
		121	* I2C bus, address and value.
		122	*
		123	* ARMCP_PACKET_I2C_RD -
		124	* Read 32-bit value from I2C device. The packet's arguments specify the
		125	* I2C bus and address.
		126	*
		127	* ARMCP_PACKET_INFO_GET -
		128	* Fetch information from the device as specified in the packet's
		129	* structure. KMD passes the max size it allows the ArmCP to write to
		130	* the structure, to prevent data corruption in case of mismatched
		131	* KMD/FW versions.
		132	*
		133	* ARMCP_PACKET_FLASH_PROGRAM_REMOVED - this packet was removed
		134	*
		135	* ARMCP_PACKET_UNMASK_RAZWI_IRQ -
		136	* Unmask the given IRQ. The IRQ number is specified in the value field.
		137	* The packet is sent after receiving an interrupt and printing its
		138	* relevant information.
		139	*
		140	* ARMCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY -
		141	* Unmask the given IRQs. The IRQs numbers are specified in an array right
		142	* after the armcp_packet structure, where its first element is the array
		143	* length. The packet is sent after a soft reset was done in order to
		144	* handle any interrupts that were sent during the reset process.
		145	*
		146	* ARMCP_PACKET_TEST -
		147	* Test packet for ArmCP connectivity. The CPU will put the fence value
		148	* in the result field.
		149	*
		150	* ARMCP_PACKET_FREQUENCY_CURR_GET -
		151	* Fetch the current frequency of a specified PLL. The packet's arguments
		152	* specify the PLL.
		153	*
		154	* ARMCP_PACKET_MAX_POWER_GET -
		155	* Fetch the maximal power of the device.
		156	*
		157	* ARMCP_PACKET_MAX_POWER_SET -
		158	* Set the maximal power of the device. The packet's arguments specify
		159	* the power.
		160	*
		161	* ARMCP_PACKET_EEPROM_DATA_GET -
		162	* Get EEPROM data from the ArmCP kernel. The buffer is specified in the
		163	* addr field. The CPU will put the returned data size in the result
		164	* field. In addition, KMD passes the max size it allows the ArmCP to
		165	* write to the structure, to prevent data corruption in case of
		166	* mismatched KMD/FW versions.
		167	*
		168	*/
		169
		170	enum armcp_packet_id {
		171	ARMCP_PACKET_DISABLE_PCI_ACCESS = 1, /* internal */
		172	ARMCP_PACKET_ENABLE_PCI_ACCESS, /* internal */
		173	ARMCP_PACKET_TEMPERATURE_GET, /* sysfs */
		174	ARMCP_PACKET_VOLTAGE_GET, /* sysfs */
		175	ARMCP_PACKET_CURRENT_GET, /* sysfs */
		176	ARMCP_PACKET_FAN_SPEED_GET, /* sysfs */
		177	ARMCP_PACKET_PWM_GET, /* sysfs */
		178	ARMCP_PACKET_PWM_SET, /* sysfs */
		179	ARMCP_PACKET_FREQUENCY_SET, /* sysfs */
		180	ARMCP_PACKET_FREQUENCY_GET, /* sysfs */
		181	ARMCP_PACKET_LED_SET, /* debugfs */
		182	ARMCP_PACKET_I2C_WR, /* debugfs */
		183	ARMCP_PACKET_I2C_RD, /* debugfs */
		184	ARMCP_PACKET_INFO_GET, /* IOCTL */
		185	ARMCP_PACKET_FLASH_PROGRAM_REMOVED,
		186	ARMCP_PACKET_UNMASK_RAZWI_IRQ, /* internal */
		187	ARMCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY, /* internal */
		188	ARMCP_PACKET_TEST, /* internal */
		189	ARMCP_PACKET_FREQUENCY_CURR_GET, /* sysfs */
		190	ARMCP_PACKET_MAX_POWER_GET, /* sysfs */
		191	ARMCP_PACKET_MAX_POWER_SET, /* sysfs */
		192	ARMCP_PACKET_EEPROM_DATA_GET, /* sysfs */
		193	};
		194
		195	#define ARMCP_PACKET_FENCE_VAL 0xFE8CE7A5
		196
		197	#define ARMCP_PKT_CTL_RC_SHIFT 12
		198	#define ARMCP_PKT_CTL_RC_MASK 0x0000F000
		199
		200	#define ARMCP_PKT_CTL_OPCODE_SHIFT 16
		201	#define ARMCP_PKT_CTL_OPCODE_MASK 0x1FFF0000
		202
		203	struct armcp_packet {
		204	union {
		205	__le64 value; /* For SET packets */
		206	__le64 result; /* For GET packets */
		207	__le64 addr; /* For PQ */
		208	};
		209
		210	__le32 ctl;
		211
		212	__le32 fence; /* Signal to KMD that message is completed */
		213
		214	union {
		215	struct {/* For temperature/current/voltage/fan/pwm get/set */
		216	__le16 sensor_index;
		217	__le16 type;
		218	};
		219
		220	struct { /* For I2C read/write */
		221	__u8 i2c_bus;
		222	__u8 i2c_addr;
		223	__u8 i2c_reg;
		224	__u8 pad; /* unused */
		225	};
		226
		227	/* For frequency get/set */
		228	__le32 pll_index;
		229
		230	/* For led set */
		231	__le32 led_index;
		232
		233	/* For get Armcp info/EEPROM data */
		234	__le32 data_max_size;
		235	};
		236	};
		237
		238	struct armcp_unmask_irq_arr_packet {
		239	struct armcp_packet armcp_pkt;
		240	__le32 length;
		241	__le32 irqs[0];
		242	};
		243
		244	enum armcp_packet_rc {
		245	armcp_packet_success,
		246	armcp_packet_invalid,
		247	armcp_packet_fault
		248	};
		249
		250	enum armcp_temp_type {
		251	armcp_temp_input,
		252	armcp_temp_max = 6,
		253	armcp_temp_max_hyst,
		254	armcp_temp_crit,
		255	armcp_temp_crit_hyst
		256	};
		257
		258	enum armcp_in_attributes {
		259	armcp_in_input,
		260	armcp_in_min,
		261	armcp_in_max
		262	};
		263
		264	enum armcp_curr_attributes {
		265	armcp_curr_input,
		266	armcp_curr_min,
		267	armcp_curr_max
		268	};
		269
		270	enum armcp_fan_attributes {
		271	armcp_fan_input,
		272	armcp_fan_min = 2,
		273	armcp_fan_max
		274	};
		275
		276	enum armcp_pwm_attributes {
		277	armcp_pwm_input,
		278	armcp_pwm_enable
		279	};
		280
		281	/* Event Queue Packets */
		282
		283	struct eq_generic_event {
		284	__le64 data[7];
		285	};
		286
13	/*	287	/*
14	* ArmCP info	288	* ArmCP info
15	*/	289	*/
16		290
17	#define VERSION_MAX_LEN 128	291	#define VERSION_MAX_LEN 128
		292	#define ARMCP_MAX_SENSORS 128
		293
		294	struct armcp_sensor {
		295	__le32 type;
		296	__le32 flags;
		297	};
		298
		299	struct armcp_info {
		300	struct armcp_sensor sensors[ARMCP_MAX_SENSORS];
		301	__u8 kernel_version[VERSION_MAX_LEN];
		302	__le32 reserved[3];
		303	__le32 cpld_version;
		304	__le32 infineon_version;
		305	__u8 fuse_version[VERSION_MAX_LEN];
		306	__u8 thermal_version[VERSION_MAX_LEN];
		307	__u8 armcp_version[VERSION_MAX_LEN];
		308	__le64 dram_size;
		309	};
18		310
19	#endif /* ARMCP_IF_H */	311	#endif /* ARMCP_IF_H */

drivers/misc/habanalabs/include/goya/goya_async_events.h+186−0

@@ -0,0 +1,186 @@
	1	/* SPDX-License-Identifier: GPL-2.0
	2	*
	3	* Copyright 2018 HabanaLabs, Ltd.
	4	* All Rights Reserved.
	5	*
	6	*/
	7
	8	#ifndef __GOYA_ASYNC_EVENTS_H_
	9	#define __GOYA_ASYNC_EVENTS_H_
	10
	11	enum goya_async_event_id {
	12	GOYA_ASYNC_EVENT_ID_PCIE_IF = 33,
	13	GOYA_ASYNC_EVENT_ID_TPC0_ECC = 36,
	14	GOYA_ASYNC_EVENT_ID_TPC1_ECC = 39,
	15	GOYA_ASYNC_EVENT_ID_TPC2_ECC = 42,
	16	GOYA_ASYNC_EVENT_ID_TPC3_ECC = 45,
	17	GOYA_ASYNC_EVENT_ID_TPC4_ECC = 48,
	18	GOYA_ASYNC_EVENT_ID_TPC5_ECC = 51,
	19	GOYA_ASYNC_EVENT_ID_TPC6_ECC = 54,
	20	GOYA_ASYNC_EVENT_ID_TPC7_ECC = 57,
	21	GOYA_ASYNC_EVENT_ID_MME_ECC = 60,
	22	GOYA_ASYNC_EVENT_ID_MME_ECC_EXT = 61,
	23	GOYA_ASYNC_EVENT_ID_MMU_ECC = 63,
	24	GOYA_ASYNC_EVENT_ID_DMA_MACRO = 64,
	25	GOYA_ASYNC_EVENT_ID_DMA_ECC = 66,
	26	GOYA_ASYNC_EVENT_ID_CPU_IF_ECC = 75,
	27	GOYA_ASYNC_EVENT_ID_PSOC_MEM = 78,
	28	GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT = 79,
	29	GOYA_ASYNC_EVENT_ID_SRAM0 = 81,
	30	GOYA_ASYNC_EVENT_ID_SRAM1 = 82,
	31	GOYA_ASYNC_EVENT_ID_SRAM2 = 83,
	32	GOYA_ASYNC_EVENT_ID_SRAM3 = 84,
	33	GOYA_ASYNC_EVENT_ID_SRAM4 = 85,
	34	GOYA_ASYNC_EVENT_ID_SRAM5 = 86,
	35	GOYA_ASYNC_EVENT_ID_SRAM6 = 87,
	36	GOYA_ASYNC_EVENT_ID_SRAM7 = 88,
	37	GOYA_ASYNC_EVENT_ID_SRAM8 = 89,
	38	GOYA_ASYNC_EVENT_ID_SRAM9 = 90,
	39	GOYA_ASYNC_EVENT_ID_SRAM10 = 91,
	40	GOYA_ASYNC_EVENT_ID_SRAM11 = 92,
	41	GOYA_ASYNC_EVENT_ID_SRAM12 = 93,
	42	GOYA_ASYNC_EVENT_ID_SRAM13 = 94,
	43	GOYA_ASYNC_EVENT_ID_SRAM14 = 95,
	44	GOYA_ASYNC_EVENT_ID_SRAM15 = 96,
	45	GOYA_ASYNC_EVENT_ID_SRAM16 = 97,
	46	GOYA_ASYNC_EVENT_ID_SRAM17 = 98,
	47	GOYA_ASYNC_EVENT_ID_SRAM18 = 99,
	48	GOYA_ASYNC_EVENT_ID_SRAM19 = 100,
	49	GOYA_ASYNC_EVENT_ID_SRAM20 = 101,
	50	GOYA_ASYNC_EVENT_ID_SRAM21 = 102,
	51	GOYA_ASYNC_EVENT_ID_SRAM22 = 103,
	52	GOYA_ASYNC_EVENT_ID_SRAM23 = 104,
	53	GOYA_ASYNC_EVENT_ID_SRAM24 = 105,
	54	GOYA_ASYNC_EVENT_ID_SRAM25 = 106,
	55	GOYA_ASYNC_EVENT_ID_SRAM26 = 107,
	56	GOYA_ASYNC_EVENT_ID_SRAM27 = 108,
	57	GOYA_ASYNC_EVENT_ID_SRAM28 = 109,
	58	GOYA_ASYNC_EVENT_ID_SRAM29 = 110,
	59	GOYA_ASYNC_EVENT_ID_GIC500 = 112,
	60	GOYA_ASYNC_EVENT_ID_PCIE_DEC = 115,
	61	GOYA_ASYNC_EVENT_ID_TPC0_DEC = 117,
	62	GOYA_ASYNC_EVENT_ID_TPC1_DEC = 120,
	63	GOYA_ASYNC_EVENT_ID_TPC2_DEC = 123,
	64	GOYA_ASYNC_EVENT_ID_TPC3_DEC = 126,
	65	GOYA_ASYNC_EVENT_ID_TPC4_DEC = 129,
	66	GOYA_ASYNC_EVENT_ID_TPC5_DEC = 132,
	67	GOYA_ASYNC_EVENT_ID_TPC6_DEC = 135,
	68	GOYA_ASYNC_EVENT_ID_TPC7_DEC = 138,
	69	GOYA_ASYNC_EVENT_ID_AXI_ECC = 139,
	70	GOYA_ASYNC_EVENT_ID_L2_RAM_ECC = 140,
	71	GOYA_ASYNC_EVENT_ID_MME_WACS = 141,
	72	GOYA_ASYNC_EVENT_ID_MME_WACSD = 142,
	73	GOYA_ASYNC_EVENT_ID_PLL0 = 143,
	74	GOYA_ASYNC_EVENT_ID_PLL1 = 144,
	75	GOYA_ASYNC_EVENT_ID_PLL3 = 146,
	76	GOYA_ASYNC_EVENT_ID_PLL4 = 147,
	77	GOYA_ASYNC_EVENT_ID_PLL5 = 148,
	78	GOYA_ASYNC_EVENT_ID_PLL6 = 149,
	79	GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER = 155,
	80	GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC = 159,
	81	GOYA_ASYNC_EVENT_ID_PSOC = 160,
	82	GOYA_ASYNC_EVENT_ID_PCIE_FLR = 171,
	83	GOYA_ASYNC_EVENT_ID_PCIE_HOT_RESET = 172,
	84	GOYA_ASYNC_EVENT_ID_PCIE_QID0_ENG0 = 174,
	85	GOYA_ASYNC_EVENT_ID_PCIE_QID0_ENG1 = 175,
	86	GOYA_ASYNC_EVENT_ID_PCIE_QID0_ENG2 = 176,
	87	GOYA_ASYNC_EVENT_ID_PCIE_QID0_ENG3 = 177,
	88	GOYA_ASYNC_EVENT_ID_PCIE_QID1_ENG0 = 178,
	89	GOYA_ASYNC_EVENT_ID_PCIE_QID1_ENG1 = 179,
	90	GOYA_ASYNC_EVENT_ID_PCIE_QID1_ENG2 = 180,
	91	GOYA_ASYNC_EVENT_ID_PCIE_QID1_ENG3 = 181,
	92	GOYA_ASYNC_EVENT_ID_PCIE_APB = 182,
	93	GOYA_ASYNC_EVENT_ID_PCIE_QDB = 183,
	94	GOYA_ASYNC_EVENT_ID_PCIE_BM_D_P_WR = 184,
	95	GOYA_ASYNC_EVENT_ID_PCIE_BM_D_RD = 185,
	96	GOYA_ASYNC_EVENT_ID_PCIE_BM_U_P_WR = 186,
	97	GOYA_ASYNC_EVENT_ID_PCIE_BM_U_RD = 187,
	98	GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU = 190,
	99	GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR = 191,
	100	GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU = 200,
	101	GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR = 201,
	102	GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU = 210,
	103	GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR = 211,
	104	GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU = 220,
	105	GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR = 221,
	106	GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU = 230,
	107	GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR = 231,
	108	GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU = 240,
	109	GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR = 241,
	110	GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU = 250,
	111	GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR = 251,
	112	GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU = 260,
	113	GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR = 261,
	114	GOYA_ASYNC_EVENT_ID_MMU_SBA_SPMU0 = 270,
	115	GOYA_ASYNC_EVENT_ID_MMU_SBA_SPMU1 = 271,
	116	GOYA_ASYNC_EVENT_ID_MME_WACS_UP = 272,
	117	GOYA_ASYNC_EVENT_ID_MME_WACS_DOWN = 273,
	118	GOYA_ASYNC_EVENT_ID_MMU_PAGE_FAULT = 280,
	119	GOYA_ASYNC_EVENT_ID_MMU_WR_PERM = 281,
	120	GOYA_ASYNC_EVENT_ID_MMU_DBG_BM = 282,
	121	GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 = 290,
	122	GOYA_ASYNC_EVENT_ID_DMA_BM_CH1 = 291,
	123	GOYA_ASYNC_EVENT_ID_DMA_BM_CH2 = 292,
	124	GOYA_ASYNC_EVENT_ID_DMA_BM_CH3 = 293,
	125	GOYA_ASYNC_EVENT_ID_DMA_BM_CH4 = 294,
	126	GOYA_ASYNC_EVENT_ID_DDR0_PHY_DFI = 300,
	127	GOYA_ASYNC_EVENT_ID_DDR0_ECC_SCRUB = 301,
	128	GOYA_ASYNC_EVENT_ID_DDR0_DB_ECC = 302,
	129	GOYA_ASYNC_EVENT_ID_DDR0_SB_ECC = 303,
	130	GOYA_ASYNC_EVENT_ID_DDR0_SB_ECC_MC = 304,
	131	GOYA_ASYNC_EVENT_ID_DDR0_AXI_RD = 305,
	132	GOYA_ASYNC_EVENT_ID_DDR0_AXI_WR = 306,
	133	GOYA_ASYNC_EVENT_ID_DDR1_PHY_DFI = 310,
	134	GOYA_ASYNC_EVENT_ID_DDR1_ECC_SCRUB = 311,
	135	GOYA_ASYNC_EVENT_ID_DDR1_DB_ECC = 312,
	136	GOYA_ASYNC_EVENT_ID_DDR1_SB_ECC = 313,
	137	GOYA_ASYNC_EVENT_ID_DDR1_SB_ECC_MC = 314,
	138	GOYA_ASYNC_EVENT_ID_DDR1_AXI_RD = 315,
	139	GOYA_ASYNC_EVENT_ID_DDR1_AXI_WR = 316,
	140	GOYA_ASYNC_EVENT_ID_CPU_BMON = 320,
	141	GOYA_ASYNC_EVENT_ID_TS_EAST = 322,
	142	GOYA_ASYNC_EVENT_ID_TS_WEST = 323,
	143	GOYA_ASYNC_EVENT_ID_TS_NORTH = 324,
	144	GOYA_ASYNC_EVENT_ID_PSOC_GPIO_U16_0 = 330,
	145	GOYA_ASYNC_EVENT_ID_PSOC_GPIO_U16_1 = 331,
	146	GOYA_ASYNC_EVENT_ID_PSOC_GPIO_U16_2 = 332,
	147	GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET = 356,
	148	GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT = 361,
	149	GOYA_ASYNC_EVENT_ID_TPC0_CMDQ = 430,
	150	GOYA_ASYNC_EVENT_ID_TPC1_CMDQ = 431,
	151	GOYA_ASYNC_EVENT_ID_TPC2_CMDQ = 432,
	152	GOYA_ASYNC_EVENT_ID_TPC3_CMDQ = 433,
	153	GOYA_ASYNC_EVENT_ID_TPC4_CMDQ = 434,
	154	GOYA_ASYNC_EVENT_ID_TPC5_CMDQ = 435,
	155	GOYA_ASYNC_EVENT_ID_TPC6_CMDQ = 436,
	156	GOYA_ASYNC_EVENT_ID_TPC7_CMDQ = 437,
	157	GOYA_ASYNC_EVENT_ID_TPC0_QM = 438,
	158	GOYA_ASYNC_EVENT_ID_TPC1_QM = 439,
	159	GOYA_ASYNC_EVENT_ID_TPC2_QM = 440,
	160	GOYA_ASYNC_EVENT_ID_TPC3_QM = 441,
	161	GOYA_ASYNC_EVENT_ID_TPC4_QM = 442,
	162	GOYA_ASYNC_EVENT_ID_TPC5_QM = 443,
	163	GOYA_ASYNC_EVENT_ID_TPC6_QM = 444,
	164	GOYA_ASYNC_EVENT_ID_TPC7_QM = 445,
	165	GOYA_ASYNC_EVENT_ID_MME_QM = 447,
	166	GOYA_ASYNC_EVENT_ID_MME_CMDQ = 448,
	167	GOYA_ASYNC_EVENT_ID_DMA0_QM = 449,
	168	GOYA_ASYNC_EVENT_ID_DMA1_QM = 450,
	169	GOYA_ASYNC_EVENT_ID_DMA2_QM = 451,
	170	GOYA_ASYNC_EVENT_ID_DMA3_QM = 452,
	171	GOYA_ASYNC_EVENT_ID_DMA4_QM = 453,
	172	GOYA_ASYNC_EVENT_ID_DMA_ON_HBW = 454,
	173	GOYA_ASYNC_EVENT_ID_DMA0_CH = 455,
	174	GOYA_ASYNC_EVENT_ID_DMA1_CH = 456,
	175	GOYA_ASYNC_EVENT_ID_DMA2_CH = 457,
	176	GOYA_ASYNC_EVENT_ID_DMA3_CH = 458,
	177	GOYA_ASYNC_EVENT_ID_DMA4_CH = 459,
	178	GOYA_ASYNC_EVENT_ID_PI_UPDATE = 484,
	179	GOYA_ASYNC_EVENT_ID_HALT_MACHINE = 485,
	180	GOYA_ASYNC_EVENT_ID_INTS_REGISTER = 486,
	181	GOYA_ASYNC_EVENT_ID_SOFT_RESET = 487,
	182	GOYA_ASYNC_EVENT_ID_LAST_VALID_ID = 1023,
	183	GOYA_ASYNC_EVENT_ID_SIZE
	184	};
	185
	186	#endif /* __GOYA_ASYNC_EVENTS_H_ */

drivers/misc/habanalabs/include/goya/goya_packets.h+129−0

@@ -0,0 +1,129 @@
	1	/* SPDX-License-Identifier: GPL-2.0
	2	*
	3	* Copyright 2017-2018 HabanaLabs, Ltd.
	4	* All Rights Reserved.
	5	*
	6	*/
	7
	8	#ifndef GOYA_PACKETS_H
	9	#define GOYA_PACKETS_H
	10
	11	#include <linux/types.h>
	12
	13	#define PACKET_HEADER_PACKET_ID_SHIFT 56
	14	#define PACKET_HEADER_PACKET_ID_MASK 0x1F00000000000000ull
	15
	16	enum packet_id {
	17	PACKET_WREG_32 = 0x1,
	18	PACKET_WREG_BULK = 0x2,
	19	PACKET_MSG_LONG = 0x3,
	20	PACKET_MSG_SHORT = 0x4,
	21	PACKET_CP_DMA = 0x5,
	22	PACKET_MSG_PROT = 0x7,
	23	PACKET_FENCE = 0x8,
	24	PACKET_LIN_DMA = 0x9,
	25	PACKET_NOP = 0xA,
	26	PACKET_STOP = 0xB,
	27	MAX_PACKET_ID = (PACKET_HEADER_PACKET_ID_MASK >>
	28	PACKET_HEADER_PACKET_ID_SHIFT) + 1
	29	};
	30
	31	enum goya_dma_direction {
	32	DMA_HOST_TO_DRAM,
	33	DMA_HOST_TO_SRAM,
	34	DMA_DRAM_TO_SRAM,
	35	DMA_SRAM_TO_DRAM,
	36	DMA_SRAM_TO_HOST,
	37	DMA_DRAM_TO_HOST,
	38	DMA_DRAM_TO_DRAM,
	39	DMA_SRAM_TO_SRAM,
	40	DMA_ENUM_MAX
	41	};
	42
	43	#define GOYA_PKT_CTL_OPCODE_SHIFT 24
	44	#define GOYA_PKT_CTL_OPCODE_MASK 0x1F000000
	45
	46	#define GOYA_PKT_CTL_EB_SHIFT 29
	47	#define GOYA_PKT_CTL_EB_MASK 0x20000000
	48
	49	#define GOYA_PKT_CTL_RB_SHIFT 30
	50	#define GOYA_PKT_CTL_RB_MASK 0x40000000
	51
	52	#define GOYA_PKT_CTL_MB_SHIFT 31
	53	#define GOYA_PKT_CTL_MB_MASK 0x80000000
	54
	55	struct packet_nop {
	56	__le32 reserved;
	57	__le32 ctl;
	58	};
	59
	60	struct packet_stop {
	61	__le32 reserved;
	62	__le32 ctl;
	63	};
	64
	65	#define GOYA_PKT_WREG32_CTL_REG_OFFSET_SHIFT 0
	66	#define GOYA_PKT_WREG32_CTL_REG_OFFSET_MASK 0x0000FFFF
	67
	68	struct packet_wreg32 {
	69	__le32 value;
	70	__le32 ctl;
	71	};
	72
	73	struct packet_wreg_bulk {
	74	__le32 size64;
	75	__le32 ctl;
	76	__le64 values[0]; /* data starts here */
	77	};
	78
	79	struct packet_msg_long {
	80	__le32 value;
	81	__le32 ctl;
	82	__le64 addr;
	83	};
	84
	85	struct packet_msg_short {
	86	__le32 value;
	87	__le32 ctl;
	88	};
	89
	90	struct packet_msg_prot {
	91	__le32 value;
	92	__le32 ctl;
	93	__le64 addr;
	94	};
	95
	96	struct packet_fence {
	97	__le32 cfg;
	98	__le32 ctl;
	99	};
	100
	101	#define GOYA_PKT_LIN_DMA_CTL_WO_SHIFT 0
	102	#define GOYA_PKT_LIN_DMA_CTL_WO_MASK 0x00000001
	103
	104	#define GOYA_PKT_LIN_DMA_CTL_RDCOMP_SHIFT 1
	105	#define GOYA_PKT_LIN_DMA_CTL_RDCOMP_MASK 0x00000002
	106
	107	#define GOYA_PKT_LIN_DMA_CTL_WRCOMP_SHIFT 2
	108	#define GOYA_PKT_LIN_DMA_CTL_WRCOMP_MASK 0x00000004
	109
	110	#define GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT 6
	111	#define GOYA_PKT_LIN_DMA_CTL_MEMSET_MASK 0x00000040
	112
	113	#define GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT 20
	114	#define GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK 0x00700000
	115
	116	struct packet_lin_dma {
	117	__le32 tsize;
	118	__le32 ctl;
	119	__le64 src_addr;
	120	__le64 dst_addr;
	121	};
	122
	123	struct packet_cp_dma {
	124	__le32 tsize;
	125	__le32 ctl;
	126	__le64 src_addr;
	127	};
	128
	129	#endif /* GOYA_PACKETS_H */

drivers/misc/habanalabs/include/qman_if.h+56−0

@@ -0,0 +1,56 @@
	1	/* SPDX-License-Identifier: GPL-2.0
	2	*
	3	* Copyright 2016-2018 HabanaLabs, Ltd.
	4	* All Rights Reserved.
	5	*
	6	*/
	7
	8	#ifndef QMAN_IF_H
	9	#define QMAN_IF_H
	10
	11	#include <linux/types.h>
	12
	13	/*
	14	* PRIMARY QUEUE
	15	*/
	16
	17	struct hl_bd {
	18	__le64 ptr;
	19	__le32 len;
	20	__le32 ctl;
	21	};
	22
	23	#define HL_BD_SIZE sizeof(struct hl_bd)
	24
	25	/*
	26	* BD_CTL_REPEAT_VALID tells the CP whether the repeat field in the BD CTL is
	27	* valid. 1 means the repeat field is valid, 0 means not-valid,
	28	* i.e. repeat == 1
	29	*/
	30	#define BD_CTL_REPEAT_VALID_SHIFT 24
	31	#define BD_CTL_REPEAT_VALID_MASK 0x01000000
	32
	33	#define BD_CTL_SHADOW_INDEX_SHIFT 0
	34	#define BD_CTL_SHADOW_INDEX_MASK 0x00000FFF
	35
	36	/*
	37	* COMPLETION QUEUE
	38	*/
	39
	40	struct hl_cq_entry {
	41	__le32 data;
	42	};
	43
	44	#define HL_CQ_ENTRY_SIZE sizeof(struct hl_cq_entry)
	45
	46	#define CQ_ENTRY_READY_SHIFT 31
	47	#define CQ_ENTRY_READY_MASK 0x80000000
	48
	49	#define CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT 30
	50	#define CQ_ENTRY_SHADOW_INDEX_VALID_MASK 0x40000000
	51
	52	#define CQ_ENTRY_SHADOW_INDEX_SHIFT BD_CTL_SHADOW_INDEX_SHIFT
	53	#define CQ_ENTRY_SHADOW_INDEX_MASK BD_CTL_SHADOW_INDEX_MASK
	54
	55
	56	#endif /* QMAN_IF_H */

drivers/misc/habanalabs/irq.c+149−0

@@ -0,0 +1,149 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	/*
	4	* Copyright 2016-2019 HabanaLabs, Ltd.
	5	* All Rights Reserved.
	6	*/
	7
	8	#include "habanalabs.h"
	9
	10	#include <linux/irqreturn.h>
	11
	12	/*
	13	* hl_cq_inc_ptr - increment ci or pi of cq
	14	*
	15	* @ptr: the current ci or pi value of the completion queue
	16	*
	17	* Increment ptr by 1. If it reaches the number of completion queue
	18	* entries, set it to 0
	19	*/
	20	inline u32 hl_cq_inc_ptr(u32 ptr)
	21	{
	22	ptr++;
	23	if (unlikely(ptr == HL_CQ_LENGTH))
	24	ptr = 0;
	25	return ptr;
	26	}
	27
	28	/*
	29	* hl_irq_handler_cq - irq handler for completion queue
	30	*
	31	* @irq: irq number
	32	* @arg: pointer to completion queue structure
	33	*
	34	*/
	35	irqreturn_t hl_irq_handler_cq(int irq, void *arg)
	36	{
	37	struct hl_cq *cq = arg;
	38	struct hl_device *hdev = cq->hdev;
	39	struct hl_hw_queue *queue;
	40	struct hl_cs_job *job;
	41	bool shadow_index_valid;
	42	u16 shadow_index;
	43	u32 *cq_entry;
	44	u32 *cq_base;
	45
	46	if (hdev->disabled) {
	47	dev_dbg(hdev->dev,
	48	"Device disabled but received IRQ %d for CQ %d\n",
	49	irq, cq->hw_queue_id);
	50	return IRQ_HANDLED;
	51	}
	52
	53	cq_base = (u32 *) (uintptr_t) cq->kernel_address;
	54
	55	while (1) {
	56	bool entry_ready = ((cq_base[cq->ci] & CQ_ENTRY_READY_MASK)
	57	>> CQ_ENTRY_READY_SHIFT);
	58
	59	if (!entry_ready)
	60	break;
	61
	62	cq_entry = (u32 *) &cq_base[cq->ci];
	63
	64	/*
	65	* Make sure we read CQ entry contents after we've
	66	* checked the ownership bit.
	67	*/
	68	dma_rmb();
	69
	70	shadow_index_valid =
	71	((*cq_entry & CQ_ENTRY_SHADOW_INDEX_VALID_MASK)
	72	>> CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT);
	73
	74	shadow_index = (u16)
	75	((*cq_entry & CQ_ENTRY_SHADOW_INDEX_MASK)
	76	>> CQ_ENTRY_SHADOW_INDEX_SHIFT);
	77
	78	queue = &hdev->kernel_queues[cq->hw_queue_id];
	79
	80	if ((shadow_index_valid) && (!hdev->disabled)) {
	81	job = queue->shadow_queue[hl_pi_2_offset(shadow_index)];
	82	queue_work(hdev->cq_wq, &job->finish_work);
	83	}
	84
	85	/*
	86	* Update ci of the context's queue. There is no
	87	* need to protect it with spinlock because this update is
	88	* done only inside IRQ and there is a different IRQ per
	89	* queue
	90	*/
	91	queue->ci = hl_queue_inc_ptr(queue->ci);
	92
	93	/* Clear CQ entry ready bit */
	94	cq_base[cq->ci] &= ~CQ_ENTRY_READY_MASK;
	95
	96	cq->ci = hl_cq_inc_ptr(cq->ci);
	97
	98	/* Increment free slots */
	99	atomic_inc(&cq->free_slots_cnt);
	100	}
	101
	102	return IRQ_HANDLED;
	103	}
	104
	105	/*
	106	* hl_cq_init - main initialization function for an cq object
	107	*
	108	* @hdev: pointer to device structure
	109	* @q: pointer to cq structure
	110	* @hw_queue_id: The H/W queue ID this completion queue belongs to
	111	*
	112	* Allocate dma-able memory for the completion queue and initialize fields
	113	* Returns 0 on success
	114	*/
	115	int hl_cq_init(struct hl_device hdev, struct hl_cq q, u32 hw_queue_id)
	116	{
	117	void *p;
	118
	119	BUILD_BUG_ON(HL_CQ_SIZE_IN_BYTES > HL_PAGE_SIZE);
	120
	121	p = hdev->asic_funcs->dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
	122	&q->bus_address, GFP_KERNEL \| __GFP_ZERO);
	123	if (!p)
	124	return -ENOMEM;
	125
	126	q->hdev = hdev;
	127	q->kernel_address = (u64) (uintptr_t) p;
	128	q->hw_queue_id = hw_queue_id;
	129	q->ci = 0;
	130	q->pi = 0;
	131
	132	atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
	133
	134	return 0;
	135	}
	136
	137	/*
	138	* hl_cq_fini - destroy completion queue
	139	*
	140	* @hdev: pointer to device structure
	141	* @q: pointer to cq structure
	142	*
	143	* Free the completion queue memory
	144	*/
	145	void hl_cq_fini(struct hl_device hdev, struct hl_cq q)
	146	{
	147	hdev->asic_funcs->dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
	148	(void *) (uintptr_t) q->kernel_address, q->bus_address);
	149	}

include/uapi/misc/habanalabs.h+29−0

@@ -17,6 +17,35 @@ · 16 unmodified lines
17	*/	17	*/
18	#define GOYA_KMD_SRAM_RESERVED_SIZE_FROM_START 0x8000 /* 32KB */	18	#define GOYA_KMD_SRAM_RESERVED_SIZE_FROM_START 0x8000 /* 32KB */
19		19
		20	/*
		21	* Queue Numbering
		22	*
		23	* The external queues (DMA channels + CPU) MUST be before the internal queues
		24	* and each group (DMA channels + CPU and internal) must be contiguous inside
		25	* itself but there can be a gap between the two groups (although not
		26	* recommended)
		27	*/
		28
		29	enum goya_queue_id {
		30	GOYA_QUEUE_ID_DMA_0 = 0,
		31	GOYA_QUEUE_ID_DMA_1,
		32	GOYA_QUEUE_ID_DMA_2,
		33	GOYA_QUEUE_ID_DMA_3,
		34	GOYA_QUEUE_ID_DMA_4,
		35	GOYA_QUEUE_ID_CPU_PQ,
		36	GOYA_QUEUE_ID_MME,
		37	GOYA_QUEUE_ID_TPC0,
		38	GOYA_QUEUE_ID_TPC1,
		39	GOYA_QUEUE_ID_TPC2,
		40	GOYA_QUEUE_ID_TPC3,
		41	GOYA_QUEUE_ID_TPC4,
		42	GOYA_QUEUE_ID_TPC5,
		43	GOYA_QUEUE_ID_TPC6,
		44	GOYA_QUEUE_ID_TPC7,
		45	GOYA_QUEUE_ID_SIZE
		46	};
		47
		48
20	/* Opcode to create a new command buffer */	49	/* Opcode to create a new command buffer */
21	#define HL_CB_OP_CREATE 0	50	#define HL_CB_OP_CREATE 0
22	/* Opcode to destroy previously created command buffer */	51	/* Opcode to destroy previously created command buffer */