Merge tag 'misc-habanalabs-next-2021-04-10' of https://git.kernel.org/pub/scm/linux/kernel/git/ogabbay/linux into char-misc-next

Oded writes: This tag contains habanalabs driver changes for v5.13: - Add support to reset device after the user closes the file descriptor. Because we support a single user, we can reset the device (if needs to) after a user closes its file descriptor to make sure the device is in idle and clean state for the next user. - Add a new feature to allow the user to wait on interrupt. This is needed for future ASICs - Replace GFP_ATOMIC with GFP_KERNEL wherever possible and add code to support failure of allocating with GFP_ATOMIC. - Update code to support the latest firmware image: - More security features are done in the firmware - Remove hard-coded assumptions and replace them with values that are sent to the firmware on loading. - Print device unusable error - Reset device in case the communication between driver and firmware gets out of sync. - Support new PCI device ids for secured GAUDI. - Expose current power draw through the INFO IOCTL. - Support resetting the device upon a request from the BMC (through F/W). - Always use only a single MSI in GAUDI, due to H/W limitation. - Improve data-path code by taking out code from spinlock protection. - Allow user to specify custom timeout per Command Submission. - Some enhancements to debugfs. - Various minor changes and improvements. * tag 'misc-habanalabs-next-2021-04-10' of https://git.kernel.org/pub/scm/linux/kernel/git/ogabbay/linux: (41 commits) habanalabs: print f/w boot unknown error habanalabs: update to latest F/W communication header habanalabs/gaudi: skip iATU if F/W security is enabled habanalabs/gaudi: derive security status from pci id habanalabs: move dram scrub to free sequence habanalabs: send dynamic msi-x indexes to f/w habanalabs/gaudi: clear QM errors only if not in stop_on_err mode habanalabs: support DEVICE_UNUSABLE error indication from FW habanalabs: use strscpy instead of sprintf and strlcpy habanalabs: remove the store jobs array from CS IOCTL habanalabs/gaudi: add debugfs to DMA from the device habanalabs/gaudi: sync stream add protection to SOB reset flow habanalabs: add custom timeout flag per cs habanalabs: improve utilization calculation habanalabs: support legacy and new pll indexes habanalabs: move relevant datapath work outside cs lock habanalabs: avoid soft lockup bug upon mapping error habanalabs/gaudi: Update async events header habanalabs/gaudi: unsecure TPC cfg status registers habanalabs/gaudi: always use single-msi mode ...

Greg Kroah-Hartman committed 5 years ago

Commit aa87e312 parents 19ab233 b575a7631 files · +2193 −517Browse files

Unified · Split

31 files

Documentation/ABI/testing/debugfs-driver-habanalabs +54 −16
drivers/misc/habanalabs/common/command_buffer.c +9 −3
drivers/misc/habanalabs/common/command_submission.c +302 −66
drivers/misc/habanalabs/common/context.c +11 −3
drivers/misc/habanalabs/common/debugfs.c +201 −23
drivers/misc/habanalabs/common/device.c +94 −127
drivers/misc/habanalabs/common/firmware_if.c +216 −22
drivers/misc/habanalabs/common/habanalabs.h +143 −41
drivers/misc/habanalabs/common/habanalabs_drv.c +24 −4
drivers/misc/habanalabs/common/habanalabs_ioctl.c +26 −9
drivers/misc/habanalabs/common/hw_queue.c +1 −9
drivers/misc/habanalabs/common/irq.c +56 −0
drivers/misc/habanalabs/common/memory.c +129 −53
drivers/misc/habanalabs/common/mmu/mmu.c +3 −0
drivers/misc/habanalabs/common/pci/pci.c +52 −0
drivers/misc/habanalabs/common/sysfs.c +24 −9
drivers/misc/habanalabs/gaudi/gaudi.c +314 −43
drivers/misc/habanalabs/gaudi/gaudiP.h +3 −0
drivers/misc/habanalabs/gaudi/gaudi_security.c +0 −8
drivers/misc/habanalabs/goya/goya.c +123 −17
drivers/misc/habanalabs/goya/goyaP.h +2 −0
drivers/misc/habanalabs/include/common/cpucp_if.h +98 −1
drivers/misc/habanalabs/include/common/hl_boot_if.h +219 −0
drivers/misc/habanalabs/include/gaudi/gaudi.h +1 −1
drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h +2 −0
drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h +27 −16
drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h +0 −14
drivers/misc/habanalabs/include/goya/goya.h +1 −1
drivers/misc/habanalabs/include/goya/goya_async_events.h +1 −0
drivers/misc/habanalabs/include/goya/goya_fw_if.h +0 −11
include/uapi/misc/habanalabs.h +57 −20

Documentation/ABI/testing/debugfs-driver-habanalabs+54−16

@@ -82,6 +82,24 @@ Description: Allows the root user to read or write 64 bit data directly · 81 unmodified lines
82	If the IOMMU is disabled, it also allows the root user to read	82	If the IOMMU is disabled, it also allows the root user to read
83	or write from the host a device VA of a host mapped memory	83	or write from the host a device VA of a host mapped memory
84		84
		85	What: /sys/kernel/debug/habanalabs/hl<n>/data_dma
		86	Date: Apr 2021
		87	KernelVersion: 5.13
		88	Contact: ogabbay@kernel.org
		89	Description: Allows the root user to read from the device's internal
		90	memory (DRAM/SRAM) through a DMA engine.
		91	This property is a binary blob that contains the result of the
		92	DMA transfer.
		93	This custom interface is needed (instead of using the generic
		94	Linux user-space PCI mapping) because the amount of internal
		95	memory is huge (>32GB) and reading it via the PCI bar will take
		96	a very long time.
		97	This interface doesn't support concurrency in the same device.
		98	In GAUDI and GOYA, this action can cause undefined behavior
		99	in case the it is done while the device is executing user
		100	workloads.
		101	Only supported on GAUDI at this stage.
		102
85	What: /sys/kernel/debug/habanalabs/hl<n>/device	103	What: /sys/kernel/debug/habanalabs/hl<n>/device
86	Date: Jan 2019	104	Date: Jan 2019
87	KernelVersion: 5.1	105	KernelVersion: 5.1
@@ -90,6 +108,24 @@ Description: Enables the root user to set the device to specific state. · 2 unmodified lines
90	Valid values are "disable", "enable", "suspend", "resume".	108	Valid values are "disable", "enable", "suspend", "resume".
91	User can read this property to see the valid values	109	User can read this property to see the valid values
92		110
		111	What: /sys/kernel/debug/habanalabs/hl<n>/dma_size
		112	Date: Apr 2021
		113	KernelVersion: 5.13
		114	Contact: ogabbay@kernel.org
		115	Description: Specify the size of the DMA transaction when using DMA to read
		116	from the device's internal memory. The value can not be larger
		117	than 128MB. Writing to this value initiates the DMA transfer.
		118	When the write is finished, the user can read the "data_dma"
		119	blob
		120
		121	What: /sys/kernel/debug/habanalabs/hl<n>/dump_security_violations
		122	Date: Jan 2021
		123	KernelVersion: 5.12
		124	Contact: ogabbay@kernel.org
		125	Description: Dumps all security violations to dmesg. This will also ack
		126	all security violations meanings those violations will not be
		127	dumped next time user calls this API
		128
93	What: /sys/kernel/debug/habanalabs/hl<n>/engines	129	What: /sys/kernel/debug/habanalabs/hl<n>/engines
94	Date: Jul 2019	130	Date: Jul 2019
95	KernelVersion: 5.3	131	KernelVersion: 5.3
@@ -154,6 +190,16 @@ Description: Displays the hop values and physical address for a given ASID · 58 unmodified lines
154	e.g. to display info about VA 0x1000 for ASID 1 you need to do:	190	e.g. to display info about VA 0x1000 for ASID 1 you need to do:
155	echo "1 0x1000" > /sys/kernel/debug/habanalabs/hl0/mmu	191	echo "1 0x1000" > /sys/kernel/debug/habanalabs/hl0/mmu
156		192
		193	What: /sys/kernel/debug/habanalabs/hl<n>/mmu_error
		194	Date: Mar 2021
		195	KernelVersion: 5.12
		196	Contact: fkassabri@habana.ai
		197	Description: Check and display page fault or access violation mmu errors for
		198	all MMUs specified in mmu_cap_mask.
		199	e.g. to display error info for MMU hw cap bit 9, you need to do:
		200	echo "0x200" > /sys/kernel/debug/habanalabs/hl0/mmu_error
		201	cat /sys/kernel/debug/habanalabs/hl0/mmu_error
		202
157	What: /sys/kernel/debug/habanalabs/hl<n>/set_power_state	203	What: /sys/kernel/debug/habanalabs/hl<n>/set_power_state
158	Date: Jan 2019	204	Date: Jan 2019
159	KernelVersion: 5.1	205	KernelVersion: 5.1
@@ -161,6 +207,13 @@ Contact: ogabbay@kernel.org · 1 unmodified line
161	Description: Sets the PCI power state. Valid values are "1" for D0 and "2"	207	Description: Sets the PCI power state. Valid values are "1" for D0 and "2"
162	for D3Hot	208	for D3Hot
163		209
		210	What: /sys/kernel/debug/habanalabs/hl<n>/stop_on_err
		211	Date: Mar 2020
		212	KernelVersion: 5.6
		213	Contact: ogabbay@kernel.org
		214	Description: Sets the stop-on_error option for the device engines. Value of
		215	"0" is for disable, otherwise enable.
		216
164	What: /sys/kernel/debug/habanalabs/hl<n>/userptr	217	What: /sys/kernel/debug/habanalabs/hl<n>/userptr
165	Date: Jan 2019	218	Date: Jan 2019
166	KernelVersion: 5.1	219	KernelVersion: 5.1
@@ -174,19 +227,4 @@ Date: Jan 2019 · 7 unmodified lines
174	KernelVersion: 5.1	227	KernelVersion: 5.1
175	Contact: ogabbay@kernel.org	228	Contact: ogabbay@kernel.org
176	Description: Displays a list with information about all the active virtual	229	Description: Displays a list with information about all the active virtual
177	address mappings per ASID	230	address mappings per ASID and all user mappings of HW blocks
178
179	What: /sys/kernel/debug/habanalabs/hl<n>/stop_on_err
180	Date: Mar 2020
181	KernelVersion: 5.6
182	Contact: ogabbay@kernel.org
183	Description: Sets the stop-on_error option for the device engines. Value of
184	"0" is for disable, otherwise enable.
185
186	What: /sys/kernel/debug/habanalabs/hl<n>/dump_security_violations
187	Date: Jan 2021
188	KernelVersion: 5.12
189	Contact: ogabbay@kernel.org
190	Description: Dumps all security violations to dmesg. This will also ack
191	all security violations meanings those violations will not be
192	dumped next time user calls this API

drivers/misc/habanalabs/common/command_buffer.c+9−3

@@ -181,7 +181,7 @@ static void cb_release(struct kref *ref) · 180 unmodified lines
181	static struct hl_cb hl_cb_alloc(struct hl_device hdev, u32 cb_size,	181	static struct hl_cb hl_cb_alloc(struct hl_device hdev, u32 cb_size,
182	int ctx_id, bool internal_cb)	182	int ctx_id, bool internal_cb)
183	{	183	{
184	struct hl_cb *cb;	184	struct hl_cb *cb = NULL;
185	u32 cb_offset;	185	u32 cb_offset;
186	void *p;	186	void *p;
187		187
@@ -193,9 +193,10 @@ static struct hl_cb hl_cb_alloc(struct hl_device hdev, u32 cb_size, · 5 unmodified lines
193	* the kernel's copy. Hence, we must never sleep in this code section	193	* the kernel's copy. Hence, we must never sleep in this code section
194	* and must use GFP_ATOMIC for all memory allocations.	194	* and must use GFP_ATOMIC for all memory allocations.
195	*/	195	*/
196	if (ctx_id == HL_KERNEL_ASID_ID)	196	if (ctx_id == HL_KERNEL_ASID_ID && !hdev->disabled)
197	cb = kzalloc(sizeof(*cb), GFP_ATOMIC);	197	cb = kzalloc(sizeof(*cb), GFP_ATOMIC);
198	else	198
		199	if (!cb)
199	cb = kzalloc(sizeof(*cb), GFP_KERNEL);	200	cb = kzalloc(sizeof(*cb), GFP_KERNEL);
200		201
201	if (!cb)	202	if (!cb)
@@ -214,6 +215,9 @@ static struct hl_cb hl_cb_alloc(struct hl_device hdev, u32 cb_size, · 12 unmodified lines
214	} else if (ctx_id == HL_KERNEL_ASID_ID) {	215	} else if (ctx_id == HL_KERNEL_ASID_ID) {
215	p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,	216	p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
216	&cb->bus_address, GFP_ATOMIC);	217	&cb->bus_address, GFP_ATOMIC);
		218	if (!p)
		219	p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
		220	cb_size, &cb->bus_address, GFP_KERNEL);
217	} else {	221	} else {
218	p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,	222	p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
219	&cb->bus_address,	223	&cb->bus_address,
@@ -310,6 +314,8 @@ int hl_cb_create(struct hl_device hdev, struct hl_cb_mgr mgr, · 90 unmodified lines
310		314
311	spin_lock(&mgr->cb_lock);	315	spin_lock(&mgr->cb_lock);
312	rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);	316	rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
		317	if (rc < 0)
		318	rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_KERNEL);
313	spin_unlock(&mgr->cb_lock);	319	spin_unlock(&mgr->cb_lock);
314		320
315	if (rc < 0) {	321	if (rc < 0) {

drivers/misc/habanalabs/common/command_submission.c+302−66

@@ -84,6 +84,38 @@ int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask) · 83 unmodified lines
84	return 0;	84	return 0;
85	}	85	}
86		86
		87	static void sob_reset_work(struct work_struct *work)
		88	{
		89	struct hl_cs_compl *hl_cs_cmpl =
		90	container_of(work, struct hl_cs_compl, sob_reset_work);
		91	struct hl_device *hdev = hl_cs_cmpl->hdev;
		92
		93	/*
		94	* A signal CS can get completion while the corresponding wait
		95	* for signal CS is on its way to the PQ. The wait for signal CS
		96	* will get stuck if the signal CS incremented the SOB to its
		97	* max value and there are no pending (submitted) waits on this
		98	* SOB.
		99	* We do the following to void this situation:
		100	* 1. The wait for signal CS must get a ref for the signal CS as
		101	* soon as possible in cs_ioctl_signal_wait() and put it
		102	* before being submitted to the PQ but after it incremented
		103	* the SOB refcnt in init_signal_wait_cs().
		104	* 2. Signal/Wait for signal CS will decrement the SOB refcnt
		105	* here.
		106	* These two measures guarantee that the wait for signal CS will
		107	* reset the SOB upon completion rather than the signal CS and
		108	* hence the above scenario is avoided.
		109	*/
		110	kref_put(&hl_cs_cmpl->hw_sob->kref, hl_sob_reset);
		111
		112	if (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT)
		113	hdev->asic_funcs->reset_sob_group(hdev,
		114	hl_cs_cmpl->sob_group);
		115
		116	kfree(hl_cs_cmpl);
		117	}
		118
87	static void hl_fence_release(struct kref *kref)	119	static void hl_fence_release(struct kref *kref)
88	{	120	{
89	struct hl_fence *fence =	121	struct hl_fence *fence =
@@ -109,28 +141,9 @@ static void hl_fence_release(struct kref *kref) · 19 unmodified lines
109	hl_cs_cmpl->hw_sob->sob_id,	141	hl_cs_cmpl->hw_sob->sob_id,
110	hl_cs_cmpl->sob_val);	142	hl_cs_cmpl->sob_val);
111		143
112	/*	144	queue_work(hdev->sob_reset_wq, &hl_cs_cmpl->sob_reset_work);
113	* A signal CS can get completion while the corresponding wait
114	* for signal CS is on its way to the PQ. The wait for signal CS
115	* will get stuck if the signal CS incremented the SOB to its
116	* max value and there are no pending (submitted) waits on this
117	* SOB.
118	* We do the following to void this situation:
119	* 1. The wait for signal CS must get a ref for the signal CS as
120	* soon as possible in cs_ioctl_signal_wait() and put it
121	* before being submitted to the PQ but after it incremented
122	* the SOB refcnt in init_signal_wait_cs().
123	* 2. Signal/Wait for signal CS will decrement the SOB refcnt
124	* here.
125	* These two measures guarantee that the wait for signal CS will
126	* reset the SOB upon completion rather than the signal CS and
127	* hence the above scenario is avoided.
128	*/
129	kref_put(&hl_cs_cmpl->hw_sob->kref, hl_sob_reset);
130		145
131	if (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT)	146	return;
132	hdev->asic_funcs->reset_sob_group(hdev,
133	hl_cs_cmpl->sob_group);
134	}	147	}
135		148
136	free:	149	free:
@@ -454,8 +467,7 @@ static void cs_handle_tdr(struct hl_device hdev, struct hl_cs cs) · 317 unmodified lines
454		467
455	if (next_entry_found && !next->tdr_active) {	468	if (next_entry_found && !next->tdr_active) {
456	next->tdr_active = true;	469	next->tdr_active = true;
457	schedule_delayed_work(&next->work_tdr,	470	schedule_delayed_work(&next->work_tdr, next->timeout_jiffies);
458	hdev->timeout_jiffies);
459	}	471	}
460		472
461	spin_unlock(&hdev->cs_mirror_lock);	473	spin_unlock(&hdev->cs_mirror_lock);
@@ -492,24 +504,6 @@ static void cs_do_release(struct kref *ref) · 30 unmodified lines
492	goto out;	504	goto out;
493	}	505	}
494		506
495	hdev->asic_funcs->hw_queues_lock(hdev);
496
497	hdev->cs_active_cnt--;
498	if (!hdev->cs_active_cnt) {
499	struct hl_device_idle_busy_ts *ts;
500
501	ts = &hdev->idle_busy_ts_arr[hdev->idle_busy_ts_idx++];
502	ts->busy_to_idle_ts = ktime_get();
503
504	if (hdev->idle_busy_ts_idx == HL_IDLE_BUSY_TS_ARR_SIZE)
505	hdev->idle_busy_ts_idx = 0;
506	} else if (hdev->cs_active_cnt < 0) {
507	dev_crit(hdev->dev, "CS active cnt %d is negative\n",
508	hdev->cs_active_cnt);
509	}
510
511	hdev->asic_funcs->hw_queues_unlock(hdev);
512
513	/* Need to update CI for all queue jobs that does not get completion */	507	/* Need to update CI for all queue jobs that does not get completion */
514	hl_hw_queue_update_ci(cs);	508	hl_hw_queue_update_ci(cs);
515		509
@@ -620,14 +614,14 @@ static void cs_timedout(struct work_struct *work) · 104 unmodified lines
620	cs_put(cs);	614	cs_put(cs);
621		615
622	if (hdev->reset_on_lockup)	616	if (hdev->reset_on_lockup)
623	hl_device_reset(hdev, false, false);	617	hl_device_reset(hdev, 0);
624	else	618	else
625	hdev->needs_reset = true;	619	hdev->needs_reset = true;
626	}	620	}
627		621
628	static int allocate_cs(struct hl_device hdev, struct hl_ctx ctx,	622	static int allocate_cs(struct hl_device hdev, struct hl_ctx ctx,
629	enum hl_cs_type cs_type, u64 user_sequence,	623	enum hl_cs_type cs_type, u64 user_sequence,
630	struct hl_cs **cs_new)	624	struct hl_cs **cs_new, u32 flags, u32 timeout)
631	{	625	{
632	struct hl_cs_counters_atomic *cntr;	626	struct hl_cs_counters_atomic *cntr;
633	struct hl_fence *other = NULL;	627	struct hl_fence *other = NULL;
@@ -638,6 +632,9 @@ static int allocate_cs(struct hl_device hdev, struct hl_ctx ctx, · 4 unmodified lines
638	cntr = &hdev->aggregated_cs_counters;	632	cntr = &hdev->aggregated_cs_counters;
639		633
640	cs = kzalloc(sizeof(*cs), GFP_ATOMIC);	634	cs = kzalloc(sizeof(*cs), GFP_ATOMIC);
		635	if (!cs)
		636	cs = kzalloc(sizeof(*cs), GFP_KERNEL);
		637
641	if (!cs) {	638	if (!cs) {
642	atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);	639	atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
643	atomic64_inc(&cntr->out_of_mem_drop_cnt);	640	atomic64_inc(&cntr->out_of_mem_drop_cnt);
@@ -651,12 +648,17 @@ static int allocate_cs(struct hl_device hdev, struct hl_ctx ctx, · 7 unmodified lines
651	cs->submitted = false;	648	cs->submitted = false;
652	cs->completed = false;	649	cs->completed = false;
653	cs->type = cs_type;	650	cs->type = cs_type;
		651	cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP);
		652	cs->timeout_jiffies = timeout;
654	INIT_LIST_HEAD(&cs->job_list);	653	INIT_LIST_HEAD(&cs->job_list);
655	INIT_DELAYED_WORK(&cs->work_tdr, cs_timedout);	654	INIT_DELAYED_WORK(&cs->work_tdr, cs_timedout);
656	kref_init(&cs->refcount);	655	kref_init(&cs->refcount);
657	spin_lock_init(&cs->job_lock);	656	spin_lock_init(&cs->job_lock);
658		657
659	cs_cmpl = kmalloc(sizeof(*cs_cmpl), GFP_ATOMIC);	658	cs_cmpl = kmalloc(sizeof(*cs_cmpl), GFP_ATOMIC);
		659	if (!cs_cmpl)
		660	cs_cmpl = kmalloc(sizeof(*cs_cmpl), GFP_KERNEL);
		661
660	if (!cs_cmpl) {	662	if (!cs_cmpl) {
661	atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);	663	atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
662	atomic64_inc(&cntr->out_of_mem_drop_cnt);	664	atomic64_inc(&cntr->out_of_mem_drop_cnt);
@@ -664,9 +666,23 @@ static int allocate_cs(struct hl_device hdev, struct hl_ctx ctx, · 1 unmodified line
664	goto free_cs;	666	goto free_cs;
665	}	667	}
666		668
		669	cs->jobs_in_queue_cnt = kcalloc(hdev->asic_prop.max_queues,
		670	sizeof(*cs->jobs_in_queue_cnt), GFP_ATOMIC);
		671	if (!cs->jobs_in_queue_cnt)
		672	cs->jobs_in_queue_cnt = kcalloc(hdev->asic_prop.max_queues,
		673	sizeof(*cs->jobs_in_queue_cnt), GFP_KERNEL);
		674
		675	if (!cs->jobs_in_queue_cnt) {
		676	atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
		677	atomic64_inc(&cntr->out_of_mem_drop_cnt);
		678	rc = -ENOMEM;
		679	goto free_cs_cmpl;
		680	}
		681
667	cs_cmpl->hdev = hdev;	682	cs_cmpl->hdev = hdev;
668	cs_cmpl->type = cs->type;	683	cs_cmpl->type = cs->type;
669	spin_lock_init(&cs_cmpl->lock);	684	spin_lock_init(&cs_cmpl->lock);
		685	INIT_WORK(&cs_cmpl->sob_reset_work, sob_reset_work);
670	cs->fence = &cs_cmpl->base_fence;	686	cs->fence = &cs_cmpl->base_fence;
671		687
672	spin_lock(&ctx->cs_lock);	688	spin_lock(&ctx->cs_lock);
@@ -696,15 +712,6 @@ static int allocate_cs(struct hl_device hdev, struct hl_ctx ctx, · 23 unmodified lines
696	goto free_fence;	712	goto free_fence;
697	}	713	}
698		714
699	cs->jobs_in_queue_cnt = kcalloc(hdev->asic_prop.max_queues,
700	sizeof(*cs->jobs_in_queue_cnt), GFP_ATOMIC);
701	if (!cs->jobs_in_queue_cnt) {
702	atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
703	atomic64_inc(&cntr->out_of_mem_drop_cnt);
704	rc = -ENOMEM;
705	goto free_fence;
706	}
707
708	/* init hl_fence */	715	/* init hl_fence */
709	hl_fence_init(&cs_cmpl->base_fence, cs_cmpl->cs_seq);	716	hl_fence_init(&cs_cmpl->base_fence, cs_cmpl->cs_seq);
710		717
@@ -727,6 +734,8 @@ static int allocate_cs(struct hl_device hdev, struct hl_ctx ctx, · 16 unmodified lines
727		734
728	free_fence:	735	free_fence:
729	spin_unlock(&ctx->cs_lock);	736	spin_unlock(&ctx->cs_lock);
		737	kfree(cs->jobs_in_queue_cnt);
		738	free_cs_cmpl:
730	kfree(cs_cmpl);	739	kfree(cs_cmpl);
731	free_cs:	740	free_cs:
732	kfree(cs);	741	kfree(cs);
@@ -749,6 +758,8 @@ void hl_cs_rollback_all(struct hl_device *hdev) · 16 unmodified lines
749	int i;	758	int i;
750	struct hl_cs cs, tmp;	759	struct hl_cs cs, tmp;
751		760
		761	flush_workqueue(hdev->sob_reset_wq);
		762
752	/* flush all completions before iterating over the CS mirror list in	763	/* flush all completions before iterating over the CS mirror list in
753	* order to avoid a race with the release functions	764	* order to avoid a race with the release functions
754	*/	765	*/
@@ -778,6 +789,44 @@ void hl_pending_cb_list_flush(struct hl_ctx *ctx) · 23 unmodified lines
778	}	789	}
779	}	790	}
780		791
		792	static void
		793	wake_pending_user_interrupt_threads(struct hl_user_interrupt *interrupt)
		794	{
		795	struct hl_user_pending_interrupt *pend;
		796
		797	spin_lock(&interrupt->wait_list_lock);
		798	list_for_each_entry(pend, &interrupt->wait_list_head, wait_list_node) {
		799	pend->fence.error = -EIO;
		800	complete_all(&pend->fence.completion);
		801	}
		802	spin_unlock(&interrupt->wait_list_lock);
		803	}
		804
		805	void hl_release_pending_user_interrupts(struct hl_device *hdev)
		806	{
		807	struct asic_fixed_properties *prop = &hdev->asic_prop;
		808	struct hl_user_interrupt *interrupt;
		809	int i;
		810
		811	if (!prop->user_interrupt_count)
		812	return;
		813
		814	/* We iterate through the user interrupt requests and waking up all
		815	* user threads waiting for interrupt completion. We iterate the
		816	* list under a lock, this is why all user threads, once awake,
		817	* will wait on the same lock and will release the waiting object upon
		818	* unlock.
		819	*/
		820
		821	for (i = 0 ; i < prop->user_interrupt_count ; i++) {
		822	interrupt = &hdev->user_interrupt[i];
		823	wake_pending_user_interrupt_threads(interrupt);
		824	}
		825
		826	interrupt = &hdev->common_user_interrupt;
		827	wake_pending_user_interrupt_threads(interrupt);
		828	}
		829
781	static void job_wq_completion(struct work_struct *work)	830	static void job_wq_completion(struct work_struct *work)
782	{	831	{
783	struct hl_cs_job *job = container_of(work, struct hl_cs_job,	832	struct hl_cs_job *job = container_of(work, struct hl_cs_job,
@@ -889,6 +938,9 @@ struct hl_cs_job hl_cs_allocate_job(struct hl_device hdev, · 105 unmodified lines
889	struct hl_cs_job *job;	938	struct hl_cs_job *job;
890		939
891	job = kzalloc(sizeof(*job), GFP_ATOMIC);	940	job = kzalloc(sizeof(*job), GFP_ATOMIC);
		941	if (!job)
		942	job = kzalloc(sizeof(*job), GFP_KERNEL);
		943
892	if (!job)	944	if (!job)
893	return NULL;	945	return NULL;
894		946
@@ -991,6 +1043,9 @@ static int hl_cs_copy_chunk_array(struct hl_device *hdev, · 96 unmodified lines
991		1043
992	cs_chunk_array = kmalloc_array(num_chunks, sizeof(*cs_chunk_array),	1044	cs_chunk_array = kmalloc_array(num_chunks, sizeof(*cs_chunk_array),
993	GFP_ATOMIC);	1045	GFP_ATOMIC);
		1046	if (!*cs_chunk_array)
		1047	*cs_chunk_array = kmalloc_array(num_chunks,
		1048	sizeof(**cs_chunk_array), GFP_KERNEL);
994	if (!*cs_chunk_array) {	1049	if (!*cs_chunk_array) {
995	atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);	1050	atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
996	atomic64_inc(&hdev->aggregated_cs_counters.out_of_mem_drop_cnt);	1051	atomic64_inc(&hdev->aggregated_cs_counters.out_of_mem_drop_cnt);
@@ -1038,7 +1093,8 @@ static int cs_staged_submission(struct hl_device hdev, struct hl_cs cs, · 41 unmodified lines
1038	}	1093	}
1039		1094
1040	static int cs_ioctl_default(struct hl_fpriv hpriv, void __user chunks,	1095	static int cs_ioctl_default(struct hl_fpriv hpriv, void __user chunks,
1041	u32 num_chunks, u64 *cs_seq, u32 flags)	1096	u32 num_chunks, u64 *cs_seq, u32 flags,
		1097	u32 timeout)
1042	{	1098	{
1043	bool staged_mid, int_queues_only = true;	1099	bool staged_mid, int_queues_only = true;
1044	struct hl_device *hdev = hpriv->hdev;	1100	struct hl_device *hdev = hpriv->hdev;
@@ -1067,11 +1123,11 @@ static int cs_ioctl_default(struct hl_fpriv hpriv, void __user chunks, · 22 unmodified lines
1067	staged_mid = false;	1123	staged_mid = false;
1068		1124
1069	rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT,	1125	rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT,
1070	staged_mid ? user_sequence : ULLONG_MAX, &cs);	1126	staged_mid ? user_sequence : ULLONG_MAX, &cs, flags,
		1127	timeout);
1071	if (rc)	1128	if (rc)
1072	goto free_cs_chunk_array;	1129	goto free_cs_chunk_array;
1073		1130
1074	cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP);
1075	*cs_seq = cs->sequence;	1131	*cs_seq = cs->sequence;
1076		1132
1077	hl_debugfs_add_cs(cs);	1133	hl_debugfs_add_cs(cs);
@@ -1269,7 +1325,8 @@ static int hl_submit_pending_cb(struct hl_fpriv *hpriv) · 191 unmodified lines
1269	list_move_tail(&pending_cb->cb_node, &local_cb_list);	1325	list_move_tail(&pending_cb->cb_node, &local_cb_list);
1270	spin_unlock(&ctx->pending_cb_lock);	1326	spin_unlock(&ctx->pending_cb_lock);
1271		1327
1272	rc = allocate_cs(hdev, ctx, CS_TYPE_DEFAULT, ULLONG_MAX, &cs);	1328	rc = allocate_cs(hdev, ctx, CS_TYPE_DEFAULT, ULLONG_MAX, &cs, 0,
		1329	hdev->timeout_jiffies);
1273	if (rc)	1330	if (rc)
1274	goto add_list_elements;	1331	goto add_list_elements;
1275		1332
@@ -1370,7 +1427,7 @@ static int hl_cs_ctx_switch(struct hl_fpriv hpriv, union hl_cs_args args, · 94 unmodified lines
1370	rc = 0;	1427	rc = 0;
1371	} else {	1428	} else {
1372	rc = cs_ioctl_default(hpriv, chunks, num_chunks,	1429	rc = cs_ioctl_default(hpriv, chunks, num_chunks,
1373	cs_seq, 0);	1430	cs_seq, 0, hdev->timeout_jiffies);
1374	}	1431	}
1375		1432
1376	mutex_unlock(&hpriv->restore_phase_mutex);	1433	mutex_unlock(&hpriv->restore_phase_mutex);
@@ -1419,7 +1476,7 @@ wait_again: · 42 unmodified lines
1419		1476
1420	out:	1477	out:
1421	if ((rc == -ETIMEDOUT \|\| rc == -EBUSY) && (need_soft_reset))	1478	if ((rc == -ETIMEDOUT \|\| rc == -EBUSY) && (need_soft_reset))
1422	hl_device_reset(hdev, false, false);	1479	hl_device_reset(hdev, 0);
1423		1480
1424	return rc;	1481	return rc;
1425	}	1482	}
@@ -1445,6 +1502,10 @@ static int cs_ioctl_extract_signal_seq(struct hl_device *hdev, · 19 unmodified lines
1445	signal_seq_arr = kmalloc_array(signal_seq_arr_len,	1502	signal_seq_arr = kmalloc_array(signal_seq_arr_len,
1446	sizeof(*signal_seq_arr),	1503	sizeof(*signal_seq_arr),
1447	GFP_ATOMIC);	1504	GFP_ATOMIC);
		1505	if (!signal_seq_arr)
		1506	signal_seq_arr = kmalloc_array(signal_seq_arr_len,
		1507	sizeof(*signal_seq_arr),
		1508	GFP_KERNEL);
1448	if (!signal_seq_arr) {	1509	if (!signal_seq_arr) {
1449	atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);	1510	atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1450	atomic64_inc(&hdev->aggregated_cs_counters.out_of_mem_drop_cnt);	1511	atomic64_inc(&hdev->aggregated_cs_counters.out_of_mem_drop_cnt);
@@ -1536,7 +1597,7 @@ static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev, · 85 unmodified lines
1536		1597
1537	static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,	1598	static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
1538	void __user *chunks, u32 num_chunks,	1599	void __user *chunks, u32 num_chunks,
1539	u64 *cs_seq, bool timestamp)	1600	u64 *cs_seq, u32 flags, u32 timeout)
1540	{	1601	{
1541	struct hl_cs_chunk cs_chunk_array, chunk;	1602	struct hl_cs_chunk cs_chunk_array, chunk;
1542	struct hw_queue_properties *hw_queue_prop;	1603	struct hw_queue_properties *hw_queue_prop;
@@ -1642,7 +1703,7 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, · 99 unmodified lines
1642	}	1703	}
1643	}	1704	}
1644		1705
1645	rc = allocate_cs(hdev, ctx, cs_type, ULLONG_MAX, &cs);	1706	rc = allocate_cs(hdev, ctx, cs_type, ULLONG_MAX, &cs, flags, timeout);
1646	if (rc) {	1707	if (rc) {
1647	if (cs_type == CS_TYPE_WAIT \|\|	1708	if (cs_type == CS_TYPE_WAIT \|\|
1648	cs_type == CS_TYPE_COLLECTIVE_WAIT)	1709	cs_type == CS_TYPE_COLLECTIVE_WAIT)
@@ -1650,8 +1711,6 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, · 1 unmodified line
1650	goto free_cs_chunk_array;	1711	goto free_cs_chunk_array;
1651	}	1712	}
1652		1713
1653	cs->timestamp = !!timestamp;
1654
1655	/*	1714	/*
1656	* Save the signal CS fence for later initialization right before	1715	* Save the signal CS fence for later initialization right before
1657	* hanging the wait CS on the queue.	1716	* hanging the wait CS on the queue.
@@ -1709,7 +1768,7 @@ int hl_cs_ioctl(struct hl_fpriv hpriv, void data) · 51 unmodified lines
1709	enum hl_cs_type cs_type;	1768	enum hl_cs_type cs_type;
1710	u64 cs_seq = ULONG_MAX;	1769	u64 cs_seq = ULONG_MAX;
1711	void __user *chunks;	1770	void __user *chunks;
1712	u32 num_chunks, flags;	1771	u32 num_chunks, flags, timeout;
1713	int rc;	1772	int rc;
1714		1773
1715	rc = hl_cs_sanity_checks(hpriv, args);	1774	rc = hl_cs_sanity_checks(hpriv, args);
@@ -1735,16 +1794,20 @@ int hl_cs_ioctl(struct hl_fpriv hpriv, void data) · 19 unmodified lines
1735	!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST))	1794	!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST))
1736	cs_seq = args->in.seq;	1795	cs_seq = args->in.seq;
1737		1796
		1797	timeout = flags & HL_CS_FLAGS_CUSTOM_TIMEOUT
		1798	? msecs_to_jiffies(args->in.timeout * 1000)
		1799	: hpriv->hdev->timeout_jiffies;
		1800
1738	switch (cs_type) {	1801	switch (cs_type) {
1739	case CS_TYPE_SIGNAL:	1802	case CS_TYPE_SIGNAL:
1740	case CS_TYPE_WAIT:	1803	case CS_TYPE_WAIT:
1741	case CS_TYPE_COLLECTIVE_WAIT:	1804	case CS_TYPE_COLLECTIVE_WAIT:
1742	rc = cs_ioctl_signal_wait(hpriv, cs_type, chunks, num_chunks,	1805	rc = cs_ioctl_signal_wait(hpriv, cs_type, chunks, num_chunks,
1743	&cs_seq, args->in.cs_flags & HL_CS_FLAGS_TIMESTAMP);	1806	&cs_seq, args->in.cs_flags, timeout);
1744	break;	1807	break;
1745	default:	1808	default:
1746	rc = cs_ioctl_default(hpriv, chunks, num_chunks, &cs_seq,	1809	rc = cs_ioctl_default(hpriv, chunks, num_chunks, &cs_seq,
1747	args->in.cs_flags);	1810	args->in.cs_flags, timeout);
1748	break;	1811	break;
1749	}	1812	}
1750		1813
@@ -1818,7 +1881,7 @@ static int _hl_cs_wait_ioctl(struct hl_device hdev, struct hl_ctx ctx, · 67 unmodified lines
1818	return rc;	1881	return rc;
1819	}	1882	}
1820		1883
1821	int hl_cs_wait_ioctl(struct hl_fpriv hpriv, void data)	1884	static int hl_cs_wait_ioctl(struct hl_fpriv hpriv, void data)
1822	{	1885	{
1823	struct hl_device *hdev = hpriv->hdev;	1886	struct hl_device *hdev = hpriv->hdev;
1824	union hl_wait_cs_args *args = data;	1887	union hl_wait_cs_args *args = data;
@@ -1873,3 +1936,176 @@ int hl_cs_wait_ioctl(struct hl_fpriv hpriv, void data) · 48 unmodified lines
1873		1936
1874	return 0;	1937	return 0;
1875	}	1938	}
		1939
		1940	static int _hl_interrupt_wait_ioctl(struct hl_device hdev, struct hl_ctx ctx,
		1941	u32 timeout_us, u64 user_address,
		1942	u32 target_value, u16 interrupt_offset,
		1943	enum hl_cs_wait_status *status)
		1944	{
		1945	struct hl_user_pending_interrupt *pend;
		1946	struct hl_user_interrupt *interrupt;
		1947	unsigned long timeout;
		1948	long completion_rc;
		1949	u32 completion_value;
		1950	int rc = 0;
		1951
		1952	if (timeout_us == U32_MAX)
		1953	timeout = timeout_us;
		1954	else
		1955	timeout = usecs_to_jiffies(timeout_us);
		1956
		1957	hl_ctx_get(hdev, ctx);
		1958
		1959	pend = kmalloc(sizeof(*pend), GFP_KERNEL);
		1960	if (!pend) {
		1961	hl_ctx_put(ctx);
		1962	return -ENOMEM;
		1963	}
		1964
		1965	hl_fence_init(&pend->fence, ULONG_MAX);
		1966
		1967	if (interrupt_offset == HL_COMMON_USER_INTERRUPT_ID)
		1968	interrupt = &hdev->common_user_interrupt;
		1969	else
		1970	interrupt = &hdev->user_interrupt[interrupt_offset];
		1971
		1972	spin_lock(&interrupt->wait_list_lock);
		1973	if (!hl_device_operational(hdev, NULL)) {
		1974	rc = -EPERM;
		1975	goto unlock_and_free_fence;
		1976	}
		1977
		1978	if (copy_from_user(&completion_value, u64_to_user_ptr(user_address), 4)) {
		1979	dev_err(hdev->dev,
		1980	"Failed to copy completion value from user\n");
		1981	rc = -EFAULT;
		1982	goto unlock_and_free_fence;
		1983	}
		1984
		1985	if (completion_value >= target_value)
		1986	*status = CS_WAIT_STATUS_COMPLETED;
		1987	else
		1988	*status = CS_WAIT_STATUS_BUSY;
		1989
		1990	if (!timeout_us \|\| (*status == CS_WAIT_STATUS_COMPLETED))
		1991	goto unlock_and_free_fence;
		1992
		1993	/* Add pending user interrupt to relevant list for the interrupt
		1994	* handler to monitor
		1995	*/
		1996	list_add_tail(&pend->wait_list_node, &interrupt->wait_list_head);
		1997	spin_unlock(&interrupt->wait_list_lock);
		1998
		1999	wait_again:
		2000	/* Wait for interrupt handler to signal completion */
		2001	completion_rc =
		2002	wait_for_completion_interruptible_timeout(
		2003	&pend->fence.completion, timeout);
		2004
		2005	/* If timeout did not expire we need to perform the comparison.
		2006	* If comparison fails, keep waiting until timeout expires
		2007	*/
		2008	if (completion_rc > 0) {
		2009	if (copy_from_user(&completion_value,
		2010	u64_to_user_ptr(user_address), 4)) {
		2011	dev_err(hdev->dev,
		2012	"Failed to copy completion value from user\n");
		2013	rc = -EFAULT;
		2014	goto remove_pending_user_interrupt;
		2015	}
		2016
		2017	if (completion_value >= target_value) {
		2018	*status = CS_WAIT_STATUS_COMPLETED;
		2019	} else {
		2020	timeout -= jiffies_to_usecs(completion_rc);
		2021	goto wait_again;
		2022	}
		2023	} else {
		2024	*status = CS_WAIT_STATUS_BUSY;
		2025	}
		2026
		2027	remove_pending_user_interrupt:
		2028	spin_lock(&interrupt->wait_list_lock);
		2029	list_del(&pend->wait_list_node);
		2030
		2031	unlock_and_free_fence:
		2032	spin_unlock(&interrupt->wait_list_lock);
		2033	kfree(pend);
		2034	hl_ctx_put(ctx);
		2035
		2036	return rc;
		2037	}
		2038
		2039	static int hl_interrupt_wait_ioctl(struct hl_fpriv hpriv, void data)
		2040	{
		2041	u16 interrupt_id, interrupt_offset, first_interrupt, last_interrupt;
		2042	struct hl_device *hdev = hpriv->hdev;
		2043	struct asic_fixed_properties *prop;
		2044	union hl_wait_cs_args *args = data;
		2045	enum hl_cs_wait_status status;
		2046	int rc;
		2047
		2048	prop = &hdev->asic_prop;
		2049
		2050	if (!prop->user_interrupt_count) {
		2051	dev_err(hdev->dev, "no user interrupts allowed");
		2052	return -EPERM;
		2053	}
		2054
		2055	interrupt_id =
		2056	FIELD_GET(HL_WAIT_CS_FLAGS_INTERRUPT_MASK, args->in.flags);
		2057
		2058	first_interrupt = prop->first_available_user_msix_interrupt;
		2059	last_interrupt = prop->first_available_user_msix_interrupt +
		2060	prop->user_interrupt_count - 1;
		2061
		2062	if ((interrupt_id < first_interrupt \|\| interrupt_id > last_interrupt) &&
		2063	interrupt_id != HL_COMMON_USER_INTERRUPT_ID) {
		2064	dev_err(hdev->dev, "invalid user interrupt %u", interrupt_id);
		2065	return -EINVAL;
		2066	}
		2067
		2068	if (interrupt_id == HL_COMMON_USER_INTERRUPT_ID)
		2069	interrupt_offset = HL_COMMON_USER_INTERRUPT_ID;
		2070	else
		2071	interrupt_offset = interrupt_id - first_interrupt;
		2072
		2073	rc = _hl_interrupt_wait_ioctl(hdev, hpriv->ctx,
		2074	args->in.interrupt_timeout_us, args->in.addr,
		2075	args->in.target, interrupt_offset, &status);
		2076
		2077	memset(args, 0, sizeof(*args));
		2078
		2079	if (rc) {
		2080	dev_err_ratelimited(hdev->dev,
		2081	"interrupt_wait_ioctl failed (%d)\n", rc);
		2082
		2083	return rc;
		2084	}
		2085
		2086	switch (status) {
		2087	case CS_WAIT_STATUS_COMPLETED:
		2088	args->out.status = HL_WAIT_CS_STATUS_COMPLETED;
		2089	break;
		2090	case CS_WAIT_STATUS_BUSY:
		2091	default:
		2092	args->out.status = HL_WAIT_CS_STATUS_BUSY;
		2093	break;
		2094	}
		2095
		2096	return 0;
		2097	}
		2098
		2099	int hl_wait_ioctl(struct hl_fpriv hpriv, void data)
		2100	{
		2101	union hl_wait_cs_args *args = data;
		2102	u32 flags = args->in.flags;
		2103	int rc;
		2104
		2105	if (flags & HL_WAIT_CS_FLAGS_INTERRUPT)
		2106	rc = hl_interrupt_wait_ioctl(hpriv, data);
		2107	else
		2108	rc = hl_cs_wait_ioctl(hpriv, data);
		2109
		2110	return rc;
		2111	}

drivers/misc/habanalabs/common/context.c+11−3

 	 */
 	hl_pending_cb_list_flush(ctx);
+	/* Release all allocated HW block mapped list entries and destroy
+	 * the mutex.
+	 */
+	hl_hw_block_mem_fini(ctx);
 	/*
 	 * If we arrived here, there are no jobs waiting for this context
 	 * on its queues so we can safely remove it.
 	if (!ctx->cs_pending)
 		return -ENOMEM;
+	hl_hw_block_mem_init(ctx);
 	if (is_kernel_ctx) {
 		ctx->asid = HL_KERNEL_ASID_ID; /* Kernel driver gets ASID 0 */
 		rc = hl_vm_ctx_init(ctx);
 		if (rc) {
 			dev_err(hdev->dev, "Failed to init mem ctx module\n");
 			rc = -ENOMEM;
-			goto err_free_cs_pending;
+			goto err_hw_block_mem_fini;
 		}
 		rc = hdev->asic_funcs->ctx_init(ctx);
 		if (!ctx->asid) {
 			dev_err(hdev->dev, "No free ASID, failed to create context\n");
 			rc = -ENOMEM;
-			goto err_free_cs_pending;
+			goto err_hw_block_mem_fini;
 		}
 		rc = hl_vm_ctx_init(ctx);
 err_asid_free:
 	if (ctx->asid != HL_KERNEL_ASID_ID)
 		hl_asid_free(hdev, ctx->asid);
-err_free_cs_pending:
+err_hw_block_mem_fini:
+	hl_hw_block_mem_fini(ctx);
 	kfree(ctx->cs_pending);
 	return rc;

drivers/misc/habanalabs/common/debugfs.c+201−23

@@ -9,8 +9,8 @@ · 8 unmodified lines
9	#include "../include/hw_ip/mmu/mmu_general.h"	9	#include "../include/hw_ip/mmu/mmu_general.h"
10		10
11	#include <linux/pci.h>	11	#include <linux/pci.h>
12	#include <linux/debugfs.h>
13	#include <linux/uaccess.h>	12	#include <linux/uaccess.h>
		13	#include <linux/vmalloc.h>
14		14
15	#define MMU_ADDR_BUF_SIZE 40	15	#define MMU_ADDR_BUF_SIZE 40
16	#define MMU_ASID_BUF_SIZE 10	16	#define MMU_ASID_BUF_SIZE 10
@@ -229,6 +229,7 @@ static int vm_show(struct seq_file s, void data) · 212 unmodified lines
229	{	229	{
230	struct hl_debugfs_entry *entry = s->private;	230	struct hl_debugfs_entry *entry = s->private;
231	struct hl_dbg_device_entry *dev_entry = entry->dev_entry;	231	struct hl_dbg_device_entry *dev_entry = entry->dev_entry;
		232	struct hl_vm_hw_block_list_node *lnode;
232	struct hl_ctx *ctx;	233	struct hl_ctx *ctx;
233	struct hl_vm *vm;	234	struct hl_vm *vm;
234	struct hl_vm_hash_node *hnode;	235	struct hl_vm_hash_node *hnode;
@@ -272,6 +273,21 @@ static int vm_show(struct seq_file s, void data) · 37 unmodified lines
272	}	273	}
273	mutex_unlock(&ctx->mem_hash_lock);	274	mutex_unlock(&ctx->mem_hash_lock);
274		275
		276	if (ctx->asid != HL_KERNEL_ASID_ID &&
		277	!list_empty(&ctx->hw_block_mem_list)) {
		278	seq_puts(s, "\nhw_block mappings:\n\n");
		279	seq_puts(s, " virtual address size HW block id\n");
		280	seq_puts(s, "-------------------------------------------\n");
		281	mutex_lock(&ctx->hw_block_list_lock);
		282	list_for_each_entry(lnode, &ctx->hw_block_mem_list,
		283	node) {
		284	seq_printf(s,
		285	" 0x%-14lx %-6u %-9u\n",
		286	lnode->vaddr, lnode->size, lnode->id);
		287	}
		288	mutex_unlock(&ctx->hw_block_list_lock);
		289	}
		290
275	vm = &ctx->hdev->vm;	291	vm = &ctx->hdev->vm;
276	spin_lock(&vm->idr_lock);	292	spin_lock(&vm->idr_lock);
277		293
@@ -441,21 +457,86 @@ out: · 163 unmodified lines
441	return false;	457	return false;
442	}	458	}
443		459
444	static int device_va_to_pa(struct hl_device *hdev, u64 virt_addr,	460	static bool hl_is_device_internal_memory_va(struct hl_device *hdev, u64 addr,
445	u64 *phys_addr)	461	u32 size)
446	{	462	{
		463	struct asic_fixed_properties *prop = &hdev->asic_prop;
		464	u64 dram_start_addr, dram_end_addr;
		465
		466	if (!hdev->mmu_enable)
		467	return false;
		468
		469	if (prop->dram_supports_virtual_memory) {
		470	dram_start_addr = prop->dmmu.start_addr;
		471	dram_end_addr = prop->dmmu.end_addr;
		472	} else {
		473	dram_start_addr = prop->dram_base_address;
		474	dram_end_addr = prop->dram_end_address;
		475	}
		476
		477	if (hl_mem_area_inside_range(addr, size, dram_start_addr,
		478	dram_end_addr))
		479	return true;
		480
		481	if (hl_mem_area_inside_range(addr, size, prop->sram_base_address,
		482	prop->sram_end_address))
		483	return true;
		484
		485	return false;
		486	}
		487
		488	static int device_va_to_pa(struct hl_device *hdev, u64 virt_addr, u32 size,
		489	u64 *phys_addr)
		490	{
		491	struct hl_vm_phys_pg_pack *phys_pg_pack;
447	struct hl_ctx *ctx = hdev->compute_ctx;	492	struct hl_ctx *ctx = hdev->compute_ctx;
448	int rc = 0;	493	struct hl_vm_hash_node *hnode;
		494	struct hl_userptr *userptr;
		495	enum vm_type_t *vm_type;
		496	bool valid = false;
		497	u64 end_address;
		498	u32 range_size;
		499	int i, rc = 0;
449		500
450	if (!ctx) {	501	if (!ctx) {
451	dev_err(hdev->dev, "no ctx available\n");	502	dev_err(hdev->dev, "no ctx available\n");
452	return -EINVAL;	503	return -EINVAL;
453	}	504	}
454		505
		506	/* Verify address is mapped */
		507	mutex_lock(&ctx->mem_hash_lock);
		508	hash_for_each(ctx->mem_hash, i, hnode, node) {
		509	vm_type = hnode->ptr;
		510
		511	if (*vm_type == VM_TYPE_USERPTR) {
		512	userptr = hnode->ptr;
		513	range_size = userptr->size;
		514	} else {
		515	phys_pg_pack = hnode->ptr;
		516	range_size = phys_pg_pack->total_size;
		517	}
		518
		519	end_address = virt_addr + size;
		520	if ((virt_addr >= hnode->vaddr) &&
		521	(end_address <= hnode->vaddr + range_size)) {
		522	valid = true;
		523	break;
		524	}
		525	}
		526	mutex_unlock(&ctx->mem_hash_lock);
		527
		528	if (!valid) {
		529	dev_err(hdev->dev,
		530	"virt addr 0x%llx is not mapped\n",
		531	virt_addr);
		532	return -EINVAL;
		533	}
		534
455	rc = hl_mmu_va_to_pa(ctx, virt_addr, phys_addr);	535	rc = hl_mmu_va_to_pa(ctx, virt_addr, phys_addr);
456	if (rc) {	536	if (rc) {
457	dev_err(hdev->dev, "virt addr 0x%llx is not mapped to phys addr\n",	537	dev_err(hdev->dev,
458	virt_addr);	538	"virt addr 0x%llx is not mapped to phys addr\n",
		539	virt_addr);
459	rc = -EINVAL;	540	rc = -EINVAL;
460	}	541	}
461		542
@@ -467,10 +548,11 @@ static ssize_t hl_data_read32(struct file f, char __user buf, · 5 unmodified lines
467	{	548	{
468	struct hl_dbg_device_entry *entry = file_inode(f)->i_private;	549	struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
469	struct hl_device *hdev = entry->hdev;	550	struct hl_device *hdev = entry->hdev;
470	char tmp_buf[32];
471	u64 addr = entry->addr;	551	u64 addr = entry->addr;
472	u32 val;	552	bool user_address;
		553	char tmp_buf[32];
473	ssize_t rc;	554	ssize_t rc;
		555	u32 val;
474		556
475	if (atomic_read(&hdev->in_reset)) {	557	if (atomic_read(&hdev->in_reset)) {
476	dev_warn_ratelimited(hdev->dev, "Can't read during reset\n");	558	dev_warn_ratelimited(hdev->dev, "Can't read during reset\n");
@@ -480,13 +562,14 @@ static ssize_t hl_data_read32(struct file f, char __user buf, · 3 unmodified lines
480	if (*ppos)	562	if (*ppos)
481	return 0;	563	return 0;
482		564
483	if (hl_is_device_va(hdev, addr)) {	565	user_address = hl_is_device_va(hdev, addr);
484	rc = device_va_to_pa(hdev, addr, &addr);	566	if (user_address) {
		567	rc = device_va_to_pa(hdev, addr, sizeof(val), &addr);
485	if (rc)	568	if (rc)
486	return rc;	569	return rc;
487	}	570	}
488		571
489	rc = hdev->asic_funcs->debugfs_read32(hdev, addr, &val);	572	rc = hdev->asic_funcs->debugfs_read32(hdev, addr, user_address, &val);
490	if (rc) {	573	if (rc) {
491	dev_err(hdev->dev, "Failed to read from 0x%010llx\n", addr);	574	dev_err(hdev->dev, "Failed to read from 0x%010llx\n", addr);
492	return rc;	575	return rc;
@@ -503,6 +586,7 @@ static ssize_t hl_data_write32(struct file f, const char __user buf, · 10 unmodified lines
503	struct hl_dbg_device_entry *entry = file_inode(f)->i_private;	586	struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
504	struct hl_device *hdev = entry->hdev;	587	struct hl_device *hdev = entry->hdev;
505	u64 addr = entry->addr;	588	u64 addr = entry->addr;
		589	bool user_address;
506	u32 value;	590	u32 value;
507	ssize_t rc;	591	ssize_t rc;
508		592
@@ -515,13 +599,14 @@ static ssize_t hl_data_write32(struct file f, const char __user buf, · 6 unmodified lines
515	if (rc)	599	if (rc)
516	return rc;	600	return rc;
517		601
518	if (hl_is_device_va(hdev, addr)) {	602	user_address = hl_is_device_va(hdev, addr);
519	rc = device_va_to_pa(hdev, addr, &addr);	603	if (user_address) {
		604	rc = device_va_to_pa(hdev, addr, sizeof(value), &addr);
520	if (rc)	605	if (rc)
521	return rc;	606	return rc;
522	}	607	}
523		608
524	rc = hdev->asic_funcs->debugfs_write32(hdev, addr, value);	609	rc = hdev->asic_funcs->debugfs_write32(hdev, addr, user_address, value);
525	if (rc) {	610	if (rc) {
526	dev_err(hdev->dev, "Failed to write 0x%08x to 0x%010llx\n",	611	dev_err(hdev->dev, "Failed to write 0x%08x to 0x%010llx\n",
527	value, addr);	612	value, addr);
@@ -536,21 +621,28 @@ static ssize_t hl_data_read64(struct file f, char __user buf, · 8 unmodified lines
536	{	621	{
537	struct hl_dbg_device_entry *entry = file_inode(f)->i_private;	622	struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
538	struct hl_device *hdev = entry->hdev;	623	struct hl_device *hdev = entry->hdev;
539	char tmp_buf[32];
540	u64 addr = entry->addr;	624	u64 addr = entry->addr;
541	u64 val;	625	bool user_address;
		626	char tmp_buf[32];
542	ssize_t rc;	627	ssize_t rc;
		628	u64 val;
		629
		630	if (atomic_read(&hdev->in_reset)) {
		631	dev_warn_ratelimited(hdev->dev, "Can't read during reset\n");
		632	return 0;
		633	}
543		634
544	if (*ppos)	635	if (*ppos)
545	return 0;	636	return 0;
546		637
547	if (hl_is_device_va(hdev, addr)) {	638	user_address = hl_is_device_va(hdev, addr);
548	rc = device_va_to_pa(hdev, addr, &addr);	639	if (user_address) {
		640	rc = device_va_to_pa(hdev, addr, sizeof(val), &addr);
549	if (rc)	641	if (rc)
550	return rc;	642	return rc;
551	}	643	}
552		644
553	rc = hdev->asic_funcs->debugfs_read64(hdev, addr, &val);	645	rc = hdev->asic_funcs->debugfs_read64(hdev, addr, user_address, &val);
554	if (rc) {	646	if (rc) {
555	dev_err(hdev->dev, "Failed to read from 0x%010llx\n", addr);	647	dev_err(hdev->dev, "Failed to read from 0x%010llx\n", addr);
556	return rc;	648	return rc;
@@ -567,20 +659,27 @@ static ssize_t hl_data_write64(struct file f, const char __user buf, · 10 unmodified lines
567	struct hl_dbg_device_entry *entry = file_inode(f)->i_private;	659	struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
568	struct hl_device *hdev = entry->hdev;	660	struct hl_device *hdev = entry->hdev;
569	u64 addr = entry->addr;	661	u64 addr = entry->addr;
		662	bool user_address;
570	u64 value;	663	u64 value;
571	ssize_t rc;	664	ssize_t rc;
572		665
		666	if (atomic_read(&hdev->in_reset)) {
		667	dev_warn_ratelimited(hdev->dev, "Can't write during reset\n");
		668	return 0;
		669	}
		670
573	rc = kstrtoull_from_user(buf, count, 16, &value);	671	rc = kstrtoull_from_user(buf, count, 16, &value);
574	if (rc)	672	if (rc)
575	return rc;	673	return rc;
576		674
577	if (hl_is_device_va(hdev, addr)) {	675	user_address = hl_is_device_va(hdev, addr);
578	rc = device_va_to_pa(hdev, addr, &addr);	676	if (user_address) {
		677	rc = device_va_to_pa(hdev, addr, sizeof(value), &addr);
579	if (rc)	678	if (rc)
580	return rc;	679	return rc;
581	}	680	}
582		681
583	rc = hdev->asic_funcs->debugfs_write64(hdev, addr, value);	682	rc = hdev->asic_funcs->debugfs_write64(hdev, addr, user_address, value);
584	if (rc) {	683	if (rc) {
585	dev_err(hdev->dev, "Failed to write 0x%016llx to 0x%010llx\n",	684	dev_err(hdev->dev, "Failed to write 0x%016llx to 0x%010llx\n",
586	value, addr);	685	value, addr);
@@ -590,6 +689,63 @@ static ssize_t hl_data_write64(struct file f, const char __user buf, · 3 unmodified lines
590	return count;	689	return count;
591	}	690	}
592		691
		692	static ssize_t hl_dma_size_write(struct file f, const char __user buf,
		693	size_t count, loff_t *ppos)
		694	{
		695	struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
		696	struct hl_device *hdev = entry->hdev;
		697	u64 addr = entry->addr;
		698	ssize_t rc;
		699	u32 size;
		700
		701	if (atomic_read(&hdev->in_reset)) {
		702	dev_warn_ratelimited(hdev->dev, "Can't DMA during reset\n");
		703	return 0;
		704	}
		705	rc = kstrtouint_from_user(buf, count, 16, &size);
		706	if (rc)
		707	return rc;
		708
		709	if (!size) {
		710	dev_err(hdev->dev, "DMA read failed. size can't be 0\n");
		711	return -EINVAL;
		712	}
		713
		714	if (size > SZ_128M) {
		715	dev_err(hdev->dev,
		716	"DMA read failed. size can't be larger than 128MB\n");
		717	return -EINVAL;
		718	}
		719
		720	if (!hl_is_device_internal_memory_va(hdev, addr, size)) {
		721	dev_err(hdev->dev,
		722	"DMA read failed. Invalid 0x%010llx + 0x%08x\n",
		723	addr, size);
		724	return -EINVAL;
		725	}
		726
		727	/* Free the previous allocation, if there was any */
		728	entry->blob_desc.size = 0;
		729	vfree(entry->blob_desc.data);
		730
		731	entry->blob_desc.data = vmalloc(size);
		732	if (!entry->blob_desc.data)
		733	return -ENOMEM;
		734
		735	rc = hdev->asic_funcs->debugfs_read_dma(hdev, addr, size,
		736	entry->blob_desc.data);
		737	if (rc) {
		738	dev_err(hdev->dev, "Failed to DMA from 0x%010llx\n", addr);
		739	vfree(entry->blob_desc.data);
		740	entry->blob_desc.data = NULL;
		741	return -EIO;
		742	}
		743
		744	entry->blob_desc.size = size;
		745
		746	return count;
		747	}
		748
593	static ssize_t hl_get_power_state(struct file f, char __user buf,	749	static ssize_t hl_get_power_state(struct file f, char __user buf,
594	size_t count, loff_t *ppos)	750	size_t count, loff_t *ppos)
595	{	751	{
@@ -871,7 +1027,7 @@ static ssize_t hl_stop_on_err_write(struct file f, const char __user buf, · 275 unmodified lines
871		1027
872	hdev->stop_on_err = value ? 1 : 0;	1028	hdev->stop_on_err = value ? 1 : 0;
873		1029
874	hl_device_reset(hdev, false, false);	1030	hl_device_reset(hdev, 0);
875		1031
876	return count;	1032	return count;
877	}	1033	}
@@ -899,6 +1055,11 @@ static const struct file_operations hl_data64b_fops = { · 21 unmodified lines
899	.write = hl_data_write64	1055	.write = hl_data_write64
900	};	1056	};
901		1057
		1058	static const struct file_operations hl_dma_size_fops = {
		1059	.owner = THIS_MODULE,
		1060	.write = hl_dma_size_write
		1061	};
		1062
902	static const struct file_operations hl_i2c_data_fops = {	1063	static const struct file_operations hl_i2c_data_fops = {
903	.owner = THIS_MODULE,	1064	.owner = THIS_MODULE,
904	.read = hl_i2c_data_read,	1065	.read = hl_i2c_data_read,
@@ -1001,6 +1162,9 @@ void hl_debugfs_add_device(struct hl_device *hdev) · 96 unmodified lines
1001	if (!dev_entry->entry_arr)	1162	if (!dev_entry->entry_arr)
1002	return;	1163	return;
1003		1164
		1165	dev_entry->blob_desc.size = 0;
		1166	dev_entry->blob_desc.data = NULL;
		1167
1004	INIT_LIST_HEAD(&dev_entry->file_list);	1168	INIT_LIST_HEAD(&dev_entry->file_list);
1005	INIT_LIST_HEAD(&dev_entry->cb_list);	1169	INIT_LIST_HEAD(&dev_entry->cb_list);
1006	INIT_LIST_HEAD(&dev_entry->cs_list);	1170	INIT_LIST_HEAD(&dev_entry->cs_list);
@@ -1103,6 +1267,17 @@ void hl_debugfs_add_device(struct hl_device *hdev) · 96 unmodified lines
1103	dev_entry,	1267	dev_entry,
1104	&hl_security_violations_fops);	1268	&hl_security_violations_fops);
1105		1269
		1270	debugfs_create_file("dma_size",
		1271	0200,
		1272	dev_entry->root,
		1273	dev_entry,
		1274	&hl_dma_size_fops);
		1275
		1276	debugfs_create_blob("data_dma",
		1277	0400,
		1278	dev_entry->root,
		1279	&dev_entry->blob_desc);
		1280
1106	for (i = 0, entry = dev_entry->entry_arr ; i < count ; i++, entry++) {	1281	for (i = 0, entry = dev_entry->entry_arr ; i < count ; i++, entry++) {
1107	debugfs_create_file(hl_debugfs_list[i].name,	1282	debugfs_create_file(hl_debugfs_list[i].name,
1108	0444,	1283	0444,
@@ -1121,6 +1296,9 @@ void hl_debugfs_remove_device(struct hl_device *hdev) · 12 unmodified lines
1121	debugfs_remove_recursive(entry->root);	1296	debugfs_remove_recursive(entry->root);
1122		1297
1123	mutex_destroy(&entry->file_mutex);	1298	mutex_destroy(&entry->file_mutex);
		1299
		1300	vfree(entry->blob_desc.data);
		1301
1124	kfree(entry->entry_arr);	1302	kfree(entry->entry_arr);
1125	}	1303	}
1126		1304

drivers/misc/habanalabs/common/device.c+94−127

 	mutex_unlock(&hdev->fpriv_list_lock);
 	kfree(hpriv);
+	if (hdev->reset_upon_device_release)
+		hl_device_reset(hdev, 0);
 }
 void hl_hpriv_get(struct hl_fpriv *hpriv)
 	kref_get(&hpriv->refcount);
 }
-void hl_hpriv_put(struct hl_fpriv *hpriv)
+int hl_hpriv_put(struct hl_fpriv *hpriv)
 {
-	kref_put(&hpriv->refcount, hpriv_release);
+	return kref_put(&hpriv->refcount, hpriv_release);
 }
 /*
 		return 0;
 	}
-	hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
+	/* Each pending user interrupt holds the user's context, hence we
-	hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
+	 * must release them all before calling hl_ctx_mgr_fini().
+	 */
+	hl_release_pending_user_interrupts(hpriv->hdev);
+	hl_cb_mgr_fini(hdev, &hpriv->cb_mgr);
+	hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);
-	hl_hpriv_put(hpriv);
+	if (!hl_hpriv_put(hpriv))
+		dev_warn(hdev->dev,
+			"Device is still in use because there are live CS and/or memory mappings\n");
 	return 0;
 }
 	struct hl_device *hdev = device_reset_work->hdev;
 	int rc;
-	rc = hl_device_reset(hdev, true, true);
+	rc = hl_device_reset(hdev, HL_RESET_HARD | HL_RESET_FROM_RESET_THREAD);
 	if ((rc == -EBUSY) && !hdev->device_fini_pending) {
 		dev_info(hdev->dev,
 			"Could not reset device. will try again in %u seconds",
 	switch (hdev->asic_type) {
 	case ASIC_GOYA:
 		goya_set_asic_funcs(hdev);
-		strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
+		strscpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
 		break;
 	case ASIC_GAUDI:
 		gaudi_set_asic_funcs(hdev);
-		sprintf(hdev->asic_name, "GAUDI");
+		strscpy(hdev->asic_name, "GAUDI", sizeof(hdev->asic_name));
+		break;
+	case ASIC_GAUDI_SEC:
+		gaudi_set_asic_funcs(hdev);
+		strscpy(hdev->asic_name, "GAUDI SEC", sizeof(hdev->asic_name));
 		break;
 	default:
 		dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
 	if (hdev->asic_prop.completion_queues_count) {
 		hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count,
 				sizeof(*hdev->cq_wq),
-				GFP_ATOMIC);
+				GFP_KERNEL);
 		if (!hdev->cq_wq) {
 			rc = -ENOMEM;
 			goto asid_fini;
 		goto free_cq_wq;
 	}
-	hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
+	hdev->sob_reset_wq = alloc_workqueue("hl-sob-reset", WQ_UNBOUND, 0);
-					GFP_KERNEL);
+	if (!hdev->sob_reset_wq) {
-	if (!hdev->hl_chip_info) {
+		dev_err(hdev->dev,
+			"Failed to allocate SOB reset workqueue\n");
 		rc = -ENOMEM;
 		goto free_eq_wq;
 	}
-	hdev->idle_busy_ts_arr = kmalloc_array(HL_IDLE_BUSY_TS_ARR_SIZE,
+	hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
-					sizeof(struct hl_device_idle_busy_ts),
+					GFP_KERNEL);
-					(GFP_KERNEL | __GFP_ZERO));
+	if (!hdev->hl_chip_info) {
-	if (!hdev->idle_busy_ts_arr) {
 		rc = -ENOMEM;
-		goto free_chip_info;
+		goto free_sob_reset_wq;
 	}
 	rc = hl_mmu_if_set_funcs(hdev);
 	if (rc)
-		goto free_idle_busy_ts_arr;
+		goto free_chip_info;
 	hl_cb_mgr_init(&hdev->kernel_cb_mgr);
 free_cb_mgr:
 	hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
-free_idle_busy_ts_arr:
-	kfree(hdev->idle_busy_ts_arr);
 free_chip_info:
 	kfree(hdev->hl_chip_info);
+free_sob_reset_wq:
+	destroy_workqueue(hdev->sob_reset_wq);
 free_eq_wq:
 	destroy_workqueue(hdev->eq_wq);
 free_cq_wq:
 	hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
-	kfree(hdev->idle_busy_ts_arr);
 	kfree(hdev->hl_chip_info);
+	destroy_workqueue(hdev->sob_reset_wq);
 	destroy_workqueue(hdev->eq_wq);
 	destroy_workqueue(hdev->device_reset_work.wq);
 		goto reschedule;
 	dev_err(hdev->dev, "Device heartbeat failed!\n");
-	hl_device_reset(hdev, true, false);
+	hl_device_reset(hdev, HL_RESET_HARD | HL_RESET_HEARTBEAT);
 	return;
 	hdev->late_init_done = false;
 }
-uint32_t hl_device_utilization(struct hl_device *hdev, uint32_t period_ms)
+int hl_device_utilization(struct hl_device *hdev, u32 *utilization)
 {
-	struct hl_device_idle_busy_ts *ts;
+	u64 max_power, curr_power, dc_power, dividend;
-	ktime_t zero_ktime, curr = ktime_get();
+	int rc;
-	u32 overlap_cnt = 0, last_index = hdev->idle_busy_ts_idx;
-	s64 period_us, last_start_us, last_end_us, last_busy_time_us,
-		total_busy_time_us = 0, total_busy_time_ms;
-	zero_ktime = ktime_set(0, 0);
-	period_us = period_ms * USEC_PER_MSEC;
-	ts = &hdev->idle_busy_ts_arr[last_index];
-	/* check case that device is currently in idle */
-	if (!ktime_compare(ts->busy_to_idle_ts, zero_ktime) &&
-			!ktime_compare(ts->idle_to_busy_ts, zero_ktime)) {
-		last_index--;
-		/* Handle case idle_busy_ts_idx was 0 */
-		if (last_index > HL_IDLE_BUSY_TS_ARR_SIZE)
-			last_index = HL_IDLE_BUSY_TS_ARR_SIZE - 1;
-		ts = &hdev->idle_busy_ts_arr[last_index];
-	}
-	while (overlap_cnt < HL_IDLE_BUSY_TS_ARR_SIZE) {
-		/* Check if we are in last sample case. i.e. if the sample
-		 * begun before the sampling period. This could be a real
-		 * sample or 0 so need to handle both cases
-		 */
-		last_start_us = ktime_to_us(
-				ktime_sub(curr, ts->idle_to_busy_ts));
-		if (last_start_us > period_us) {
-			/* First check two cases:
-			 * 1. If the device is currently busy
-			 * 2. If the device was idle during the whole sampling
-			 *    period
-			 */
-			if (!ktime_compare(ts->busy_to_idle_ts, zero_ktime)) {
-				/* Check if the device is currently busy */
-				if (ktime_compare(ts->idle_to_busy_ts,
-						zero_ktime))
-					return 100;
-				/* We either didn't have any activity or we
-				 * reached an entry which is 0. Either way,
-				 * exit and return what was accumulated so far
-				 */
-				break;
-			}
-			/* If sample has finished, check it is relevant */
-			last_end_us = ktime_to_us(
-					ktime_sub(curr, ts->busy_to_idle_ts));
-			if (last_end_us > period_us)
-				break;
-			/* It is relevant so add it but with adjustment */
-			last_busy_time_us = ktime_to_us(
-						ktime_sub(ts->busy_to_idle_ts,
-						ts->idle_to_busy_ts));
-			total_busy_time_us += last_busy_time_us -
-					(last_start_us - period_us);
-			break;
-		}
-		/* Check if the sample is finished or still open */
-		if (ktime_compare(ts->busy_to_idle_ts, zero_ktime))
-			last_busy_time_us = ktime_to_us(
-						ktime_sub(ts->busy_to_idle_ts,
-						ts->idle_to_busy_ts));
-		else
-			last_busy_time_us = ktime_to_us(
-					ktime_sub(curr, ts->idle_to_busy_ts));
-		total_busy_time_us += last_busy_time_us;
-		last_index--;
+	max_power = hdev->asic_prop.max_power_default;
-		/* Handle case idle_busy_ts_idx was 0 */
+	dc_power = hdev->asic_prop.dc_power_default;
-		if (last_index > HL_IDLE_BUSY_TS_ARR_SIZE)
+	rc = hl_fw_cpucp_power_get(hdev, &curr_power);
-			last_index = HL_IDLE_BUSY_TS_ARR_SIZE - 1;
-		ts = &hdev->idle_busy_ts_arr[last_index];
+	if (rc)
+		return rc;
-		overlap_cnt++;
+	curr_power = clamp(curr_power, dc_power, max_power);
-	}
-	total_busy_time_ms = DIV_ROUND_UP_ULL(total_busy_time_us,
+	dividend = (curr_power - dc_power) * 100;
-						USEC_PER_MSEC);
+	*utilization = (u32) div_u64(dividend, (max_power - dc_power));
-	return DIV_ROUND_UP_ULL(total_busy_time_ms * 100, period_ms);
+	return 0;
 }
 /*
 	hdev->disabled = false;
 	atomic_set(&hdev->in_reset, 0);
-	rc = hl_device_reset(hdev, true, false);
+	rc = hl_device_reset(hdev, HL_RESET_HARD);
 	if (rc) {
 		dev_err(hdev->dev, "Failed to reset device during resume\n");
 		goto disable_device;
  * hl_device_reset - reset the device
  *
  * @hdev: pointer to habanalabs device structure
- * @hard_reset: should we do hard reset to all engines or just reset the
+ * @flags: reset flags.
- *              compute/dma engines
- * @from_hard_reset_thread: is the caller the hard-reset thread
  *
  * Block future CS and wait for pending CS to be enqueued
  * Call ASIC H/W fini
  *
  * Returns 0 for success or an error on failure.
  */
-int hl_device_reset(struct hl_device *hdev, bool hard_reset,
+int hl_device_reset(struct hl_device *hdev, u32 flags)
-			bool from_hard_reset_thread)
 {
+	u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
+	bool hard_reset, from_hard_reset_thread;
 	int i, rc;
 	if (!hdev->init_done) {
 		return 0;
 	}
+	hard_reset = (flags & HL_RESET_HARD) != 0;
+	from_hard_reset_thread = (flags & HL_RESET_FROM_RESET_THREAD) != 0;
 	if ((!hard_reset) && (!hdev->supports_soft_reset)) {
 		dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n");
 		hard_reset = true;
 		if (rc)
 			return 0;
-		if (hard_reset) {
+		/*
+		 * if reset is due to heartbeat, device CPU is no responsive in
+		 * which case no point sending PCI disable message to it
+		 */
+		if (hard_reset && !(flags & HL_RESET_HEARTBEAT)) {
 			/* Disable PCI access from device F/W so he won't send
 			 * us additional interrupts. We disable MSI/MSI-X at
 			 * the halt_engines function and we can't have the F/W
 	/* Go over all the queues, release all CS and their jobs */
 	hl_cs_rollback_all(hdev);
+	/* Release all pending user interrupts, each pending user interrupt
+	 * holds a reference to user context
+	 */
+	hl_release_pending_user_interrupts(hdev);
 kill_processes:
 	if (hard_reset) {
 		/* Kill processes here after CS rollback. This is because the
 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
 		hl_cq_reset(hdev, &hdev->completion_queue[i]);
-	hdev->idle_busy_ts_idx = 0;
-	hdev->idle_busy_ts_arr[0].busy_to_idle_ts = ktime_set(0, 0);
-	hdev->idle_busy_ts_arr[0].idle_to_busy_ts = ktime_set(0, 0);
-	if (hdev->cs_active_cnt)
-		dev_crit(hdev->dev, "CS active cnt %d is not 0 during reset\n",
-			hdev->cs_active_cnt);
 	mutex_lock(&hdev->fpriv_list_lock);
 	/* Make sure the context switch phase will run again */
 		goto out_err;
 	}
+	/* If device is not idle fail the reset process */
+	if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask,
+			HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL)) {
+		dev_err(hdev->dev,
+			"device is not idle (mask %#llx %#llx) after reset\n",
+			idle_mask[0], idle_mask[1]);
+		rc = -EIO;
+		goto out_err;
+	}
 	/* Check that the communication with the device is working */
 	rc = hdev->asic_funcs->test_queues(hdev);
 	if (rc) {
  */
 int hl_device_init(struct hl_device *hdev, struct class *hclass)
 {
-	int i, rc, cq_cnt, cq_ready_cnt;
+	int i, rc, cq_cnt, user_interrupt_cnt, cq_ready_cnt;
 	char *name;
 	bool add_cdev_sysfs_on_err = false;
 	if (rc)
 		goto free_dev_ctrl;
+	user_interrupt_cnt = hdev->asic_prop.user_interrupt_count;
+	if (user_interrupt_cnt) {
+		hdev->user_interrupt = kcalloc(user_interrupt_cnt,
+				sizeof(*hdev->user_interrupt),
+				GFP_KERNEL);
+		if (!hdev->user_interrupt) {
+			rc = -ENOMEM;
+			goto early_fini;
+		}
+	}
 	/*
 	 * Start calling ASIC initialization. First S/W then H/W and finally
 	 * late init
 	 */
 	rc = hdev->asic_funcs->sw_init(hdev);
 	if (rc)
-		goto early_fini;
+		goto user_interrupts_fini;
 	/*
 	 * Initialize the H/W queues. Must be done before hw_init, because
 	hl_hw_queues_destroy(hdev);
 sw_fini:
 	hdev->asic_funcs->sw_fini(hdev);
+user_interrupts_fini:
+	kfree(hdev->user_interrupt);
 early_fini:
 	device_early_fini(hdev);
 free_dev_ctrl:
 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
 		hl_cq_fini(hdev, &hdev->completion_queue[i]);
 	kfree(hdev->completion_queue);
+	kfree(hdev->user_interrupt);
 	hl_hw_queues_destroy(hdev);

drivers/misc/habanalabs/common/firmware_if.c+216−22

@@ -293,6 +293,7 @@ static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, · 292 unmodified lines
293	u32 cpu_security_boot_status_reg)	293	u32 cpu_security_boot_status_reg)
294	{	294	{
295	u32 err_val, security_val;	295	u32 err_val, security_val;
		296	bool err_exists = false;
296		297
297	/* Some of the firmware status codes are deprecated in newer f/w	298	/* Some of the firmware status codes are deprecated in newer f/w
298	* versions. In those versions, the errors are reported	299	* versions. In those versions, the errors are reported
@@ -307,48 +308,102 @@ static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, · 8 unmodified lines
307	if (!(err_val & CPU_BOOT_ERR0_ENABLED))	308	if (!(err_val & CPU_BOOT_ERR0_ENABLED))
308	return 0;	309	return 0;
309		310
310	if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL)	311	if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
311	dev_err(hdev->dev,	312	dev_err(hdev->dev,
312	"Device boot error - DRAM initialization failed\n");	313	"Device boot error - DRAM initialization failed\n");
313	if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED)	314	err_exists = true;
		315	}
		316
		317	if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) {
314	dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");	318	dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
315	if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL)	319	err_exists = true;
		320	}
		321
		322	if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) {
316	dev_err(hdev->dev,	323	dev_err(hdev->dev,
317	"Device boot error - Thermal Sensor initialization failed\n");	324	"Device boot error - Thermal Sensor initialization failed\n");
318	if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED)	325	err_exists = true;
		326	}
		327
		328	if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) {
319	dev_warn(hdev->dev,	329	dev_warn(hdev->dev,
320	"Device boot warning - Skipped DRAM initialization\n");	330	"Device boot warning - Skipped DRAM initialization\n");
		331	/* This is a warning so we don't want it to disable the
		332	* device
		333	*/
		334	err_val &= ~CPU_BOOT_ERR0_DRAM_SKIPPED;
		335	}
321		336
322	if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {	337	if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {
323	if (hdev->bmc_enable)	338	if (hdev->bmc_enable) {
324	dev_warn(hdev->dev,	339	dev_err(hdev->dev,
325	"Device boot error - Skipped waiting for BMC\n");	340	"Device boot error - Skipped waiting for BMC\n");
326	else	341	err_exists = true;
		342	} else {
		343	dev_info(hdev->dev,
		344	"Device boot message - Skipped waiting for BMC\n");
		345	/* This is an info so we don't want it to disable the
		346	* device
		347	*/
327	err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;	348	err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;
		349	}
328	}	350	}
329		351
330	if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY)	352	if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) {
331	dev_err(hdev->dev,	353	dev_err(hdev->dev,
332	"Device boot error - Serdes data from BMC not available\n");	354	"Device boot error - Serdes data from BMC not available\n");
333	if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL)	355	err_exists = true;
		356	}
		357
		358	if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) {
334	dev_err(hdev->dev,	359	dev_err(hdev->dev,
335	"Device boot error - NIC F/W initialization failed\n");	360	"Device boot error - NIC F/W initialization failed\n");
336	if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY)	361	err_exists = true;
		362	}
		363
		364	if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) {
337	dev_warn(hdev->dev,	365	dev_warn(hdev->dev,
338	"Device boot warning - security not ready\n");	366	"Device boot warning - security not ready\n");
339	if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL)	367	/* This is a warning so we don't want it to disable the
		368	* device
		369	*/
		370	err_val &= ~CPU_BOOT_ERR0_SECURITY_NOT_RDY;
		371	}
		372
		373	if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) {
340	dev_err(hdev->dev, "Device boot error - security failure\n");	374	dev_err(hdev->dev, "Device boot error - security failure\n");
341	if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL)	375	err_exists = true;
		376	}
		377
		378	if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) {
342	dev_err(hdev->dev, "Device boot error - eFuse failure\n");	379	dev_err(hdev->dev, "Device boot error - eFuse failure\n");
343	if (err_val & CPU_BOOT_ERR0_PLL_FAIL)	380	err_exists = true;
		381	}
		382
		383	if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
344	dev_err(hdev->dev, "Device boot error - PLL failure\n");	384	dev_err(hdev->dev, "Device boot error - PLL failure\n");
		385	err_exists = true;
		386	}
		387
		388	if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
		389	dev_err(hdev->dev,
		390	"Device boot error - device unusable\n");
		391	err_exists = true;
		392	}
345		393
346	security_val = RREG32(cpu_security_boot_status_reg);	394	security_val = RREG32(cpu_security_boot_status_reg);
347	if (security_val & CPU_BOOT_DEV_STS0_ENABLED)	395	if (security_val & CPU_BOOT_DEV_STS0_ENABLED)
348	dev_dbg(hdev->dev, "Device security status %#x\n",	396	dev_dbg(hdev->dev, "Device security status %#x\n",
349	security_val);	397	security_val);
350		398
351	if (err_val & ~CPU_BOOT_ERR0_ENABLED)	399	if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
		400	dev_err(hdev->dev,
		401	"Device boot error - unknown error 0x%08x\n",
		402	err_val);
		403	err_exists = true;
		404	}
		405
		406	if (err_exists)
352	return -EIO;	407	return -EIO;
353		408
354	return 0;	409	return 0;
@@ -419,6 +474,73 @@ out: · 64 unmodified lines
419	return rc;	474	return rc;
420	}	475	}
421		476
		477	static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
		478	{
		479	struct cpucp_array_data_packet *pkt;
		480	size_t total_pkt_size, data_size;
		481	u64 result;
		482	int rc;
		483
		484	/* skip sending this info for unsupported ASICs */
		485	if (!hdev->asic_funcs->get_msi_info)
		486	return 0;
		487
		488	data_size = CPUCP_NUM_OF_MSI_TYPES * sizeof(u32);
		489	total_pkt_size = sizeof(struct cpucp_array_data_packet) + data_size;
		490
		491	/* data should be aligned to 8 bytes in order to CPU-CP to copy it */
		492	total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
		493
		494	/* total_pkt_size is casted to u16 later on */
		495	if (total_pkt_size > USHRT_MAX) {
		496	dev_err(hdev->dev, "CPUCP array data is too big\n");
		497	return -EINVAL;
		498	}
		499
		500	pkt = kzalloc(total_pkt_size, GFP_KERNEL);
		501	if (!pkt)
		502	return -ENOMEM;
		503
		504	pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);
		505
		506	hdev->asic_funcs->get_msi_info((u32 *)&pkt->data);
		507
		508	pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<
		509	CPUCP_PKT_CTL_OPCODE_SHIFT);
		510
		511	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)pkt,
		512	total_pkt_size, 0, &result);
		513
		514	/*
		515	* in case packet result is invalid it means that FW does not support
		516	* this feature and will use default/hard coded MSI values. no reason
		517	* to stop the boot
		518	*/
		519	if (rc && result == cpucp_packet_invalid)
		520	rc = 0;
		521
		522	if (rc)
		523	dev_err(hdev->dev, "failed to send CPUCP array data\n");
		524
		525	kfree(pkt);
		526
		527	return rc;
		528	}
		529
		530	int hl_fw_cpucp_handshake(struct hl_device *hdev,
		531	u32 cpu_security_boot_status_reg,
		532	u32 boot_err0_reg)
		533	{
		534	int rc;
		535
		536	rc = hl_fw_cpucp_info_get(hdev, cpu_security_boot_status_reg,
		537	boot_err0_reg);
		538	if (rc)
		539	return rc;
		540
		541	return hl_fw_send_msi_info_msg(hdev);
		542	}
		543
422	int hl_fw_get_eeprom_data(struct hl_device hdev, void data, size_t max_size)	544	int hl_fw_get_eeprom_data(struct hl_device hdev, void data, size_t max_size)
423	{	545	{
424	struct cpucp_packet pkt = {};	546	struct cpucp_packet pkt = {};
@@ -539,18 +661,63 @@ int hl_fw_cpucp_total_energy_get(struct hl_device hdev, u64 total_energy) · 114 unmodified lines
539	return rc;	661	return rc;
540	}	662	}
541		663
542	int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u16 pll_index,	664	int get_used_pll_index(struct hl_device *hdev, enum pll_index input_pll_index,
		665	enum pll_index *pll_index)
		666	{
		667	struct asic_fixed_properties *prop = &hdev->asic_prop;
		668	u8 pll_byte, pll_bit_off;
		669	bool dynamic_pll;
		670
		671	if (input_pll_index >= PLL_MAX) {
		672	dev_err(hdev->dev, "PLL index %d is out of range\n",
		673	input_pll_index);
		674	return -EINVAL;
		675	}
		676
		677	dynamic_pll = prop->fw_security_status_valid &&
		678	(prop->fw_app_security_map & CPU_BOOT_DEV_STS0_DYN_PLL_EN);
		679
		680	if (!dynamic_pll) {
		681	/*
		682	* in case we are working with legacy FW (each asic has unique
		683	* PLL numbering) extract the legacy numbering
		684	*/
		685	*pll_index = hdev->legacy_pll_map[input_pll_index];
		686	return 0;
		687	}
		688
		689	/* PLL map is a u8 array */
		690	pll_byte = prop->cpucp_info.pll_map[input_pll_index >> 3];
		691	pll_bit_off = input_pll_index & 0x7;
		692
		693	if (!(pll_byte & BIT(pll_bit_off))) {
		694	dev_err(hdev->dev, "PLL index %d is not supported\n",
		695	input_pll_index);
		696	return -EINVAL;
		697	}
		698
		699	*pll_index = input_pll_index;
		700
		701	return 0;
		702	}
		703
		704	int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, enum pll_index pll_index,
543	u16 *pll_freq_arr)	705	u16 *pll_freq_arr)
544	{	706	{
545	struct cpucp_packet pkt;	707	struct cpucp_packet pkt;
		708	enum pll_index used_pll_idx;
546	u64 result;	709	u64 result;
547	int rc;	710	int rc;
548		711
		712	rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
		713	if (rc)
		714	return rc;
		715
549	memset(&pkt, 0, sizeof(pkt));	716	memset(&pkt, 0, sizeof(pkt));
550		717
551	pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<	718	pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<
552	CPUCP_PKT_CTL_OPCODE_SHIFT);	719	CPUCP_PKT_CTL_OPCODE_SHIFT);
553	pkt.pll_type = __cpu_to_le16(pll_index);	720	pkt.pll_type = __cpu_to_le16((u16)used_pll_idx);
554		721
555	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),	722	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
556	HL_CPUCP_INFO_TIMEOUT_USEC, &result);	723	HL_CPUCP_INFO_TIMEOUT_USEC, &result);
@@ -565,6 +732,29 @@ int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u16 pll_index, · 8 unmodified lines
565	return rc;	732	return rc;
566	}	733	}
567		734
		735	int hl_fw_cpucp_power_get(struct hl_device hdev, u64 power)
		736	{
		737	struct cpucp_packet pkt;
		738	u64 result;
		739	int rc;
		740
		741	memset(&pkt, 0, sizeof(pkt));
		742
		743	pkt.ctl = cpu_to_le32(CPUCP_PACKET_POWER_GET <<
		744	CPUCP_PKT_CTL_OPCODE_SHIFT);
		745
		746	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
		747	HL_CPUCP_INFO_TIMEOUT_USEC, &result);
		748	if (rc) {
		749	dev_err(hdev->dev, "Failed to read power, error %d\n", rc);
		750	return rc;
		751	}
		752
		753	*power = result;
		754
		755	return rc;
		756	}
		757
568	static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)	758	static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
569	{	759	{
570	/* Some of the status codes below are deprecated in newer f/w	760	/* Some of the status codes below are deprecated in newer f/w
@@ -623,7 +813,11 @@ int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, · 52 unmodified lines
623	u32 status, security_status;	813	u32 status, security_status;
624	int rc;	814	int rc;
625		815
626	if (!hdev->cpu_enable)	816	/* pldm was added for cases in which we use preboot on pldm and want
		817	* to load boot fit, but we can't wait for preboot because it runs
		818	* very slowly
		819	*/
		820	if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) \|\| hdev->pldm)
627	return 0;	821	return 0;
628		822
629	/* Need to check two possible scenarios:	823	/* Need to check two possible scenarios:
@@ -677,16 +871,16 @@ int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, · 47 unmodified lines
677	if (security_status & CPU_BOOT_DEV_STS0_ENABLED) {	871	if (security_status & CPU_BOOT_DEV_STS0_ENABLED) {
678	prop->fw_security_status_valid = 1;	872	prop->fw_security_status_valid = 1;
679		873
		874	/* FW security should be derived from PCI ID, we keep this
		875	* check for backward compatibility
		876	*/
680	if (security_status & CPU_BOOT_DEV_STS0_SECURITY_EN)	877	if (security_status & CPU_BOOT_DEV_STS0_SECURITY_EN)
681	prop->fw_security_disabled = false;	878	prop->fw_security_disabled = false;
682	else
683	prop->fw_security_disabled = true;
684		879
685	if (security_status & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)	880	if (security_status & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
686	prop->hard_reset_done_by_fw = true;	881	prop->hard_reset_done_by_fw = true;
687	} else {	882	} else {
688	prop->fw_security_status_valid = 0;	883	prop->fw_security_status_valid = 0;
689	prop->fw_security_disabled = true;
690	}	884	}
691		885
692	dev_dbg(hdev->dev, "Firmware preboot security status %#x\n",	886	dev_dbg(hdev->dev, "Firmware preboot security status %#x\n",
@@ -710,7 +904,7 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, · 17 unmodified lines
710	u32 status;	904	u32 status;
711	int rc;	905	int rc;
712		906
713	if (!(hdev->fw_loading & FW_TYPE_BOOT_CPU))	907	if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))
714	return 0;	908	return 0;
715		909
716	dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",	910	dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
@@ -801,7 +995,7 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, · 84 unmodified lines
801	goto out;	995	goto out;
802	}	996	}
803		997
804	if (!(hdev->fw_loading & FW_TYPE_LINUX)) {	998	if (!(hdev->fw_components & FW_TYPE_LINUX)) {
805	dev_info(hdev->dev, "Skip loading Linux F/W\n");	999	dev_info(hdev->dev, "Skip loading Linux F/W\n");
806	goto out;	1000	goto out;
807	}	1001	}

drivers/misc/habanalabs/common/habanalabs.h+143−41

 #include <linux/dma-direction.h>
 #include <linux/scatterlist.h>
 #include <linux/hashtable.h>
+#include <linux/debugfs.h>
 #include <linux/bitfield.h>
 #include <linux/genalloc.h>
 #include <linux/sched/signal.h>
 #define HL_SIM_MAX_TIMEOUT_US		10000000 /* 10s */
-#define HL_IDLE_BUSY_TS_ARR_SIZE	4096
+#define HL_COMMON_USER_INTERRUPT_ID	0xFFF
 /* Memory */
 #define MEM_HASH_TABLE_BITS		7 /* 1 << 7 buckets */
 #define HL_MAX_DCORES			4
+/*
+ * Reset Flags
+ *
+ * - HL_RESET_HARD
+ *       If set do hard reset to all engines. If not set reset just
+ *       compute/DMA engines.
+ *
+ * - HL_RESET_FROM_RESET_THREAD
+ *       Set if the caller is the hard-reset thread
+ *
+ * - HL_RESET_HEARTBEAT
+ *       Set if reset is due to heartbeat
+ */
+#define HL_RESET_HARD			(1 << 0)
+#define HL_RESET_FROM_RESET_THREAD	(1 << 1)
+#define HL_RESET_HEARTBEAT		(1 << 2)
 #define HL_MAX_SOBS_PER_MONITOR	8
 /**
 };
 /**
- * enum hl_fw_types - F/W types to load
+ * enum hl_fw_types - F/W types present in the system
  * @FW_TYPE_LINUX: Linux image for device CPU
  * @FW_TYPE_BOOT_CPU: Boot image for device CPU
+ * @FW_TYPE_PREBOOT_CPU: Indicates pre-loaded CPUs are present in the system
+ *                       (preboot, ppboot etc...)
  * @FW_TYPE_ALL_TYPES: Mask for all types
  */
 enum hl_fw_types {
 	FW_TYPE_LINUX = 0x1,
 	FW_TYPE_BOOT_CPU = 0x2,
-	FW_TYPE_ALL_TYPES = (FW_TYPE_LINUX | FW_TYPE_BOOT_CPU)
+	FW_TYPE_PREBOOT_CPU = 0x4,
+	FW_TYPE_ALL_TYPES =
+		(FW_TYPE_LINUX | FW_TYPE_BOOT_CPU | FW_TYPE_PREBOOT_CPU)
 };
 /**
  * @dram_size: DRAM total size.
  * @dram_pci_bar_size: size of PCI bar towards DRAM.
  * @max_power_default: max power of the device after reset
+ * @dc_power_default: power consumed by the device in mode idle.
  * @dram_size_for_default_page_mapping: DRAM size needed to map to avoid page
  *                                      fault.
  * @pcie_dbi_base_address: Base address of the PCIE_DBI block.
  * @first_available_user_msix_interrupt: first available msix interrupt
  *                                       reserved for the user
  * @first_available_cq: first available CQ for the user.
+ * @user_interrupt_count: number of user interrupts.
  * @tpc_enabled_mask: which TPCs are enabled.
  * @completion_queues_count: number of completion queues.
  * @fw_security_disabled: true if security measures are disabled in firmware,
  * @dram_supports_virtual_memory: is there an MMU towards the DRAM
  * @hard_reset_done_by_fw: true if firmware is handling hard reset flow
  * @num_functional_hbms: number of functional HBMs in each DCORE.
+ * @iatu_done_by_fw: true if iATU configuration is being done by FW.
  */
 struct asic_fixed_properties {
 	struct hw_queue_properties	*hw_queues_props;
 	u64				dram_size;
 	u64				dram_pci_bar_size;
 	u64				max_power_default;
+	u64				dc_power_default;
 	u64				dram_size_for_default_page_mapping;
 	u64				pcie_dbi_base_address;
 	u64				pcie_aux_dbi_reg_addr;
 	u16				first_available_user_mon[HL_MAX_DCORES];
 	u16				first_available_user_msix_interrupt;
 	u16				first_available_cq[HL_MAX_DCORES];
+	u16				user_interrupt_count;
 	u8				tpc_enabled_mask;
 	u8				completion_queues_count;
 	u8				fw_security_disabled;
 	u8				dram_supports_virtual_memory;
 	u8				hard_reset_done_by_fw;
 	u8				num_functional_hbms;
+	u8				iatu_done_by_fw;
 };
 /**
 /**
  * struct hl_cs_compl - command submission completion object.
+ * @sob_reset_work: workqueue object to run SOB reset flow.
  * @base_fence: hl fence object.
  * @lock: spinlock to protect fence.
  * @hdev: habanalabs device structure.
  * @sob_group: the SOB group that is used in this collective wait CS.
  */
 struct hl_cs_compl {
+	struct work_struct	sob_reset_work;
 	struct hl_fence		base_fence;
 	spinlock_t		lock;
 	struct hl_device	*hdev;
 	atomic_t		free_slots_cnt;
 };
+/**
+ * struct hl_user_interrupt - holds user interrupt information
+ * @hdev: pointer to the device structure
+ * @wait_list_head: head to the list of user threads pending on this interrupt
+ * @wait_list_lock: protects wait_list_head
+ * @interrupt_id: msix interrupt id
+ */
+struct hl_user_interrupt {
+	struct hl_device	*hdev;
+	struct list_head	wait_list_head;
+	spinlock_t		wait_list_lock;
+	u32			interrupt_id;
+};
+/**
+ * struct hl_user_pending_interrupt - holds a context to a user thread
+ *                                    pending on an interrupt
+ * @wait_list_node: node in the list of user threads pending on an interrupt
+ * @fence: hl fence object for interrupt completion
+ */
+struct hl_user_pending_interrupt {
+	struct list_head	wait_list_node;
+	struct hl_fence		fence;
+};
 /**
  * struct hl_eq - describes the event queue (single one per device)
  * @hdev: pointer to the device structure
  * @ASIC_INVALID: Invalid ASIC type.
  * @ASIC_GOYA: Goya device.
  * @ASIC_GAUDI: Gaudi device.
+ * @ASIC_GAUDI_SEC: Gaudi secured device (HL-2000).
  */
 enum hl_asic_type {
 	ASIC_INVALID,
 	ASIC_GOYA,
-	ASIC_GAUDI
+	ASIC_GAUDI,
+	ASIC_GAUDI_SEC
 };
 struct hl_cs_parser;
  * @update_eq_ci: update event queue CI.
  * @context_switch: called upon ASID context switch.
  * @restore_phase_topology: clear all SOBs amd MONs.
- * @debugfs_read32: debug interface for reading u32 from DRAM/SRAM.
+ * @debugfs_read32: debug interface for reading u32 from DRAM/SRAM/Host memory.
- * @debugfs_write32: debug interface for writing u32 to DRAM/SRAM.
+ * @debugfs_write32: debug interface for writing u32 to DRAM/SRAM/Host memory.
+ * @debugfs_read64: debug interface for reading u64 from DRAM/SRAM/Host memory.
+ * @debugfs_write64: debug interface for writing u64 to DRAM/SRAM/Host memory.
+ * @debugfs_read_dma: debug interface for reading up to 2MB from the device's
+ *                    internal memory via DMA engine.
  * @add_device_attr: add ASIC specific device attributes.
  * @handle_eqe: handle event queue entry (IRQ) from CPU-CP.
  * @set_pll_profile: change PLL profile (manual/automatic).
 	void (*update_eq_ci)(struct hl_device *hdev, u32 val);
 	int (*context_switch)(struct hl_device *hdev, u32 asid);
 	void (*restore_phase_topology)(struct hl_device *hdev);
-	int (*debugfs_read32)(struct hl_device *hdev, u64 addr, u32 *val);
+	int (*debugfs_read32)(struct hl_device *hdev, u64 addr,
-	int (*debugfs_write32)(struct hl_device *hdev, u64 addr, u32 val);
+				bool user_address, u32 *val);
-	int (*debugfs_read64)(struct hl_device *hdev, u64 addr, u64 *val);
+	int (*debugfs_write32)(struct hl_device *hdev, u64 addr,
-	int (*debugfs_write64)(struct hl_device *hdev, u64 addr, u64 val);
+				bool user_address, u32 val);
+	int (*debugfs_read64)(struct hl_device *hdev, u64 addr,
+				bool user_address, u64 *val);
+	int (*debugfs_write64)(struct hl_device *hdev, u64 addr,
+				bool user_address, u64 val);
+	int (*debugfs_read_dma)(struct hl_device *hdev, u64 addr, u32 size,
+				void *blob_addr);
 	void (*add_device_attr)(struct hl_device *hdev,
 				struct attribute_group *dev_attr_grp);
 	void (*handle_eqe)(struct hl_device *hdev,
 	int (*hw_block_mmap)(struct hl_device *hdev, struct vm_area_struct *vma,
 			u32 block_id, u32 block_size);
 	void (*enable_events_from_fw)(struct hl_device *hdev);
+	void (*get_msi_info)(u32 *table);
 };
  * @mem_hash_lock: protects the mem_hash.
  * @mmu_lock: protects the MMU page tables. Any change to the PGT, modifying the
  *            MMU hash or walking the PGT requires talking this lock.
+ * @hw_block_list_lock: protects the HW block memory list.
  * @debugfs_list: node in debugfs list of contexts.
  * pending_cb_list: list of pending command buffers waiting to be sent upon
  *                  next user command submission context.
+ * @hw_block_mem_list: list of HW block virtual mapped addresses.
  * @cs_counters: context command submission counters.
  * @cb_va_pool: device VA pool for command buffers which are mapped to the
  *              device's MMU.
 	struct hl_va_range		*va_range[HL_VA_RANGE_TYPE_MAX];
 	struct mutex			mem_hash_lock;
 	struct mutex			mmu_lock;
+	struct mutex			hw_block_list_lock;
 	struct list_head		debugfs_list;
 	struct list_head		pending_cb_list;
+	struct list_head		hw_block_mem_list;
 	struct hl_cs_counters_atomic	cs_counters;
 	struct gen_pool			*cb_va_pool;
 	u64				cs_sequence;
  * @sequence: the sequence number of this CS.
  * @staged_sequence: the sequence of the staged submission this CS is part of,
  *                   relevant only if staged_cs is set.
+ * @timeout_jiffies: cs timeout in jiffies.
  * @type: CS_TYPE_*.
  * @submitted: true if CS was submitted to H/W.
  * @completed: true if CS was completed by device.
 	struct list_head	debugfs_list;
 	u64			sequence;
 	u64			staged_sequence;
+	u64			timeout_jiffies;
 	enum hl_cs_type		type;
 	u8			submitted;
 	u8			completed;
 	void			*ptr;
 };
+/**
+ * struct hl_vm_hw_block_list_node - list element from user virtual address to
+ *				HW block id.
+ * @node: node to hang on the list in context object.
+ * @ctx: the context this node belongs to.
+ * @vaddr: virtual address of the HW block.
+ * @size: size of the block.
+ * @id: HW block id (handle).
+ */
+struct hl_vm_hw_block_list_node {
+	struct list_head	node;
+	struct hl_ctx		*ctx;
+	unsigned long		vaddr;
+	u32			size;
+	u32			id;
+};
 /**
  * struct hl_vm_phys_pg_pack - physical page pack.
  * @vm_type: describes the type of the virtual area descriptor.
  * @userptr_spinlock: protects userptr_list.
  * @ctx_mem_hash_list: list of available contexts with MMU mappings.
  * @ctx_mem_hash_spinlock: protects cb_list.
+ * @blob_desc: descriptor of blob
  * @addr: next address to read/write from/to in read/write32.
  * @mmu_addr: next virtual address to translate to physical address in mmu_show.
  * @mmu_asid: ASID to use while translating in mmu_show.
  * @i2c_bus: generic u8 debugfs file for bus value to use in i2c_data_read.
- * @i2c_bus: generic u8 debugfs file for address value to use in i2c_data_read.
+ * @i2c_addr: generic u8 debugfs file for address value to use in i2c_data_read.
- * @i2c_bus: generic u8 debugfs file for register value to use in i2c_data_read.
+ * @i2c_reg: generic u8 debugfs file for register value to use in i2c_data_read.
  */
 struct hl_dbg_device_entry {
 	struct dentry			*root;
 	spinlock_t			userptr_spinlock;
 	struct list_head		ctx_mem_hash_list;
 	spinlock_t			ctx_mem_hash_spinlock;
+	struct debugfs_blob_wrapper	blob_desc;
 	u64				addr;
 	u64				mmu_addr;
 	u32				mmu_asid;
 	struct hl_device		*hdev;
 };
-/**
- * struct hl_device_idle_busy_ts - used for calculating device utilization rate.
- * @idle_to_busy_ts: timestamp where device changed from idle to busy.
- * @busy_to_idle_ts: timestamp where device changed from busy to idle.
- */
-struct hl_device_idle_busy_ts {
-	ktime_t				idle_to_busy_ts;
-	ktime_t				busy_to_idle_ts;
-};
 /**
  * struct hr_mmu_hop_addrs - used for holding per-device host-resident mmu hop
  * information.
  * @asic_name: ASIC specific name.
  * @asic_type: ASIC specific type.
  * @completion_queue: array of hl_cq.
+ * @user_interrupt: array of hl_user_interrupt. upon the corresponding user
+ *                  interrupt, driver will monitor the list of fences
+ *                  registered to this interrupt.
+ * @common_user_interrupt: common user interrupt for all user interrupts.
+ *                         upon any user interrupt, driver will monitor the
+ *                         list of fences registered to this common structure.
  * @cq_wq: work queues of completion queues for executing work in process
  *         context.
  * @eq_wq: work queue of event queue for executing work in process context.
+ * @sob_reset_wq: work queue for sob reset executions.
  * @kernel_ctx: Kernel driver context structure.
  * @kernel_queues: array of hl_hw_queue.
  * @cs_mirror_list: CS mirror list for TDR.
  *              when a user opens the device
  * @fpriv_list_lock: protects the fpriv_list
  * @compute_ctx: current compute context executing.
- * @idle_busy_ts_arr: array to hold time stamps of transitions from idle to busy
- *                    and vice-versa
  * @aggregated_cs_counters: aggregated cs counters among all contexts
  * @mmu_priv: device-specific MMU data.
  * @mmu_func: device-related MMU functions.
+ * @legacy_pll_map: map holding map between dynamic (common) PLL indexes and
+ *                  static (asic specific) PLL indexes.
  * @dram_used_mem: current DRAM memory consumption.
  * @timeout_jiffies: device CS timeout value.
  * @max_power: the max power of the device, as configured by the sysadmin. This
  * @curr_pll_profile: current PLL profile.
  * @card_type: Various ASICs have several card types. This indicates the card
  *             type of the current device.
- * @cs_active_cnt: number of active command submissions on this device (active
- *                 means already in H/W queues)
  * @major: habanalabs kernel driver major.
  * @high_pll: high PLL profile frequency.
  * @soft_reset_cnt: number of soft reset since the driver was loaded.
  * @hard_reset_cnt: number of hard reset since the driver was loaded.
- * @idle_busy_ts_idx: index of current entry in idle_busy_ts_arr
  * @clk_throttling_reason: bitmask represents the current clk throttling reasons
  * @id: device minor.
  * @id_control: minor of the control device
 	char				status[HL_DEV_STS_MAX][HL_STR_MAX];
 	enum hl_asic_type		asic_type;
 	struct hl_cq			*completion_queue;
+	struct hl_user_interrupt	*user_interrupt;
+	struct hl_user_interrupt	common_user_interrupt;
 	struct workqueue_struct		**cq_wq;
 	struct workqueue_struct		*eq_wq;
+	struct workqueue_struct		*sob_reset_wq;
 	struct hl_ctx			*kernel_ctx;
 	struct hl_hw_queue		*kernel_queues;
 	struct list_head		cs_mirror_list;
 	struct hl_ctx			*compute_ctx;
-	struct hl_device_idle_busy_ts	*idle_busy_ts_arr;
 	struct hl_cs_counters_atomic	aggregated_cs_counters;
 	struct hl_mmu_priv		mmu_priv;
 	struct hl_mmu_funcs		mmu_func[MMU_NUM_PGT_LOCATIONS];
+	enum pll_index			*legacy_pll_map;
 	atomic64_t			dram_used_mem;
 	u64				timeout_jiffies;
 	u64				max_power;
 	atomic_t			in_reset;
 	enum hl_pll_frequency		curr_pll_profile;
 	enum cpucp_card_types		card_type;
-	int				cs_active_cnt;
 	u32				major;
 	u32				high_pll;
 	u32				soft_reset_cnt;
 	u32				hard_reset_cnt;
-	u32				idle_busy_ts_idx;
 	u32				clk_throttling_reason;
 	u16				id;
 	u16				id_control;
 	/* Parameters for bring-up */
 	u64				nic_ports_mask;
-	u64				fw_loading;
+	u64				fw_components;
 	u8				mmu_enable;
 	u8				mmu_huge_page_opt;
-	u8				cpu_enable;
 	u8				reset_pcilink;
 	u8				cpu_queues_enable;
 	u8				pldm;
 	u8				bmc_enable;
 	u8				rl_enable;
 	u8				reset_on_preboot_fail;
+	u8				reset_upon_device_release;
 };
 void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q);
 irqreturn_t hl_irq_handler_cq(int irq, void *arg);
 irqreturn_t hl_irq_handler_eq(int irq, void *arg);
+irqreturn_t hl_irq_handler_user_cq(int irq, void *arg);
+irqreturn_t hl_irq_handler_default(int irq, void *arg);
 u32 hl_cq_inc_ptr(u32 ptr);
 int hl_asid_init(struct hl_device *hdev);
 void hl_device_fini(struct hl_device *hdev);
 int hl_device_suspend(struct hl_device *hdev);
 int hl_device_resume(struct hl_device *hdev);
-int hl_device_reset(struct hl_device *hdev, bool hard_reset,
+int hl_device_reset(struct hl_device *hdev, u32 flags);
-			bool from_hard_reset_thread);
 void hl_hpriv_get(struct hl_fpriv *hpriv);
-void hl_hpriv_put(struct hl_fpriv *hpriv);
+int hl_hpriv_put(struct hl_fpriv *hpriv);
 int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq);
-uint32_t hl_device_utilization(struct hl_device *hdev, uint32_t period_ms);
+int hl_device_utilization(struct hl_device *hdev, u32 *utilization);
 int hl_build_hwmon_channel_info(struct hl_device *hdev,
 		struct cpucp_sensor *sensors_arr);
 int hl_vm_init(struct hl_device *hdev);
 void hl_vm_fini(struct hl_device *hdev);
+void hl_hw_block_mem_init(struct hl_ctx *ctx);
+void hl_hw_block_mem_fini(struct hl_ctx *ctx);
 u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
 		enum hl_va_range_type type, u32 size, u32 alignment);
 int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
 int hl_fw_cpucp_info_get(struct hl_device *hdev,
 			u32 cpu_security_boot_status_reg,
 			u32 boot_err0_reg);
+int hl_fw_cpucp_handshake(struct hl_device *hdev,
+			u32 cpu_security_boot_status_reg,
+			u32 boot_err0_reg);
 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size);
 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
 		struct hl_info_pci_counters *counters);
 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev,
 			u64 *total_energy);
-int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u16 pll_index,
+int get_used_pll_index(struct hl_device *hdev, enum pll_index input_pll_index,
+						enum pll_index *pll_index);
+int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, enum pll_index pll_index,
 		u16 *pll_freq_arr);
+int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power);
 int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
 			u32 msg_to_cpu_reg, u32 cpu_msg_status_reg,
 			u32 cpu_security_boot_status_reg, u32 boot_err0_reg,
 int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
 			bool is_wc[3]);
+int hl_pci_elbi_read(struct hl_device *hdev, u64 addr, u32 *data);
 int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data);
 int hl_pci_set_inbound_region(struct hl_device *hdev, u8 region,
 		struct hl_inbound_pci_region *pci_region);
 int hl_pci_init(struct hl_device *hdev);
 void hl_pci_fini(struct hl_device *hdev);
-long hl_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr);
+long hl_get_frequency(struct hl_device *hdev, enum pll_index pll_index,
-void hl_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq);
+								bool curr);
+void hl_set_frequency(struct hl_device *hdev, enum pll_index pll_index,
+								u64 freq);
 int hl_get_temperature(struct hl_device *hdev,
 		       int sensor_index, u32 attr, long *value);
 int hl_set_temperature(struct hl_device *hdev,
 			int sensor_index, u32 attr, long value);
 int hl_set_current(struct hl_device *hdev,
 			int sensor_index, u32 attr, long value);
+void hl_release_pending_user_interrupts(struct hl_device *hdev);
 #ifdef CONFIG_DEBUG_FS
 long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg);
 int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data);
 int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data);
-int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data);
+int hl_wait_ioctl(struct hl_fpriv *hpriv, void *data);
 int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data);
 #endif /* HABANALABSP_H_ */

drivers/misc/habanalabs/common/habanalabs_drv.c+24−4

 static DEFINE_IDR(hl_devs_idr);
 static DEFINE_MUTEX(hl_devs_idr_lock);
-static int timeout_locked = 5;
+static int timeout_locked = 30;
 static int reset_on_lockup = 1;
 static int memory_scrub = 1;
 module_param(timeout_locked, int, 0444);
 MODULE_PARM_DESC(timeout_locked,
-	"Device lockup timeout in seconds (0 = disabled, default 5s)");
+	"Device lockup timeout in seconds (0 = disabled, default 30s)");
 module_param(reset_on_lockup, int, 0444);
 MODULE_PARM_DESC(reset_on_lockup,
 #define PCI_IDS_GOYA			0x0001
 #define PCI_IDS_GAUDI			0x1000
+#define PCI_IDS_GAUDI_SEC		0x1010
 static const struct pci_device_id ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GOYA), },
 	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI), },
+	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI_SEC), },
 	{ 0, }
 };
 MODULE_DEVICE_TABLE(pci, ids);
 	case PCI_IDS_GAUDI:
 		asic_type = ASIC_GAUDI;
 		break;
+	case PCI_IDS_GAUDI_SEC:
+		asic_type = ASIC_GAUDI_SEC;
+		break;
 	default:
 		asic_type = ASIC_INVALID;
 		break;
 	return asic_type;
 }
+static bool is_asic_secured(enum hl_asic_type asic_type)
+{
+	switch (asic_type) {
+	case ASIC_GAUDI_SEC:
+		return true;
+	default:
+		return false;
+	}
+}
 /*
  * hl_device_open - open function for habanalabs device
  *
 static void set_driver_behavior_per_device(struct hl_device *hdev)
 {
-	hdev->cpu_enable = 1;
+	hdev->fw_components = FW_TYPE_ALL_TYPES;
-	hdev->fw_loading = FW_TYPE_ALL_TYPES;
 	hdev->cpu_queues_enable = 1;
 	hdev->heartbeat = 1;
 	hdev->mmu_enable = 1;
 		hdev->asic_type = asic_type;
 	}
+	if (pdev)
+		hdev->asic_prop.fw_security_disabled =
+				!is_asic_secured(pdev->device);
+	else
+		hdev->asic_prop.fw_security_disabled = true;
 	/* Assign status description string */
 	strncpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION],
 					"disabled", HL_STR_MAX);

drivers/misc/habanalabs/common/habanalabs_ioctl.c+26−9

 	struct hl_info_device_utilization device_util = {0};
 	u32 max_size = args->return_size;
 	void __user *out = (void __user *) (uintptr_t) args->return_pointer;
+	int rc;
 	if ((!max_size) || (!out))
 		return -EINVAL;
-	if ((args->period_ms < 100) || (args->period_ms > 1000) ||
+	rc = hl_device_utilization(hdev, &device_util.utilization);
-		(args->period_ms % 100)) {
+	if (rc)
-		dev_err(hdev->dev,
-			"period %u must be between 100 - 1000 and must be divisible by 100\n",
-			args->period_ms);
 		return -EINVAL;
-	}
-	device_util.utilization = hl_device_utilization(hdev, args->period_ms);
 	return copy_to_user(out, &device_util,
 		min((size_t) max_size, sizeof(device_util))) ? -EFAULT : 0;
 		min((size_t) max_size, sizeof(freq_info))) ? -EFAULT : 0;
 }
+static int power_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
+{
+	struct hl_device *hdev = hpriv->hdev;
+	u32 max_size = args->return_size;
+	struct hl_power_info power_info = {0};
+	void __user *out = (void __user *) (uintptr_t) args->return_pointer;
+	int rc;
+	if ((!max_size) || (!out))
+		return -EINVAL;
+	rc = hl_fw_cpucp_power_get(hdev, &power_info.power);
+	if (rc)
+		return rc;
+	return copy_to_user(out, &power_info,
+		min((size_t) max_size, sizeof(power_info))) ? -EFAULT : 0;
+}
 static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
 				struct device *dev)
 {
 	case HL_INFO_PLL_FREQUENCY:
 		return pll_frequency_info(hpriv, args);
+	case HL_INFO_POWER:
+		return power_info(hpriv, args);
 	default:
 		dev_err(dev, "Invalid request %d\n", args->op);
 		rc = -ENOTTY;
 	HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl),
 	HL_IOCTL_DEF(HL_IOCTL_CB, hl_cb_ioctl),
 	HL_IOCTL_DEF(HL_IOCTL_CS, hl_cs_ioctl),
-	HL_IOCTL_DEF(HL_IOCTL_WAIT_CS, hl_cs_wait_ioctl),
+	HL_IOCTL_DEF(HL_IOCTL_WAIT_CS, hl_wait_ioctl),
 	HL_IOCTL_DEF(HL_IOCTL_MEMORY, hl_mem_ioctl),
 	HL_IOCTL_DEF(HL_IOCTL_DEBUG, hl_debug_ioctl)
 };

drivers/misc/habanalabs/common/hw_queue.c+1−9

 	if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) &&
 				first_entry && cs_needs_timeout(cs)) {
 		cs->tdr_active = true;
-		schedule_delayed_work(&cs->work_tdr, hdev->timeout_jiffies);
+		schedule_delayed_work(&cs->work_tdr, cs->timeout_jiffies);
 	}
 	spin_unlock(&hdev->cs_mirror_lock);
-	if (!hdev->cs_active_cnt++) {
-		struct hl_device_idle_busy_ts *ts;
-		ts = &hdev->idle_busy_ts_arr[hdev->idle_busy_ts_idx];
-		ts->busy_to_idle_ts = ktime_set(0, 0);
-		ts->idle_to_busy_ts = ktime_get();
-	}
 	list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)
 		switch (job->queue_type) {
 		case QUEUE_TYPE_EXT:

drivers/misc/habanalabs/common/irq.c+56−0

@@ -137,6 +137,62 @@ irqreturn_t hl_irq_handler_cq(int irq, void *arg) · 136 unmodified lines
137	return IRQ_HANDLED;	137	return IRQ_HANDLED;
138	}	138	}
139		139
		140	static void handle_user_cq(struct hl_device *hdev,
		141	struct hl_user_interrupt *user_cq)
		142	{
		143	struct hl_user_pending_interrupt *pend;
		144
		145	spin_lock(&user_cq->wait_list_lock);
		146	list_for_each_entry(pend, &user_cq->wait_list_head, wait_list_node)
		147	complete_all(&pend->fence.completion);
		148	spin_unlock(&user_cq->wait_list_lock);
		149	}
		150
		151	/**
		152	* hl_irq_handler_user_cq - irq handler for user completion queues
		153	*
		154	* @irq: irq number
		155	* @arg: pointer to user interrupt structure
		156	*
		157	*/
		158	irqreturn_t hl_irq_handler_user_cq(int irq, void *arg)
		159	{
		160	struct hl_user_interrupt *user_cq = arg;
		161	struct hl_device *hdev = user_cq->hdev;
		162
		163	dev_dbg(hdev->dev,
		164	"got user completion interrupt id %u",
		165	user_cq->interrupt_id);
		166
		167	/* Handle user cq interrupts registered on all interrupts */
		168	handle_user_cq(hdev, &hdev->common_user_interrupt);
		169
		170	/* Handle user cq interrupts registered on this specific interrupt */
		171	handle_user_cq(hdev, user_cq);
		172
		173	return IRQ_HANDLED;
		174	}
		175
		176	/**
		177	* hl_irq_handler_default - default irq handler
		178	*
		179	* @irq: irq number
		180	* @arg: pointer to user interrupt structure
		181	*
		182	*/
		183	irqreturn_t hl_irq_handler_default(int irq, void *arg)
		184	{
		185	struct hl_user_interrupt *user_interrupt = arg;
		186	struct hl_device *hdev = user_interrupt->hdev;
		187	u32 interrupt_id = user_interrupt->interrupt_id;
		188
		189	dev_err(hdev->dev,
		190	"got invalid user interrupt %u",
		191	interrupt_id);
		192
		193	return IRQ_HANDLED;
		194	}
		195
140	/**	196	/**
141	* hl_irq_handler_eq - irq handler for event queue	197	* hl_irq_handler_eq - irq handler for event queue
142	*	198	*

drivers/misc/habanalabs/common/memory.c+129−53

@@ -81,16 +81,6 @@ static int alloc_device_memory(struct hl_ctx ctx, struct hl_mem_in args, · 80 unmodified lines
81	num_pgs, total_size);	81	num_pgs, total_size);
82	return -ENOMEM;	82	return -ENOMEM;
83	}	83	}
84
85	if (hdev->memory_scrub) {
86	rc = hdev->asic_funcs->scrub_device_mem(hdev, paddr,
87	total_size);
88	if (rc) {
89	dev_err(hdev->dev,
90	"Failed to scrub contiguous device memory\n");
91	goto pages_pack_err;
92	}
93	}
94	}	84	}
95		85
96	phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);	86	phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);
@@ -128,24 +118,13 @@ static int alloc_device_memory(struct hl_ctx ctx, struct hl_mem_in args, · 31 unmodified lines
128	goto page_err;	118	goto page_err;
129	}	119	}
130		120
131	if (hdev->memory_scrub) {
132	rc = hdev->asic_funcs->scrub_device_mem(hdev,
133	phys_pg_pack->pages[i],
134	page_size);
135	if (rc) {
136	dev_err(hdev->dev,
137	"Failed to scrub device memory\n");
138	goto page_err;
139	}
140	}
141
142	num_curr_pgs++;	121	num_curr_pgs++;
143	}	122	}
144	}	123	}
145		124
146	spin_lock(&vm->idr_lock);	125	spin_lock(&vm->idr_lock);
147	handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0,	126	handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0,
148	GFP_ATOMIC);	127	GFP_KERNEL);
149	spin_unlock(&vm->idr_lock);	128	spin_unlock(&vm->idr_lock);
150		129
151	if (handle < 0) {	130	if (handle < 0) {
@@ -280,37 +259,67 @@ static void dram_pg_pool_do_release(struct kref *ref) · 128 unmodified lines
280	* @phys_pg_pack: physical page pack to free.	259	* @phys_pg_pack: physical page pack to free.
281	*	260	*
282	* This function does the following:	261	* This function does the following:
283	* - For DRAM memory only, iterate over the pack and free each physical block	262	* - For DRAM memory only
284	* structure by returning it to the general pool.	263	* - iterate over the pack, scrub and free each physical block structure by
		264	* returning it to the general pool.
		265	* In case of error during scrubbing, initiate hard reset.
		266	* Once hard reset is triggered, scrubbing is bypassed while freeing the
		267	* memory continues.
285	* - Free the hl_vm_phys_pg_pack structure.	268	* - Free the hl_vm_phys_pg_pack structure.
286	*/	269	*/
287	static void free_phys_pg_pack(struct hl_device *hdev,	270	static int free_phys_pg_pack(struct hl_device *hdev,
288	struct hl_vm_phys_pg_pack *phys_pg_pack)	271	struct hl_vm_phys_pg_pack *phys_pg_pack)
289	{	272	{
290	struct hl_vm *vm = &hdev->vm;	273	struct hl_vm *vm = &hdev->vm;
291	u64 i;	274	u64 i;
		275	int rc = 0;
292		276
293	if (!phys_pg_pack->created_from_userptr) {	277	if (phys_pg_pack->created_from_userptr)
294	if (phys_pg_pack->contiguous) {	278	goto end;
295	gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[0],	279
		280	if (phys_pg_pack->contiguous) {
		281	if (hdev->memory_scrub && !hdev->disabled) {
		282	rc = hdev->asic_funcs->scrub_device_mem(hdev,
		283	phys_pg_pack->pages[0],
296	phys_pg_pack->total_size);	284	phys_pg_pack->total_size);
		285	if (rc)
		286	dev_err(hdev->dev,
		287	"Failed to scrub contiguous device memory\n");
		288	}
297		289
298	for (i = 0; i < phys_pg_pack->npages ; i++)	290	gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[0],
299	kref_put(&vm->dram_pg_pool_refcount,	291	phys_pg_pack->total_size);
300	dram_pg_pool_do_release);	292
301	} else {	293	for (i = 0; i < phys_pg_pack->npages ; i++)
302	for (i = 0 ; i < phys_pg_pack->npages ; i++) {	294	kref_put(&vm->dram_pg_pool_refcount,
303	gen_pool_free(vm->dram_pg_pool,	295	dram_pg_pool_do_release);
		296	} else {
		297	for (i = 0 ; i < phys_pg_pack->npages ; i++) {
		298	if (hdev->memory_scrub && !hdev->disabled && rc == 0) {
		299	rc = hdev->asic_funcs->scrub_device_mem(
		300	hdev,
304	phys_pg_pack->pages[i],	301	phys_pg_pack->pages[i],
305	phys_pg_pack->page_size);	302	phys_pg_pack->page_size);
306	kref_put(&vm->dram_pg_pool_refcount,	303	if (rc)
307	dram_pg_pool_do_release);	304	dev_err(hdev->dev,
		305	"Failed to scrub device memory\n");
308	}	306	}
		307	gen_pool_free(vm->dram_pg_pool,
		308	phys_pg_pack->pages[i],
		309	phys_pg_pack->page_size);
		310	kref_put(&vm->dram_pg_pool_refcount,
		311	dram_pg_pool_do_release);
309	}	312	}
310	}	313	}
311		314
		315	if (rc && !hdev->disabled)
		316	hl_device_reset(hdev, HL_RESET_HARD);
		317
		318	end:
312	kvfree(phys_pg_pack->pages);	319	kvfree(phys_pg_pack->pages);
313	kfree(phys_pg_pack);	320	kfree(phys_pg_pack);
		321
		322	return rc;
314	}	323	}
315		324
316	/**	325	/**
@@ -349,7 +358,7 @@ static int free_device_memory(struct hl_ctx ctx, struct hl_mem_in args) · 32 unmodified lines
349	atomic64_sub(phys_pg_pack->total_size, &ctx->dram_phys_mem);	358	atomic64_sub(phys_pg_pack->total_size, &ctx->dram_phys_mem);
350	atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem);	359	atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem);
351		360
352	free_phys_pg_pack(hdev, phys_pg_pack);	361	return free_phys_pg_pack(hdev, phys_pg_pack);
353	} else {	362	} else {
354	spin_unlock(&vm->idr_lock);	363	spin_unlock(&vm->idr_lock);
355	dev_err(hdev->dev,	364	dev_err(hdev->dev,
@@ -857,6 +866,7 @@ static int map_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr, · 501 unmodified lines
857	u64 next_vaddr = vaddr, paddr, mapped_pg_cnt = 0, i;	866	u64 next_vaddr = vaddr, paddr, mapped_pg_cnt = 0, i;
858	u32 page_size = phys_pg_pack->page_size;	867	u32 page_size = phys_pg_pack->page_size;
859	int rc = 0;	868	int rc = 0;
		869	bool is_host_addr;
860		870
861	for (i = 0 ; i < phys_pg_pack->npages ; i++) {	871	for (i = 0 ; i < phys_pg_pack->npages ; i++) {
862	paddr = phys_pg_pack->pages[i];	872	paddr = phys_pg_pack->pages[i];
@@ -878,6 +888,8 @@ static int map_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr, · 15 unmodified lines
878	return 0;	888	return 0;
879		889
880	err:	890	err:
		891	is_host_addr = !hl_is_dram_va(hdev, vaddr);
		892
881	next_vaddr = vaddr;	893	next_vaddr = vaddr;
882	for (i = 0 ; i < mapped_pg_cnt ; i++) {	894	for (i = 0 ; i < mapped_pg_cnt ; i++) {
883	if (hl_mmu_unmap_page(ctx, next_vaddr, page_size,	895	if (hl_mmu_unmap_page(ctx, next_vaddr, page_size,
@@ -888,6 +900,17 @@ err: · 4 unmodified lines
888	phys_pg_pack->pages[i], page_size);	900	phys_pg_pack->pages[i], page_size);
889		901
890	next_vaddr += page_size;	902	next_vaddr += page_size;
		903
		904	/*
		905	* unmapping on Palladium can be really long, so avoid a CPU
		906	* soft lockup bug by sleeping a little between unmapping pages
		907	*
		908	* In addition, on host num of pages could be huge,
		909	* because page size could be 4KB, so when unmapping host
		910	* pages sleep every 32K pages to avoid soft lockup
		911	*/
		912	if (hdev->pldm \|\| (is_host_addr && (i & 0x7FFF) == 0))
		913	usleep_range(50, 200);
891	}	914	}
892		915
893	return rc;	916	return rc;
@@ -921,9 +944,9 @@ static void unmap_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr, · 27 unmodified lines
921	* unmapping on Palladium can be really long, so avoid a CPU	944	* unmapping on Palladium can be really long, so avoid a CPU
922	* soft lockup bug by sleeping a little between unmapping pages	945	* soft lockup bug by sleeping a little between unmapping pages
923	*	946	*
924	* In addition, when unmapping host memory we pass through	947	* In addition, on host num of pages could be huge,
925	* the Linux kernel to unpin the pages and that takes a long	948	* because page size could be 4KB, so when unmapping host
926	* time. Therefore, sleep every 32K pages to avoid soft lockup	949	* pages sleep every 32K pages to avoid soft lockup
927	*/	950	*/
928	if (hdev->pldm \|\| (is_host_addr && (i & 0x7FFF) == 0))	951	if (hdev->pldm \|\| (is_host_addr && (i & 0x7FFF) == 0))
929	usleep_range(50, 200);	952	usleep_range(50, 200);
@@ -1117,9 +1140,9 @@ static int map_device_va(struct hl_ctx ctx, struct hl_mem_in args, · 187 unmodified lines
1117	*device_addr = ret_vaddr;	1140	*device_addr = ret_vaddr;
1118		1141
1119	if (is_userptr)	1142	if (is_userptr)
1120	free_phys_pg_pack(hdev, phys_pg_pack);	1143	rc = free_phys_pg_pack(hdev, phys_pg_pack);
1121		1144
1122	return 0;	1145	return rc;
1123		1146
1124	map_err:	1147	map_err:
1125	if (add_va_block(hdev, va_range, ret_vaddr,	1148	if (add_va_block(hdev, va_range, ret_vaddr,
@@ -1272,7 +1295,7 @@ static int unmap_device_va(struct hl_ctx ctx, struct hl_mem_in args, · 146 unmodified lines
1272	kfree(hnode);	1295	kfree(hnode);
1273		1296
1274	if (is_userptr) {	1297	if (is_userptr) {
1275	free_phys_pg_pack(hdev, phys_pg_pack);	1298	rc = free_phys_pg_pack(hdev, phys_pg_pack);
1276	dma_unmap_host_va(hdev, userptr);	1299	dma_unmap_host_va(hdev, userptr);
1277	}	1300	}
1278		1301
@@ -1305,9 +1328,15 @@ static int map_block(struct hl_device hdev, u64 address, u64 handle, · 26 unmodified lines
1305		1328
1306	static void hw_block_vm_close(struct vm_area_struct *vma)	1329	static void hw_block_vm_close(struct vm_area_struct *vma)
1307	{	1330	{
1308	struct hl_ctx ctx = (struct hl_ctx ) vma->vm_private_data;	1331	struct hl_vm_hw_block_list_node *lnode =
		1332	(struct hl_vm_hw_block_list_node *) vma->vm_private_data;
		1333	struct hl_ctx *ctx = lnode->ctx;
1309		1334
		1335	mutex_lock(&ctx->hw_block_list_lock);
		1336	list_del(&lnode->node);
		1337	mutex_unlock(&ctx->hw_block_list_lock);
1310	hl_ctx_put(ctx);	1338	hl_ctx_put(ctx);
		1339	kfree(lnode);
1311	vma->vm_private_data = NULL;	1340	vma->vm_private_data = NULL;
1312	}	1341	}
1313		1342
@@ -1325,7 +1354,9 @@ static const struct vm_operations_struct hw_block_vm_ops = { · 11 unmodified lines
1325	*/	1354	*/
1326	int hl_hw_block_mmap(struct hl_fpriv hpriv, struct vm_area_struct vma)	1355	int hl_hw_block_mmap(struct hl_fpriv hpriv, struct vm_area_struct vma)
1327	{	1356	{
		1357	struct hl_vm_hw_block_list_node *lnode;
1328	struct hl_device *hdev = hpriv->hdev;	1358	struct hl_device *hdev = hpriv->hdev;
		1359	struct hl_ctx *ctx = hpriv->ctx;
1329	u32 block_id, block_size;	1360	u32 block_id, block_size;
1330	int rc;	1361	int rc;
1331		1362
@@ -1351,17 +1382,31 @@ int hl_hw_block_mmap(struct hl_fpriv hpriv, struct vm_area_struct vma) · 19 unmodified lines
1351	return -EINVAL;	1382	return -EINVAL;
1352	}	1383	}
1353		1384
		1385	lnode = kzalloc(sizeof(*lnode), GFP_KERNEL);
		1386	if (!lnode)
		1387	return -ENOMEM;
		1388
1354	vma->vm_ops = &hw_block_vm_ops;	1389	vma->vm_ops = &hw_block_vm_ops;
1355	vma->vm_private_data = hpriv->ctx;	1390	vma->vm_private_data = lnode;
1356		1391
1357	hl_ctx_get(hdev, hpriv->ctx);	1392	hl_ctx_get(hdev, ctx);
1358		1393
1359	rc = hdev->asic_funcs->hw_block_mmap(hdev, vma, block_id, block_size);	1394	rc = hdev->asic_funcs->hw_block_mmap(hdev, vma, block_id, block_size);
1360	if (rc) {	1395	if (rc) {
1361	hl_ctx_put(hpriv->ctx);	1396	hl_ctx_put(ctx);
		1397	kfree(lnode);
1362	return rc;	1398	return rc;
1363	}	1399	}
1364		1400
		1401	lnode->ctx = ctx;
		1402	lnode->vaddr = vma->vm_start;
		1403	lnode->size = block_size;
		1404	lnode->id = block_id;
		1405
		1406	mutex_lock(&ctx->hw_block_list_lock);
		1407	list_add_tail(&lnode->node, &ctx->hw_block_mem_list);
		1408	mutex_unlock(&ctx->hw_block_list_lock);
		1409
1365	vma->vm_pgoff = block_id;	1410	vma->vm_pgoff = block_id;
1366		1411
1367	return 0;	1412	return 0;
@@ -1574,7 +1619,7 @@ static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size, · 206 unmodified lines
1574		1619
1575	rc = sg_alloc_table_from_pages(userptr->sgt,	1620	rc = sg_alloc_table_from_pages(userptr->sgt,
1576	userptr->pages,	1621	userptr->pages,
1577	npages, offset, size, GFP_ATOMIC);	1622	npages, offset, size, GFP_KERNEL);
1578	if (rc < 0) {	1623	if (rc < 0) {
1579	dev_err(hdev->dev, "failed to create SG table from pages\n");	1624	dev_err(hdev->dev, "failed to create SG table from pages\n");
1580	goto put_pages;	1625	goto put_pages;
@@ -1624,11 +1669,7 @@ int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size, · 43 unmodified lines
1624	return -EINVAL;	1669	return -EINVAL;
1625	}	1670	}
1626		1671
1627	/*	1672	userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_KERNEL);
1628	* This function can be called also from data path, hence use atomic
1629	* always as it is not a big allocation.
1630	*/
1631	userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_ATOMIC);
1632	if (!userptr->sgt)	1673	if (!userptr->sgt)
1633	return -ENOMEM;	1674	return -ENOMEM;
1634		1675
@@ -2122,3 +2163,38 @@ void hl_vm_fini(struct hl_device *hdev) · 487 unmodified lines
2122		2163
2123	vm->init_done = false;	2164	vm->init_done = false;
2124	}	2165	}
		2166
		2167	/**
		2168	* hl_hw_block_mem_init() - HW block memory initialization.
		2169	* @ctx: pointer to the habanalabs context structure.
		2170	*
		2171	* This function initializes the HW block virtual mapped addresses list and
		2172	* it's lock.
		2173	*/
		2174	void hl_hw_block_mem_init(struct hl_ctx *ctx)
		2175	{
		2176	mutex_init(&ctx->hw_block_list_lock);
		2177	INIT_LIST_HEAD(&ctx->hw_block_mem_list);
		2178	}
		2179
		2180	/**
		2181	* hl_hw_block_mem_fini() - HW block memory teardown.
		2182	* @ctx: pointer to the habanalabs context structure.
		2183	*
		2184	* This function clears the HW block virtual mapped addresses list and destroys
		2185	* it's lock.
		2186	*/
		2187	void hl_hw_block_mem_fini(struct hl_ctx *ctx)
		2188	{
		2189	struct hl_vm_hw_block_list_node lnode, tmp;
		2190
		2191	if (!list_empty(&ctx->hw_block_mem_list))
		2192	dev_crit(ctx->hdev->dev, "HW block mem list isn't empty\n");
		2193
		2194	list_for_each_entry_safe(lnode, tmp, &ctx->hw_block_mem_list, node) {
		2195	list_del(&lnode->node);
		2196	kfree(lnode);
		2197	}
		2198
		2199	mutex_destroy(&ctx->hw_block_list_lock);
		2200	}

drivers/misc/habanalabs/common/mmu/mmu.c+3−0

 	struct hl_mmu_hop_info hops;
 	int rc;
+	memset(&hops, 0, sizeof(hops));
 	rc = hl_mmu_get_tlb_info(ctx, virt_addr, &hops);
 	if (rc)
 		return rc;
 	switch (hdev->asic_type) {
 	case ASIC_GOYA:
 	case ASIC_GAUDI:
+	case ASIC_GAUDI_SEC:
 		hl_mmu_v1_set_funcs(hdev, &hdev->mmu_func[MMU_DR_PGT]);
 		break;
 	default:

drivers/misc/habanalabs/common/pci/pci.c+52−0

@@ -85,6 +85,58 @@ static void hl_pci_bars_unmap(struct hl_device *hdev) · 84 unmodified lines
85	pci_release_regions(pdev);	85	pci_release_regions(pdev);
86	}	86	}
87		87
		88	int hl_pci_elbi_read(struct hl_device hdev, u64 addr, u32 data)
		89	{
		90	struct pci_dev *pdev = hdev->pdev;
		91	ktime_t timeout;
		92	u64 msec;
		93	u32 val;
		94
		95	if (hdev->pldm)
		96	msec = HL_PLDM_PCI_ELBI_TIMEOUT_MSEC;
		97	else
		98	msec = HL_PCI_ELBI_TIMEOUT_MSEC;
		99
		100	/* Clear previous status */
		101	pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, 0);
		102
		103	pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_ADDR, (u32) addr);
		104	pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_CTRL, 0);
		105
		106	timeout = ktime_add_ms(ktime_get(), msec);
		107	for (;;) {
		108	pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, &val);
		109	if (val & PCI_CONFIG_ELBI_STS_MASK)
		110	break;
		111	if (ktime_compare(ktime_get(), timeout) > 0) {
		112	pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_STS,
		113	&val);
		114	break;
		115	}
		116
		117	usleep_range(300, 500);
		118	}
		119
		120	if ((val & PCI_CONFIG_ELBI_STS_MASK) == PCI_CONFIG_ELBI_STS_DONE) {
		121	pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_DATA, data);
		122
		123	return 0;
		124	}
		125
		126	if (val & PCI_CONFIG_ELBI_STS_ERR) {
		127	dev_err(hdev->dev, "Error reading from ELBI\n");
		128	return -EIO;
		129	}
		130
		131	if (!(val & PCI_CONFIG_ELBI_STS_MASK)) {
		132	dev_err(hdev->dev, "ELBI read didn't finish in time\n");
		133	return -EIO;
		134	}
		135
		136	dev_err(hdev->dev, "ELBI read has undefined bits in status\n");
		137	return -EIO;
		138	}
		139
88	/**	140	/**
89	* hl_pci_elbi_write() - Write through the ELBI interface.	141	* hl_pci_elbi_write() - Write through the ELBI interface.
90	* @hdev: Pointer to hl_device structure.	142	* @hdev: Pointer to hl_device structure.

drivers/misc/habanalabs/common/sysfs.c+24−9

 #include <linux/pci.h>
-long hl_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr)
+long hl_get_frequency(struct hl_device *hdev, enum pll_index pll_index,
+								bool curr)
 {
 	struct cpucp_packet pkt;
+	u32 used_pll_idx;
 	u64 result;
 	int rc;
+	rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
+	if (rc)
+		return rc;
 	memset(&pkt, 0, sizeof(pkt));
 	if (curr)
 	else
 		pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_GET <<
 						CPUCP_PKT_CTL_OPCODE_SHIFT);
-	pkt.pll_index = cpu_to_le32(pll_index);
+	pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 						0, &result);
 	if (rc) {
 		dev_err(hdev->dev,
 			"Failed to get frequency of PLL %d, error %d\n",
-			pll_index, rc);
+			used_pll_idx, rc);
 		return rc;
 	}
 	return (long) result;
 }
-void hl_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq)
+void hl_set_frequency(struct hl_device *hdev, enum pll_index pll_index,
+								u64 freq)
 {
 	struct cpucp_packet pkt;
+	u32 used_pll_idx;
 	int rc;
+	rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
+	if (rc)
+		return;
 	memset(&pkt, 0, sizeof(pkt));
 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_SET <<
 					CPUCP_PKT_CTL_OPCODE_SHIFT);
-	pkt.pll_index = cpu_to_le32(pll_index);
+	pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
 	pkt.value = cpu_to_le64(freq);
 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 	if (rc)
 		dev_err(hdev->dev,
 			"Failed to set frequency to PLL %d, error %d\n",
-			pll_index, rc);
+			used_pll_idx, rc);
 }
 u64 hl_get_max_power(struct hl_device *hdev)
 	dev_warn(hdev->dev, "Soft-Reset requested through sysfs\n");
-	hl_device_reset(hdev, false, false);
+	hl_device_reset(hdev, 0);
 out:
 	return count;
 	dev_warn(hdev->dev, "Hard-Reset requested through sysfs\n");
-	hl_device_reset(hdev, true, false);
+	hl_device_reset(hdev, HL_RESET_HARD);
 out:
 	return count;
 	case ASIC_GAUDI:
 		str = "GAUDI";
 		break;
+	case ASIC_GAUDI_SEC:
+		str = "GAUDI SEC";
+		break;
 	default:
 		dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
 				hdev->asic_type);
 			struct bin_attribute *attr, char *buf, loff_t offset,
 			size_t max_size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct hl_device *hdev = dev_get_drvdata(dev);
 	char *data;
 	int rc;

drivers/misc/habanalabs/gaudi/gaudi.c+314−43

@@ -105,6 +105,36 @@ · 104 unmodified lines
105		105
106	#define GAUDI_PLL_MAX 10	106	#define GAUDI_PLL_MAX 10
107		107
		108	/*
		109	* this enum kept here for compatibility with old FW (in which each asic has
		110	* unique PLL numbering
		111	*/
		112	enum gaudi_pll_index {
		113	GAUDI_CPU_PLL = 0,
		114	GAUDI_PCI_PLL,
		115	GAUDI_SRAM_PLL,
		116	GAUDI_HBM_PLL,
		117	GAUDI_NIC_PLL,
		118	GAUDI_DMA_PLL,
		119	GAUDI_MESH_PLL,
		120	GAUDI_MME_PLL,
		121	GAUDI_TPC_PLL,
		122	GAUDI_IF_PLL,
		123	};
		124
		125	static enum pll_index gaudi_pll_map[PLL_MAX] = {
		126	[CPU_PLL] = GAUDI_CPU_PLL,
		127	[PCI_PLL] = GAUDI_PCI_PLL,
		128	[SRAM_PLL] = GAUDI_SRAM_PLL,
		129	[HBM_PLL] = GAUDI_HBM_PLL,
		130	[NIC_PLL] = GAUDI_NIC_PLL,
		131	[DMA_PLL] = GAUDI_DMA_PLL,
		132	[MESH_PLL] = GAUDI_MESH_PLL,
		133	[MME_PLL] = GAUDI_MME_PLL,
		134	[TPC_PLL] = GAUDI_TPC_PLL,
		135	[IF_PLL] = GAUDI_IF_PLL,
		136	};
		137
108	static const char gaudi_irq_name[GAUDI_MSI_ENTRIES][GAUDI_MAX_STRING_LEN] = {	138	static const char gaudi_irq_name[GAUDI_MSI_ENTRIES][GAUDI_MAX_STRING_LEN] = {
109	"gaudi cq 0_0", "gaudi cq 0_1", "gaudi cq 0_2", "gaudi cq 0_3",	139	"gaudi cq 0_0", "gaudi cq 0_1", "gaudi cq 0_2", "gaudi cq 0_3",
110	"gaudi cq 1_0", "gaudi cq 1_1", "gaudi cq 1_2", "gaudi cq 1_3",	140	"gaudi cq 1_0", "gaudi cq 1_1", "gaudi cq 1_2", "gaudi cq 1_3",
@@ -396,6 +426,19 @@ get_collective_mode(struct hl_device *hdev, u32 queue_id) · 285 unmodified lines
396	return HL_COLLECTIVE_NOT_SUPPORTED;	426	return HL_COLLECTIVE_NOT_SUPPORTED;
397	}	427	}
398		428
		429	static inline void set_default_power_values(struct hl_device *hdev)
		430	{
		431	struct asic_fixed_properties *prop = &hdev->asic_prop;
		432
		433	if (hdev->card_type == cpucp_card_type_pmc) {
		434	prop->max_power_default = MAX_POWER_DEFAULT_PMC;
		435	prop->dc_power_default = DC_POWER_DEFAULT_PMC;
		436	} else {
		437	prop->max_power_default = MAX_POWER_DEFAULT_PCI;
		438	prop->dc_power_default = DC_POWER_DEFAULT_PCI;
		439	}
		440	}
		441
399	static int gaudi_get_fixed_properties(struct hl_device *hdev)	442	static int gaudi_get_fixed_properties(struct hl_device *hdev)
400	{	443	{
401	struct asic_fixed_properties *prop = &hdev->asic_prop;	444	struct asic_fixed_properties *prop = &hdev->asic_prop;
@@ -507,7 +550,7 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev) · 105 unmodified lines
507	prop->num_of_events = GAUDI_EVENT_SIZE;	550	prop->num_of_events = GAUDI_EVENT_SIZE;
508	prop->tpc_enabled_mask = TPC_ENABLED_MASK;	551	prop->tpc_enabled_mask = TPC_ENABLED_MASK;
509		552
510	prop->max_power_default = MAX_POWER_DEFAULT_PCI;	553	set_default_power_values(hdev);
511		554
512	prop->cb_pool_cb_cnt = GAUDI_CB_POOL_CB_CNT;	555	prop->cb_pool_cb_cnt = GAUDI_CB_POOL_CB_CNT;
513	prop->cb_pool_cb_size = GAUDI_CB_POOL_CB_SIZE;	556	prop->cb_pool_cb_size = GAUDI_CB_POOL_CB_SIZE;
@@ -532,8 +575,6 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev) · 18 unmodified lines
532	for (i = 0 ; i < HL_MAX_DCORES ; i++)	575	for (i = 0 ; i < HL_MAX_DCORES ; i++)
533	prop->first_available_cq[i] = USHRT_MAX;	576	prop->first_available_cq[i] = USHRT_MAX;
534		577
535	/* disable fw security for now, set it in a later stage */
536	prop->fw_security_disabled = true;
537	prop->fw_security_status_valid = false;	578	prop->fw_security_status_valid = false;
538	prop->hard_reset_done_by_fw = false;	579	prop->hard_reset_done_by_fw = false;
539		580
@@ -588,6 +629,11 @@ static int gaudi_init_iatu(struct hl_device *hdev) · 48 unmodified lines
588	struct hl_outbound_pci_region outbound_region;	629	struct hl_outbound_pci_region outbound_region;
589	int rc;	630	int rc;
590		631
		632	if (hdev->asic_prop.iatu_done_by_fw) {
		633	hdev->asic_funcs->set_dma_mask_from_fw(hdev);
		634	return 0;
		635	}
		636
591	/* Inbound Region 0 - Bar 0 - Point to SRAM + CFG */	637	/* Inbound Region 0 - Bar 0 - Point to SRAM + CFG */
592	inbound_region.mode = PCI_BAR_MATCH_MODE;	638	inbound_region.mode = PCI_BAR_MATCH_MODE;
593	inbound_region.bar = SRAM_BAR_ID;	639	inbound_region.bar = SRAM_BAR_ID;
@@ -632,6 +678,7 @@ static int gaudi_early_init(struct hl_device *hdev) · 38 unmodified lines
632	{	678	{
633	struct asic_fixed_properties *prop = &hdev->asic_prop;	679	struct asic_fixed_properties *prop = &hdev->asic_prop;
634	struct pci_dev *pdev = hdev->pdev;	680	struct pci_dev *pdev = hdev->pdev;
		681	u32 fw_boot_status;
635	int rc;	682	int rc;
636		683
637	rc = gaudi_get_fixed_properties(hdev);	684	rc = gaudi_get_fixed_properties(hdev);
@@ -665,6 +712,23 @@ static int gaudi_early_init(struct hl_device *hdev) · 27 unmodified lines
665		712
666	prop->dram_pci_bar_size = pci_resource_len(pdev, HBM_BAR_ID);	713	prop->dram_pci_bar_size = pci_resource_len(pdev, HBM_BAR_ID);
667		714
		715	/* If FW security is enabled at this point it means no access to ELBI */
		716	if (!hdev->asic_prop.fw_security_disabled) {
		717	hdev->asic_prop.iatu_done_by_fw = true;
		718	goto pci_init;
		719	}
		720
		721	rc = hl_pci_elbi_read(hdev, CFG_BASE + mmCPU_BOOT_DEV_STS0,
		722	&fw_boot_status);
		723	if (rc)
		724	goto free_queue_props;
		725
		726	/* Check whether FW is configuring iATU */
		727	if ((fw_boot_status & CPU_BOOT_DEV_STS0_ENABLED) &&
		728	(fw_boot_status & CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN))
		729	hdev->asic_prop.iatu_done_by_fw = true;
		730
		731	pci_init:
668	rc = hl_pci_init(hdev);	732	rc = hl_pci_init(hdev);
669	if (rc)	733	if (rc)
670	goto free_queue_props;	734	goto free_queue_props;
@@ -1588,6 +1652,9 @@ static int gaudi_sw_init(struct hl_device *hdev) · 917 unmodified lines
1588		1652
1589	hdev->asic_specific = gaudi;	1653	hdev->asic_specific = gaudi;
1590		1654
		1655	/* store legacy PLL map */
		1656	hdev->legacy_pll_map = gaudi_pll_map;
		1657
1591	/* Create DMA pool for small allocations */	1658	/* Create DMA pool for small allocations */
1592	hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),	1659	hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),
1593	&hdev->pdev->dev, GAUDI_DMA_POOL_BLK_SIZE, 8, 0);	1660	&hdev->pdev->dev, GAUDI_DMA_POOL_BLK_SIZE, 8, 0);
@@ -1766,8 +1833,7 @@ static int gaudi_enable_msi(struct hl_device *hdev) · 172 unmodified lines
1766	if (gaudi->hw_cap_initialized & HW_CAP_MSI)	1833	if (gaudi->hw_cap_initialized & HW_CAP_MSI)
1767	return 0;	1834	return 0;
1768		1835
1769	rc = pci_alloc_irq_vectors(hdev->pdev, 1, GAUDI_MSI_ENTRIES,	1836	rc = pci_alloc_irq_vectors(hdev->pdev, 1, 1, PCI_IRQ_MSI);
1770	PCI_IRQ_MSI);
1771	if (rc < 0) {	1837	if (rc < 0) {
1772	dev_err(hdev->dev, "MSI: Failed to enable support %d\n", rc);	1838	dev_err(hdev->dev, "MSI: Failed to enable support %d\n", rc);
1773	return rc;	1839	return rc;
@@ -3701,7 +3767,7 @@ static int gaudi_init_cpu(struct hl_device *hdev) · 1927 unmodified lines
3701	struct gaudi_device *gaudi = hdev->asic_specific;	3767	struct gaudi_device *gaudi = hdev->asic_specific;
3702	int rc;	3768	int rc;
3703		3769
3704	if (!hdev->cpu_enable)	3770	if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
3705	return 0;	3771	return 0;
3706		3772
3707	if (gaudi->hw_cap_initialized & HW_CAP_CPU)	3773	if (gaudi->hw_cap_initialized & HW_CAP_CPU)
@@ -4873,7 +4939,7 @@ static int gaudi_pin_memory_before_cs(struct hl_device *hdev, · 1165 unmodified lines
4873	parser->job_userptr_list, &userptr))	4939	parser->job_userptr_list, &userptr))
4874	goto already_pinned;	4940	goto already_pinned;
4875		4941
4876	userptr = kzalloc(sizeof(*userptr), GFP_ATOMIC);	4942	userptr = kzalloc(sizeof(*userptr), GFP_KERNEL);
4877	if (!userptr)	4943	if (!userptr)
4878	return -ENOMEM;	4944	return -ENOMEM;
4879		4945
@@ -5684,18 +5750,26 @@ release_cb: · 804 unmodified lines
5684	static int gaudi_schedule_register_memset(struct hl_device *hdev,	5750	static int gaudi_schedule_register_memset(struct hl_device *hdev,
5685	u32 hw_queue_id, u64 reg_base, u32 num_regs, u32 val)	5751	u32 hw_queue_id, u64 reg_base, u32 num_regs, u32 val)
5686	{	5752	{
5687	struct hl_ctx *ctx = hdev->compute_ctx;	5753	struct hl_ctx *ctx;
5688	struct hl_pending_cb *pending_cb;	5754	struct hl_pending_cb *pending_cb;
5689	struct packet_msg_long *pkt;	5755	struct packet_msg_long *pkt;
5690	u32 cb_size, ctl;	5756	u32 cb_size, ctl;
5691	struct hl_cb *cb;	5757	struct hl_cb *cb;
5692	int i;	5758	int i, rc;
		5759
		5760	mutex_lock(&hdev->fpriv_list_lock);
		5761	ctx = hdev->compute_ctx;
5693		5762
5694	/* If no compute context available or context is going down	5763	/* If no compute context available or context is going down
5695	* memset registers directly	5764	* memset registers directly
5696	*/	5765	*/
5697	if (!ctx \|\| kref_read(&ctx->refcount) == 0)	5766	if (!ctx \|\| kref_read(&ctx->refcount) == 0) {
5698	return gaudi_memset_registers(hdev, reg_base, num_regs, val);	5767	rc = gaudi_memset_registers(hdev, reg_base, num_regs, val);
		5768	mutex_unlock(&hdev->fpriv_list_lock);
		5769	return rc;
		5770	}
		5771
		5772	mutex_unlock(&hdev->fpriv_list_lock);
5699		5773
5700	cb_size = (sizeof(pkt) num_regs) +	5774	cb_size = (sizeof(pkt) num_regs) +
5701	sizeof(struct packet_msg_prot) * 2;	5775	sizeof(struct packet_msg_prot) * 2;
@@ -5911,13 +5985,16 @@ static void gaudi_restore_phase_topology(struct hl_device *hdev) · 209 unmodified lines
5911		5985
5912	}	5986	}
5913		5987
5914	static int gaudi_debugfs_read32(struct hl_device hdev, u64 addr, u32 val)	5988	static int gaudi_debugfs_read32(struct hl_device *hdev, u64 addr,
		5989	bool user_address, u32 *val)
5915	{	5990	{
5916	struct asic_fixed_properties *prop = &hdev->asic_prop;	5991	struct asic_fixed_properties *prop = &hdev->asic_prop;
5917	struct gaudi_device *gaudi = hdev->asic_specific;	5992	struct gaudi_device *gaudi = hdev->asic_specific;
5918	u64 hbm_bar_addr;	5993	u64 hbm_bar_addr, host_phys_end;
5919	int rc = 0;	5994	int rc = 0;
5920		5995
		5996	host_phys_end = HOST_PHYS_BASE + HOST_PHYS_SIZE;
		5997
5921	if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {	5998	if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
5922		5999
5923	if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&	6000	if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&
@@ -5949,6 +6026,9 @@ static int gaudi_debugfs_read32(struct hl_device hdev, u64 addr, u32 val) · 25 unmodified lines
5949	}	6026	}
5950	if (hbm_bar_addr == U64_MAX)	6027	if (hbm_bar_addr == U64_MAX)
5951	rc = -EIO;	6028	rc = -EIO;
		6029	} else if (addr >= HOST_PHYS_BASE && addr < host_phys_end &&
		6030	user_address && !iommu_present(&pci_bus_type)) {
		6031	val = (u32 *) phys_to_virt(addr - HOST_PHYS_BASE);
5952	} else {	6032	} else {
5953	rc = -EFAULT;	6033	rc = -EFAULT;
5954	}	6034	}
@@ -5956,13 +6036,16 @@ static int gaudi_debugfs_read32(struct hl_device hdev, u64 addr, u32 val) · 1 unmodified line
5956	return rc;	6036	return rc;
5957	}	6037	}
5958		6038
5959	static int gaudi_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val)	6039	static int gaudi_debugfs_write32(struct hl_device *hdev, u64 addr,
		6040	bool user_address, u32 val)
5960	{	6041	{
5961	struct asic_fixed_properties *prop = &hdev->asic_prop;	6042	struct asic_fixed_properties *prop = &hdev->asic_prop;
5962	struct gaudi_device *gaudi = hdev->asic_specific;	6043	struct gaudi_device *gaudi = hdev->asic_specific;
5963	u64 hbm_bar_addr;	6044	u64 hbm_bar_addr, host_phys_end;
5964	int rc = 0;	6045	int rc = 0;
5965		6046
		6047	host_phys_end = HOST_PHYS_BASE + HOST_PHYS_SIZE;
		6048
5966	if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {	6049	if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
5967		6050
5968	if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&	6051	if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&
@@ -5994,6 +6077,9 @@ static int gaudi_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val) · 25 unmodified lines
5994	}	6077	}
5995	if (hbm_bar_addr == U64_MAX)	6078	if (hbm_bar_addr == U64_MAX)
5996	rc = -EIO;	6079	rc = -EIO;
		6080	} else if (addr >= HOST_PHYS_BASE && addr < host_phys_end &&
		6081	user_address && !iommu_present(&pci_bus_type)) {
		6082	(u32 ) phys_to_virt(addr - HOST_PHYS_BASE) = val;
5997	} else {	6083	} else {
5998	rc = -EFAULT;	6084	rc = -EFAULT;
5999	}	6085	}
@@ -6001,13 +6087,16 @@ static int gaudi_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val) · 1 unmodified line
6001	return rc;	6087	return rc;
6002	}	6088	}
6003		6089
6004	static int gaudi_debugfs_read64(struct hl_device hdev, u64 addr, u64 val)	6090	static int gaudi_debugfs_read64(struct hl_device *hdev, u64 addr,
		6091	bool user_address, u64 *val)
6005	{	6092	{
6006	struct asic_fixed_properties *prop = &hdev->asic_prop;	6093	struct asic_fixed_properties *prop = &hdev->asic_prop;
6007	struct gaudi_device *gaudi = hdev->asic_specific;	6094	struct gaudi_device *gaudi = hdev->asic_specific;
6008	u64 hbm_bar_addr;	6095	u64 hbm_bar_addr, host_phys_end;
6009	int rc = 0;	6096	int rc = 0;
6010		6097
		6098	host_phys_end = HOST_PHYS_BASE + HOST_PHYS_SIZE;
		6099
6011	if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) {	6100	if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) {
6012		6101
6013	if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&	6102	if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&
@@ -6043,6 +6132,9 @@ static int gaudi_debugfs_read64(struct hl_device hdev, u64 addr, u64 val) · 29 unmodified lines
6043	}	6132	}
6044	if (hbm_bar_addr == U64_MAX)	6133	if (hbm_bar_addr == U64_MAX)
6045	rc = -EIO;	6134	rc = -EIO;
		6135	} else if (addr >= HOST_PHYS_BASE && addr < host_phys_end &&
		6136	user_address && !iommu_present(&pci_bus_type)) {
		6137	val = (u64 *) phys_to_virt(addr - HOST_PHYS_BASE);
6046	} else {	6138	} else {
6047	rc = -EFAULT;	6139	rc = -EFAULT;
6048	}	6140	}
@@ -6050,13 +6142,16 @@ static int gaudi_debugfs_read64(struct hl_device hdev, u64 addr, u64 val) · 1 unmodified line
6050	return rc;	6142	return rc;
6051	}	6143	}
6052		6144
6053	static int gaudi_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val)	6145	static int gaudi_debugfs_write64(struct hl_device *hdev, u64 addr,
		6146	bool user_address, u64 val)
6054	{	6147	{
6055	struct asic_fixed_properties *prop = &hdev->asic_prop;	6148	struct asic_fixed_properties *prop = &hdev->asic_prop;
6056	struct gaudi_device *gaudi = hdev->asic_specific;	6149	struct gaudi_device *gaudi = hdev->asic_specific;
6057	u64 hbm_bar_addr;	6150	u64 hbm_bar_addr, host_phys_end;
6058	int rc = 0;	6151	int rc = 0;
6059		6152
		6153	host_phys_end = HOST_PHYS_BASE + HOST_PHYS_SIZE;
		6154
6060	if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) {	6155	if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) {
6061		6156
6062	if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&	6157	if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&
@@ -6091,6 +6186,9 @@ static int gaudi_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val) · 28 unmodified lines
6091	}	6186	}
6092	if (hbm_bar_addr == U64_MAX)	6187	if (hbm_bar_addr == U64_MAX)
6093	rc = -EIO;	6188	rc = -EIO;
		6189	} else if (addr >= HOST_PHYS_BASE && addr < host_phys_end &&
		6190	user_address && !iommu_present(&pci_bus_type)) {
		6191	(u64 ) phys_to_virt(addr - HOST_PHYS_BASE) = val;
6094	} else {	6192	} else {
6095	rc = -EFAULT;	6193	rc = -EFAULT;
6096	}	6194	}
@@ -6098,6 +6196,164 @@ static int gaudi_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val) · 1 unmodified line
6098	return rc;	6196	return rc;
6099	}	6197	}
6100		6198
		6199	static int gaudi_dma_core_transfer(struct hl_device *hdev, int dma_id, u64 addr,
		6200	u32 size_to_dma, dma_addr_t dma_addr)
		6201	{
		6202	u32 err_cause, val;
		6203	u64 dma_offset;
		6204	int rc;
		6205
		6206	dma_offset = dma_id * DMA_CORE_OFFSET;
		6207
		6208	WREG32(mmDMA0_CORE_SRC_BASE_LO + dma_offset, lower_32_bits(addr));
		6209	WREG32(mmDMA0_CORE_SRC_BASE_HI + dma_offset, upper_32_bits(addr));
		6210	WREG32(mmDMA0_CORE_DST_BASE_LO + dma_offset, lower_32_bits(dma_addr));
		6211	WREG32(mmDMA0_CORE_DST_BASE_HI + dma_offset, upper_32_bits(dma_addr));
		6212	WREG32(mmDMA0_CORE_DST_TSIZE_0 + dma_offset, size_to_dma);
		6213	WREG32(mmDMA0_CORE_COMMIT + dma_offset,
		6214	(1 << DMA0_CORE_COMMIT_LIN_SHIFT));
		6215
		6216	rc = hl_poll_timeout(
		6217	hdev,
		6218	mmDMA0_CORE_STS0 + dma_offset,
		6219	val,
		6220	((val & DMA0_CORE_STS0_BUSY_MASK) == 0),
		6221	0,
		6222	1000000);
		6223
		6224	if (rc) {
		6225	dev_err(hdev->dev,
		6226	"DMA %d timed-out during reading of 0x%llx\n",
		6227	dma_id, addr);
		6228	return -EIO;
		6229	}
		6230
		6231	/* Verify DMA is OK */
		6232	err_cause = RREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset);
		6233	if (err_cause) {
		6234	dev_err(hdev->dev, "DMA Failed, cause 0x%x\n", err_cause);
		6235	dev_dbg(hdev->dev,
		6236	"Clearing DMA0 engine from errors (cause 0x%x)\n",
		6237	err_cause);
		6238	WREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset, err_cause);
		6239
		6240	return -EIO;
		6241	}
		6242
		6243	return 0;
		6244	}
		6245
		6246	static int gaudi_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size,
		6247	void *blob_addr)
		6248	{
		6249	u32 dma_core_sts0, err_cause, cfg1, size_left, pos, size_to_dma;
		6250	struct gaudi_device *gaudi = hdev->asic_specific;
		6251	u64 dma_offset, qm_offset;
		6252	dma_addr_t dma_addr;
		6253	void *kernel_addr;
		6254	bool is_eng_idle;
		6255	int rc, dma_id;
		6256
		6257	kernel_addr = hdev->asic_funcs->asic_dma_alloc_coherent(
		6258	hdev, SZ_2M,
		6259	&dma_addr,
		6260	GFP_KERNEL \| __GFP_ZERO);
		6261
		6262	if (!kernel_addr)
		6263	return -ENOMEM;
		6264
		6265	mutex_lock(&gaudi->clk_gate_mutex);
		6266
		6267	hdev->asic_funcs->disable_clock_gating(hdev);
		6268
		6269	hdev->asic_funcs->hw_queues_lock(hdev);
		6270
		6271	dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_1];
		6272	dma_offset = dma_id * DMA_CORE_OFFSET;
		6273	qm_offset = dma_id * DMA_QMAN_OFFSET;
		6274	dma_core_sts0 = RREG32(mmDMA0_CORE_STS0 + dma_offset);
		6275	is_eng_idle = IS_DMA_IDLE(dma_core_sts0);
		6276
		6277	if (!is_eng_idle) {
		6278	dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_2];
		6279	dma_offset = dma_id * DMA_CORE_OFFSET;
		6280	qm_offset = dma_id * DMA_QMAN_OFFSET;
		6281	dma_core_sts0 = RREG32(mmDMA0_CORE_STS0 + dma_offset);
		6282	is_eng_idle = IS_DMA_IDLE(dma_core_sts0);
		6283
		6284	if (!is_eng_idle) {
		6285	dev_err_ratelimited(hdev->dev,
		6286	"Can't read via DMA because it is BUSY\n");
		6287	rc = -EAGAIN;
		6288	goto out;
		6289	}
		6290	}
		6291
		6292	cfg1 = RREG32(mmDMA0_QM_GLBL_CFG1 + qm_offset);
		6293	WREG32(mmDMA0_QM_GLBL_CFG1 + qm_offset,
		6294	0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
		6295
		6296	/* TODO: remove this by mapping the DMA temporary buffer to the MMU
		6297	* using the compute ctx ASID, if exists. If not, use the kernel ctx
		6298	* ASID
		6299	*/
		6300	WREG32_OR(mmDMA0_CORE_PROT + dma_offset, BIT(DMA0_CORE_PROT_VAL_SHIFT));
		6301
		6302	/* Verify DMA is OK */
		6303	err_cause = RREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset);
		6304	if (err_cause) {
		6305	dev_dbg(hdev->dev,
		6306	"Clearing DMA0 engine from errors (cause 0x%x)\n",
		6307	err_cause);
		6308	WREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset, err_cause);
		6309	}
		6310
		6311	pos = 0;
		6312	size_left = size;
		6313	size_to_dma = SZ_2M;
		6314
		6315	while (size_left > 0) {
		6316
		6317	if (size_left < SZ_2M)
		6318	size_to_dma = size_left;
		6319
		6320	rc = gaudi_dma_core_transfer(hdev, dma_id, addr, size_to_dma,
		6321	dma_addr);
		6322	if (rc)
		6323	break;
		6324
		6325	memcpy(blob_addr + pos, kernel_addr, size_to_dma);
		6326
		6327	if (size_left <= SZ_2M)
		6328	break;
		6329
		6330	pos += SZ_2M;
		6331	addr += SZ_2M;
		6332	size_left -= SZ_2M;
		6333	}
		6334
		6335	/* TODO: remove this by mapping the DMA temporary buffer to the MMU
		6336	* using the compute ctx ASID, if exists. If not, use the kernel ctx
		6337	* ASID
		6338	*/
		6339	WREG32_AND(mmDMA0_CORE_PROT + dma_offset,
		6340	~BIT(DMA0_CORE_PROT_VAL_SHIFT));
		6341
		6342	WREG32(mmDMA0_QM_GLBL_CFG1 + qm_offset, cfg1);
		6343
		6344	out:
		6345	hdev->asic_funcs->hw_queues_unlock(hdev);
		6346
		6347	hdev->asic_funcs->set_clock_gating(hdev);
		6348
		6349	mutex_unlock(&gaudi->clk_gate_mutex);
		6350
		6351	hdev->asic_funcs->asic_dma_free_coherent(hdev, SZ_2M, kernel_addr,
		6352	dma_addr);
		6353
		6354	return rc;
		6355	}
		6356
6101	static u64 gaudi_read_pte(struct hl_device *hdev, u64 addr)	6357	static u64 gaudi_read_pte(struct hl_device *hdev, u64 addr)
6102	{	6358	{
6103	struct gaudi_device *gaudi = hdev->asic_specific;	6359	struct gaudi_device *gaudi = hdev->asic_specific;
@@ -6851,7 +7107,8 @@ static void gaudi_handle_qman_err_generic(struct hl_device *hdev, · 747 unmodified lines
6851	}	7107	}
6852		7108
6853	/* Write 1 clear errors */	7109	/* Write 1 clear errors */
6854	WREG32(glbl_sts_addr + 4 * i, glbl_sts_clr_val);	7110	if (!hdev->stop_on_err)
		7111	WREG32(glbl_sts_addr + 4 * i, glbl_sts_clr_val);
6855	}	7112	}
6856		7113
6857	arb_err_val = RREG32(arb_err_addr);	7114	arb_err_val = RREG32(arb_err_addr);
@@ -7097,6 +7354,15 @@ static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type, · 239 unmodified lines
7097	}	7354	}
7098	}	7355	}
7099		7356
		7357	static void gaudi_print_out_of_sync_info(struct hl_device *hdev,
		7358	struct cpucp_pkt_sync_err *sync_err)
		7359	{
		7360	struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ];
		7361
		7362	dev_err(hdev->dev, "Out of sync with FW, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%u\n",
		7363	sync_err->pi, sync_err->ci, q->pi, atomic_read(&q->ci));
		7364	}
		7365
7100	static int gaudi_soft_reset_late_init(struct hl_device *hdev)	7366	static int gaudi_soft_reset_late_init(struct hl_device *hdev)
7101	{	7367	{
7102	struct gaudi_device *gaudi = hdev->asic_specific;	7368	struct gaudi_device *gaudi = hdev->asic_specific;
@@ -7371,18 +7637,14 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 268 unmodified lines
7371	case GAUDI_EVENT_MMU_DERR:	7637	case GAUDI_EVENT_MMU_DERR:
7372	gaudi_print_irq_info(hdev, event_type, true);	7638	gaudi_print_irq_info(hdev, event_type, true);
7373	gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);	7639	gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);
7374	if (hdev->hard_reset_on_fw_events)	7640	goto reset_device;
7375	hl_device_reset(hdev, true, false);
7376	break;
7377		7641
7378	case GAUDI_EVENT_GIC500:	7642	case GAUDI_EVENT_GIC500:
7379	case GAUDI_EVENT_AXI_ECC:	7643	case GAUDI_EVENT_AXI_ECC:
7380	case GAUDI_EVENT_L2_RAM_ECC:	7644	case GAUDI_EVENT_L2_RAM_ECC:
7381	case GAUDI_EVENT_PLL0 ... GAUDI_EVENT_PLL17:	7645	case GAUDI_EVENT_PLL0 ... GAUDI_EVENT_PLL17:
7382	gaudi_print_irq_info(hdev, event_type, false);	7646	gaudi_print_irq_info(hdev, event_type, false);
7383	if (hdev->hard_reset_on_fw_events)	7647	goto reset_device;
7384	hl_device_reset(hdev, true, false);
7385	break;
7386		7648
7387	case GAUDI_EVENT_HBM0_SPI_0:	7649	case GAUDI_EVENT_HBM0_SPI_0:
7388	case GAUDI_EVENT_HBM1_SPI_0:	7650	case GAUDI_EVENT_HBM1_SPI_0:
@@ -7392,9 +7654,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 3 unmodified lines
7392	gaudi_hbm_read_interrupts(hdev,	7654	gaudi_hbm_read_interrupts(hdev,
7393	gaudi_hbm_event_to_dev(event_type),	7655	gaudi_hbm_event_to_dev(event_type),
7394	&eq_entry->hbm_ecc_data);	7656	&eq_entry->hbm_ecc_data);
7395	if (hdev->hard_reset_on_fw_events)	7657	goto reset_device;
7396	hl_device_reset(hdev, true, false);
7397	break;
7398		7658
7399	case GAUDI_EVENT_HBM0_SPI_1:	7659	case GAUDI_EVENT_HBM0_SPI_1:
7400	case GAUDI_EVENT_HBM1_SPI_1:	7660	case GAUDI_EVENT_HBM1_SPI_1:
@@ -7423,8 +7683,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 22 unmodified lines
7423	dev_err(hdev->dev, "hard reset required due to %s\n",	7683	dev_err(hdev->dev, "hard reset required due to %s\n",
7424	gaudi_irq_map_table[event_type].name);	7684	gaudi_irq_map_table[event_type].name);
7425		7685
7426	if (hdev->hard_reset_on_fw_events)	7686	goto reset_device;
7427	hl_device_reset(hdev, true, false);
7428	} else {	7687	} else {
7429	hl_fw_unmask_irq(hdev, event_type);	7688	hl_fw_unmask_irq(hdev, event_type);
7430	}	7689	}
@@ -7446,8 +7705,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 15 unmodified lines
7446	dev_err(hdev->dev, "hard reset required due to %s\n",	7705	dev_err(hdev->dev, "hard reset required due to %s\n",
7447	gaudi_irq_map_table[event_type].name);	7706	gaudi_irq_map_table[event_type].name);
7448		7707
7449	if (hdev->hard_reset_on_fw_events)	7708	goto reset_device;
7450	hl_device_reset(hdev, true, false);
7451	} else {	7709	} else {
7452	hl_fw_unmask_irq(hdev, event_type);	7710	hl_fw_unmask_irq(hdev, event_type);
7453	}	7711	}
@@ -7516,9 +7774,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 62 unmodified lines
7516		7774
7517	case GAUDI_EVENT_RAZWI_OR_ADC_SW:	7775	case GAUDI_EVENT_RAZWI_OR_ADC_SW:
7518	gaudi_print_irq_info(hdev, event_type, true);	7776	gaudi_print_irq_info(hdev, event_type, true);
7519	if (hdev->hard_reset_on_fw_events)	7777	goto reset_device;
7520	hl_device_reset(hdev, true, false);
7521	break;
7522		7778
7523	case GAUDI_EVENT_TPC0_BMON_SPMU:	7779	case GAUDI_EVENT_TPC0_BMON_SPMU:
7524	case GAUDI_EVENT_TPC1_BMON_SPMU:	7780	case GAUDI_EVENT_TPC1_BMON_SPMU:
@@ -7552,11 +7808,28 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 27 unmodified lines
7552	event_type, cause);	7808	event_type, cause);
7553	break;	7809	break;
7554		7810
		7811	case GAUDI_EVENT_DEV_RESET_REQ:
		7812	gaudi_print_irq_info(hdev, event_type, false);
		7813	goto reset_device;
		7814
		7815	case GAUDI_EVENT_PKT_QUEUE_OUT_SYNC:
		7816	gaudi_print_irq_info(hdev, event_type, false);
		7817	gaudi_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err);
		7818	goto reset_device;
		7819
7555	default:	7820	default:
7556	dev_err(hdev->dev, "Received invalid H/W interrupt %d\n",	7821	dev_err(hdev->dev, "Received invalid H/W interrupt %d\n",
7557	event_type);	7822	event_type);
7558	break;	7823	break;
7559	}	7824	}
		7825
		7826	return;
		7827
		7828	reset_device:
		7829	if (hdev->hard_reset_on_fw_events)
		7830	hl_device_reset(hdev, HL_RESET_HARD);
		7831	else
		7832	hl_fw_unmask_irq(hdev, event_type);
7560	}	7833	}
7561		7834
7562	static void gaudi_get_events_stat(struct hl_device hdev, bool aggregate,	7835	static void gaudi_get_events_stat(struct hl_device hdev, bool aggregate,
@@ -7607,7 +7880,7 @@ static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, · 44 unmodified lines
7607	if (rc) {	7880	if (rc) {
7608	dev_err_ratelimited(hdev->dev,	7881	dev_err_ratelimited(hdev->dev,
7609	"MMU cache invalidation timeout\n");	7882	"MMU cache invalidation timeout\n");
7610	hl_device_reset(hdev, true, false);	7883	hl_device_reset(hdev, HL_RESET_HARD);
7611	}	7884	}
7612		7885
7613	return rc;	7886	return rc;
@@ -7656,7 +7929,7 @@ static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev, · 42 unmodified lines
7656	if (rc) {	7929	if (rc) {
7657	dev_err_ratelimited(hdev->dev,	7930	dev_err_ratelimited(hdev->dev,
7658	"MMU cache invalidation timeout\n");	7931	"MMU cache invalidation timeout\n");
7659	hl_device_reset(hdev, true, false);	7932	hl_device_reset(hdev, HL_RESET_HARD);
7660	}	7933	}
7661		7934
7662	return rc;	7935	return rc;
@@ -7714,7 +7987,7 @@ static int gaudi_cpucp_info_get(struct hl_device *hdev) · 51 unmodified lines
7714	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))	7987	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))
7715	return 0;	7988	return 0;
7716		7989
7717	rc = hl_fw_cpucp_info_get(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0);	7990	rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0);
7718	if (rc)	7991	if (rc)
7719	return rc;	7992	return rc;
7720		7993
@@ -7724,10 +7997,7 @@ static int gaudi_cpucp_info_get(struct hl_device *hdev) · 3 unmodified lines
7724		7997
7725	hdev->card_type = le32_to_cpu(hdev->asic_prop.cpucp_info.card_type);	7998	hdev->card_type = le32_to_cpu(hdev->asic_prop.cpucp_info.card_type);
7726		7999
7727	if (hdev->card_type == cpucp_card_type_pci)	8000	set_default_power_values(hdev);
7728	prop->max_power_default = MAX_POWER_DEFAULT_PCI;
7729	else if (hdev->card_type == cpucp_card_type_pmc)
7730	prop->max_power_default = MAX_POWER_DEFAULT_PMC;
7731		8001
7732	hdev->max_power = prop->max_power_default;	8002	hdev->max_power = prop->max_power_default;
7733		8003
@@ -8549,6 +8819,7 @@ static const struct hl_asic_funcs gaudi_funcs = { · 815 unmodified lines
8549	.debugfs_write32 = gaudi_debugfs_write32,	8819	.debugfs_write32 = gaudi_debugfs_write32,
8550	.debugfs_read64 = gaudi_debugfs_read64,	8820	.debugfs_read64 = gaudi_debugfs_read64,
8551	.debugfs_write64 = gaudi_debugfs_write64,	8821	.debugfs_write64 = gaudi_debugfs_write64,
		8822	.debugfs_read_dma = gaudi_debugfs_read_dma,
8552	.add_device_attr = gaudi_add_device_attr,	8823	.add_device_attr = gaudi_add_device_attr,
8553	.handle_eqe = gaudi_handle_eqe,	8824	.handle_eqe = gaudi_handle_eqe,
8554	.set_pll_profile = gaudi_set_pll_profile,	8825	.set_pll_profile = gaudi_set_pll_profile,

drivers/misc/habanalabs/gaudi/gaudiP.h+3−0

@@ -47,6 +47,9 @@ · 46 unmodified lines
47	#define MAX_POWER_DEFAULT_PCI 200000 /* 200W */	47	#define MAX_POWER_DEFAULT_PCI 200000 /* 200W */
48	#define MAX_POWER_DEFAULT_PMC 350000 /* 350W */	48	#define MAX_POWER_DEFAULT_PMC 350000 /* 350W */
49		49
		50	#define DC_POWER_DEFAULT_PCI 60000 /* 60W */
		51	#define DC_POWER_DEFAULT_PMC 60000 /* 60W */
		52
50	#define GAUDI_CPU_TIMEOUT_USEC 30000000 /* 30s */	53	#define GAUDI_CPU_TIMEOUT_USEC 30000000 /* 30s */
51		54
52	#define TPC_ENABLED_MASK 0xFF	55	#define TPC_ENABLED_MASK 0xFF

drivers/misc/habanalabs/gaudi/gaudi_security.c+0−8

 	mask = 1U << ((mmTPC0_CFG_PROT & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC0_CFG_VFLAGS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC0_CFG_SFLAGS & 0x7F) >> 2);
-	mask |= 1U << ((mmTPC0_CFG_STATUS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC0_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC0_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC0_CFG_TPC_STALL & 0x7F) >> 2);
 	mask = 1U << ((mmTPC1_CFG_PROT & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC1_CFG_VFLAGS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC1_CFG_SFLAGS & 0x7F) >> 2);
-	mask |= 1U << ((mmTPC1_CFG_STATUS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC1_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC1_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC1_CFG_TPC_STALL & 0x7F) >> 2);
 	mask = 1U << ((mmTPC2_CFG_PROT & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC2_CFG_VFLAGS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC2_CFG_SFLAGS & 0x7F) >> 2);
-	mask |= 1U << ((mmTPC2_CFG_STATUS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC2_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC2_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC2_CFG_TPC_STALL & 0x7F) >> 2);
 	mask = 1U << ((mmTPC3_CFG_PROT & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC3_CFG_VFLAGS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC3_CFG_SFLAGS & 0x7F) >> 2);
-	mask |= 1U << ((mmTPC3_CFG_STATUS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC3_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC3_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC3_CFG_TPC_STALL & 0x7F) >> 2);
 	mask = 1U << ((mmTPC4_CFG_PROT & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC4_CFG_VFLAGS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC4_CFG_SFLAGS & 0x7F) >> 2);
-	mask |= 1U << ((mmTPC4_CFG_STATUS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC4_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC4_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC4_CFG_TPC_STALL & 0x7F) >> 2);
 	mask = 1U << ((mmTPC5_CFG_PROT & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC5_CFG_VFLAGS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC5_CFG_SFLAGS & 0x7F) >> 2);
-	mask |= 1U << ((mmTPC5_CFG_STATUS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC5_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC5_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC5_CFG_TPC_STALL & 0x7F) >> 2);
 	mask = 1U << ((mmTPC6_CFG_PROT & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC6_CFG_VFLAGS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC6_CFG_SFLAGS & 0x7F) >> 2);
-	mask |= 1U << ((mmTPC6_CFG_STATUS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC6_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC6_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC6_CFG_TPC_STALL & 0x7F) >> 2);
 	mask = 1U << ((mmTPC7_CFG_PROT & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC7_CFG_VFLAGS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC7_CFG_SFLAGS & 0x7F) >> 2);
-	mask |= 1U << ((mmTPC7_CFG_STATUS & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC7_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC7_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1U << ((mmTPC7_CFG_TPC_STALL & 0x7F) >> 2);

drivers/misc/habanalabs/goya/goya.c+123−17

 #define IS_MME_IDLE(mme_arch_sts) \
 	(((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK)
+/*
+ * this enum kept here for compatibility with old FW (in which each asic has
+ * unique PLL numbering
+ */
+enum goya_pll_index {
+	GOYA_CPU_PLL = 0,
+	GOYA_IC_PLL,
+	GOYA_MC_PLL,
+	GOYA_MME_PLL,
+	GOYA_PCI_PLL,
+	GOYA_EMMC_PLL,
+	GOYA_TPC_PLL,
+};
+static enum pll_index goya_pll_map[PLL_MAX] = {
+	[CPU_PLL] = GOYA_CPU_PLL,
+	[IC_PLL] = GOYA_IC_PLL,
+	[MC_PLL] = GOYA_MC_PLL,
+	[MME_PLL] = GOYA_MME_PLL,
+	[PCI_PLL] = GOYA_PCI_PLL,
+	[EMMC_PLL] = GOYA_EMMC_PLL,
+	[TPC_PLL] = GOYA_TPC_PLL,
+};
 static const char goya_irq_name[GOYA_MSIX_ENTRIES][GOYA_MAX_STRING_LEN] = {
 		"goya cq 0", "goya cq 1", "goya cq 2", "goya cq 3",
 	prop->cb_pool_cb_cnt = GOYA_CB_POOL_CB_CNT;
 	prop->cb_pool_cb_size = GOYA_CB_POOL_CB_SIZE;
 	prop->max_power_default = MAX_POWER_DEFAULT;
+	prop->dc_power_default = DC_POWER_DEFAULT;
 	prop->tpc_enabled_mask = TPC_ENABLED_MASK;
 	prop->pcie_dbi_base_address = mmPCIE_DBI_BASE;
 	prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
 	for (i = 0 ; i < HL_MAX_DCORES ; i++)
 		prop->first_available_cq[i] = USHRT_MAX;
-	/* disable fw security for now, set it in a later stage */
-	prop->fw_security_disabled = true;
 	prop->fw_security_status_valid = false;
 	prop->hard_reset_done_by_fw = false;
 	struct hl_outbound_pci_region outbound_region;
 	int rc;
+	if (hdev->asic_prop.iatu_done_by_fw) {
+		hdev->asic_funcs->set_dma_mask_from_fw(hdev);
+		return 0;
+	}
 	/* Inbound Region 0 - Bar 0 - Point to SRAM and CFG */
 	inbound_region.mode = PCI_BAR_MATCH_MODE;
 	inbound_region.bar = SRAM_CFG_BAR_ID;
 {
 	struct asic_fixed_properties *prop = &hdev->asic_prop;
 	struct pci_dev *pdev = hdev->pdev;
-	u32 val;
+	u32 fw_boot_status, val;
 	int rc;
 	rc = goya_get_fixed_properties(hdev);
 	prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID);
+	/* If FW security is enabled at this point it means no access to ELBI */
+	if (!hdev->asic_prop.fw_security_disabled) {
+		hdev->asic_prop.iatu_done_by_fw = true;
+		goto pci_init;
+	}
+	rc = hl_pci_elbi_read(hdev, CFG_BASE + mmCPU_BOOT_DEV_STS0,
+				&fw_boot_status);
+	if (rc)
+		goto free_queue_props;
+	/* Check whether FW is configuring iATU */
+	if ((fw_boot_status & CPU_BOOT_DEV_STS0_ENABLED) &&
+			(fw_boot_status & CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN))
+		hdev->asic_prop.iatu_done_by_fw = true;
+pci_init:
 	rc = hl_pci_init(hdev);
 	if (rc)
 		goto free_queue_props;
 	hdev->asic_specific = goya;
+	/* store legacy PLL map */
+	hdev->legacy_pll_map = goya_pll_map;
 	/* Create DMA pool for small allocations */
 	hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),
 			&hdev->pdev->dev, GOYA_DMA_POOL_BLK_SIZE, 8, 0);
 	struct goya_device *goya = hdev->asic_specific;
 	int rc;
-	if (!hdev->cpu_enable)
+	if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
 		return 0;
 	if (goya->hw_cap_initialized & HW_CAP_CPU)
 			parser->job_userptr_list, &userptr))
 		goto already_pinned;
-	userptr = kzalloc(sizeof(*userptr), GFP_ATOMIC);
+	userptr = kzalloc(sizeof(*userptr), GFP_KERNEL);
 	if (!userptr)
 		return -ENOMEM;
  * lead to undefined behavior and therefore, should be done with extreme care
  *
  */
-static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val)
+static int goya_debugfs_read32(struct hl_device *hdev, u64 addr,
+			bool user_address, u32 *val)
 {
 	struct asic_fixed_properties *prop = &hdev->asic_prop;
-	u64 ddr_bar_addr;
+	u64 ddr_bar_addr, host_phys_end;
 	int rc = 0;
+	host_phys_end = HOST_PHYS_BASE + HOST_PHYS_SIZE;
 	if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
 		*val = RREG32(addr - CFG_BASE);
 		if (ddr_bar_addr == U64_MAX)
 			rc = -EIO;
+	} else if (addr >= HOST_PHYS_BASE && addr < host_phys_end &&
+			user_address && !iommu_present(&pci_bus_type)) {
+		*val = *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE);
 	} else {
 		rc = -EFAULT;
 	}
  * lead to undefined behavior and therefore, should be done with extreme care
  *
  */
-static int goya_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val)
+static int goya_debugfs_write32(struct hl_device *hdev, u64 addr,
+			bool user_address, u32 val)
 {
 	struct asic_fixed_properties *prop = &hdev->asic_prop;
-	u64 ddr_bar_addr;
+	u64 ddr_bar_addr, host_phys_end;
 	int rc = 0;
+	host_phys_end = HOST_PHYS_BASE + HOST_PHYS_SIZE;
 	if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
 		WREG32(addr - CFG_BASE, val);
 		if (ddr_bar_addr == U64_MAX)
 			rc = -EIO;
+	} else if (addr >= HOST_PHYS_BASE && addr < host_phys_end &&
+			user_address && !iommu_present(&pci_bus_type)) {
+		*(u32 *) phys_to_virt(addr - HOST_PHYS_BASE) = val;
 	} else {
 		rc = -EFAULT;
 	}
 	return rc;
 }
-static int goya_debugfs_read64(struct hl_device *hdev, u64 addr, u64 *val)
+static int goya_debugfs_read64(struct hl_device *hdev, u64 addr,
+			bool user_address, u64 *val)
 {
 	struct asic_fixed_properties *prop = &hdev->asic_prop;
-	u64 ddr_bar_addr;
+	u64 ddr_bar_addr, host_phys_end;
 	int rc = 0;
+	host_phys_end = HOST_PHYS_BASE + HOST_PHYS_SIZE;
 	if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) {
 		u32 val_l = RREG32(addr - CFG_BASE);
 		u32 val_h = RREG32(addr + sizeof(u32) - CFG_BASE);
 		if (ddr_bar_addr == U64_MAX)
 			rc = -EIO;
+	} else if (addr >= HOST_PHYS_BASE && addr < host_phys_end &&
+			user_address && !iommu_present(&pci_bus_type)) {
+		*val = *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE);
 	} else {
 		rc = -EFAULT;
 	}
 	return rc;
 }
-static int goya_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val)
+static int goya_debugfs_write64(struct hl_device *hdev, u64 addr,
+				bool user_address, u64 val)
 {
 	struct asic_fixed_properties *prop = &hdev->asic_prop;
-	u64 ddr_bar_addr;
+	u64 ddr_bar_addr, host_phys_end;
 	int rc = 0;
+	host_phys_end = HOST_PHYS_BASE + HOST_PHYS_SIZE;
 	if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) {
 		WREG32(addr - CFG_BASE, lower_32_bits(val));
 		WREG32(addr + sizeof(u32) - CFG_BASE, upper_32_bits(val));
 		if (ddr_bar_addr == U64_MAX)
 			rc = -EIO;
+	} else if (addr >= HOST_PHYS_BASE && addr < host_phys_end &&
+			user_address && !iommu_present(&pci_bus_type)) {
+		*(u64 *) phys_to_virt(addr - HOST_PHYS_BASE) = val;
 	} else {
 		rc = -EFAULT;
 	}
 	return rc;
 }
+static int goya_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size,
+				void *blob_addr)
+{
+	dev_err(hdev->dev, "Reading via DMA is unimplemented yet\n");
+	return -EPERM;
+}
 static u64 goya_read_pte(struct hl_device *hdev, u64 addr)
 {
 	struct goya_device *goya = hdev->asic_specific;
 		return "THERMAL_ENV_S";
 	case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E:
 		return "THERMAL_ENV_E";
+	case GOYA_ASYNC_EVENT_PKT_QUEUE_OUT_SYNC:
+		return "QUEUE_OUT_OF_SYNC";
 	default:
 		return "N/A";
 	}
 		index = event_type - GOYA_ASYNC_EVENT_ID_DMA_BM_CH0;
 		snprintf(desc, size, _goya_get_event_desc(event_type), index);
 		break;
+	case GOYA_ASYNC_EVENT_PKT_QUEUE_OUT_SYNC:
+		snprintf(desc, size, _goya_get_event_desc(event_type));
+		break;
 	default:
 		snprintf(desc, size, _goya_get_event_desc(event_type));
 		break;
 	}
 }
+static void goya_print_out_of_sync_info(struct hl_device *hdev,
+					struct cpucp_pkt_sync_err *sync_err)
+{
+	struct hl_hw_queue *q = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ];
+	dev_err(hdev->dev, "Out of sync with FW, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%u\n",
+			sync_err->pi, sync_err->ci, q->pi, atomic_read(&q->ci));
+}
 static void goya_print_irq_info(struct hl_device *hdev, u16 event_type,
 				bool razwi)
 {
 	case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET:
 		goya_print_irq_info(hdev, event_type, false);
 		if (hdev->hard_reset_on_fw_events)
-			hl_device_reset(hdev, true, false);
+			hl_device_reset(hdev, HL_RESET_HARD);
 		break;
 	case GOYA_ASYNC_EVENT_ID_PCIE_DEC:
 		goya_unmask_irq(hdev, event_type);
 		break;
+	case GOYA_ASYNC_EVENT_PKT_QUEUE_OUT_SYNC:
+		goya_print_irq_info(hdev, event_type, false);
+		goya_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err);
+		if (hdev->hard_reset_on_fw_events)
+			hl_device_reset(hdev, HL_RESET_HARD);
+		else
+			hl_fw_unmask_irq(hdev, event_type);
+		break;
 	default:
 		dev_err(hdev->dev, "Received invalid H/W interrupt %d\n",
 				event_type);
 	if (rc) {
 		dev_err_ratelimited(hdev->dev,
 					"MMU cache invalidation timeout\n");
-		hl_device_reset(hdev, true, false);
+		hl_device_reset(hdev, HL_RESET_HARD);
 	}
 	return rc;
 	if (rc) {
 		dev_err_ratelimited(hdev->dev,
 					"MMU cache invalidation timeout\n");
-		hl_device_reset(hdev, true, false);
+		hl_device_reset(hdev, HL_RESET_HARD);
 	}
 	return rc;
 	if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
 		return 0;
-	rc = hl_fw_cpucp_info_get(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0);
+	rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0);
 	if (rc)
 		return rc;
 	.debugfs_write32 = goya_debugfs_write32,
 	.debugfs_read64 = goya_debugfs_read64,
 	.debugfs_write64 = goya_debugfs_write64,
+	.debugfs_read_dma = goya_debugfs_read_dma,
 	.add_device_attr = goya_add_device_attr,
 	.handle_eqe = goya_handle_eqe,
 	.set_pll_profile = goya_set_pll_profile,

drivers/misc/habanalabs/goya/goyaP.h+2−0

@@ -49,6 +49,8 @@ · 48 unmodified lines
49		49
50	#define MAX_POWER_DEFAULT 200000 /* 200W */	50	#define MAX_POWER_DEFAULT 200000 /* 200W */
51		51
		52	#define DC_POWER_DEFAULT 20000 /* 20W */
		53
52	#define DRAM_PHYS_DEFAULT_SIZE 0x100000000ull /* 4GB */	54	#define DRAM_PHYS_DEFAULT_SIZE 0x100000000ull /* 4GB */
53		55
54	#define GOYA_DEFAULT_CARD_NAME "HL1000"	56	#define GOYA_DEFAULT_CARD_NAME "HL1000"

drivers/misc/habanalabs/include/common/cpucp_if.h+98−1

@@ -11,6 +11,8 @@ · 10 unmodified lines
11	#include <linux/types.h>	11	#include <linux/types.h>
12	#include <linux/if_ether.h>	12	#include <linux/if_ether.h>
13		13
		14	#include "hl_boot_if.h"
		15
14	#define NUM_HBM_PSEUDO_CH 2	16	#define NUM_HBM_PSEUDO_CH 2
15	#define NUM_HBM_CH_PER_DEV 8	17	#define NUM_HBM_CH_PER_DEV 8
16	#define CPUCP_PKT_HBM_ECC_INFO_WR_PAR_SHIFT 0	18	#define CPUCP_PKT_HBM_ECC_INFO_WR_PAR_SHIFT 0
@@ -28,6 +30,17 @@ · 11 unmodified lines
28	#define CPUCP_PKT_HBM_ECC_INFO_HBM_CH_SHIFT 6	30	#define CPUCP_PKT_HBM_ECC_INFO_HBM_CH_SHIFT 6
29	#define CPUCP_PKT_HBM_ECC_INFO_HBM_CH_MASK 0x000007C0	31	#define CPUCP_PKT_HBM_ECC_INFO_HBM_CH_MASK 0x000007C0
30		32
		33	#define PLL_MAP_MAX_BITS 128
		34	#define PLL_MAP_LEN (PLL_MAP_MAX_BITS / 8)
		35
		36	/*
		37	* info of the pkt queue pointers in the first async occurrence
		38	*/
		39	struct cpucp_pkt_sync_err {
		40	__le32 pi;
		41	__le32 ci;
		42	};
		43
31	struct hl_eq_hbm_ecc_data {	44	struct hl_eq_hbm_ecc_data {
32	/* SERR counter */	45	/* SERR counter */
33	__le32 sec_cnt;	46	__le32 sec_cnt;
@@ -77,6 +90,7 @@ struct hl_eq_entry { · 43 unmodified lines
77	struct hl_eq_ecc_data ecc_data;	90	struct hl_eq_ecc_data ecc_data;
78	struct hl_eq_hbm_ecc_data hbm_ecc_data;	91	struct hl_eq_hbm_ecc_data hbm_ecc_data;
79	struct hl_eq_sm_sei_data sm_sei_data;	92	struct hl_eq_sm_sei_data sm_sei_data;
		93	struct cpucp_pkt_sync_err pkt_sync_err;
80	__le64 data[7];	94	__le64 data[7];
81	};	95	};
82	};	96	};
@@ -287,6 +301,30 @@ enum pq_init_status { · 204 unmodified lines
287	* The result is composed of 4 outputs, each is 16-bit	301	* The result is composed of 4 outputs, each is 16-bit
288	* frequency in MHz.	302	* frequency in MHz.
289	*	303	*
		304	* CPUCP_PACKET_POWER_GET
		305	* Fetch the present power consumption of the device (Current * Voltage).
		306	*
		307	* CPUCP_PACKET_NIC_PFC_SET -
		308	* Enable/Disable the NIC PFC feature. The packet's arguments specify the
		309	* NIC port, relevant lanes to configure and one bit indication for
		310	* enable/disable.
		311	*
		312	* CPUCP_PACKET_NIC_FAULT_GET -
		313	* Fetch the current indication for local/remote faults from the NIC MAC.
		314	* The result is 32-bit value of the relevant register.
		315	*
		316	* CPUCP_PACKET_NIC_LPBK_SET -
		317	* Enable/Disable the MAC loopback feature. The packet's arguments specify
		318	* the NIC port, relevant lanes to configure and one bit indication for
		319	* enable/disable.
		320	*
		321	* CPUCP_PACKET_NIC_MAC_INIT -
		322	* Configure the NIC MAC channels. The packet's arguments specify the
		323	* NIC port and the speed.
		324	*
		325	* CPUCP_PACKET_MSI_INFO_SET -
		326	* set the index number for each supported msi type going from
		327	* host to device
290	*/	328	*/
291		329
292	enum cpucp_packet_id {	330	enum cpucp_packet_id {
@@ -320,6 +358,13 @@ enum cpucp_packet_id { · 27 unmodified lines
320	CPUCP_PACKET_PCIE_REPLAY_CNT_GET, /* internal */	358	CPUCP_PACKET_PCIE_REPLAY_CNT_GET, /* internal */
321	CPUCP_PACKET_TOTAL_ENERGY_GET, /* internal */	359	CPUCP_PACKET_TOTAL_ENERGY_GET, /* internal */
322	CPUCP_PACKET_PLL_INFO_GET, /* internal */	360	CPUCP_PACKET_PLL_INFO_GET, /* internal */
		361	CPUCP_PACKET_NIC_STATUS, /* internal */
		362	CPUCP_PACKET_POWER_GET, /* internal */
		363	CPUCP_PACKET_NIC_PFC_SET, /* internal */
		364	CPUCP_PACKET_NIC_FAULT_GET, /* internal */
		365	CPUCP_PACKET_NIC_LPBK_SET, /* internal */
		366	CPUCP_PACKET_NIC_MAC_CFG, /* internal */
		367	CPUCP_PACKET_MSI_INFO_SET, /* internal */
323	};	368	};
324		369
325	#define CPUCP_PACKET_FENCE_VAL 0xFE8CE7A5	370	#define CPUCP_PACKET_FENCE_VAL 0xFE8CE7A5
@@ -391,6 +436,12 @@ struct cpucp_unmask_irq_arr_packet { · 65 unmodified lines
391	__le32 irqs[0];	436	__le32 irqs[0];
392	};	437	};
393		438
		439	struct cpucp_array_data_packet {
		440	struct cpucp_packet cpucp_pkt;
		441	__le32 length;
		442	__le32 data[0];
		443	};
		444
394	enum cpucp_packet_rc {	445	enum cpucp_packet_rc {
395	cpucp_packet_success,	446	cpucp_packet_success,
396	cpucp_packet_invalid,	447	cpucp_packet_invalid,
@@ -459,6 +510,51 @@ enum cpucp_pll_type_attributes { · 62 unmodified lines
459	cpucp_pll_pci,	510	cpucp_pll_pci,
460	};	511	};
461		512
		513	/*
		514	* MSI type enumeration table for all ASICs and future SW versions.
		515	* For future ASIC-LKD compatibility, we can only add new enumerations.
		516	* at the end of the table (before CPUCP_NUM_OF_MSI_TYPES).
		517	* Changing the order of entries or removing entries is not allowed.
		518	*/
		519	enum cpucp_msi_type {
		520	CPUCP_EVENT_QUEUE_MSI_TYPE,
		521	CPUCP_NIC_PORT1_MSI_TYPE,
		522	CPUCP_NIC_PORT3_MSI_TYPE,
		523	CPUCP_NIC_PORT5_MSI_TYPE,
		524	CPUCP_NIC_PORT7_MSI_TYPE,
		525	CPUCP_NIC_PORT9_MSI_TYPE,
		526	CPUCP_NUM_OF_MSI_TYPES
		527	};
		528
		529	/*
		530	* PLL enumeration table used for all ASICs and future SW versions.
		531	* For future ASIC-LKD compatibility, we can only add new enumerations.
		532	* at the end of the table.
		533	* Changing the order of entries or removing entries is not allowed.
		534	*/
		535	enum pll_index {
		536	CPU_PLL = 0,
		537	PCI_PLL = 1,
		538	NIC_PLL = 2,
		539	DMA_PLL = 3,
		540	MESH_PLL = 4,
		541	MME_PLL = 5,
		542	TPC_PLL = 6,
		543	IF_PLL = 7,
		544	SRAM_PLL = 8,
		545	NS_PLL = 9,
		546	HBM_PLL = 10,
		547	MSS_PLL = 11,
		548	DDR_PLL = 12,
		549	VID_PLL = 13,
		550	BANK_PLL = 14,
		551	MMU_PLL = 15,
		552	IC_PLL = 16,
		553	MC_PLL = 17,
		554	EMMC_PLL = 18,
		555	PLL_MAX
		556	};
		557
462	/* Event Queue Packets */	558	/* Event Queue Packets */
463		559
464	struct eq_generic_event {	560	struct eq_generic_event {
@@ -470,7 +566,6 @@ struct eq_generic_event { · 5 unmodified lines
470	*/	566	*/
471		567
472	#define CARD_NAME_MAX_LEN 16	568	#define CARD_NAME_MAX_LEN 16
473	#define VERSION_MAX_LEN 128
474	#define CPUCP_MAX_SENSORS 128	569	#define CPUCP_MAX_SENSORS 128
475	#define CPUCP_MAX_NICS 128	570	#define CPUCP_MAX_NICS 128
476	#define CPUCP_LANES_PER_NIC 4	571	#define CPUCP_LANES_PER_NIC 4
@@ -533,6 +628,7 @@ struct cpucp_security_info { · 56 unmodified lines
533	* @dram_size: available DRAM size.	628	* @dram_size: available DRAM size.
534	* @card_name: card name that will be displayed in HWMON subsystem on the host	629	* @card_name: card name that will be displayed in HWMON subsystem on the host
535	* @sec_info: security information	630	* @sec_info: security information
		631	* @pll_map: Bit map of supported PLLs for current ASIC version.
536	*/	632	*/
537	struct cpucp_info {	633	struct cpucp_info {
538	struct cpucp_sensor sensors[CPUCP_MAX_SENSORS];	634	struct cpucp_sensor sensors[CPUCP_MAX_SENSORS];
@@ -554,6 +650,7 @@ struct cpucp_info { · 15 unmodified lines
554	__u8 pad[7];	650	__u8 pad[7];
555	struct cpucp_security_info sec_info;	651	struct cpucp_security_info sec_info;
556	__le32 reserved6;	652	__le32 reserved6;
		653	__u8 pll_map[PLL_MAP_LEN];
557	};	654	};
558		655
559	struct cpucp_mac_addr {	656	struct cpucp_mac_addr {

drivers/misc/habanalabs/include/common/hl_boot_if.h+219−0

@@ -13,6 +13,8 @@ · 12 unmodified lines
13		13
14	#define BOOT_FIT_SRAM_OFFSET 0x200000	14	#define BOOT_FIT_SRAM_OFFSET 0x200000
15		15
		16	#define VERSION_MAX_LEN 128
		17
16	/*	18	/*
17	* CPU error bits in BOOT_ERROR registers	19	* CPU error bits in BOOT_ERROR registers
18	*	20	*
@@ -73,6 +75,9 @@ · 54 unmodified lines
73	* CPU_BOOT_ERR0_PLL_FAIL PLL settings failed, meaning that one	75	* CPU_BOOT_ERR0_PLL_FAIL PLL settings failed, meaning that one
74	* of the PLLs remains in REF_CLK	76	* of the PLLs remains in REF_CLK
75	*	77	*
		78	* CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL Device is unusable and customer support
		79	* should be contacted.
		80	*
76	* CPU_BOOT_ERR0_ENABLED Error registers enabled.	81	* CPU_BOOT_ERR0_ENABLED Error registers enabled.
77	* This is a main indication that the	82	* This is a main indication that the
78	* running FW populates the error	83	* running FW populates the error
@@ -92,6 +97,7 @@ · 13 unmodified lines
92	#define CPU_BOOT_ERR0_PRI_IMG_VER_FAIL (1 << 10)	97	#define CPU_BOOT_ERR0_PRI_IMG_VER_FAIL (1 << 10)
93	#define CPU_BOOT_ERR0_SEC_IMG_VER_FAIL (1 << 11)	98	#define CPU_BOOT_ERR0_SEC_IMG_VER_FAIL (1 << 11)
94	#define CPU_BOOT_ERR0_PLL_FAIL (1 << 12)	99	#define CPU_BOOT_ERR0_PLL_FAIL (1 << 12)
		100	#define CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL (1 << 13)
95	#define CPU_BOOT_ERR0_ENABLED (1 << 31)	101	#define CPU_BOOT_ERR0_ENABLED (1 << 31)
96		102
97	/*	103	/*
@@ -170,6 +176,20 @@ · 72 unmodified lines
170	* is set to the PI counter.	176	* is set to the PI counter.
171	* Initialized in: linux	177	* Initialized in: linux
172	*	178	*
		179	* CPU_BOOT_DEV_STS0_FW_LD_COM_EN Flexible FW loading communication
		180	* protocol is enabled.
		181	* Initialized in: preboot
		182	*
		183	* CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN FW iATU configuration is enabled.
		184	* This bit if set, means the iATU has been
		185	* configured and is ready for use.
		186	* Initialized in: ppboot
		187	*
		188	* CPU_BOOT_DEV_STS0_DYN_PLL_EN Dynamic PLL configuration is enabled.
		189	* FW sends to host a bitmap of supported
		190	* PLLs.
		191	* Initialized in: linux
		192	*
173	* CPU_BOOT_DEV_STS0_ENABLED Device status register enabled.	193	* CPU_BOOT_DEV_STS0_ENABLED Device status register enabled.
174	* This is a main indication that the	194	* This is a main indication that the
175	* running FW populates the device status	195	* running FW populates the device status
@@ -195,6 +215,9 @@ · 19 unmodified lines
195	#define CPU_BOOT_DEV_STS0_CLK_GATE_EN (1 << 13)	215	#define CPU_BOOT_DEV_STS0_CLK_GATE_EN (1 << 13)
196	#define CPU_BOOT_DEV_STS0_HBM_ECC_EN (1 << 14)	216	#define CPU_BOOT_DEV_STS0_HBM_ECC_EN (1 << 14)
197	#define CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN (1 << 15)	217	#define CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN (1 << 15)
		218	#define CPU_BOOT_DEV_STS0_FW_LD_COM_EN (1 << 16)
		219	#define CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN (1 << 17)
		220	#define CPU_BOOT_DEV_STS0_DYN_PLL_EN (1 << 19)
198	#define CPU_BOOT_DEV_STS0_ENABLED (1 << 31)	221	#define CPU_BOOT_DEV_STS0_ENABLED (1 << 31)
199		222
200	enum cpu_boot_status {	223	enum cpu_boot_status {
@@ -230,6 +253,7 @@ enum kmd_msg { · 29 unmodified lines
230	KMD_MSG_SKIP_BMC,	253	KMD_MSG_SKIP_BMC,
231	RESERVED,	254	RESERVED,
232	KMD_MSG_RST_DEV,	255	KMD_MSG_RST_DEV,
		256	KMD_MSG_LAST
233	};	257	};
234		258
235	enum cpu_msg_status {	259	enum cpu_msg_status {
@@ -238,4 +262,199 @@ enum cpu_msg_status { · 2 unmodified lines
238	CPU_MSG_ERR,	262	CPU_MSG_ERR,
239	};	263	};
240		264
		265	/* communication registers mapping - consider ABI when changing */
		266	struct cpu_dyn_regs {
		267	uint32_t cpu_pq_base_addr_low;
		268	uint32_t cpu_pq_base_addr_high;
		269	uint32_t cpu_pq_length;
		270	uint32_t cpu_pq_init_status;
		271	uint32_t cpu_eq_base_addr_low;
		272	uint32_t cpu_eq_base_addr_high;
		273	uint32_t cpu_eq_length;
		274	uint32_t cpu_eq_ci;
		275	uint32_t cpu_cq_base_addr_low;
		276	uint32_t cpu_cq_base_addr_high;
		277	uint32_t cpu_cq_length;
		278	uint32_t cpu_pf_pq_pi;
		279	uint32_t cpu_boot_dev_sts0;
		280	uint32_t cpu_boot_dev_sts1;
		281	uint32_t cpu_boot_err0;
		282	uint32_t cpu_boot_err1;
		283	uint32_t cpu_boot_status;
		284	uint32_t fw_upd_sts;
		285	uint32_t fw_upd_cmd;
		286	uint32_t fw_upd_pending_sts;
		287	uint32_t fuse_ver_offset;
		288	uint32_t preboot_ver_offset;
		289	uint32_t uboot_ver_offset;
		290	uint32_t hw_state;
		291	uint32_t kmd_msg_to_cpu;
		292	uint32_t cpu_cmd_status_to_host;
		293	uint32_t reserved1[32]; /* reserve for future use */
		294	};
		295
		296	/* HCDM - Habana Communications Descriptor Magic */
		297	#define HL_COMMS_DESC_MAGIC 0x4843444D
		298	#define HL_COMMS_DESC_VER 1
		299
		300	/* this is the comms descriptor header - meta data */
		301	struct comms_desc_header {
		302	uint32_t magic; /* magic for validation */
		303	uint32_t crc32; /* CRC32 of the descriptor w/o header */
		304	uint16_t size; /* size of the descriptor w/o header */
		305	uint8_t version; /* descriptor version */
		306	uint8_t reserved[5]; /* pad to 64 bit */
		307	};
		308
		309	/* this is the main FW descriptor - consider ABI when changing */
		310	struct lkd_fw_comms_desc {
		311	struct comms_desc_header header;
		312	struct cpu_dyn_regs cpu_dyn_regs;
		313	char fuse_ver[VERSION_MAX_LEN];
		314	char cur_fw_ver[VERSION_MAX_LEN];
		315	/* can be used for 1 more version w/o ABI change */
		316	char reserved0[VERSION_MAX_LEN];
		317	uint64_t img_addr; /* address for next FW component load */
		318	};
		319
		320	/*
		321	* LKD commands:
		322	*
		323	* COMMS_NOOP Used to clear the command register and no actual
		324	* command is send.
		325	*
		326	* COMMS_CLR_STS Clear status command - FW should clear the
		327	* status register. Used for synchronization
		328	* between the commands as part of the race free
		329	* protocol.
		330	*
		331	* COMMS_RST_STATE Reset the current communication state which is
		332	* kept by FW for proper responses.
		333	* Should be used in the beginning of the
		334	* communication cycle to clean any leftovers from
		335	* previous communication attempts.
		336	*
		337	* COMMS_PREP_DESC Prepare descriptor for setting up the
		338	* communication and other dynamic data:
		339	* struct lkd_fw_comms_desc.
		340	* This command has a parameter stating the next FW
		341	* component size, so the FW can actually prepare a
		342	* space for it and in the status response provide
		343	* the descriptor offset. The Offset of the next FW
		344	* data component is a part of the descriptor
		345	* structure.
		346	*
		347	* COMMS_DATA_RDY The FW data has been uploaded and is ready for
		348	* validation.
		349	*
		350	* COMMS_EXEC Execute the next FW component.
		351	*
		352	* COMMS_RST_DEV Reset the device.
		353	*
		354	* COMMS_GOTO_WFE Execute WFE command. Allowed only on non-secure
		355	* devices.
		356	*
		357	* COMMS_SKIP_BMC Perform actions required for BMC-less servers.
		358	* Do not wait for BMC response.
		359	*
		360	* COMMS_LOW_PLL_OPP Initialize PLLs for low OPP.
		361	*/
		362	enum comms_cmd {
		363	COMMS_NOOP = 0,
		364	COMMS_CLR_STS = 1,
		365	COMMS_RST_STATE = 2,
		366	COMMS_PREP_DESC = 3,
		367	COMMS_DATA_RDY = 4,
		368	COMMS_EXEC = 5,
		369	COMMS_RST_DEV = 6,
		370	COMMS_GOTO_WFE = 7,
		371	COMMS_SKIP_BMC = 8,
		372	COMMS_LOW_PLL_OPP = 9,
		373	COMMS_INVLD_LAST
		374	};
		375
		376	#define COMMS_COMMAND_SIZE_SHIFT 0
		377	#define COMMS_COMMAND_SIZE_MASK 0x1FFFFFF
		378	#define COMMS_COMMAND_CMD_SHIFT 27
		379	#define COMMS_COMMAND_CMD_MASK 0xF8000000
		380
		381	/*
		382	* LKD command to FW register structure
		383	* @size - FW component size
		384	* @cmd - command from enum comms_cmd
		385	*/
		386	struct comms_command {
		387	union { /* bit fields are only for FW use */
		388	struct {
		389	unsigned int size :25; /* 32MB max. */
		390	unsigned int reserved :2;
		391	enum comms_cmd cmd :5; /* 32 commands */
		392	};
		393	unsigned int val;
		394	};
		395	};
		396
		397	/*
		398	* FW status
		399	*
		400	* COMMS_STS_NOOP Used to clear the status register and no actual
		401	* status is provided.
		402	*
		403	* COMMS_STS_ACK Command has been received and recognized.
		404	*
		405	* COMMS_STS_OK Command execution has finished successfully.
		406	*
		407	* COMMS_STS_ERR Command execution was unsuccessful and resulted
		408	* in error.
		409	*
		410	* COMMS_STS_VALID_ERR FW validation has failed.
		411	*
		412	* COMMS_STS_TIMEOUT_ERR Command execution has timed out.
		413	*/
		414	enum comms_sts {
		415	COMMS_STS_NOOP = 0,
		416	COMMS_STS_ACK = 1,
		417	COMMS_STS_OK = 2,
		418	COMMS_STS_ERR = 3,
		419	COMMS_STS_VALID_ERR = 4,
		420	COMMS_STS_TIMEOUT_ERR = 5,
		421	COMMS_STS_INVLD_LAST
		422	};
		423
		424	/* RAM types for FW components loading - defines the base address */
		425	enum comms_ram_types {
		426	COMMS_SRAM = 0,
		427	COMMS_DRAM = 1,
		428	};
		429
		430	#define COMMS_STATUS_OFFSET_SHIFT 0
		431	#define COMMS_STATUS_OFFSET_MASK 0x03FFFFFF
		432	#define COMMS_STATUS_OFFSET_ALIGN_SHIFT 2
		433	#define COMMS_STATUS_RAM_TYPE_SHIFT 26
		434	#define COMMS_STATUS_RAM_TYPE_MASK 0x0C000000
		435	#define COMMS_STATUS_STATUS_SHIFT 28
		436	#define COMMS_STATUS_STATUS_MASK 0xF0000000
		437
		438	/*
		439	* FW status to LKD register structure
		440	* @offset - an offset from the base of the ram_type shifted right by
		441	* 2 bits (always aligned to 32 bits).
		442	* Allows a maximum addressable offset of 256MB from RAM base.
		443	* Example: for real offset in RAM of 0x800000 (8MB), the value
		444	* in offset field is (0x800000 >> 2) = 0x200000.
		445	* @ram_type - the RAM type that should be used for offset from
		446	* enum comms_ram_types
		447	* @status - status from enum comms_sts
		448	*/
		449	struct comms_status {
		450	union { /* bit fields are only for FW use */
		451	struct {
		452	unsigned int offset :26;
		453	unsigned int ram_type :2;
		454	enum comms_sts status :4; /* 16 statuses */
		455	};
		456	unsigned int val;
		457	};
		458	};
		459
241	#endif /* HL_BOOT_IF_H */	460	#endif /* HL_BOOT_IF_H */

drivers/misc/habanalabs/include/gaudi/gaudi.h+1−1

@@ -38,7 +38,7 @@ · 37 unmodified lines
38		38
39	#define QMAN_PQ_ENTRY_SIZE 16 /* Bytes */	39	#define QMAN_PQ_ENTRY_SIZE 16 /* Bytes */
40		40
41	#define MAX_ASID 1024	41	#define MAX_ASID 2
42		42
43	#define PROT_BITS_OFFS 0xF80	43	#define PROT_BITS_OFFS 0xF80
44		44

drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h+2−0

 	GAUDI_EVENT_NIC3_QP1 = 619,
 	GAUDI_EVENT_NIC4_QP0 = 620,
 	GAUDI_EVENT_NIC4_QP1 = 621,
+	GAUDI_EVENT_DEV_RESET_REQ = 646,
+	GAUDI_EVENT_PKT_QUEUE_OUT_SYNC = 647,
 	GAUDI_EVENT_FIX_POWER_ENV_S = 658,
 	GAUDI_EVENT_FIX_POWER_ENV_E = 659,
 	GAUDI_EVENT_FIX_THERMAL_ENV_S = 660,

drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h+27−16

@@ -301,10 +301,10 @@ static struct gaudi_async_events_ids_map gaudi_irq_map_table[] = { · 300 unmodified lines
301	{ .fc_id = 274, .cpu_id = 128, .valid = 0, .name = "" },	301	{ .fc_id = 274, .cpu_id = 128, .valid = 0, .name = "" },
302	{ .fc_id = 275, .cpu_id = 128, .valid = 0, .name = "" },	302	{ .fc_id = 275, .cpu_id = 128, .valid = 0, .name = "" },
303	{ .fc_id = 276, .cpu_id = 128, .valid = 0, .name = "" },	303	{ .fc_id = 276, .cpu_id = 128, .valid = 0, .name = "" },
304	{ .fc_id = 277, .cpu_id = 129, .valid = 0, .name = "" },	304	{ .fc_id = 277, .cpu_id = 129, .valid = 1, .name = "DMA_IF_SEI_0" },
305	{ .fc_id = 278, .cpu_id = 129, .valid = 0, .name = "" },	305	{ .fc_id = 278, .cpu_id = 129, .valid = 1, .name = "DMA_IF_SEI_1" },
306	{ .fc_id = 279, .cpu_id = 129, .valid = 0, .name = "" },	306	{ .fc_id = 279, .cpu_id = 129, .valid = 1, .name = "DMA_IF_SEI_2" },
307	{ .fc_id = 280, .cpu_id = 129, .valid = 0, .name = "" },	307	{ .fc_id = 280, .cpu_id = 129, .valid = 1, .name = "DMA_IF_SEI_3" },
308	{ .fc_id = 281, .cpu_id = 130, .valid = 0, .name = "" },	308	{ .fc_id = 281, .cpu_id = 130, .valid = 0, .name = "" },
309	{ .fc_id = 282, .cpu_id = 131, .valid = 0, .name = "" },	309	{ .fc_id = 282, .cpu_id = 131, .valid = 0, .name = "" },
310	{ .fc_id = 283, .cpu_id = 132, .valid = 0, .name = "" },	310	{ .fc_id = 283, .cpu_id = 132, .valid = 0, .name = "" },
@@ -670,18 +670,29 @@ static struct gaudi_async_events_ids_map gaudi_irq_map_table[] = { · 359 unmodified lines
670	{ .fc_id = 643, .cpu_id = 492, .valid = 0, .name = "" },	670	{ .fc_id = 643, .cpu_id = 492, .valid = 0, .name = "" },
671	{ .fc_id = 644, .cpu_id = 493, .valid = 0, .name = "" },	671	{ .fc_id = 644, .cpu_id = 493, .valid = 0, .name = "" },
672	{ .fc_id = 645, .cpu_id = 494, .valid = 0, .name = "" },	672	{ .fc_id = 645, .cpu_id = 494, .valid = 0, .name = "" },
673	{ .fc_id = 646, .cpu_id = 495, .valid = 0, .name = "" },	673	{ .fc_id = 646, .cpu_id = 495, .valid = 1, .name = "DEV_RESET_REQ" },
674	{ .fc_id = 647, .cpu_id = 496, .valid = 0, .name = "" },	674	{ .fc_id = 647, .cpu_id = 496, .valid = 1,
675	{ .fc_id = 648, .cpu_id = 497, .valid = 0, .name = "" },	675	.name = "PKT_QUEUE_OUT_SYNC" },
676	{ .fc_id = 649, .cpu_id = 498, .valid = 0, .name = "" },	676	{ .fc_id = 648, .cpu_id = 497, .valid = 1,
677	{ .fc_id = 650, .cpu_id = 499, .valid = 0, .name = "" },	677	.name = "STATUS_NIC0_ENG0" },
678	{ .fc_id = 651, .cpu_id = 500, .valid = 0, .name = "" },	678	{ .fc_id = 649, .cpu_id = 498, .valid = 1,
679	{ .fc_id = 652, .cpu_id = 501, .valid = 0, .name = "" },	679	.name = "STATUS_NIC0_ENG1" },
680	{ .fc_id = 653, .cpu_id = 502, .valid = 0, .name = "" },	680	{ .fc_id = 650, .cpu_id = 499, .valid = 1,
681	{ .fc_id = 654, .cpu_id = 503, .valid = 0, .name = "" },	681	.name = "STATUS_NIC1_ENG0" },
682	{ .fc_id = 655, .cpu_id = 504, .valid = 0, .name = "" },	682	{ .fc_id = 651, .cpu_id = 500, .valid = 1,
683	{ .fc_id = 656, .cpu_id = 505, .valid = 0, .name = "" },	683	.name = "STATUS_NIC1_ENG1" },
684	{ .fc_id = 657, .cpu_id = 506, .valid = 0, .name = "" },	684	{ .fc_id = 652, .cpu_id = 501, .valid = 1,
		685	.name = "STATUS_NIC2_ENG0" },
		686	{ .fc_id = 653, .cpu_id = 502, .valid = 1,
		687	.name = "STATUS_NIC2_ENG1" },
		688	{ .fc_id = 654, .cpu_id = 503, .valid = 1,
		689	.name = "STATUS_NIC3_ENG0" },
		690	{ .fc_id = 655, .cpu_id = 504, .valid = 1,
		691	.name = "STATUS_NIC3_ENG1" },
		692	{ .fc_id = 656, .cpu_id = 505, .valid = 1,
		693	.name = "STATUS_NIC4_ENG0" },
		694	{ .fc_id = 657, .cpu_id = 506, .valid = 1,
		695	.name = "STATUS_NIC4_ENG1" },
685	{ .fc_id = 658, .cpu_id = 507, .valid = 1, .name = "FIX_POWER_ENV_S" },	696	{ .fc_id = 658, .cpu_id = 507, .valid = 1, .name = "FIX_POWER_ENV_S" },
686	{ .fc_id = 659, .cpu_id = 508, .valid = 1, .name = "FIX_POWER_ENV_E" },	697	{ .fc_id = 659, .cpu_id = 508, .valid = 1, .name = "FIX_POWER_ENV_E" },
687	{ .fc_id = 660, .cpu_id = 509, .valid = 1,	698	{ .fc_id = 660, .cpu_id = 509, .valid = 1,

drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h+0−14

@@ -20,20 +20,6 @@ · 19 unmodified lines
20	#define UBOOT_FW_OFFSET 0x100000 /* 1MB in SRAM */	20	#define UBOOT_FW_OFFSET 0x100000 /* 1MB in SRAM */
21	#define LINUX_FW_OFFSET 0x800000 /* 8MB in HBM */	21	#define LINUX_FW_OFFSET 0x800000 /* 8MB in HBM */
22		22
23	enum gaudi_pll_index {
24	CPU_PLL = 0,
25	PCI_PLL,
26	SRAM_PLL,
27	HBM_PLL,
28	NIC_PLL,
29	DMA_PLL,
30	MESH_PLL,
31	MME_PLL,
32	TPC_PLL,
33	IF_PLL,
34	PLL_MAX
35	};
36
37	enum gaudi_nic_axi_error {	23	enum gaudi_nic_axi_error {
38	RXB,	24	RXB,
39	RXE,	25	RXE,

drivers/misc/habanalabs/include/goya/goya.h+1−1

@@ -30,7 +30,7 @@ · 29 unmodified lines
30		30
31	#define QMAN_PQ_ENTRY_SIZE 16 /* Bytes */	31	#define QMAN_PQ_ENTRY_SIZE 16 /* Bytes */
32		32
33	#define MAX_ASID 1024	33	#define MAX_ASID 2
34		34
35	#define PROT_BITS_OFFS 0xF80	35	#define PROT_BITS_OFFS 0xF80
36		36

drivers/misc/habanalabs/include/goya/goya_async_events.h+1−0

@@ -188,6 +188,7 @@ enum goya_async_event_id { · 187 unmodified lines
188	GOYA_ASYNC_EVENT_ID_HALT_MACHINE = 485,	188	GOYA_ASYNC_EVENT_ID_HALT_MACHINE = 485,
189	GOYA_ASYNC_EVENT_ID_INTS_REGISTER = 486,	189	GOYA_ASYNC_EVENT_ID_INTS_REGISTER = 486,
190	GOYA_ASYNC_EVENT_ID_SOFT_RESET = 487,	190	GOYA_ASYNC_EVENT_ID_SOFT_RESET = 487,
		191	GOYA_ASYNC_EVENT_PKT_QUEUE_OUT_SYNC = 506,
191	GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_S = 507,	192	GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_S = 507,
192	GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_E = 508,	193	GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_E = 508,
193	GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_S = 509,	194	GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_S = 509,

drivers/misc/habanalabs/include/goya/goya_fw_if.h+0−11

@@ -15,17 +15,6 @@ · 14 unmodified lines
15	#define UBOOT_FW_OFFSET 0x100000 /* 1MB in SRAM */	15	#define UBOOT_FW_OFFSET 0x100000 /* 1MB in SRAM */
16	#define LINUX_FW_OFFSET 0x800000 /* 8MB in DDR */	16	#define LINUX_FW_OFFSET 0x800000 /* 8MB in DDR */
17		17
18	enum goya_pll_index {
19	CPU_PLL = 0,
20	IC_PLL,
21	MC_PLL,
22	MME_PLL,
23	PCI_PLL,
24	EMMC_PLL,
25	TPC_PLL,
26	PLL_MAX
27	};
28
29	#define GOYA_PLL_FREQ_LOW 50000000 /* 50 MHz */	18	#define GOYA_PLL_FREQ_LOW 50000000 /* 50 MHz */
30		19
31	#endif /* GOYA_FW_IF_H */	20	#endif /* GOYA_FW_IF_H */

include/uapi/misc/habanalabs.h+57−20

 #define HL_INFO_SYNC_MANAGER		14
 #define HL_INFO_TOTAL_ENERGY		15
 #define HL_INFO_PLL_FREQUENCY		16
+#define HL_INFO_POWER			17
 #define HL_INFO_VERSION_MAX_LEN	128
 #define HL_INFO_CARD_NAME_MAX_LEN	16
 	__u16 output[HL_PLL_NUM_OUTPUTS];
 };
+/**
+ * struct hl_power_info - power information
+ * @power: power consumption
+ */
+struct hl_power_info {
+	__u64 power;
+};
 /**
  * struct hl_info_sync_manager - sync manager information
  * @first_available_sync_object: first available sob
 #define HL_CS_FLAGS_STAGED_SUBMISSION		0x40
 #define HL_CS_FLAGS_STAGED_SUBMISSION_FIRST	0x80
 #define HL_CS_FLAGS_STAGED_SUBMISSION_LAST	0x100
+#define HL_CS_FLAGS_CUSTOM_TIMEOUT		0x200
 #define HL_CS_STATUS_SUCCESS		0
 	/* holds address of array of hl_cs_chunk for execution phase */
 	__u64 chunks_execute;
-	union {
+	/* Sequence number of a staged submission CS
-		/* this holds address of array of hl_cs_chunk for store phase -
+	 * valid only if HL_CS_FLAGS_STAGED_SUBMISSION is set
-		 * Currently not in use
+	 */
-		 */
+	__u64 seq;
-		__u64 chunks_store;
-		/* Sequence number of a staged submission CS
-		 * valid only if HL_CS_FLAGS_STAGED_SUBMISSION is set
-		 */
-		__u64 seq;
-	};
 	/* Number of chunks in restore phase array. Maximum number is
 	 * HL_MAX_JOBS_PER_CS
 	 */
 	__u32 num_chunks_execute;
-	/* Number of chunks in restore phase array - Currently not in use */
+	/* timeout in seconds - valid only if HL_CS_FLAGS_CUSTOM_TIMEOUT
-	__u32 num_chunks_store;
+	 * is set
+	 */
+	__u32 timeout;
 	/* HL_CS_FLAGS_* */
 	__u32 cs_flags;
 	struct hl_cs_out out;
 };
+#define HL_WAIT_CS_FLAGS_INTERRUPT	0x2
+#define HL_WAIT_CS_FLAGS_INTERRUPT_MASK 0xFFF00000
 struct hl_wait_cs_in {
-	/* Command submission sequence number */
+	union {
-	__u64 seq;
+		struct {
-	/* Absolute timeout to wait in microseconds */
+			/* Command submission sequence number */
-	__u64 timeout_us;
+			__u64 seq;
+			/* Absolute timeout to wait for command submission
+			 * in microseconds
+			 */
+			__u64 timeout_us;
+		};
+		struct {
+			/* User address for completion comparison.
+			 * upon interrupt, driver will compare the value pointed
+			 * by this address with the supplied target value.
+			 * in order not to perform any comparison, set address
+			 * to all 1s.
+			 * Relevant only when HL_WAIT_CS_FLAGS_INTERRUPT is set
+			 */
+			__u64 addr;
+			/* Target value for completion comparison */
+			__u32 target;
+			/* Absolute timeout to wait for interrupt
+			 * in microseconds
+			 */
+			__u32 interrupt_timeout_us;
+		};
+	};
 	/* Context ID - Currently not in use */
 	__u32 ctx_id;
-	__u32 pad;
+	/* HL_WAIT_CS_FLAGS_*
+	 * If HL_WAIT_CS_FLAGS_INTERRUPT is set, this field should include
+	 * interrupt id according to HL_WAIT_CS_FLAGS_INTERRUPT_MASK, in order
+	 * not to specify an interrupt id ,set mask to all 1s.
+	 */
+	__u32 flags;
 };
 #define HL_WAIT_CS_STATUS_COMPLETED	0
  * Each JOB will be enqueued on a specific queue, according to the user's input.
  * There can be more then one JOB per queue.
  *
- * The CS IOCTL will receive three sets of JOBS. One set is for "restore" phase,
+ * The CS IOCTL will receive two sets of JOBS. One set is for "restore" phase
- * a second set is for "execution" phase and a third set is for "store" phase.
+ * and a second set is for "execution" phase.
  * The JOBS on the "restore" phase are enqueued only after context-switch
  * (or if its the first CS for this context). The user can also order the
  * driver to run the "restore" phase explicitly