Merge tag 'char-misc-5.15-rc1-2' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc

Pull habanalabs updates from Greg KH: "Here is another round of misc driver patches for 5.15-rc1. In here is only updates for the Habanalabs driver. This request is late because the previously-objected-to dma-buf patches are all removed and some fixes that you and others found are now included in here as well. All of these have been in linux-next for well over a week with no reports of problems, and they are all self-contained to only this one driver. Full details are in the shortlog" * tag 'char-misc-5.15-rc1-2' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc: (61 commits) habanalabs/gaudi: hwmon default card name habanalabs: add support for f/w reset habanalabs/gaudi: block ICACHE_BASE_ADDERESS_HIGH in TPC habanalabs: cannot sleep while holding spinlock habanalabs: never copy_from_user inside spinlock habanalabs: remove unnecessary device status check habanalabs: disable IRQ in user interrupts spinlock habanalabs: add "in device creation" status habanalabs/gaudi: invalidate PMMU mem cache on init habanalabs/gaudi: size should be printed in decimal habanalabs/gaudi: define DC POWER for secured PMC habanalabs/gaudi: unmask out of bounds SLM access interrupt habanalabs: add userptr_lookup node in debugfs habanalabs/gaudi: fetch TPC/MME ECC errors from F/W habanalabs: modify multi-CS to wait on stream masters habanalabs/gaudi: add monitored SOBs to state dump habanalabs/gaudi: restore user registers when context opens habanalabs/gaudi: increase boot fit timeout habanalabs: update to latest firmware headers habanalabs/gaudi: minimize number of register reads ...

Linus Torvalds committed 5 years ago

Commit 5ffc06e2 parents a668acb 4cd67ad28 files · +3918 −834Browse files

Unified · Split

28 files

Documentation/ABI/testing/debugfs-driver-habanalabs +19 −0
drivers/misc/habanalabs/common/Makefile +2 −1
drivers/misc/habanalabs/common/command_buffer.c +1 −3
drivers/misc/habanalabs/common/command_submission.c +1049 −338
drivers/misc/habanalabs/common/context.c +126 −20
drivers/misc/habanalabs/common/debugfs.c +174 −10
drivers/misc/habanalabs/common/device.c +86 −77
drivers/misc/habanalabs/common/firmware_if.c +32 −24
drivers/misc/habanalabs/common/habanalabs.h +375 −46
drivers/misc/habanalabs/common/habanalabs_drv.c +8 −5
drivers/misc/habanalabs/common/habanalabs_ioctl.c +2 −0
drivers/misc/habanalabs/common/hw_queue.c +155 −43
drivers/misc/habanalabs/common/memory.c +123 −46
drivers/misc/habanalabs/common/mmu/mmu_v1.c +6 −6
drivers/misc/habanalabs/common/pci/pci.c +2 −0
drivers/misc/habanalabs/common/state_dump.c +718 −0
drivers/misc/habanalabs/common/sysfs.c +7 −13
drivers/misc/habanalabs/gaudi/gaudi.c +571 −145
drivers/misc/habanalabs/gaudi/gaudiP.h +18 −1
drivers/misc/habanalabs/gaudi/gaudi_coresight.c +0 −5
drivers/misc/habanalabs/gaudi/gaudi_security.c +8 −0
drivers/misc/habanalabs/goya/goya.c +84 −18
drivers/misc/habanalabs/include/common/cpucp_if.h +109 −6
drivers/misc/habanalabs/include/common/hl_boot_if.h +57 −5
drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_regs.h +3 −0
drivers/misc/habanalabs/include/gaudi/gaudi_masks.h +17 −0
drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h +0 −2
include/uapi/misc/habanalabs.h +166 −20

Documentation/ABI/testing/debugfs-driver-habanalabs+19−0

@@ -215,6 +215,17 @@ Description: Sets the skip reset on timeout option for the device. Value of · 214 unmodified lines
215	215	"0" means device will be reset in case some CS has timed out,
216	216	otherwise it will not be reset.
217	217
	218	+What: /sys/kernel/debug/habanalabs/hl<n>/state_dump
	219	+Date: Oct 2021
	220	+KernelVersion: 5.15
	221	+Contact: ynudelman@habana.ai
	222	+Description: Gets the state dump occurring on a CS timeout or failure.
	223	+ State dump is used for debug and is created each time in case of
	224	+ a problem in a CS execution, before reset.
	225	+ Reading from the node returns the newest state dump available.
	226	+ Writing an integer X discards X state dumps, so that the
	227	+ next read would return X+1-st newest state dump.
	228	+
218	229	What: /sys/kernel/debug/habanalabs/hl<n>/stop_on_err
219	230	Date: Mar 2020
220	231	KernelVersion: 5.6
@@ -230,6 +241,14 @@ Description: Displays a list with information about the currently user · 9 unmodified lines
230	241	pointers (user virtual addresses) that are pinned and mapped
231	242	to DMA addresses
232	243
	244	+What: /sys/kernel/debug/habanalabs/hl<n>/userptr_lookup
	245	+Date: Aug 2021
	246	+KernelVersion: 5.15
	247	+Contact: ogabbay@kernel.org
	248	+Description: Allows to search for specific user pointers (user virtual
	249	+ addresses) that are pinned and mapped to DMA addresses, and see
	250	+ their resolution to the specific dma address.
	251	+
233	252	What: /sys/kernel/debug/habanalabs/hl<n>/vm
234	253	Date: Jan 2019
235	254	KernelVersion: 5.1

drivers/misc/habanalabs/common/Makefile+2−1

@@ -10,4 +10,5 @@ HL_COMMON_FILES := common/habanalabs_drv.o common/device.o common/context.o \ · 9 unmodified lines
10	10	common/asid.o common/habanalabs_ioctl.o \
11	11	common/command_buffer.o common/hw_queue.o common/irq.o \
12	12	common/sysfs.o common/hwmon.o common/memory.o \
13		− common/command_submission.o common/firmware_if.o
	13	+ common/command_submission.o common/firmware_if.o \
	14	+ common/state_dump.o

drivers/misc/habanalabs/common/command_buffer.c+1−3

@@ -314,8 +314,6 @@ int hl_cb_create(struct hl_device hdev, struct hl_cb_mgr mgr, · 313 unmodified lines
314	314
315	315	spin_lock(&mgr->cb_lock);
316	316	rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
317		− if (rc < 0)
318		− rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_KERNEL);
319	317	spin_unlock(&mgr->cb_lock);
320	318
321	319	if (rc < 0) {
@@ -552,7 +550,7 @@ int hl_cb_mmap(struct hl_fpriv hpriv, struct vm_area_struct vma) · 230 unmodified lines
552	550
553	551	vma->vm_private_data = cb;
554	552
555		− rc = hdev->asic_funcs->cb_mmap(hdev, vma, cb->kernel_address,
	553	+ rc = hdev->asic_funcs->mmap(hdev, vma, cb->kernel_address,
556	554	cb->bus_address, cb->size);
557	555	if (rc) {
558	556	spin_lock(&cb->lock);

drivers/misc/habanalabs/common/command_submission.c+1049−338

@@ -38,7 +38,11 @@ static void hl_sob_reset(struct kref *ref) · 37 unmodified lines
38	38	kref);
39	39	struct hl_device *hdev = hw_sob->hdev;
40	40
	41	+ dev_dbg(hdev->dev, "reset sob id %u\n", hw_sob->sob_id);
	42	+
41	43	hdev->asic_funcs->reset_sob(hdev, hw_sob);
	44	+
	45	+ hw_sob->need_reset = false;
42	46	}
43	47
44	48	void hl_sob_reset_error(struct kref *ref)
@@ -52,6 +56,24 @@ void hl_sob_reset_error(struct kref *ref) · 7 unmodified lines
52	56	hw_sob->q_idx, hw_sob->sob_id);
53	57	}
54	58
	59	+void hw_sob_put(struct hl_hw_sob *hw_sob)
	60	+{
	61	+ if (hw_sob)
	62	+ kref_put(&hw_sob->kref, hl_sob_reset);
	63	+}
	64	+
	65	+static void hw_sob_put_err(struct hl_hw_sob *hw_sob)
	66	+{
	67	+ if (hw_sob)
	68	+ kref_put(&hw_sob->kref, hl_sob_reset_error);
	69	+}
	70	+
	71	+void hw_sob_get(struct hl_hw_sob *hw_sob)
	72	+{
	73	+ if (hw_sob)
	74	+ kref_get(&hw_sob->kref);
	75	+}
	76	+
55	77	/**
56	78	* hl_gen_sob_mask() - Generates a sob mask to be used in a monitor arm packet
57	79	* @sob_base: sob base id
@@ -84,76 +106,29 @@ int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask) · 26 unmodified lines
84	106	return 0;
85	107	}
86	108
87		−static void sob_reset_work(struct work_struct *work)
88		−{
89		− struct hl_cs_compl *hl_cs_cmpl =
90		− container_of(work, struct hl_cs_compl, sob_reset_work);
91		− struct hl_device *hdev = hl_cs_cmpl->hdev;
92		−
93		− /*
94		− * A signal CS can get completion while the corresponding wait
95		− * for signal CS is on its way to the PQ. The wait for signal CS
96		− * will get stuck if the signal CS incremented the SOB to its
97		− * max value and there are no pending (submitted) waits on this
98		− * SOB.
99		− * We do the following to void this situation:
100		− * 1. The wait for signal CS must get a ref for the signal CS as
101		− * soon as possible in cs_ioctl_signal_wait() and put it
102		− * before being submitted to the PQ but after it incremented
103		− * the SOB refcnt in init_signal_wait_cs().
104		− * 2. Signal/Wait for signal CS will decrement the SOB refcnt
105		− * here.
106		− * These two measures guarantee that the wait for signal CS will
107		− * reset the SOB upon completion rather than the signal CS and
108		− * hence the above scenario is avoided.
109		− */
110		− kref_put(&hl_cs_cmpl->hw_sob->kref, hl_sob_reset);
111		−
112		− if (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT)
113		− hdev->asic_funcs->reset_sob_group(hdev,
114		− hl_cs_cmpl->sob_group);
115		−
116		− kfree(hl_cs_cmpl);
117		−}
118		−
119	109	static void hl_fence_release(struct kref *kref)
120	110	{
121	111	struct hl_fence *fence =
122	112	container_of(kref, struct hl_fence, refcount);
123	113	struct hl_cs_compl *hl_cs_cmpl =
124	114	container_of(fence, struct hl_cs_compl, base_fence);
125		− struct hl_device *hdev = hl_cs_cmpl->hdev;
126		−
127		− /* EBUSY means the CS was never submitted and hence we don't have
128		− * an attached hw_sob object that we should handle here
129		− */
130		− if (fence->error == -EBUSY)
131		− goto free;
132		−
133		− if ((hl_cs_cmpl->type == CS_TYPE_SIGNAL) \|\|
134		− (hl_cs_cmpl->type == CS_TYPE_WAIT) \|\|
135		− (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT)) {
136		−
137		− dev_dbg(hdev->dev,
138		− "CS 0x%llx type %d finished, sob_id: %d, sob_val: 0x%x\n",
139		− hl_cs_cmpl->cs_seq,
140		− hl_cs_cmpl->type,
141		− hl_cs_cmpl->hw_sob->sob_id,
142		− hl_cs_cmpl->sob_val);
143		−
144		− queue_work(hdev->sob_reset_wq, &hl_cs_cmpl->sob_reset_work);
145	115
146		− return;
147		− }
148		−
149		−free:
150	116	kfree(hl_cs_cmpl);
151	117	}
152	118
153	119	void hl_fence_put(struct hl_fence *fence)
154	120	{
155		− if (fence)
156		− kref_put(&fence->refcount, hl_fence_release);
	121	+ if (IS_ERR_OR_NULL(fence))
	122	+ return;
	123	+ kref_put(&fence->refcount, hl_fence_release);
	124	+}
	125	+
	126	+void hl_fences_put(struct hl_fence **fence, int len)
	127	+{
	128	+ int i;
	129	+
	130	+ for (i = 0; i < len; i++, fence++)
	131	+ hl_fence_put(*fence);
157	132	}
158	133
159	134	void hl_fence_get(struct hl_fence *fence)
@@ -473,11 +448,139 @@ static void cs_handle_tdr(struct hl_device hdev, struct hl_cs cs) · 313 unmodified lines
473	448	spin_unlock(&hdev->cs_mirror_lock);
474	449	}
475	450
	451	+/*
	452	+ * force_complete_multi_cs - complete all contexts that wait on multi-CS
	453	+ *
	454	+ * @hdev: pointer to habanalabs device structure
	455	+ */
	456	+static void force_complete_multi_cs(struct hl_device *hdev)
	457	+{
	458	+ int i;
	459	+
	460	+ for (i = 0; i < MULTI_CS_MAX_USER_CTX; i++) {
	461	+ struct multi_cs_completion *mcs_compl;
	462	+
	463	+ mcs_compl = &hdev->multi_cs_completion[i];
	464	+
	465	+ spin_lock(&mcs_compl->lock);
	466	+
	467	+ if (!mcs_compl->used) {
	468	+ spin_unlock(&mcs_compl->lock);
	469	+ continue;
	470	+ }
	471	+
	472	+ /* when calling force complete no context should be waiting on
	473	+ * multi-cS.
	474	+ * We are calling the function as a protection for such case
	475	+ * to free any pending context and print error message
	476	+ */
	477	+ dev_err(hdev->dev,
	478	+ "multi-CS completion context %d still waiting when calling force completion\n",
	479	+ i);
	480	+ complete_all(&mcs_compl->completion);
	481	+ spin_unlock(&mcs_compl->lock);
	482	+ }
	483	+}
	484	+
	485	+/*
	486	+ * complete_multi_cs - complete all waiting entities on multi-CS
	487	+ *
	488	+ * @hdev: pointer to habanalabs device structure
	489	+ * @cs: CS structure
	490	+ * The function signals a waiting entity that has an overlapping stream masters
	491	+ * with the completed CS.
	492	+ * For example:
	493	+ * - a completed CS worked on stream master QID 4, multi CS completion
	494	+ * is actively waiting on stream master QIDs 3, 5. don't send signal as no
	495	+ * common stream master QID
	496	+ * - a completed CS worked on stream master QID 4, multi CS completion
	497	+ * is actively waiting on stream master QIDs 3, 4. send signal as stream
	498	+ * master QID 4 is common
	499	+ */
	500	+static void complete_multi_cs(struct hl_device hdev, struct hl_cs cs)
	501	+{
	502	+ struct hl_fence *fence = cs->fence;
	503	+ int i;
	504	+
	505	+ /* in case of multi CS check for completion only for the first CS */
	506	+ if (cs->staged_cs && !cs->staged_first)
	507	+ return;
	508	+
	509	+ for (i = 0; i < MULTI_CS_MAX_USER_CTX; i++) {
	510	+ struct multi_cs_completion *mcs_compl;
	511	+
	512	+ mcs_compl = &hdev->multi_cs_completion[i];
	513	+ if (!mcs_compl->used)
	514	+ continue;
	515	+
	516	+ spin_lock(&mcs_compl->lock);
	517	+
	518	+ /*
	519	+ * complete if:
	520	+ * 1. still waiting for completion
	521	+ * 2. the completed CS has at least one overlapping stream
	522	+ * master with the stream masters in the completion
	523	+ */
	524	+ if (mcs_compl->used &&
	525	+ (fence->stream_master_qid_map &
	526	+ mcs_compl->stream_master_qid_map)) {
	527	+ /* extract the timestamp only of first completed CS */
	528	+ if (!mcs_compl->timestamp)
	529	+ mcs_compl->timestamp =
	530	+ ktime_to_ns(fence->timestamp);
	531	+ complete_all(&mcs_compl->completion);
	532	+ }
	533	+
	534	+ spin_unlock(&mcs_compl->lock);
	535	+ }
	536	+}
	537	+
	538	+static inline void cs_release_sob_reset_handler(struct hl_device *hdev,
	539	+ struct hl_cs *cs,
	540	+ struct hl_cs_compl *hl_cs_cmpl)
	541	+{
	542	+ /* Skip this handler if the cs wasn't submitted, to avoid putting
	543	+ * the hw_sob twice, since this case already handled at this point,
	544	+ * also skip if the hw_sob pointer wasn't set.
	545	+ */
	546	+ if (!hl_cs_cmpl->hw_sob \|\| !cs->submitted)
	547	+ return;
	548	+
	549	+ spin_lock(&hl_cs_cmpl->lock);
	550	+
	551	+ /*
	552	+ * we get refcount upon reservation of signals or signal/wait cs for the
	553	+ * hw_sob object, and need to put it when the first staged cs
	554	+ * (which cotains the encaps signals) or cs signal/wait is completed.
	555	+ */
	556	+ if ((hl_cs_cmpl->type == CS_TYPE_SIGNAL) \|\|
	557	+ (hl_cs_cmpl->type == CS_TYPE_WAIT) \|\|
	558	+ (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT) \|\|
	559	+ (!!hl_cs_cmpl->encaps_signals)) {
	560	+ dev_dbg(hdev->dev,
	561	+ "CS 0x%llx type %d finished, sob_id: %d, sob_val: %u\n",
	562	+ hl_cs_cmpl->cs_seq,
	563	+ hl_cs_cmpl->type,
	564	+ hl_cs_cmpl->hw_sob->sob_id,
	565	+ hl_cs_cmpl->sob_val);
	566	+
	567	+ hw_sob_put(hl_cs_cmpl->hw_sob);
	568	+
	569	+ if (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT)
	570	+ hdev->asic_funcs->reset_sob_group(hdev,
	571	+ hl_cs_cmpl->sob_group);
	572	+ }
	573	+
	574	+ spin_unlock(&hl_cs_cmpl->lock);
	575	+}
	576	+
476	577	static void cs_do_release(struct kref *ref)
477	578	{
478	579	struct hl_cs *cs = container_of(ref, struct hl_cs, refcount);
479	580	struct hl_device *hdev = cs->ctx->hdev;
480	581	struct hl_cs_job job, tmp;
	582	+ struct hl_cs_compl *hl_cs_cmpl =
	583	+ container_of(cs->fence, struct hl_cs_compl, base_fence);
481	584
482	585	cs->completed = true;
483	586
@@ -493,8 +596,9 @@ static void cs_do_release(struct kref *ref) · 9 unmodified lines
493	596	complete_job(hdev, job);
494	597
495	598	if (!cs->submitted) {
496		− /* In case the wait for signal CS was submitted, the put occurs
497		− * in init_signal_wait_cs() or collective_wait_init_cs()
	599	+ /*
	600	+ * In case the wait for signal CS was submitted, the fence put
	601	+ * occurs in init_signal_wait_cs() or collective_wait_init_cs()
498	602	* right before hanging on the PQ.
499	603	*/
500	604	if (cs->type == CS_TYPE_WAIT \|\|
@@ -535,8 +639,20 @@ static void cs_do_release(struct kref *ref) · 34 unmodified lines
535	639	list_del(&cs->staged_cs_node);
536	640	spin_unlock(&hdev->cs_mirror_lock);
537	641	}
	642	+
	643	+ /* decrement refcount to handle when first staged cs
	644	+ * with encaps signals is completed.
	645	+ */
	646	+ if (hl_cs_cmpl->encaps_signals)
	647	+ kref_put(&hl_cs_cmpl->encaps_sig_hdl->refcount,
	648	+ hl_encaps_handle_do_release);
538	649	}
539	650
	651	+ if ((cs->type == CS_TYPE_WAIT \|\| cs->type == CS_TYPE_COLLECTIVE_WAIT)
	652	+ && cs->encaps_signals)
	653	+ kref_put(&cs->encaps_sig_hdl->refcount,
	654	+ hl_encaps_handle_do_release);
	655	+
540	656	out:
541	657	/* Must be called before hl_ctx_put because inside we use ctx to get
542	658	* the device
@@ -566,6 +682,10 @@ out: · 23 unmodified lines
566	682	if (cs->timestamp)
567	683	cs->fence->timestamp = ktime_get();
568	684	complete_all(&cs->fence->completion);
	685	+ complete_multi_cs(hdev, cs);
	686	+
	687	+ cs_release_sob_reset_handler(hdev, cs, hl_cs_cmpl);
	688	+
569	689	hl_fence_put(cs->fence);
570	690
571	691	kfree(cs->jobs_in_queue_cnt);
@@ -621,6 +741,10 @@ static void cs_timedout(struct work_struct *work) · 49 unmodified lines
621	741	break;
622	742	}
623	743
	744	+ rc = hl_state_dump(hdev);
	745	+ if (rc)
	746	+ dev_err(hdev->dev, "Error during system state dump %d\n", rc);
	747	+
624	748	cs_put(cs);
625	749
626	750	if (likely(!skip_reset_on_timeout)) {
@@ -661,6 +785,7 @@ static int allocate_cs(struct hl_device hdev, struct hl_ctx ctx, · 34 unmodified lines
661	785	cs->completed = false;
662	786	cs->type = cs_type;
663	787	cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP);
	788	+ cs->encaps_signals = !!(flags & HL_CS_FLAGS_ENCAP_SIGNALS);
664	789	cs->timeout_jiffies = timeout;
665	790	cs->skip_reset_on_timeout =
666	791	hdev->skip_reset_on_timeout \|\|
@@ -671,9 +796,9 @@ static int allocate_cs(struct hl_device hdev, struct hl_ctx ctx, · 4 unmodified lines
671	796	kref_init(&cs->refcount);
672	797	spin_lock_init(&cs->job_lock);
673	798
674		− cs_cmpl = kmalloc(sizeof(*cs_cmpl), GFP_ATOMIC);
	799	+ cs_cmpl = kzalloc(sizeof(*cs_cmpl), GFP_ATOMIC);
675	800	if (!cs_cmpl)
676		− cs_cmpl = kmalloc(sizeof(*cs_cmpl), GFP_KERNEL);
	801	+ cs_cmpl = kzalloc(sizeof(*cs_cmpl), GFP_KERNEL);
677	802
678	803	if (!cs_cmpl) {
679	804	atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
@@ -698,7 +823,6 @@ static int allocate_cs(struct hl_device hdev, struct hl_ctx ctx, · 18 unmodified lines
698	823	cs_cmpl->hdev = hdev;
699	824	cs_cmpl->type = cs->type;
700	825	spin_lock_init(&cs_cmpl->lock);
701		− INIT_WORK(&cs_cmpl->sob_reset_work, sob_reset_work);
702	826	cs->fence = &cs_cmpl->base_fence;
703	827
704	828	spin_lock(&ctx->cs_lock);
@@ -791,31 +915,22 @@ void hl_cs_rollback_all(struct hl_device *hdev) · 86 unmodified lines
791	915	cs_rollback(hdev, cs);
792	916	cs_put(cs);
793	917	}
794		−}
795		−
796		−void hl_pending_cb_list_flush(struct hl_ctx *ctx)
797		−{
798		− struct hl_pending_cb pending_cb, tmp;
799	918
800		− list_for_each_entry_safe(pending_cb, tmp,
801		− &ctx->pending_cb_list, cb_node) {
802		− list_del(&pending_cb->cb_node);
803		− hl_cb_put(pending_cb->cb);
804		− kfree(pending_cb);
805		− }
	919	+ force_complete_multi_cs(hdev);
806	920	}
807	921
808	922	static void
809	923	wake_pending_user_interrupt_threads(struct hl_user_interrupt *interrupt)
810	924	{
811	925	struct hl_user_pending_interrupt *pend;
	926	+ unsigned long flags;
812	927
813		− spin_lock(&interrupt->wait_list_lock);
	928	+ spin_lock_irqsave(&interrupt->wait_list_lock, flags);
814	929	list_for_each_entry(pend, &interrupt->wait_list_head, wait_list_node) {
815	930	pend->fence.error = -EIO;
816	931	complete_all(&pend->fence.completion);
817	932	}
818		− spin_unlock(&interrupt->wait_list_lock);
	933	+ spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
819	934	}
820	935
821	936	void hl_release_pending_user_interrupts(struct hl_device *hdev)
@@ -981,6 +1096,10 @@ static enum hl_cs_type hl_cs_get_cs_type(u32 cs_type_flags) · 159 unmodified lines
981	1096	return CS_TYPE_WAIT;
982	1097	else if (cs_type_flags & HL_CS_FLAGS_COLLECTIVE_WAIT)
983	1098	return CS_TYPE_COLLECTIVE_WAIT;
	1099	+ else if (cs_type_flags & HL_CS_FLAGS_RESERVE_SIGNALS_ONLY)
	1100	+ return CS_RESERVE_SIGNALS;
	1101	+ else if (cs_type_flags & HL_CS_FLAGS_UNRESERVE_SIGNALS_ONLY)
	1102	+ return CS_UNRESERVE_SIGNALS;
984	1103	else
985	1104	return CS_TYPE_DEFAULT;
986	1105	}
@@ -1081,7 +1200,8 @@ static int hl_cs_copy_chunk_array(struct hl_device *hdev, · 94 unmodified lines
1081	1200	}
1082	1201
1083	1202	static int cs_staged_submission(struct hl_device hdev, struct hl_cs cs,
1084		− u64 sequence, u32 flags)
	1203	+ u64 sequence, u32 flags,
	1204	+ u32 encaps_signal_handle)
1085	1205	{
1086	1206	if (!(flags & HL_CS_FLAGS_STAGED_SUBMISSION))
1087	1207	return 0;
@@ -1093,6 +1213,9 @@ static int cs_staged_submission(struct hl_device hdev, struct hl_cs cs, · 5 unmodified lines
1093	1213	/* Staged CS sequence is the first CS sequence */
1094	1214	INIT_LIST_HEAD(&cs->staged_cs_node);
1095	1215	cs->staged_sequence = cs->sequence;
	1216	+
	1217	+ if (cs->encaps_signals)
	1218	+ cs->encaps_sig_hdl_id = encaps_signal_handle;
1096	1219	} else {
1097	1220	/* User sequence will be validated in 'hl_hw_queue_schedule_cs'
1098	1221	* under the cs_mirror_lock
@@ -1108,9 +1231,20 @@ static int cs_staged_submission(struct hl_device hdev, struct hl_cs cs, · 9 unmodified lines
1108	1231	return 0;
1109	1232	}
1110	1233
	1234	+static u32 get_stream_master_qid_mask(struct hl_device *hdev, u32 qid)
	1235	+{
	1236	+ int i;
	1237	+
	1238	+ for (i = 0; i < hdev->stream_master_qid_arr_size; i++)
	1239	+ if (qid == hdev->stream_master_qid_arr[i])
	1240	+ return BIT(i);
	1241	+
	1242	+ return 0;
	1243	+}
	1244	+
1111	1245	static int cs_ioctl_default(struct hl_fpriv hpriv, void __user chunks,
1112	1246	u32 num_chunks, u64 *cs_seq, u32 flags,
1113		− u32 timeout)
	1247	+ u32 encaps_signals_handle, u32 timeout)
1114	1248	{
1115	1249	bool staged_mid, int_queues_only = true;
1116	1250	struct hl_device *hdev = hpriv->hdev;
@@ -1121,6 +1255,7 @@ static int cs_ioctl_default(struct hl_fpriv hpriv, void __user chunks, · 4 unmodified lines
1121	1255	struct hl_cs *cs;
1122	1256	struct hl_cb *cb;
1123	1257	u64 user_sequence;
	1258	+ u8 stream_master_qid_map = 0;
1124	1259	int rc, i;
1125	1260
1126	1261	cntr = &hdev->aggregated_cs_counters;
@@ -1148,7 +1283,8 @@ static int cs_ioctl_default(struct hl_fpriv hpriv, void __user chunks, · 21 unmodified lines
1148	1283
1149	1284	hl_debugfs_add_cs(cs);
1150	1285
1151		− rc = cs_staged_submission(hdev, cs, user_sequence, flags);
	1286	+ rc = cs_staged_submission(hdev, cs, user_sequence, flags,
	1287	+ encaps_signals_handle);
1152	1288	if (rc)
1153	1289	goto free_cs_object;
1154	1290
@@ -1179,9 +1315,20 @@ static int cs_ioctl_default(struct hl_fpriv hpriv, void __user chunks, · 24 unmodified lines
1179	1315	cb = (struct hl_cb *) (uintptr_t) chunk->cb_handle;
1180	1316	}
1181	1317
1182		− if (queue_type == QUEUE_TYPE_EXT \|\| queue_type == QUEUE_TYPE_HW)
	1318	+ if (queue_type == QUEUE_TYPE_EXT \|\|
	1319	+ queue_type == QUEUE_TYPE_HW) {
1183	1320	int_queues_only = false;
1184	1321
	1322	+ /*
	1323	+ * store which stream are being used for external/HW
	1324	+ * queues of this CS
	1325	+ */
	1326	+ if (hdev->supports_wait_for_multi_cs)
	1327	+ stream_master_qid_map \|=
	1328	+ get_stream_master_qid_mask(hdev,
	1329	+ chunk->queue_index);
	1330	+ }
	1331	+
1185	1332	job = hl_cs_allocate_job(hdev, queue_type,
1186	1333	is_kernel_allocated_cb);
1187	1334	if (!job) {
@@ -1242,6 +1389,13 @@ static int cs_ioctl_default(struct hl_fpriv hpriv, void __user chunks, · 54 unmodified lines
1242	1389	goto free_cs_object;
1243	1390	}
1244	1391
	1392	+ /*
	1393	+ * store the (external/HW queues) streams used by the CS in the
	1394	+ * fence object for multi-CS completion
	1395	+ */
	1396	+ if (hdev->supports_wait_for_multi_cs)
	1397	+ cs->fence->stream_master_qid_map = stream_master_qid_map;
	1398	+
1245	1399	rc = hl_hw_queue_schedule_cs(cs);
1246	1400	if (rc) {
1247	1401	if (rc != -EAGAIN)
@@ -1270,130 +1424,6 @@ out: · 22 unmodified lines
1270	1424	return rc;
1271	1425	}
1272	1426
1273		−static int pending_cb_create_job(struct hl_device hdev, struct hl_ctx ctx,
1274		− struct hl_cs cs, struct hl_cb cb, u32 size, u32 hw_queue_id)
1275		−{
1276		− struct hw_queue_properties *hw_queue_prop;
1277		− struct hl_cs_counters_atomic *cntr;
1278		− struct hl_cs_job *job;
1279		−
1280		− hw_queue_prop = &hdev->asic_prop.hw_queues_props[hw_queue_id];
1281		− cntr = &hdev->aggregated_cs_counters;
1282		−
1283		− job = hl_cs_allocate_job(hdev, hw_queue_prop->type, true);
1284		− if (!job) {
1285		− atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1286		− atomic64_inc(&cntr->out_of_mem_drop_cnt);
1287		− dev_err(hdev->dev, "Failed to allocate a new job\n");
1288		− return -ENOMEM;
1289		− }
1290		−
1291		− job->id = 0;
1292		− job->cs = cs;
1293		− job->user_cb = cb;
1294		− atomic_inc(&job->user_cb->cs_cnt);
1295		− job->user_cb_size = size;
1296		− job->hw_queue_id = hw_queue_id;
1297		− job->patched_cb = job->user_cb;
1298		− job->job_cb_size = job->user_cb_size;
1299		−
1300		− /* increment refcount as for external queues we get completion */
1301		− cs_get(cs);
1302		−
1303		− cs->jobs_in_queue_cnt[job->hw_queue_id]++;
1304		−
1305		− list_add_tail(&job->cs_node, &cs->job_list);
1306		−
1307		− hl_debugfs_add_job(hdev, job);
1308		−
1309		− return 0;
1310		−}
1311		−
1312		−static int hl_submit_pending_cb(struct hl_fpriv *hpriv)
1313		−{
1314		− struct hl_device *hdev = hpriv->hdev;
1315		− struct hl_ctx *ctx = hpriv->ctx;
1316		− struct hl_pending_cb pending_cb, tmp;
1317		− struct list_head local_cb_list;
1318		− struct hl_cs *cs;
1319		− struct hl_cb *cb;
1320		− u32 hw_queue_id;
1321		− u32 cb_size;
1322		− int process_list, rc = 0;
1323		−
1324		− if (list_empty(&ctx->pending_cb_list))
1325		− return 0;
1326		−
1327		− process_list = atomic_cmpxchg(&ctx->thread_pending_cb_token, 1, 0);
1328		−
1329		− /* Only a single thread is allowed to process the list */
1330		− if (!process_list)
1331		− return 0;
1332		−
1333		− if (list_empty(&ctx->pending_cb_list))
1334		− goto free_pending_cb_token;
1335		−
1336		− /* move all list elements to a local list */
1337		− INIT_LIST_HEAD(&local_cb_list);
1338		− spin_lock(&ctx->pending_cb_lock);
1339		− list_for_each_entry_safe(pending_cb, tmp, &ctx->pending_cb_list,
1340		− cb_node)
1341		− list_move_tail(&pending_cb->cb_node, &local_cb_list);
1342		− spin_unlock(&ctx->pending_cb_lock);
1343		−
1344		− rc = allocate_cs(hdev, ctx, CS_TYPE_DEFAULT, ULLONG_MAX, &cs, 0,
1345		− hdev->timeout_jiffies);
1346		− if (rc)
1347		− goto add_list_elements;
1348		−
1349		− hl_debugfs_add_cs(cs);
1350		−
1351		− /* Iterate through pending cb list, create jobs and add to CS */
1352		− list_for_each_entry(pending_cb, &local_cb_list, cb_node) {
1353		− cb = pending_cb->cb;
1354		− cb_size = pending_cb->cb_size;
1355		− hw_queue_id = pending_cb->hw_queue_id;
1356		−
1357		− rc = pending_cb_create_job(hdev, ctx, cs, cb, cb_size,
1358		− hw_queue_id);
1359		− if (rc)
1360		− goto free_cs_object;
1361		− }
1362		−
1363		− rc = hl_hw_queue_schedule_cs(cs);
1364		− if (rc) {
1365		− if (rc != -EAGAIN)
1366		− dev_err(hdev->dev,
1367		− "Failed to submit CS %d.%llu (%d)\n",
1368		− ctx->asid, cs->sequence, rc);
1369		− goto free_cs_object;
1370		− }
1371		−
1372		− /* pending cb was scheduled successfully */
1373		− list_for_each_entry_safe(pending_cb, tmp, &local_cb_list, cb_node) {
1374		− list_del(&pending_cb->cb_node);
1375		− kfree(pending_cb);
1376		− }
1377		−
1378		− cs_put(cs);
1379		−
1380		− goto free_pending_cb_token;
1381		−
1382		−free_cs_object:
1383		− cs_rollback(hdev, cs);
1384		− cs_put(cs);
1385		−add_list_elements:
1386		− spin_lock(&ctx->pending_cb_lock);
1387		− list_for_each_entry_safe_reverse(pending_cb, tmp, &local_cb_list,
1388		− cb_node)
1389		− list_move(&pending_cb->cb_node, &ctx->pending_cb_list);
1390		− spin_unlock(&ctx->pending_cb_lock);
1391		−free_pending_cb_token:
1392		− atomic_set(&ctx->thread_pending_cb_token, 1);
1393		−
1394		− return rc;
1395		−}
1396		−
1397	1427	static int hl_cs_ctx_switch(struct hl_fpriv hpriv, union hl_cs_args args,
1398	1428	u64 *cs_seq)
1399	1429	{
@@ -1443,7 +1473,7 @@ static int hl_cs_ctx_switch(struct hl_fpriv hpriv, union hl_cs_args args, · 43 unmodified lines
1443	1473	rc = 0;
1444	1474	} else {
1445	1475	rc = cs_ioctl_default(hpriv, chunks, num_chunks,
1446		− cs_seq, 0, hdev->timeout_jiffies);
	1476	+ cs_seq, 0, 0, hdev->timeout_jiffies);
1447	1477	}
1448	1478
1449	1479	mutex_unlock(&hpriv->restore_phase_mutex);
@@ -1501,10 +1531,17 @@ out: · 51 unmodified lines
1501	1531	* hl_cs_signal_sob_wraparound_handler: handle SOB value wrapaound case.
1502	1532	* if the SOB value reaches the max value move to the other SOB reserved
1503	1533	* to the queue.
	1534	+ * @hdev: pointer to device structure
	1535	+ * @q_idx: stream queue index
	1536	+ * @hw_sob: the H/W SOB used in this signal CS.
	1537	+ * @count: signals count
	1538	+ * @encaps_sig: tells whether it's reservation for encaps signals or not.
	1539	+ *
1504	1540	* Note that this function must be called while hw_queues_lock is taken.
1505	1541	*/
1506	1542	int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,
1507		− struct hl_hw_sob **hw_sob, u32 count)
	1543	+ struct hl_hw_sob **hw_sob, u32 count, bool encaps_sig)
	1544	+
1508	1545	{
1509	1546	struct hl_sync_stream_properties *prop;
1510	1547	struct hl_hw_sob sob = hw_sob, *other_sob;
@@ -1512,7 +1549,7 @@ int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, · 1 unmodified line
1512	1549
1513	1550	prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
1514	1551
1515		− kref_get(&sob->kref);
	1552	+ hw_sob_get(sob);
1516	1553
1517	1554	/* check for wraparound */
1518	1555	if (prop->next_sob_val + count >= HL_MAX_SOB_VAL) {
@@ -1522,7 +1559,7 @@ int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, · 3 unmodified lines
1522	1559	* just incremented the refcount right before calling this
1523	1560	* function.
1524	1561	*/
1525		− kref_put(&sob->kref, hl_sob_reset_error);
	1562	+ hw_sob_put_err(sob);
1526	1563
1527	1564	/*
1528	1565	* check the other sob value, if it still in use then fail
@@ -1537,12 +1574,42 @@ int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, · 8 unmodified lines
1537	1574	return -EINVAL;
1538	1575	}
1539	1576
1540		− prop->next_sob_val = 1;
	1577	+ /*
	1578	+ * next_sob_val always points to the next available signal
	1579	+ * in the sob, so in encaps signals it will be the next one
	1580	+ * after reserving the required amount.
	1581	+ */
	1582	+ if (encaps_sig)
	1583	+ prop->next_sob_val = count + 1;
	1584	+ else
	1585	+ prop->next_sob_val = count;
1541	1586
1542	1587	/* only two SOBs are currently in use */
1543	1588	prop->curr_sob_offset = other_sob_offset;
1544	1589	*hw_sob = other_sob;
1545	1590
	1591	+ /*
	1592	+ * check if other_sob needs reset, then do it before using it
	1593	+ * for the reservation or the next signal cs.
	1594	+ * we do it here, and for both encaps and regular signal cs
	1595	+ * cases in order to avoid possible races of two kref_put
	1596	+ * of the sob which can occur at the same time if we move the
	1597	+ * sob reset(kref_put) to cs_do_release function.
	1598	+ * in addition, if we have combination of cs signal and
	1599	+ * encaps, and at the point we need to reset the sob there was
	1600	+ * no more reservations and only signal cs keep coming,
	1601	+ * in such case we need signal_cs to put the refcount and
	1602	+ * reset the sob.
	1603	+ */
	1604	+ if (other_sob->need_reset)
	1605	+ hw_sob_put(other_sob);
	1606	+
	1607	+ if (encaps_sig) {
	1608	+ /* set reset indication for the sob */
	1609	+ sob->need_reset = true;
	1610	+ hw_sob_get(other_sob);
	1611	+ }
	1612	+
1546	1613	dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n",
1547	1614	prop->curr_sob_offset, q_idx);
1548	1615	} else {
@@ -1553,12 +1620,18 @@ int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, · 4 unmodified lines
1553	1620	}
1554	1621
1555	1622	static int cs_ioctl_extract_signal_seq(struct hl_device *hdev,
1556		− struct hl_cs_chunk chunk, u64 signal_seq, struct hl_ctx *ctx)
	1623	+ struct hl_cs_chunk chunk, u64 signal_seq, struct hl_ctx *ctx,
	1624	+ bool encaps_signals)
1557	1625	{
1558	1626	u64 *signal_seq_arr = NULL;
1559	1627	u32 size_to_copy, signal_seq_arr_len;
1560	1628	int rc = 0;
1561	1629
	1630	+ if (encaps_signals) {
	1631	+ *signal_seq = chunk->encaps_signal_seq;
	1632	+ return 0;
	1633	+ }
	1634	+
1562	1635	signal_seq_arr_len = chunk->num_signal_seq_arr;
1563	1636
1564	1637	/* currently only one signal seq is supported */
@@ -1583,7 +1656,7 @@ static int cs_ioctl_extract_signal_seq(struct hl_device *hdev, · 18 unmodified lines
1583	1656	return -ENOMEM;
1584	1657	}
1585	1658
1586		− size_to_copy = chunk->num_signal_seq_arr * sizeof(*signal_seq_arr);
	1659	+ size_to_copy = signal_seq_arr_len * sizeof(*signal_seq_arr);
1587	1660	if (copy_from_user(signal_seq_arr,
1588	1661	u64_to_user_ptr(chunk->signal_seq_arr),
1589	1662	size_to_copy)) {
@@ -1605,8 +1678,8 @@ out: · 15 unmodified lines
1605	1678	}
1606	1679
1607	1680	static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev,
1608		− struct hl_ctx ctx, struct hl_cs cs, enum hl_queue_type q_type,
1609		− u32 q_idx)
	1681	+ struct hl_ctx ctx, struct hl_cs cs,
	1682	+ enum hl_queue_type q_type, u32 q_idx, u32 encaps_signal_offset)
1610	1683	{
1611	1684	struct hl_cs_counters_atomic *cntr;
1612	1685	struct hl_cs_job *job;
@@ -1644,6 +1717,9 @@ static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev, · 31 unmodified lines
1644	1717	job->user_cb_size = cb_size;
1645	1718	job->hw_queue_id = q_idx;
1646	1719
	1720	+ if ((cs->type == CS_TYPE_WAIT \|\| cs->type == CS_TYPE_COLLECTIVE_WAIT)
	1721	+ && cs->encaps_signals)
	1722	+ job->encaps_sig_wait_offset = encaps_signal_offset;
1647	1723	/*
1648	1724	* No need in parsing, user CB is the patched CB.
1649	1725	* We call hl_cb_destroy() out of two reasons - we don't need the CB in
@@ -1666,75 +1742,307 @@ static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev, · 16 unmodified lines
1666	1742	return 0;
1667	1743	}
1668	1744
1669		−static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
1670		− void __user *chunks, u32 num_chunks,
1671		− u64 *cs_seq, u32 flags, u32 timeout)
	1745	+static int cs_ioctl_reserve_signals(struct hl_fpriv *hpriv,
	1746	+ u32 q_idx, u32 count,
	1747	+ u32 handle_id, u32 sob_addr,
	1748	+ u32 *signals_count)
1672	1749	{
1673		− struct hl_cs_chunk cs_chunk_array, chunk;
1674	1750	struct hw_queue_properties *hw_queue_prop;
	1751	+ struct hl_sync_stream_properties *prop;
1675	1752	struct hl_device *hdev = hpriv->hdev;
1676		− struct hl_cs_compl *sig_waitcs_cmpl;
1677		− u32 q_idx, collective_engine_id = 0;
1678		− struct hl_cs_counters_atomic *cntr;
1679		− struct hl_fence *sig_fence = NULL;
1680		− struct hl_ctx *ctx = hpriv->ctx;
1681		− enum hl_queue_type q_type;
1682		− struct hl_cs *cs;
1683		− u64 signal_seq;
1684		− int rc;
1685		−
1686		− cntr = &hdev->aggregated_cs_counters;
1687		− *cs_seq = ULLONG_MAX;
	1753	+ struct hl_cs_encaps_sig_handle *handle;
	1754	+ struct hl_encaps_signals_mgr *mgr;
	1755	+ struct hl_hw_sob *hw_sob;
	1756	+ int hdl_id;
	1757	+ int rc = 0;
1688	1758
1689		− rc = hl_cs_copy_chunk_array(hdev, &cs_chunk_array, chunks, num_chunks,
1690		− ctx);
1691		− if (rc)
	1759	+ if (count >= HL_MAX_SOB_VAL) {
	1760	+ dev_err(hdev->dev, "signals count(%u) exceeds the max SOB value\n",
	1761	+ count);
	1762	+ rc = -EINVAL;
1692	1763	goto out;
	1764	+ }
1693	1765
1694		− /* currently it is guaranteed to have only one chunk */
1695		− chunk = &cs_chunk_array[0];
1696		−
1697		− if (chunk->queue_index >= hdev->asic_prop.max_queues) {
1698		− atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1699		− atomic64_inc(&cntr->validation_drop_cnt);
	1766	+ if (q_idx >= hdev->asic_prop.max_queues) {
1700	1767	dev_err(hdev->dev, "Queue index %d is invalid\n",
1701		− chunk->queue_index);
	1768	+ q_idx);
1702	1769	rc = -EINVAL;
1703		− goto free_cs_chunk_array;
	1770	+ goto out;
1704	1771	}
1705	1772
1706		− q_idx = chunk->queue_index;
1707	1773	hw_queue_prop = &hdev->asic_prop.hw_queues_props[q_idx];
1708		− q_type = hw_queue_prop->type;
1709	1774
1710	1775	if (!hw_queue_prop->supports_sync_stream) {
1711		− atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1712		− atomic64_inc(&cntr->validation_drop_cnt);
1713	1776	dev_err(hdev->dev,
1714	1777	"Queue index %d does not support sync stream operations\n",
1715		− q_idx);
	1778	+ q_idx);
1716	1779	rc = -EINVAL;
1717		− goto free_cs_chunk_array;
	1780	+ goto out;
1718	1781	}
1719	1782
1720		− if (cs_type == CS_TYPE_COLLECTIVE_WAIT) {
1721		− if (!(hw_queue_prop->collective_mode == HL_COLLECTIVE_MASTER)) {
1722		− atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1723		− atomic64_inc(&cntr->validation_drop_cnt);
1724		− dev_err(hdev->dev,
1725		− "Queue index %d is invalid\n", q_idx);
1726		− rc = -EINVAL;
1727		− goto free_cs_chunk_array;
1728		− }
	1783	+ prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
1729	1784
1730		− collective_engine_id = chunk->collective_engine_id;
1731		− }
	1785	+ handle = kzalloc(sizeof(*handle), GFP_KERNEL);
	1786	+ if (!handle) {
	1787	+ rc = -ENOMEM;
	1788	+ goto out;
	1789	+ }
	1790	+
	1791	+ handle->count = count;
	1792	+ mgr = &hpriv->ctx->sig_mgr;
	1793	+
	1794	+ spin_lock(&mgr->lock);
	1795	+ hdl_id = idr_alloc(&mgr->handles, handle, 1, 0, GFP_ATOMIC);
	1796	+ spin_unlock(&mgr->lock);
	1797	+
	1798	+ if (hdl_id < 0) {
	1799	+ dev_err(hdev->dev, "Failed to allocate IDR for a new signal reservation\n");
	1800	+ rc = -EINVAL;
	1801	+ goto out;
	1802	+ }
	1803	+
	1804	+ handle->id = hdl_id;
	1805	+ handle->q_idx = q_idx;
	1806	+ handle->hdev = hdev;
	1807	+ kref_init(&handle->refcount);
	1808	+
	1809	+ hdev->asic_funcs->hw_queues_lock(hdev);
	1810	+
	1811	+ hw_sob = &prop->hw_sob[prop->curr_sob_offset];
	1812	+
	1813	+ /*
	1814	+ * Increment the SOB value by count by user request
	1815	+ * to reserve those signals
	1816	+ * check if the signals amount to reserve is not exceeding the max sob
	1817	+ * value, if yes then switch sob.
	1818	+ */
	1819	+ rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, count,
	1820	+ true);
	1821	+ if (rc) {
	1822	+ dev_err(hdev->dev, "Failed to switch SOB\n");
	1823	+ hdev->asic_funcs->hw_queues_unlock(hdev);
	1824	+ rc = -EINVAL;
	1825	+ goto remove_idr;
	1826	+ }
	1827	+ /* set the hw_sob to the handle after calling the sob wraparound handler
	1828	+ * since sob could have changed.
	1829	+ */
	1830	+ handle->hw_sob = hw_sob;
	1831	+
	1832	+ /* store the current sob value for unreserve validity check, and
	1833	+ * signal offset support
	1834	+ */
	1835	+ handle->pre_sob_val = prop->next_sob_val - handle->count;
	1836	+
	1837	+ *signals_count = prop->next_sob_val;
	1838	+ hdev->asic_funcs->hw_queues_unlock(hdev);
	1839	+
	1840	+ *sob_addr = handle->hw_sob->sob_addr;
	1841	+ *handle_id = hdl_id;
	1842	+
	1843	+ dev_dbg(hdev->dev,
	1844	+ "Signals reserved, sob_id: %d, sob addr: 0x%x, last sob_val: %u, q_idx: %d, hdl_id: %d\n",
	1845	+ hw_sob->sob_id, handle->hw_sob->sob_addr,
	1846	+ prop->next_sob_val - 1, q_idx, hdl_id);
	1847	+ goto out;
	1848	+
	1849	+remove_idr:
	1850	+ spin_lock(&mgr->lock);
	1851	+ idr_remove(&mgr->handles, hdl_id);
	1852	+ spin_unlock(&mgr->lock);
	1853	+
	1854	+ kfree(handle);
	1855	+out:
	1856	+ return rc;
	1857	+}
	1858	+
	1859	+static int cs_ioctl_unreserve_signals(struct hl_fpriv *hpriv, u32 handle_id)
	1860	+{
	1861	+ struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
	1862	+ struct hl_sync_stream_properties *prop;
	1863	+ struct hl_device *hdev = hpriv->hdev;
	1864	+ struct hl_encaps_signals_mgr *mgr;
	1865	+ struct hl_hw_sob *hw_sob;
	1866	+ u32 q_idx, sob_addr;
	1867	+ int rc = 0;
	1868	+
	1869	+ mgr = &hpriv->ctx->sig_mgr;
	1870	+
	1871	+ spin_lock(&mgr->lock);
	1872	+ encaps_sig_hdl = idr_find(&mgr->handles, handle_id);
	1873	+ if (encaps_sig_hdl) {
	1874	+ dev_dbg(hdev->dev, "unreserve signals, handle: %u, SOB:0x%x, count: %u\n",
	1875	+ handle_id, encaps_sig_hdl->hw_sob->sob_addr,
	1876	+ encaps_sig_hdl->count);
	1877	+
	1878	+ hdev->asic_funcs->hw_queues_lock(hdev);
	1879	+
	1880	+ q_idx = encaps_sig_hdl->q_idx;
	1881	+ prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
	1882	+ hw_sob = &prop->hw_sob[prop->curr_sob_offset];
	1883	+ sob_addr = hdev->asic_funcs->get_sob_addr(hdev, hw_sob->sob_id);
	1884	+
	1885	+ /* Check if sob_val got out of sync due to other
	1886	+ * signal submission requests which were handled
	1887	+ * between the reserve-unreserve calls or SOB switch
	1888	+ * upon reaching SOB max value.
	1889	+ */
	1890	+ if (encaps_sig_hdl->pre_sob_val + encaps_sig_hdl->count
	1891	+ != prop->next_sob_val \|\|
	1892	+ sob_addr != encaps_sig_hdl->hw_sob->sob_addr) {
	1893	+ dev_err(hdev->dev, "Cannot unreserve signals, SOB val ran out of sync, expected: %u, actual val: %u\n",
	1894	+ encaps_sig_hdl->pre_sob_val,
	1895	+ (prop->next_sob_val - encaps_sig_hdl->count));
	1896	+
	1897	+ hdev->asic_funcs->hw_queues_unlock(hdev);
	1898	+ rc = -EINVAL;
	1899	+ goto out;
	1900	+ }
	1901	+
	1902	+ /*
	1903	+ * Decrement the SOB value by count by user request
	1904	+ * to unreserve those signals
	1905	+ */
	1906	+ prop->next_sob_val -= encaps_sig_hdl->count;
1732	1907
1733		− if (cs_type == CS_TYPE_WAIT \|\| cs_type == CS_TYPE_COLLECTIVE_WAIT) {
1734		− rc = cs_ioctl_extract_signal_seq(hdev, chunk, &signal_seq, ctx);
	1908	+ hdev->asic_funcs->hw_queues_unlock(hdev);
	1909	+
	1910	+ hw_sob_put(hw_sob);
	1911	+
	1912	+ /* Release the id and free allocated memory of the handle */
	1913	+ idr_remove(&mgr->handles, handle_id);
	1914	+ kfree(encaps_sig_hdl);
	1915	+ } else {
	1916	+ rc = -EINVAL;
	1917	+ dev_err(hdev->dev, "failed to unreserve signals, cannot find handler\n");
	1918	+ }
	1919	+out:
	1920	+ spin_unlock(&mgr->lock);
	1921	+
	1922	+ return rc;
	1923	+}
	1924	+
	1925	+static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
	1926	+ void __user *chunks, u32 num_chunks,
	1927	+ u64 *cs_seq, u32 flags, u32 timeout)
	1928	+{
	1929	+ struct hl_cs_encaps_sig_handle *encaps_sig_hdl = NULL;
	1930	+ bool handle_found = false, is_wait_cs = false,
	1931	+ wait_cs_submitted = false,
	1932	+ cs_encaps_signals = false;
	1933	+ struct hl_cs_chunk cs_chunk_array, chunk;
	1934	+ bool staged_cs_with_encaps_signals = false;
	1935	+ struct hw_queue_properties *hw_queue_prop;
	1936	+ struct hl_device *hdev = hpriv->hdev;
	1937	+ struct hl_cs_compl *sig_waitcs_cmpl;
	1938	+ u32 q_idx, collective_engine_id = 0;
	1939	+ struct hl_cs_counters_atomic *cntr;
	1940	+ struct hl_fence *sig_fence = NULL;
	1941	+ struct hl_ctx *ctx = hpriv->ctx;
	1942	+ enum hl_queue_type q_type;
	1943	+ struct hl_cs *cs;
	1944	+ u64 signal_seq;
	1945	+ int rc;
	1946	+
	1947	+ cntr = &hdev->aggregated_cs_counters;
	1948	+ *cs_seq = ULLONG_MAX;
	1949	+
	1950	+ rc = hl_cs_copy_chunk_array(hdev, &cs_chunk_array, chunks, num_chunks,
	1951	+ ctx);
	1952	+ if (rc)
	1953	+ goto out;
	1954	+
	1955	+ /* currently it is guaranteed to have only one chunk */
	1956	+ chunk = &cs_chunk_array[0];
	1957	+
	1958	+ if (chunk->queue_index >= hdev->asic_prop.max_queues) {
	1959	+ atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
	1960	+ atomic64_inc(&cntr->validation_drop_cnt);
	1961	+ dev_err(hdev->dev, "Queue index %d is invalid\n",
	1962	+ chunk->queue_index);
	1963	+ rc = -EINVAL;
	1964	+ goto free_cs_chunk_array;
	1965	+ }
	1966	+
	1967	+ q_idx = chunk->queue_index;
	1968	+ hw_queue_prop = &hdev->asic_prop.hw_queues_props[q_idx];
	1969	+ q_type = hw_queue_prop->type;
	1970	+
	1971	+ if (!hw_queue_prop->supports_sync_stream) {
	1972	+ atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
	1973	+ atomic64_inc(&cntr->validation_drop_cnt);
	1974	+ dev_err(hdev->dev,
	1975	+ "Queue index %d does not support sync stream operations\n",
	1976	+ q_idx);
	1977	+ rc = -EINVAL;
	1978	+ goto free_cs_chunk_array;
	1979	+ }
	1980	+
	1981	+ if (cs_type == CS_TYPE_COLLECTIVE_WAIT) {
	1982	+ if (!(hw_queue_prop->collective_mode == HL_COLLECTIVE_MASTER)) {
	1983	+ atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
	1984	+ atomic64_inc(&cntr->validation_drop_cnt);
	1985	+ dev_err(hdev->dev,
	1986	+ "Queue index %d is invalid\n", q_idx);
	1987	+ rc = -EINVAL;
	1988	+ goto free_cs_chunk_array;
	1989	+ }
	1990	+
	1991	+ collective_engine_id = chunk->collective_engine_id;
	1992	+ }
	1993	+
	1994	+ is_wait_cs = !!(cs_type == CS_TYPE_WAIT \|\|
	1995	+ cs_type == CS_TYPE_COLLECTIVE_WAIT);
	1996	+
	1997	+ cs_encaps_signals = !!(flags & HL_CS_FLAGS_ENCAP_SIGNALS);
	1998	+
	1999	+ if (is_wait_cs) {
	2000	+ rc = cs_ioctl_extract_signal_seq(hdev, chunk, &signal_seq,
	2001	+ ctx, cs_encaps_signals);
1735	2002	if (rc)
1736	2003	goto free_cs_chunk_array;
1737	2004
	2005	+ if (cs_encaps_signals) {
	2006	+ /* check if cs sequence has encapsulated
	2007	+ * signals handle
	2008	+ */
	2009	+ struct idr *idp;
	2010	+ u32 id;
	2011	+
	2012	+ spin_lock(&ctx->sig_mgr.lock);
	2013	+ idp = &ctx->sig_mgr.handles;
	2014	+ idr_for_each_entry(idp, encaps_sig_hdl, id) {
	2015	+ if (encaps_sig_hdl->cs_seq == signal_seq) {
	2016	+ handle_found = true;
	2017	+ /* get refcount to protect removing
	2018	+ * this handle from idr, needed when
	2019	+ * multiple wait cs are used with offset
	2020	+ * to wait on reserved encaps signals.
	2021	+ */
	2022	+ kref_get(&encaps_sig_hdl->refcount);
	2023	+ break;
	2024	+ }
	2025	+ }
	2026	+ spin_unlock(&ctx->sig_mgr.lock);
	2027	+
	2028	+ if (!handle_found) {
	2029	+ dev_err(hdev->dev, "Cannot find encapsulated signals handle for seq 0x%llx\n",
	2030	+ signal_seq);
	2031	+ rc = -EINVAL;
	2032	+ goto free_cs_chunk_array;
	2033	+ }
	2034	+
	2035	+ /* validate also the signal offset value */
	2036	+ if (chunk->encaps_signal_offset >
	2037	+ encaps_sig_hdl->count) {
	2038	+ dev_err(hdev->dev, "offset(%u) value exceed max reserved signals count(%u)!\n",
	2039	+ chunk->encaps_signal_offset,
	2040	+ encaps_sig_hdl->count);
	2041	+ rc = -EINVAL;
	2042	+ goto free_cs_chunk_array;
	2043	+ }
	2044	+ }
	2045	+
1738	2046	sig_fence = hl_ctx_get_fence(ctx, signal_seq);
1739	2047	if (IS_ERR(sig_fence)) {
1740	2048	atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
@@ -1755,11 +2063,16 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, · 14 unmodified lines
1755	2063	sig_waitcs_cmpl =
1756	2064	container_of(sig_fence, struct hl_cs_compl, base_fence);
1757	2065
1758		− if (sig_waitcs_cmpl->type != CS_TYPE_SIGNAL) {
	2066	+ staged_cs_with_encaps_signals = !!
	2067	+ (sig_waitcs_cmpl->type == CS_TYPE_DEFAULT &&
	2068	+ (flags & HL_CS_FLAGS_ENCAP_SIGNALS));
	2069	+
	2070	+ if (sig_waitcs_cmpl->type != CS_TYPE_SIGNAL &&
	2071	+ !staged_cs_with_encaps_signals) {
1759	2072	atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1760	2073	atomic64_inc(&cntr->validation_drop_cnt);
1761	2074	dev_err(hdev->dev,
1762		− "CS seq 0x%llx is not of a signal CS\n",
	2075	+ "CS seq 0x%llx is not of a signal/encaps-signal CS\n",
1763	2076	signal_seq);
1764	2077	hl_fence_put(sig_fence);
1765	2078	rc = -EINVAL;
@@ -1776,18 +2089,27 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, · 10 unmodified lines
1776	2089
1777	2090	rc = allocate_cs(hdev, ctx, cs_type, ULLONG_MAX, &cs, flags, timeout);
1778	2091	if (rc) {
1779		− if (cs_type == CS_TYPE_WAIT \|\|
1780		− cs_type == CS_TYPE_COLLECTIVE_WAIT)
	2092	+ if (is_wait_cs)
1781	2093	hl_fence_put(sig_fence);
	2094	+
1782	2095	goto free_cs_chunk_array;
1783	2096	}
1784	2097
1785	2098	/*
1786	2099	* Save the signal CS fence for later initialization right before
1787	2100	* hanging the wait CS on the queue.
	2101	+ * for encaps signals case, we save the cs sequence and handle pointer
	2102	+ * for later initialization.
1788	2103	*/
1789		− if (cs_type == CS_TYPE_WAIT \|\| cs_type == CS_TYPE_COLLECTIVE_WAIT)
	2104	+ if (is_wait_cs) {
1790	2105	cs->signal_fence = sig_fence;
	2106	+ /* store the handle pointer, so we don't have to
	2107	+ * look for it again, later on the flow
	2108	+ * when we need to set SOB info in hw_queue.
	2109	+ */
	2110	+ if (cs->encaps_signals)
	2111	+ cs->encaps_sig_hdl = encaps_sig_hdl;
	2112	+ }
1791	2113
1792	2114	hl_debugfs_add_cs(cs);
1793	2115
@@ -1795,10 +2117,11 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, · 1 unmodified line
1795	2117
1796	2118	if (cs_type == CS_TYPE_WAIT \|\| cs_type == CS_TYPE_SIGNAL)
1797	2119	rc = cs_ioctl_signal_wait_create_jobs(hdev, ctx, cs, q_type,
1798		− q_idx);
	2120	+ q_idx, chunk->encaps_signal_offset);
1799	2121	else if (cs_type == CS_TYPE_COLLECTIVE_WAIT)
1800	2122	rc = hdev->asic_funcs->collective_wait_create_jobs(hdev, ctx,
1801		− cs, q_idx, collective_engine_id);
	2123	+ cs, q_idx, collective_engine_id,
	2124	+ chunk->encaps_signal_offset);
1802	2125	else {
1803	2126	atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1804	2127	atomic64_inc(&cntr->validation_drop_cnt);
@@ -1810,7 +2133,13 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, · 5 unmodified lines
1810	2133
1811	2134	rc = hl_hw_queue_schedule_cs(cs);
1812	2135	if (rc) {
1813		− if (rc != -EAGAIN)
	2136	+ /* In case wait cs failed here, it means the signal cs
	2137	+ * already completed. we want to free all it's related objects
	2138	+ * but we don't want to fail the ioctl.
	2139	+ */
	2140	+ if (is_wait_cs)
	2141	+ rc = 0;
	2142	+ else if (rc != -EAGAIN)
1814	2143	dev_err(hdev->dev,
1815	2144	"Failed to submit CS %d.%llu to H/W queues, error %d\n",
1816	2145	ctx->asid, cs->sequence, rc);
@@ -1818,6 +2147,8 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, · 1 unmodified line
1818	2147	}
1819	2148
1820	2149	rc = HL_CS_STATUS_SUCCESS;
	2150	+ if (is_wait_cs)
	2151	+ wait_cs_submitted = true;
1821	2152	goto put_cs;
1822	2153
1823	2154	free_cs_object:
@@ -1828,6 +2159,10 @@ put_cs: · 4 unmodified lines
1828	2159	/* We finished with the CS in this function, so put the ref */
1829	2160	cs_put(cs);
1830	2161	free_cs_chunk_array:
	2162	+ if (!wait_cs_submitted && cs_encaps_signals && handle_found &&
	2163	+ is_wait_cs)
	2164	+ kref_put(&encaps_sig_hdl->refcount,
	2165	+ hl_encaps_handle_do_release);
1831	2166	kfree(cs_chunk_array);
1832	2167	out:
1833	2168	return rc;
@@ -1836,10 +2171,11 @@ out: · 2 unmodified lines
1836	2171	int hl_cs_ioctl(struct hl_fpriv hpriv, void data)
1837	2172	{
1838	2173	union hl_cs_args *args = data;
1839		− enum hl_cs_type cs_type;
	2174	+ enum hl_cs_type cs_type = 0;
1840	2175	u64 cs_seq = ULONG_MAX;
1841	2176	void __user *chunks;
1842		− u32 num_chunks, flags, timeout;
	2177	+ u32 num_chunks, flags, timeout,
	2178	+ signals_count = 0, sob_addr = 0, handle_id = 0;
1843	2179	int rc;
1844	2180
1845	2181	rc = hl_cs_sanity_checks(hpriv, args);
@@ -1850,10 +2186,6 @@ int hl_cs_ioctl(struct hl_fpriv hpriv, void data) · 4 unmodified lines
1850	2186	if (rc)
1851	2187	goto out;
1852	2188
1853		− rc = hl_submit_pending_cb(hpriv);
1854		− if (rc)
1855		− goto out;
1856		−
1857	2189	cs_type = hl_cs_get_cs_type(args->in.cs_flags &
1858	2190	~HL_CS_FLAGS_FORCE_RESTORE);
1859	2191	chunks = (void __user *) (uintptr_t) args->in.chunks_execute;
@@ -1876,80 +2208,448 @@ int hl_cs_ioctl(struct hl_fpriv hpriv, void data) · 16 unmodified lines
1876	2208	rc = cs_ioctl_signal_wait(hpriv, cs_type, chunks, num_chunks,
1877	2209	&cs_seq, args->in.cs_flags, timeout);
1878	2210	break;
	2211	+ case CS_RESERVE_SIGNALS:
	2212	+ rc = cs_ioctl_reserve_signals(hpriv,
	2213	+ args->in.encaps_signals_q_idx,
	2214	+ args->in.encaps_signals_count,
	2215	+ &handle_id, &sob_addr, &signals_count);
	2216	+ break;
	2217	+ case CS_UNRESERVE_SIGNALS:
	2218	+ rc = cs_ioctl_unreserve_signals(hpriv,
	2219	+ args->in.encaps_sig_handle_id);
	2220	+ break;
1879	2221	default:
1880	2222	rc = cs_ioctl_default(hpriv, chunks, num_chunks, &cs_seq,
1881		− args->in.cs_flags, timeout);
	2223	+ args->in.cs_flags,
	2224	+ args->in.encaps_sig_handle_id,
	2225	+ timeout);
1882	2226	break;
1883	2227	}
1884		−
1885	2228	out:
1886	2229	if (rc != -EAGAIN) {
1887	2230	memset(args, 0, sizeof(*args));
	2231	+
	2232	+ if (cs_type == CS_RESERVE_SIGNALS) {
	2233	+ args->out.handle_id = handle_id;
	2234	+ args->out.sob_base_addr_offset = sob_addr;
	2235	+ args->out.count = signals_count;
	2236	+ } else {
	2237	+ args->out.seq = cs_seq;
	2238	+ }
1888	2239	args->out.status = rc;
1889		− args->out.seq = cs_seq;
1890	2240	}
1891	2241
1892	2242	return rc;
1893	2243	}
1894	2244
	2245	+static int hl_wait_for_fence(struct hl_ctx ctx, u64 seq, struct hl_fence fence,
	2246	+ enum hl_cs_wait_status *status, u64 timeout_us,
	2247	+ s64 *timestamp)
	2248	+{
	2249	+ struct hl_device *hdev = ctx->hdev;
	2250	+ long completion_rc;
	2251	+ int rc = 0;
	2252	+
	2253	+ if (IS_ERR(fence)) {
	2254	+ rc = PTR_ERR(fence);
	2255	+ if (rc == -EINVAL)
	2256	+ dev_notice_ratelimited(hdev->dev,
	2257	+ "Can't wait on CS %llu because current CS is at seq %llu\n",
	2258	+ seq, ctx->cs_sequence);
	2259	+ return rc;
	2260	+ }
	2261	+
	2262	+ if (!fence) {
	2263	+ dev_dbg(hdev->dev,
	2264	+ "Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n",
	2265	+ seq, ctx->cs_sequence);
	2266	+
	2267	+ *status = CS_WAIT_STATUS_GONE;
	2268	+ return 0;
	2269	+ }
	2270	+
	2271	+ if (!timeout_us) {
	2272	+ completion_rc = completion_done(&fence->completion);
	2273	+ } else {
	2274	+ unsigned long timeout;
	2275	+
	2276	+ timeout = (timeout_us == MAX_SCHEDULE_TIMEOUT) ?
	2277	+ timeout_us : usecs_to_jiffies(timeout_us);
	2278	+ completion_rc =
	2279	+ wait_for_completion_interruptible_timeout(
	2280	+ &fence->completion, timeout);
	2281	+ }
	2282	+
	2283	+ if (completion_rc > 0) {
	2284	+ *status = CS_WAIT_STATUS_COMPLETED;
	2285	+ if (timestamp)
	2286	+ *timestamp = ktime_to_ns(fence->timestamp);
	2287	+ } else {
	2288	+ *status = CS_WAIT_STATUS_BUSY;
	2289	+ }
	2290	+
	2291	+ if (fence->error == -ETIMEDOUT)
	2292	+ rc = -ETIMEDOUT;
	2293	+ else if (fence->error == -EIO)
	2294	+ rc = -EIO;
	2295	+
	2296	+ return rc;
	2297	+}
	2298	+
	2299	+/*
	2300	+ * hl_cs_poll_fences - iterate CS fences to check for CS completion
	2301	+ *
	2302	+ * @mcs_data: multi-CS internal data
	2303	+ *
	2304	+ * @return 0 on success, otherwise non 0 error code
	2305	+ *
	2306	+ * The function iterates on all CS sequence in the list and set bit in
	2307	+ * completion_bitmap for each completed CS.
	2308	+ * while iterating, the function can extracts the stream map to be later
	2309	+ * used by the waiting function.
	2310	+ * this function shall be called after taking context ref
	2311	+ */
	2312	+static int hl_cs_poll_fences(struct multi_cs_data *mcs_data)
	2313	+{
	2314	+ struct hl_fence **fence_ptr = mcs_data->fence_arr;
	2315	+ struct hl_device *hdev = mcs_data->ctx->hdev;
	2316	+ int i, rc, arr_len = mcs_data->arr_len;
	2317	+ u64 *seq_arr = mcs_data->seq_arr;
	2318	+ ktime_t max_ktime, first_cs_time;
	2319	+ enum hl_cs_wait_status status;
	2320	+
	2321	+ memset(fence_ptr, 0, arr_len * sizeof(*fence_ptr));
	2322	+
	2323	+ /* get all fences under the same lock */
	2324	+ rc = hl_ctx_get_fences(mcs_data->ctx, seq_arr, fence_ptr, arr_len);
	2325	+ if (rc)
	2326	+ return rc;
	2327	+
	2328	+ /*
	2329	+ * set to maximum time to verify timestamp is valid: if at the end
	2330	+ * this value is maintained- no timestamp was updated
	2331	+ */
	2332	+ max_ktime = ktime_set(KTIME_SEC_MAX, 0);
	2333	+ first_cs_time = max_ktime;
	2334	+
	2335	+ for (i = 0; i < arr_len; i++, fence_ptr++) {
	2336	+ struct hl_fence fence = fence_ptr;
	2337	+
	2338	+ /*
	2339	+ * function won't sleep as it is called with timeout 0 (i.e.
	2340	+ * poll the fence)
	2341	+ */
	2342	+ rc = hl_wait_for_fence(mcs_data->ctx, seq_arr[i], fence,
	2343	+ &status, 0, NULL);
	2344	+ if (rc) {
	2345	+ dev_err(hdev->dev,
	2346	+ "wait_for_fence error :%d for CS seq %llu\n",
	2347	+ rc, seq_arr[i]);
	2348	+ break;
	2349	+ }
	2350	+
	2351	+ mcs_data->stream_master_qid_map \|= fence->stream_master_qid_map;
	2352	+
	2353	+ if (status == CS_WAIT_STATUS_BUSY)
	2354	+ continue;
	2355	+
	2356	+ mcs_data->completion_bitmap \|= BIT(i);
	2357	+
	2358	+ /*
	2359	+ * best effort to extract timestamp. few notes:
	2360	+ * - if even single fence is gone we cannot extract timestamp
	2361	+ * (as fence not exist anymore)
	2362	+ * - for all completed CSs we take the earliest timestamp.
	2363	+ * for this we have to validate that:
	2364	+ * 1. given timestamp was indeed set
	2365	+ * 2. the timestamp is earliest of all timestamps so far
	2366	+ */
	2367	+
	2368	+ if (status == CS_WAIT_STATUS_GONE) {
	2369	+ mcs_data->update_ts = false;
	2370	+ mcs_data->gone_cs = true;
	2371	+ } else if (mcs_data->update_ts &&
	2372	+ (ktime_compare(fence->timestamp,
	2373	+ ktime_set(0, 0)) > 0) &&
	2374	+ (ktime_compare(fence->timestamp, first_cs_time) < 0)) {
	2375	+ first_cs_time = fence->timestamp;
	2376	+ }
	2377	+ }
	2378	+
	2379	+ hl_fences_put(mcs_data->fence_arr, arr_len);
	2380	+
	2381	+ if (mcs_data->update_ts &&
	2382	+ (ktime_compare(first_cs_time, max_ktime) != 0))
	2383	+ mcs_data->timestamp = ktime_to_ns(first_cs_time);
	2384	+
	2385	+ return rc;
	2386	+}
	2387	+
1895	2388	static int _hl_cs_wait_ioctl(struct hl_device hdev, struct hl_ctx ctx,
1896	2389	u64 timeout_us, u64 seq,
1897	2390	enum hl_cs_wait_status status, s64 timestamp)
1898	2391	{
1899	2392	struct hl_fence *fence;
1900		− unsigned long timeout;
1901	2393	int rc = 0;
1902		− long completion_rc;
1903	2394
1904	2395	if (timestamp)
1905	2396	*timestamp = 0;
1906	2397
1907		− if (timeout_us == MAX_SCHEDULE_TIMEOUT)
1908		− timeout = timeout_us;
1909		− else
1910		− timeout = usecs_to_jiffies(timeout_us);
1911		−
1912	2398	hl_ctx_get(hdev, ctx);
1913	2399
1914	2400	fence = hl_ctx_get_fence(ctx, seq);
1915		− if (IS_ERR(fence)) {
1916		− rc = PTR_ERR(fence);
1917		− if (rc == -EINVAL)
1918		− dev_notice_ratelimited(hdev->dev,
1919		− "Can't wait on CS %llu because current CS is at seq %llu\n",
1920		− seq, ctx->cs_sequence);
1921		− } else if (fence) {
1922		− if (!timeout_us)
1923		− completion_rc = completion_done(&fence->completion);
1924		− else
1925		− completion_rc =
1926		− wait_for_completion_interruptible_timeout(
1927		− &fence->completion, timeout);
1928	2401
1929		− if (completion_rc > 0) {
1930		− *status = CS_WAIT_STATUS_COMPLETED;
1931		− if (timestamp)
1932		− *timestamp = ktime_to_ns(fence->timestamp);
1933		− } else {
1934		− *status = CS_WAIT_STATUS_BUSY;
	2402	+ rc = hl_wait_for_fence(ctx, seq, fence, status, timeout_us, timestamp);
	2403	+ hl_fence_put(fence);
	2404	+ hl_ctx_put(ctx);
	2405	+
	2406	+ return rc;
	2407	+}
	2408	+
	2409	+/*
	2410	+ * hl_wait_multi_cs_completion_init - init completion structure
	2411	+ *
	2412	+ * @hdev: pointer to habanalabs device structure
	2413	+ * @stream_master_bitmap: stream master QIDs map, set bit indicates stream
	2414	+ * master QID to wait on
	2415	+ *
	2416	+ * @return valid completion struct pointer on success, otherwise error pointer
	2417	+ *
	2418	+ * up to MULTI_CS_MAX_USER_CTX calls can be done concurrently to the driver.
	2419	+ * the function gets the first available completion (by marking it "used")
	2420	+ * and initialize its values.
	2421	+ */
	2422	+static struct multi_cs_completion *hl_wait_multi_cs_completion_init(
	2423	+ struct hl_device *hdev,
	2424	+ u8 stream_master_bitmap)
	2425	+{
	2426	+ struct multi_cs_completion *mcs_compl;
	2427	+ int i;
	2428	+
	2429	+ /* find free multi_cs completion structure */
	2430	+ for (i = 0; i < MULTI_CS_MAX_USER_CTX; i++) {
	2431	+ mcs_compl = &hdev->multi_cs_completion[i];
	2432	+ spin_lock(&mcs_compl->lock);
	2433	+ if (!mcs_compl->used) {
	2434	+ mcs_compl->used = 1;
	2435	+ mcs_compl->timestamp = 0;
	2436	+ mcs_compl->stream_master_qid_map = stream_master_bitmap;
	2437	+ reinit_completion(&mcs_compl->completion);
	2438	+ spin_unlock(&mcs_compl->lock);
	2439	+ break;
1935	2440	}
	2441	+ spin_unlock(&mcs_compl->lock);
	2442	+ }
1936	2443
1937		− if (fence->error == -ETIMEDOUT)
1938		− rc = -ETIMEDOUT;
1939		− else if (fence->error == -EIO)
1940		− rc = -EIO;
	2444	+ if (i == MULTI_CS_MAX_USER_CTX) {
	2445	+ dev_err(hdev->dev,
	2446	+ "no available multi-CS completion structure\n");
	2447	+ return ERR_PTR(-ENOMEM);
	2448	+ }
	2449	+ return mcs_compl;
	2450	+}
1941	2451
1942		− hl_fence_put(fence);
1943		− } else {
1944		− dev_dbg(hdev->dev,
1945		− "Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n",
1946		− seq, ctx->cs_sequence);
1947		− *status = CS_WAIT_STATUS_GONE;
	2452	+/*
	2453	+ * hl_wait_multi_cs_completion_fini - return completion structure and set as
	2454	+ * unused
	2455	+ *
	2456	+ * @mcs_compl: pointer to the completion structure
	2457	+ */
	2458	+static void hl_wait_multi_cs_completion_fini(
	2459	+ struct multi_cs_completion *mcs_compl)
	2460	+{
	2461	+ /*
	2462	+ * free completion structure, do it under lock to be in-sync with the
	2463	+ * thread that signals completion
	2464	+ */
	2465	+ spin_lock(&mcs_compl->lock);
	2466	+ mcs_compl->used = 0;
	2467	+ spin_unlock(&mcs_compl->lock);
	2468	+}
	2469	+
	2470	+/*
	2471	+ * hl_wait_multi_cs_completion - wait for first CS to complete
	2472	+ *
	2473	+ * @mcs_data: multi-CS internal data
	2474	+ *
	2475	+ * @return 0 on success, otherwise non 0 error code
	2476	+ */
	2477	+static int hl_wait_multi_cs_completion(struct multi_cs_data *mcs_data)
	2478	+{
	2479	+ struct hl_device *hdev = mcs_data->ctx->hdev;
	2480	+ struct multi_cs_completion *mcs_compl;
	2481	+ long completion_rc;
	2482	+
	2483	+ mcs_compl = hl_wait_multi_cs_completion_init(hdev,
	2484	+ mcs_data->stream_master_qid_map);
	2485	+ if (IS_ERR(mcs_compl))
	2486	+ return PTR_ERR(mcs_compl);
	2487	+
	2488	+ completion_rc = wait_for_completion_interruptible_timeout(
	2489	+ &mcs_compl->completion,
	2490	+ usecs_to_jiffies(mcs_data->timeout_us));
	2491	+
	2492	+ /* update timestamp */
	2493	+ if (completion_rc > 0)
	2494	+ mcs_data->timestamp = mcs_compl->timestamp;
	2495	+
	2496	+ hl_wait_multi_cs_completion_fini(mcs_compl);
	2497	+
	2498	+ mcs_data->wait_status = completion_rc;
	2499	+
	2500	+ return 0;
	2501	+}
	2502	+
	2503	+/*
	2504	+ * hl_multi_cs_completion_init - init array of multi-CS completion structures
	2505	+ *
	2506	+ * @hdev: pointer to habanalabs device structure
	2507	+ */
	2508	+void hl_multi_cs_completion_init(struct hl_device *hdev)
	2509	+{
	2510	+ struct multi_cs_completion *mcs_cmpl;
	2511	+ int i;
	2512	+
	2513	+ for (i = 0; i < MULTI_CS_MAX_USER_CTX; i++) {
	2514	+ mcs_cmpl = &hdev->multi_cs_completion[i];
	2515	+ mcs_cmpl->used = 0;
	2516	+ spin_lock_init(&mcs_cmpl->lock);
	2517	+ init_completion(&mcs_cmpl->completion);
	2518	+ }
	2519	+}
	2520	+
	2521	+/*
	2522	+ * hl_multi_cs_wait_ioctl - implementation of the multi-CS wait ioctl
	2523	+ *
	2524	+ * @hpriv: pointer to the private data of the fd
	2525	+ * @data: pointer to multi-CS wait ioctl in/out args
	2526	+ *
	2527	+ */
	2528	+static int hl_multi_cs_wait_ioctl(struct hl_fpriv hpriv, void data)
	2529	+{
	2530	+ struct hl_device *hdev = hpriv->hdev;
	2531	+ struct multi_cs_data mcs_data = {0};
	2532	+ union hl_wait_cs_args *args = data;
	2533	+ struct hl_ctx *ctx = hpriv->ctx;
	2534	+ struct hl_fence **fence_arr;
	2535	+ void __user *seq_arr;
	2536	+ u32 size_to_copy;
	2537	+ u64 *cs_seq_arr;
	2538	+ u8 seq_arr_len;
	2539	+ int rc;
	2540	+
	2541	+ if (!hdev->supports_wait_for_multi_cs) {
	2542	+ dev_err(hdev->dev, "Wait for multi CS is not supported\n");
	2543	+ return -EPERM;
	2544	+ }
	2545	+
	2546	+ seq_arr_len = args->in.seq_arr_len;
	2547	+
	2548	+ if (seq_arr_len > HL_WAIT_MULTI_CS_LIST_MAX_LEN) {
	2549	+ dev_err(hdev->dev, "Can wait only up to %d CSs, input sequence is of length %u\n",
	2550	+ HL_WAIT_MULTI_CS_LIST_MAX_LEN, seq_arr_len);
	2551	+ return -EINVAL;
	2552	+ }
	2553	+
	2554	+ /* allocate memory for sequence array */
	2555	+ cs_seq_arr =
	2556	+ kmalloc_array(seq_arr_len, sizeof(*cs_seq_arr), GFP_KERNEL);
	2557	+ if (!cs_seq_arr)
	2558	+ return -ENOMEM;
	2559	+
	2560	+ /* copy CS sequence array from user */
	2561	+ seq_arr = (void __user *) (uintptr_t) args->in.seq;
	2562	+ size_to_copy = seq_arr_len * sizeof(*cs_seq_arr);
	2563	+ if (copy_from_user(cs_seq_arr, seq_arr, size_to_copy)) {
	2564	+ dev_err(hdev->dev, "Failed to copy multi-cs sequence array from user\n");
	2565	+ rc = -EFAULT;
	2566	+ goto free_seq_arr;
	2567	+ }
	2568	+
	2569	+ /* allocate array for the fences */
	2570	+ fence_arr = kmalloc_array(seq_arr_len, sizeof(*fence_arr), GFP_KERNEL);
	2571	+ if (!fence_arr) {
	2572	+ rc = -ENOMEM;
	2573	+ goto free_seq_arr;
	2574	+ }
	2575	+
	2576	+ /* initialize the multi-CS internal data */
	2577	+ mcs_data.ctx = ctx;
	2578	+ mcs_data.seq_arr = cs_seq_arr;
	2579	+ mcs_data.fence_arr = fence_arr;
	2580	+ mcs_data.arr_len = seq_arr_len;
	2581	+
	2582	+ hl_ctx_get(hdev, ctx);
	2583	+
	2584	+ /* poll all CS fences, extract timestamp */
	2585	+ mcs_data.update_ts = true;
	2586	+ rc = hl_cs_poll_fences(&mcs_data);
	2587	+ /*
	2588	+ * skip wait for CS completion when one of the below is true:
	2589	+ * - an error on the poll function
	2590	+ * - one or more CS in the list completed
	2591	+ * - the user called ioctl with timeout 0
	2592	+ */
	2593	+ if (rc \|\| mcs_data.completion_bitmap \|\| !args->in.timeout_us)
	2594	+ goto put_ctx;
	2595	+
	2596	+ /* wait (with timeout) for the first CS to be completed */
	2597	+ mcs_data.timeout_us = args->in.timeout_us;
	2598	+ rc = hl_wait_multi_cs_completion(&mcs_data);
	2599	+ if (rc)
	2600	+ goto put_ctx;
	2601	+
	2602	+ if (mcs_data.wait_status > 0) {
	2603	+ /*
	2604	+ * poll fences once again to update the CS map.
	2605	+ * no timestamp should be updated this time.
	2606	+ */
	2607	+ mcs_data.update_ts = false;
	2608	+ rc = hl_cs_poll_fences(&mcs_data);
	2609	+
	2610	+ /*
	2611	+ * if hl_wait_multi_cs_completion returned before timeout (i.e.
	2612	+ * it got a completion) we expect to see at least one CS
	2613	+ * completed after the poll function.
	2614	+ */
	2615	+ if (!mcs_data.completion_bitmap) {
	2616	+ dev_err(hdev->dev, "Multi-CS got completion on wait but no CS completed\n");
	2617	+ rc = -EFAULT;
	2618	+ }
1948	2619	}
1949	2620
	2621	+put_ctx:
1950	2622	hl_ctx_put(ctx);
	2623	+ kfree(fence_arr);
1951	2624
1952		− return rc;
	2625	+free_seq_arr:
	2626	+ kfree(cs_seq_arr);
	2627	+
	2628	+ /* update output args */
	2629	+ memset(args, 0, sizeof(*args));
	2630	+ if (rc)
	2631	+ return rc;
	2632	+
	2633	+ if (mcs_data.completion_bitmap) {
	2634	+ args->out.status = HL_WAIT_CS_STATUS_COMPLETED;
	2635	+ args->out.cs_completion_map = mcs_data.completion_bitmap;
	2636	+
	2637	+ /* if timestamp not 0- it's valid */
	2638	+ if (mcs_data.timestamp) {
	2639	+ args->out.timestamp_nsec = mcs_data.timestamp;
	2640	+ args->out.flags \|= HL_WAIT_CS_STATUS_FLAG_TIMESTAMP_VLD;
	2641	+ }
	2642	+
	2643	+ /* update if some CS was gone */
	2644	+ if (mcs_data.timestamp)
	2645	+ args->out.flags \|= HL_WAIT_CS_STATUS_FLAG_GONE;
	2646	+ } else if (mcs_data.wait_status == -ERESTARTSYS) {
	2647	+ args->out.status = HL_WAIT_CS_STATUS_INTERRUPTED;
	2648	+ } else {
	2649	+ args->out.status = HL_WAIT_CS_STATUS_BUSY;
	2650	+ }
	2651	+
	2652	+ return 0;
1953	2653	}
1954	2654
1955	2655	static int hl_cs_wait_ioctl(struct hl_fpriv hpriv, void data)
@@ -2015,9 +2715,9 @@ static int _hl_interrupt_wait_ioctl(struct hl_device hdev, struct hl_ctx ctx, · 59 unmodified lines
2015	2715	{
2016	2716	struct hl_user_pending_interrupt *pend;
2017	2717	struct hl_user_interrupt *interrupt;
2018		− unsigned long timeout;
2019		− long completion_rc;
	2718	+ unsigned long timeout, flags;
2020	2719	u32 completion_value;
	2720	+ long completion_rc;
2021	2721	int rc = 0;
2022	2722
2023	2723	if (timeout_us == U32_MAX)
@@ -2040,17 +2740,10 @@ static int _hl_interrupt_wait_ioctl(struct hl_device hdev, struct hl_ctx ctx, · 16 unmodified lines
2040	2740	else
2041	2741	interrupt = &hdev->user_interrupt[interrupt_offset];
2042	2742
2043		− spin_lock(&interrupt->wait_list_lock);
2044		− if (!hl_device_operational(hdev, NULL)) {
2045		− rc = -EPERM;
2046		− goto unlock_and_free_fence;
2047		− }
2048		−
2049	2743	if (copy_from_user(&completion_value, u64_to_user_ptr(user_address), 4)) {
2050		− dev_err(hdev->dev,
2051		− "Failed to copy completion value from user\n");
	2744	+ dev_err(hdev->dev, "Failed to copy completion value from user\n");
2052	2745	rc = -EFAULT;
2053		− goto unlock_and_free_fence;
	2746	+ goto free_fence;
2054	2747	}
2055	2748
2056	2749	if (completion_value >= target_value)
@@ -2059,48 +2752,57 @@ static int _hl_interrupt_wait_ioctl(struct hl_device hdev, struct hl_ctx ctx, · 2 unmodified lines
2059	2752	*status = CS_WAIT_STATUS_BUSY;
2060	2753
2061	2754	if (!timeout_us \|\| (*status == CS_WAIT_STATUS_COMPLETED))
2062		− goto unlock_and_free_fence;
	2755	+ goto free_fence;
2063	2756
2064	2757	/* Add pending user interrupt to relevant list for the interrupt
2065	2758	* handler to monitor
2066	2759	*/
	2760	+ spin_lock_irqsave(&interrupt->wait_list_lock, flags);
2067	2761	list_add_tail(&pend->wait_list_node, &interrupt->wait_list_head);
2068		− spin_unlock(&interrupt->wait_list_lock);
	2762	+ spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
2069	2763
2070	2764	wait_again:
2071	2765	/* Wait for interrupt handler to signal completion */
2072		− completion_rc =
2073		− wait_for_completion_interruptible_timeout(
2074		− &pend->fence.completion, timeout);
	2766	+ completion_rc = wait_for_completion_interruptible_timeout(&pend->fence.completion,
	2767	+ timeout);
2075	2768
2076	2769	/* If timeout did not expire we need to perform the comparison.
2077	2770	* If comparison fails, keep waiting until timeout expires
2078	2771	*/
2079	2772	if (completion_rc > 0) {
2080		− if (copy_from_user(&completion_value,
2081		− u64_to_user_ptr(user_address), 4)) {
2082		− dev_err(hdev->dev,
2083		− "Failed to copy completion value from user\n");
	2773	+ if (copy_from_user(&completion_value, u64_to_user_ptr(user_address), 4)) {
	2774	+ dev_err(hdev->dev, "Failed to copy completion value from user\n");
2084	2775	rc = -EFAULT;
	2776	+
2085	2777	goto remove_pending_user_interrupt;
2086	2778	}
2087	2779
2088	2780	if (completion_value >= target_value) {
2089	2781	*status = CS_WAIT_STATUS_COMPLETED;
2090	2782	} else {
	2783	+ spin_lock_irqsave(&interrupt->wait_list_lock, flags);
	2784	+ reinit_completion(&pend->fence.completion);
2091	2785	timeout = completion_rc;
	2786	+
	2787	+ spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
2092	2788	goto wait_again;
2093	2789	}
	2790	+ } else if (completion_rc == -ERESTARTSYS) {
	2791	+ dev_err_ratelimited(hdev->dev,
	2792	+ "user process got signal while waiting for interrupt ID %d\n",
	2793	+ interrupt->interrupt_id);
	2794	+ *status = HL_WAIT_CS_STATUS_INTERRUPTED;
	2795	+ rc = -EINTR;
2094	2796	} else {
2095	2797	*status = CS_WAIT_STATUS_BUSY;
2096	2798	}
2097	2799
2098	2800	remove_pending_user_interrupt:
2099		− spin_lock(&interrupt->wait_list_lock);
	2801	+ spin_lock_irqsave(&interrupt->wait_list_lock, flags);
2100	2802	list_del(&pend->wait_list_node);
	2803	+ spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
2101	2804
2102		−unlock_and_free_fence:
2103		− spin_unlock(&interrupt->wait_list_lock);
	2805	+free_fence:
2104	2806	kfree(pend);
2105	2807	hl_ctx_put(ctx);
2106	2808
@@ -2148,8 +2850,9 @@ static int hl_interrupt_wait_ioctl(struct hl_fpriv hpriv, void data) · 41 unmodified lines
2148	2850	memset(args, 0, sizeof(*args));
2149	2851
2150	2852	if (rc) {
2151		− dev_err_ratelimited(hdev->dev,
2152		− "interrupt_wait_ioctl failed (%d)\n", rc);
	2853	+ if (rc != -EINTR)
	2854	+ dev_err_ratelimited(hdev->dev,
	2855	+ "interrupt_wait_ioctl failed (%d)\n", rc);
2153	2856
2154	2857	return rc;
2155	2858	}
@@ -2173,8 +2876,16 @@ int hl_wait_ioctl(struct hl_fpriv hpriv, void data) · 17 unmodified lines
2173	2876	u32 flags = args->in.flags;
2174	2877	int rc;
2175	2878
	2879	+ /* If the device is not operational, no point in waiting for any command submission or
	2880	+ * user interrupt
	2881	+ */
	2882	+ if (!hl_device_operational(hpriv->hdev, NULL))
	2883	+ return -EPERM;
	2884	+
2176	2885	if (flags & HL_WAIT_CS_FLAGS_INTERRUPT)
2177	2886	rc = hl_interrupt_wait_ioctl(hpriv, data);
	2887	+ else if (flags & HL_WAIT_CS_FLAGS_MULTI_CS)
	2888	+ rc = hl_multi_cs_wait_ioctl(hpriv, data);
2178	2889	else
2179	2890	rc = hl_cs_wait_ioctl(hpriv, data);
2180	2891

drivers/misc/habanalabs/common/context.c+126−20

@@ -9,16 +9,70 @@ · 8 unmodified lines
9	9
10	10	#include <linux/slab.h>
11	11
	12	+void hl_encaps_handle_do_release(struct kref *ref)
	13	+{
	14	+ struct hl_cs_encaps_sig_handle *handle =
	15	+ container_of(ref, struct hl_cs_encaps_sig_handle, refcount);
	16	+ struct hl_ctx *ctx = handle->hdev->compute_ctx;
	17	+ struct hl_encaps_signals_mgr *mgr = &ctx->sig_mgr;
	18	+
	19	+ spin_lock(&mgr->lock);
	20	+ idr_remove(&mgr->handles, handle->id);
	21	+ spin_unlock(&mgr->lock);
	22	+
	23	+ kfree(handle);
	24	+}
	25	+
	26	+static void hl_encaps_handle_do_release_sob(struct kref *ref)
	27	+{
	28	+ struct hl_cs_encaps_sig_handle *handle =
	29	+ container_of(ref, struct hl_cs_encaps_sig_handle, refcount);
	30	+ struct hl_ctx *ctx = handle->hdev->compute_ctx;
	31	+ struct hl_encaps_signals_mgr *mgr = &ctx->sig_mgr;
	32	+
	33	+ /* if we're here, then there was a signals reservation but cs with
	34	+ * encaps signals wasn't submitted, so need to put refcount
	35	+ * to hw_sob taken at the reservation.
	36	+ */
	37	+ hw_sob_put(handle->hw_sob);
	38	+
	39	+ spin_lock(&mgr->lock);
	40	+ idr_remove(&mgr->handles, handle->id);
	41	+ spin_unlock(&mgr->lock);
	42	+
	43	+ kfree(handle);
	44	+}
	45	+
	46	+static void hl_encaps_sig_mgr_init(struct hl_encaps_signals_mgr *mgr)
	47	+{
	48	+ spin_lock_init(&mgr->lock);
	49	+ idr_init(&mgr->handles);
	50	+}
	51	+
	52	+static void hl_encaps_sig_mgr_fini(struct hl_device *hdev,
	53	+ struct hl_encaps_signals_mgr *mgr)
	54	+{
	55	+ struct hl_cs_encaps_sig_handle *handle;
	56	+ struct idr *idp;
	57	+ u32 id;
	58	+
	59	+ idp = &mgr->handles;
	60	+
	61	+ if (!idr_is_empty(idp)) {
	62	+ dev_warn(hdev->dev, "device released while some encaps signals handles are still allocated\n");
	63	+ idr_for_each_entry(idp, handle, id)
	64	+ kref_put(&handle->refcount,
	65	+ hl_encaps_handle_do_release_sob);
	66	+ }
	67	+
	68	+ idr_destroy(&mgr->handles);
	69	+}
	70	+
12	71	static void hl_ctx_fini(struct hl_ctx *ctx)
13	72	{
14	73	struct hl_device *hdev = ctx->hdev;
15	74	int i;
16	75
17		− /* Release all allocated pending cb's, those cb's were never
18		− * scheduled so it is safe to release them here
19		− */
20		− hl_pending_cb_list_flush(ctx);
21		−
22	76	/* Release all allocated HW block mapped list entries and destroy
23	77	* the mutex.
24	78	*/
@@ -53,6 +107,7 @@ static void hl_ctx_fini(struct hl_ctx *ctx) · 28 unmodified lines
53	107	hl_cb_va_pool_fini(ctx);
54	108	hl_vm_ctx_fini(ctx);
55	109	hl_asid_free(hdev, ctx->asid);
	110	+ hl_encaps_sig_mgr_fini(hdev, &ctx->sig_mgr);
56	111
57	112	/* Scrub both SRAM and DRAM */
58	113	hdev->asic_funcs->scrub_device_mem(hdev, 0, 0);
@@ -130,9 +185,6 @@ void hl_ctx_free(struct hl_device hdev, struct hl_ctx ctx) · 71 unmodified lines
130	185	{
131	186	if (kref_put(&ctx->refcount, hl_ctx_do_release) == 1)
132	187	return;
133		−
134		− dev_warn(hdev->dev,
135		− "user process released device but its command submissions are still executing\n");
136	188	}
137	189
138	190	int hl_ctx_init(struct hl_device hdev, struct hl_ctx ctx, bool is_kernel_ctx)
@@ -144,11 +196,8 @@ int hl_ctx_init(struct hl_device hdev, struct hl_ctx ctx, bool is_kernel_ctx) · 5 unmodified lines
144	196	kref_init(&ctx->refcount);
145	197
146	198	ctx->cs_sequence = 1;
147		− INIT_LIST_HEAD(&ctx->pending_cb_list);
148		− spin_lock_init(&ctx->pending_cb_lock);
149	199	spin_lock_init(&ctx->cs_lock);
150	200	atomic_set(&ctx->thread_ctx_switch_token, 1);
151		− atomic_set(&ctx->thread_pending_cb_token, 1);
152	201	ctx->thread_ctx_switch_wait_token = 0;
153	202	ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs,
154	203	sizeof(struct hl_fence *),
@@ -200,6 +249,8 @@ int hl_ctx_init(struct hl_device hdev, struct hl_ctx ctx, bool is_kernel_ctx) · 45 unmodified lines
200	249	goto err_cb_va_pool_fini;
201	250	}
202	251
	252	+ hl_encaps_sig_mgr_init(&ctx->sig_mgr);
	253	+
203	254	dev_dbg(hdev->dev, "create user context %d\n", ctx->asid);
204	255	}
205	256
@@ -229,31 +280,86 @@ int hl_ctx_put(struct hl_ctx *ctx) · 23 unmodified lines
229	280	return kref_put(&ctx->refcount, hl_ctx_do_release);
230	281	}
231	282
232		−struct hl_fence hl_ctx_get_fence(struct hl_ctx ctx, u64 seq)
	283	+/*
	284	+ * hl_ctx_get_fence_locked - get CS fence under CS lock
	285	+ *
	286	+ * @ctx: pointer to the context structure.
	287	+ * @seq: CS sequences number
	288	+ *
	289	+ * @return valid fence pointer on success, NULL if fence is gone, otherwise
	290	+ * error pointer.
	291	+ *
	292	+ * NOTE: this function shall be called with cs_lock locked
	293	+ */
	294	+static struct hl_fence hl_ctx_get_fence_locked(struct hl_ctx ctx, u64 seq)
233	295	{
234	296	struct asic_fixed_properties *asic_prop = &ctx->hdev->asic_prop;
235	297	struct hl_fence *fence;
236	298
237		− spin_lock(&ctx->cs_lock);
238		−
239		− if (seq >= ctx->cs_sequence) {
240		− spin_unlock(&ctx->cs_lock);
	299	+ if (seq >= ctx->cs_sequence)
241	300	return ERR_PTR(-EINVAL);
242		− }
243	301
244		− if (seq + asic_prop->max_pending_cs < ctx->cs_sequence) {
245		− spin_unlock(&ctx->cs_lock);
	302	+ if (seq + asic_prop->max_pending_cs < ctx->cs_sequence)
246	303	return NULL;
247		− }
248	304
249	305	fence = ctx->cs_pending[seq & (asic_prop->max_pending_cs - 1)];
250	306	hl_fence_get(fence);
	307	+ return fence;
	308	+}
	309	+
	310	+struct hl_fence hl_ctx_get_fence(struct hl_ctx ctx, u64 seq)
	311	+{
	312	+ struct hl_fence *fence;
	313	+
	314	+ spin_lock(&ctx->cs_lock);
	315	+
	316	+ fence = hl_ctx_get_fence_locked(ctx, seq);
251	317
252	318	spin_unlock(&ctx->cs_lock);
253	319
254	320	return fence;
255	321	}
256	322
	323	+/*
	324	+ * hl_ctx_get_fences - get multiple CS fences under the same CS lock
	325	+ *
	326	+ * @ctx: pointer to the context structure.
	327	+ * @seq_arr: array of CS sequences to wait for
	328	+ * @fence: fence array to store the CS fences
	329	+ * @arr_len: length of seq_arr and fence_arr
	330	+ *
	331	+ * @return 0 on success, otherwise non 0 error code
	332	+ */
	333	+int hl_ctx_get_fences(struct hl_ctx ctx, u64 seq_arr,
	334	+ struct hl_fence **fence, u32 arr_len)
	335	+{
	336	+ struct hl_fence **fence_arr_base = fence;
	337	+ int i, rc = 0;
	338	+
	339	+ spin_lock(&ctx->cs_lock);
	340	+
	341	+ for (i = 0; i < arr_len; i++, fence++) {
	342	+ u64 seq = seq_arr[i];
	343	+
	344	+ *fence = hl_ctx_get_fence_locked(ctx, seq);
	345	+
	346	+ if (IS_ERR(*fence)) {
	347	+ dev_err(ctx->hdev->dev,
	348	+ "Failed to get fence for CS with seq 0x%llx\n",
	349	+ seq);
	350	+ rc = PTR_ERR(*fence);
	351	+ break;
	352	+ }
	353	+ }
	354	+
	355	+ spin_unlock(&ctx->cs_lock);
	356	+
	357	+ if (rc)
	358	+ hl_fences_put(fence_arr_base, i);
	359	+
	360	+ return rc;
	361	+}
	362	+
257	363	/*
258	364	* hl_ctx_mgr_init - initialize the context manager
259	365	*

drivers/misc/habanalabs/common/debugfs.c+174−10

@@ -209,12 +209,12 @@ static int userptr_show(struct seq_file s, void data) · 208 unmodified lines
209	209	if (first) {
210	210	first = false;
211	211	seq_puts(s, "\n");
212		− seq_puts(s, " user virtual address size dma dir\n");
	212	+ seq_puts(s, " pid user virtual address size dma dir\n");
213	213	seq_puts(s, "----------------------------------------------------------\n");
214	214	}
215		− seq_printf(s,
216		− " 0x%-14llx %-10u %-30s\n",
217		− userptr->addr, userptr->size, dma_dir[userptr->dir]);
	215	+ seq_printf(s, " %-7d 0x%-14llx %-10llu %-30s\n",
	216	+ userptr->pid, userptr->addr, userptr->size,
	217	+ dma_dir[userptr->dir]);
218	218	}
219	219
220	220	spin_unlock(&dev_entry->userptr_spinlock);
@@ -235,7 +235,7 @@ static int vm_show(struct seq_file s, void data) · 14 unmodified lines
235	235	struct hl_vm_hash_node *hnode;
236	236	struct hl_userptr *userptr;
237	237	struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;
238		− enum vm_type_t *vm_type;
	238	+ enum vm_type *vm_type;
239	239	bool once = true;
240	240	u64 j;
241	241	int i;
@@ -261,7 +261,7 @@ static int vm_show(struct seq_file s, void data) · 19 unmodified lines
261	261	if (*vm_type == VM_TYPE_USERPTR) {
262	262	userptr = hnode->ptr;
263	263	seq_printf(s,
264		− " 0x%-14llx %-10u\n",
	264	+ " 0x%-14llx %-10llu\n",
265	265	hnode->vaddr, userptr->size);
266	266	} else {
267	267	phys_pg_pack = hnode->ptr;
@@ -320,6 +320,77 @@ static int vm_show(struct seq_file s, void data) · 52 unmodified lines
320	320	return 0;
321	321	}
322	322
	323	+static int userptr_lookup_show(struct seq_file s, void data)
	324	+{
	325	+ struct hl_debugfs_entry *entry = s->private;
	326	+ struct hl_dbg_device_entry *dev_entry = entry->dev_entry;
	327	+ struct scatterlist *sg;
	328	+ struct hl_userptr *userptr;
	329	+ bool first = true;
	330	+ u64 total_npages, npages, sg_start, sg_end;
	331	+ dma_addr_t dma_addr;
	332	+ int i;
	333	+
	334	+ spin_lock(&dev_entry->userptr_spinlock);
	335	+
	336	+ list_for_each_entry(userptr, &dev_entry->userptr_list, debugfs_list) {
	337	+ if (dev_entry->userptr_lookup >= userptr->addr &&
	338	+ dev_entry->userptr_lookup < userptr->addr + userptr->size) {
	339	+ total_npages = 0;
	340	+ for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents,
	341	+ i) {
	342	+ npages = hl_get_sg_info(sg, &dma_addr);
	343	+ sg_start = userptr->addr +
	344	+ total_npages * PAGE_SIZE;
	345	+ sg_end = userptr->addr +
	346	+ (total_npages + npages) * PAGE_SIZE;
	347	+
	348	+ if (dev_entry->userptr_lookup >= sg_start &&
	349	+ dev_entry->userptr_lookup < sg_end) {
	350	+ dma_addr += (dev_entry->userptr_lookup -
	351	+ sg_start);
	352	+ if (first) {
	353	+ first = false;
	354	+ seq_puts(s, "\n");
	355	+ seq_puts(s, " user virtual address dma address pid region start region size\n");
	356	+ seq_puts(s, "---------------------------------------------------------------------------------------\n");
	357	+ }
	358	+ seq_printf(s, " 0x%-18llx 0x%-16llx %-8u 0x%-16llx %-12llu\n",
	359	+ dev_entry->userptr_lookup,
	360	+ (u64)dma_addr, userptr->pid,
	361	+ userptr->addr, userptr->size);
	362	+ }
	363	+ total_npages += npages;
	364	+ }
	365	+ }
	366	+ }
	367	+
	368	+ spin_unlock(&dev_entry->userptr_spinlock);
	369	+
	370	+ if (!first)
	371	+ seq_puts(s, "\n");
	372	+
	373	+ return 0;
	374	+}
	375	+
	376	+static ssize_t userptr_lookup_write(struct file file, const char __user buf,
	377	+ size_t count, loff_t *f_pos)
	378	+{
	379	+ struct seq_file *s = file->private_data;
	380	+ struct hl_debugfs_entry *entry = s->private;
	381	+ struct hl_dbg_device_entry *dev_entry = entry->dev_entry;
	382	+ ssize_t rc;
	383	+ u64 value;
	384	+
	385	+ rc = kstrtoull_from_user(buf, count, 16, &value);
	386	+ if (rc)
	387	+ return rc;
	388	+
	389	+ dev_entry->userptr_lookup = value;
	390	+
	391	+ return count;
	392	+}
	393	+
323	394	static int mmu_show(struct seq_file s, void data)
324	395	{
325	396	struct hl_debugfs_entry *entry = s->private;
@@ -349,7 +420,7 @@ static int mmu_show(struct seq_file s, void data) · 23 unmodified lines
349	420	return 0;
350	421	}
351	422
352		− phys_addr = hops_info.hop_info[hops_info.used_hops - 1].hop_pte_val;
	423	+ hl_mmu_va_to_pa(ctx, virt_addr, &phys_addr);
353	424
354	425	if (hops_info.scrambled_vaddr &&
355	426	(dev_entry->mmu_addr != hops_info.scrambled_vaddr))
@@ -491,11 +562,10 @@ static int device_va_to_pa(struct hl_device *hdev, u64 virt_addr, u32 size, · 135 unmodified lines
491	562	struct hl_vm_phys_pg_pack *phys_pg_pack;
492	563	struct hl_ctx *ctx = hdev->compute_ctx;
493	564	struct hl_vm_hash_node *hnode;
	565	+ u64 end_address, range_size;
494	566	struct hl_userptr *userptr;
495		− enum vm_type_t *vm_type;
	567	+ enum vm_type *vm_type;
496	568	bool valid = false;
497		− u64 end_address;
498		− u32 range_size;
499	569	int i, rc = 0;
500	570
501	571	if (!ctx) {
@@ -1043,6 +1113,60 @@ static ssize_t hl_security_violations_read(struct file f, char __user buf, · 541 unmodified lines
1043	1113	return 0;
1044	1114	}
1045	1115
	1116	+static ssize_t hl_state_dump_read(struct file f, char __user buf,
	1117	+ size_t count, loff_t *ppos)
	1118	+{
	1119	+ struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
	1120	+ ssize_t rc;
	1121	+
	1122	+ down_read(&entry->state_dump_sem);
	1123	+ if (!entry->state_dump[entry->state_dump_head])
	1124	+ rc = 0;
	1125	+ else
	1126	+ rc = simple_read_from_buffer(
	1127	+ buf, count, ppos,
	1128	+ entry->state_dump[entry->state_dump_head],
	1129	+ strlen(entry->state_dump[entry->state_dump_head]));
	1130	+ up_read(&entry->state_dump_sem);
	1131	+
	1132	+ return rc;
	1133	+}
	1134	+
	1135	+static ssize_t hl_state_dump_write(struct file f, const char __user buf,
	1136	+ size_t count, loff_t *ppos)
	1137	+{
	1138	+ struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
	1139	+ struct hl_device *hdev = entry->hdev;
	1140	+ ssize_t rc;
	1141	+ u32 size;
	1142	+ int i;
	1143	+
	1144	+ rc = kstrtouint_from_user(buf, count, 10, &size);
	1145	+ if (rc)
	1146	+ return rc;
	1147	+
	1148	+ if (size <= 0 \|\| size >= ARRAY_SIZE(entry->state_dump)) {
	1149	+ dev_err(hdev->dev, "Invalid number of dumps to skip\n");
	1150	+ return -EINVAL;
	1151	+ }
	1152	+
	1153	+ if (entry->state_dump[entry->state_dump_head]) {
	1154	+ down_write(&entry->state_dump_sem);
	1155	+ for (i = 0; i < size; ++i) {
	1156	+ vfree(entry->state_dump[entry->state_dump_head]);
	1157	+ entry->state_dump[entry->state_dump_head] = NULL;
	1158	+ if (entry->state_dump_head > 0)
	1159	+ entry->state_dump_head--;
	1160	+ else
	1161	+ entry->state_dump_head =
	1162	+ ARRAY_SIZE(entry->state_dump) - 1;
	1163	+ }
	1164	+ up_write(&entry->state_dump_sem);
	1165	+ }
	1166	+
	1167	+ return count;
	1168	+}
	1169	+
1046	1170	static const struct file_operations hl_data32b_fops = {
1047	1171	.owner = THIS_MODULE,
1048	1172	.read = hl_data_read32,
@@ -1110,12 +1234,19 @@ static const struct file_operations hl_security_violations_fops = { · 61 unmodified lines
1110	1234	.read = hl_security_violations_read
1111	1235	};
1112	1236
	1237	+static const struct file_operations hl_state_dump_fops = {
	1238	+ .owner = THIS_MODULE,
	1239	+ .read = hl_state_dump_read,
	1240	+ .write = hl_state_dump_write
	1241	+};
	1242	+
1113	1243	static const struct hl_info_list hl_debugfs_list[] = {
1114	1244	{"command_buffers", command_buffers_show, NULL},
1115	1245	{"command_submission", command_submission_show, NULL},
1116	1246	{"command_submission_jobs", command_submission_jobs_show, NULL},
1117	1247	{"userptr", userptr_show, NULL},
1118	1248	{"vm", vm_show, NULL},
	1249	+ {"userptr_lookup", userptr_lookup_show, userptr_lookup_write},
1119	1250	{"mmu", mmu_show, mmu_asid_va_write},
1120	1251	{"engines", engines_show, NULL}
1121	1252	};
@@ -1172,6 +1303,7 @@ void hl_debugfs_add_device(struct hl_device *hdev) · 50 unmodified lines
1172	1303	INIT_LIST_HEAD(&dev_entry->userptr_list);
1173	1304	INIT_LIST_HEAD(&dev_entry->ctx_mem_hash_list);
1174	1305	mutex_init(&dev_entry->file_mutex);
	1306	+ init_rwsem(&dev_entry->state_dump_sem);
1175	1307	spin_lock_init(&dev_entry->cb_spinlock);
1176	1308	spin_lock_init(&dev_entry->cs_spinlock);
1177	1309	spin_lock_init(&dev_entry->cs_job_spinlock);
@@ -1283,6 +1415,12 @@ void hl_debugfs_add_device(struct hl_device *hdev) · 105 unmodified lines
1283	1415	dev_entry->root,
1284	1416	&hdev->skip_reset_on_timeout);
1285	1417
	1418	+ debugfs_create_file("state_dump",
	1419	+ 0600,
	1420	+ dev_entry->root,
	1421	+ dev_entry,
	1422	+ &hl_state_dump_fops);
	1423	+
1286	1424	for (i = 0, entry = dev_entry->entry_arr ; i < count ; i++, entry++) {
1287	1425	debugfs_create_file(hl_debugfs_list[i].name,
1288	1426	0444,
@@ -1297,6 +1435,7 @@ void hl_debugfs_add_device(struct hl_device *hdev) · 8 unmodified lines
1297	1435	void hl_debugfs_remove_device(struct hl_device *hdev)
1298	1436	{
1299	1437	struct hl_dbg_device_entry *entry = &hdev->hl_debugfs;
	1438	+ int i;
1300	1439
1301	1440	debugfs_remove_recursive(entry->root);
1302	1441
@@ -1304,6 +1443,9 @@ void hl_debugfs_remove_device(struct hl_device *hdev) · 1 unmodified line
1304	1443
1305	1444	vfree(entry->blob_desc.data);
1306	1445
	1446	+ for (i = 0; i < ARRAY_SIZE(entry->state_dump); ++i)
	1447	+ vfree(entry->state_dump[i]);
	1448	+
1307	1449	kfree(entry->entry_arr);
1308	1450	}
1309	1451
@@ -1416,6 +1558,28 @@ void hl_debugfs_remove_ctx_mem_hash(struct hl_device hdev, struct hl_ctx ctx) · 106 unmodified lines
1416	1558	spin_unlock(&dev_entry->ctx_mem_hash_spinlock);
1417	1559	}
1418	1560
	1561	+/**
	1562	+ * hl_debugfs_set_state_dump - register state dump making it accessible via
	1563	+ * debugfs
	1564	+ * @hdev: pointer to the device structure
	1565	+ * @data: the actual dump data
	1566	+ * @length: the length of the data
	1567	+ */
	1568	+void hl_debugfs_set_state_dump(struct hl_device hdev, char data,
	1569	+ unsigned long length)
	1570	+{
	1571	+ struct hl_dbg_device_entry *dev_entry = &hdev->hl_debugfs;
	1572	+
	1573	+ down_write(&dev_entry->state_dump_sem);
	1574	+
	1575	+ dev_entry->state_dump_head = (dev_entry->state_dump_head + 1) %
	1576	+ ARRAY_SIZE(dev_entry->state_dump);
	1577	+ vfree(dev_entry->state_dump[dev_entry->state_dump_head]);
	1578	+ dev_entry->state_dump[dev_entry->state_dump_head] = data;
	1579	+
	1580	+ up_write(&dev_entry->state_dump_sem);
	1581	+}
	1582	+
1419	1583	void __init hl_debugfs_init(void)
1420	1584	{
1421	1585	hl_debug_root = debugfs_create_dir("habanalabs", NULL);

drivers/misc/habanalabs/common/device.c+86−77

@@ -7,11 +7,11 @@ · 6 unmodified lines
7	7
8	8	#define pr_fmt(fmt) "habanalabs: " fmt
9	9
	10	+#include <uapi/misc/habanalabs.h>
10	11	#include "habanalabs.h"
11	12
12	13	#include <linux/pci.h>
13	14	#include <linux/hwmon.h>
14		−#include <uapi/misc/habanalabs.h>
15	15
16	16	enum hl_device_status hl_device_status(struct hl_device *hdev)
17	17	{
@@ -23,6 +23,8 @@ enum hl_device_status hl_device_status(struct hl_device *hdev) · 5 unmodified lines
23	23	status = HL_DEVICE_STATUS_NEEDS_RESET;
24	24	else if (hdev->disabled)
25	25	status = HL_DEVICE_STATUS_MALFUNCTION;
	26	+ else if (!hdev->init_done)
	27	+ status = HL_DEVICE_STATUS_IN_DEVICE_CREATION;
26	28	else
27	29	status = HL_DEVICE_STATUS_OPERATIONAL;
28	30
@@ -44,6 +46,7 @@ bool hl_device_operational(struct hl_device *hdev, · 15 unmodified lines
44	46	case HL_DEVICE_STATUS_NEEDS_RESET:
45	47	return false;
46	48	case HL_DEVICE_STATUS_OPERATIONAL:
	49	+ case HL_DEVICE_STATUS_IN_DEVICE_CREATION:
47	50	default:
48	51	return true;
49	52	}
@@ -129,8 +132,8 @@ static int hl_device_release(struct inode inode, struct file filp) · 79 unmodified lines
129	132	hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);
130	133
131	134	if (!hl_hpriv_put(hpriv))
132		− dev_warn(hdev->dev,
133		− "Device is still in use because there are live CS and/or memory mappings\n");
	135	+ dev_notice(hdev->dev,
	136	+ "User process closed FD but device still in use\n");
134	137
135	138	hdev->last_open_session_duration_jif =
136	139	jiffies - hdev->last_successful_open_jif;
@@ -308,9 +311,15 @@ static void device_hard_reset_pending(struct work_struct *work) · 171 unmodified lines
308	311	container_of(work, struct hl_device_reset_work,
309	312	reset_work.work);
310	313	struct hl_device *hdev = device_reset_work->hdev;
	314	+ u32 flags;
311	315	int rc;
312	316
313		− rc = hl_device_reset(hdev, HL_RESET_HARD \| HL_RESET_FROM_RESET_THREAD);
	317	+ flags = HL_RESET_HARD \| HL_RESET_FROM_RESET_THREAD;
	318	+
	319	+ if (device_reset_work->fw_reset)
	320	+ flags \|= HL_RESET_FW;
	321	+
	322	+ rc = hl_device_reset(hdev, flags);
314	323	if ((rc == -EBUSY) && !hdev->device_fini_pending) {
315	324	dev_info(hdev->dev,
316	325	"Could not reset device. will try again in %u seconds",
@@ -682,6 +691,44 @@ out: · 365 unmodified lines
682	691	return rc;
683	692	}
684	693
	694	+static void take_release_locks(struct hl_device *hdev)
	695	+{
	696	+ /* Flush anyone that is inside the critical section of enqueue
	697	+ * jobs to the H/W
	698	+ */
	699	+ hdev->asic_funcs->hw_queues_lock(hdev);
	700	+ hdev->asic_funcs->hw_queues_unlock(hdev);
	701	+
	702	+ /* Flush processes that are sending message to CPU */
	703	+ mutex_lock(&hdev->send_cpu_message_lock);
	704	+ mutex_unlock(&hdev->send_cpu_message_lock);
	705	+
	706	+ /* Flush anyone that is inside device open */
	707	+ mutex_lock(&hdev->fpriv_list_lock);
	708	+ mutex_unlock(&hdev->fpriv_list_lock);
	709	+}
	710	+
	711	+static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_reset)
	712	+{
	713	+ if (hard_reset)
	714	+ device_late_fini(hdev);
	715	+
	716	+ /*
	717	+ * Halt the engines and disable interrupts so we won't get any more
	718	+ * completions from H/W and we won't have any accesses from the
	719	+ * H/W to the host machine
	720	+ */
	721	+ hdev->asic_funcs->halt_engines(hdev, hard_reset, fw_reset);
	722	+
	723	+ /* Go over all the queues, release all CS and their jobs */
	724	+ hl_cs_rollback_all(hdev);
	725	+
	726	+ /* Release all pending user interrupts, each pending user interrupt
	727	+ * holds a reference to user context
	728	+ */
	729	+ hl_release_pending_user_interrupts(hdev);
	730	+}
	731	+
685	732	/*
686	733	* hl_device_suspend - initiate device suspend
687	734	*
@@ -707,16 +754,7 @@ int hl_device_suspend(struct hl_device *hdev) · 19 unmodified lines
707	754	/* This blocks all other stuff that is not blocked by in_reset */
708	755	hdev->disabled = true;
709	756
710		− /*
711		− * Flush anyone that is inside the critical section of enqueue
712		− * jobs to the H/W
713		− */
714		− hdev->asic_funcs->hw_queues_lock(hdev);
715		− hdev->asic_funcs->hw_queues_unlock(hdev);
716		−
717		− /* Flush processes that are sending message to CPU */
718		− mutex_lock(&hdev->send_cpu_message_lock);
719		− mutex_unlock(&hdev->send_cpu_message_lock);
	757	+ take_release_locks(hdev);
720	758
721	759	rc = hdev->asic_funcs->suspend(hdev);
722	760	if (rc)
@@ -819,6 +857,11 @@ static int device_kill_open_processes(struct hl_device *hdev, u32 timeout) · 96 unmodified lines
819	857	usleep_range(1000, 10000);
820	858
821	859	put_task_struct(task);
	860	+ } else {
	861	+ dev_warn(hdev->dev,
	862	+ "Can't get task struct for PID so giving up on killing process\n");
	863	+ mutex_unlock(&hdev->fpriv_list_lock);
	864	+ return -ETIME;
822	865	}
823	866	}
824	867
@@ -885,7 +928,7 @@ static void device_disable_open_processes(struct hl_device *hdev) · 60 unmodified lines
885	928	int hl_device_reset(struct hl_device *hdev, u32 flags)
886	929	{
887	930	u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
888		− bool hard_reset, from_hard_reset_thread, hard_instead_soft = false;
	931	+ bool hard_reset, from_hard_reset_thread, fw_reset, hard_instead_soft = false;
889	932	int i, rc;
890	933
891	934	if (!hdev->init_done) {
@@ -894,8 +937,9 @@ int hl_device_reset(struct hl_device *hdev, u32 flags) · 2 unmodified lines
894	937	return 0;
895	938	}
896	939
897		− hard_reset = (flags & HL_RESET_HARD) != 0;
898		− from_hard_reset_thread = (flags & HL_RESET_FROM_RESET_THREAD) != 0;
	940	+ hard_reset = !!(flags & HL_RESET_HARD);
	941	+ from_hard_reset_thread = !!(flags & HL_RESET_FROM_RESET_THREAD);
	942	+ fw_reset = !!(flags & HL_RESET_FW);
899	943
900	944	if (!hard_reset && !hdev->supports_soft_reset) {
901	945	hard_instead_soft = true;
@@ -947,11 +991,13 @@ do_reset: · 45 unmodified lines
947	991	else
948	992	hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
949	993
950		− /*
951		− * if reset is due to heartbeat, device CPU is no responsive in
952		− * which case no point sending PCI disable message to it
	994	+ /* If reset is due to heartbeat, device CPU is no responsive in
	995	+ * which case no point sending PCI disable message to it.
	996	+ *
	997	+ * If F/W is performing the reset, no need to send it a message to disable
	998	+ * PCI access
953	999	*/
954		− if (hard_reset && !(flags & HL_RESET_HEARTBEAT)) {
	1000	+ if (hard_reset && !(flags & (HL_RESET_HEARTBEAT \| HL_RESET_FW))) {
955	1001	/* Disable PCI access from device F/W so he won't send
956	1002	* us additional interrupts. We disable MSI/MSI-X at
957	1003	* the halt_engines function and we can't have the F/W
@@ -970,15 +1016,7 @@ do_reset: · 12 unmodified lines
970	1016	/* This also blocks future CS/VM/JOB completion operations */
971	1017	hdev->disabled = true;
972	1018
973		− /* Flush anyone that is inside the critical section of enqueue
974		− * jobs to the H/W
975		− */
976		− hdev->asic_funcs->hw_queues_lock(hdev);
977		− hdev->asic_funcs->hw_queues_unlock(hdev);
978		−
979		− /* Flush anyone that is inside device open */
980		− mutex_lock(&hdev->fpriv_list_lock);
981		− mutex_unlock(&hdev->fpriv_list_lock);
	1019	+ take_release_locks(hdev);
982	1020
983	1021	dev_err(hdev->dev, "Going to RESET device!\n");
984	1022	}
@@ -989,6 +1027,8 @@ again: · 4 unmodified lines
989	1027
990	1028	hdev->process_kill_trial_cnt = 0;
991	1029
	1030	+ hdev->device_reset_work.fw_reset = fw_reset;
	1031	+
992	1032	/*
993	1033	* Because the reset function can't run from heartbeat work,
994	1034	* we need to call the reset function from a dedicated work.
@@ -999,31 +1039,7 @@ again: · 4 unmodified lines
999	1039	return 0;
1000	1040	}
1001	1041
1002		− if (hard_reset) {
1003		− device_late_fini(hdev);
1004		−
1005		− /*
1006		− * Now that the heartbeat thread is closed, flush processes
1007		− * which are sending messages to CPU
1008		− */
1009		− mutex_lock(&hdev->send_cpu_message_lock);
1010		− mutex_unlock(&hdev->send_cpu_message_lock);
1011		− }
1012		−
1013		− /*
1014		− * Halt the engines and disable interrupts so we won't get any more
1015		− * completions from H/W and we won't have any accesses from the
1016		− * H/W to the host machine
1017		− */
1018		− hdev->asic_funcs->halt_engines(hdev, hard_reset);
1019		−
1020		− /* Go over all the queues, release all CS and their jobs */
1021		− hl_cs_rollback_all(hdev);
1022		−
1023		− /* Release all pending user interrupts, each pending user interrupt
1024		− * holds a reference to user context
1025		− */
1026		− hl_release_pending_user_interrupts(hdev);
	1042	+ cleanup_resources(hdev, hard_reset, fw_reset);
1027	1043
1028	1044	kill_processes:
1029	1045	if (hard_reset) {
@@ -1057,12 +1073,15 @@ kill_processes: · 27 unmodified lines
1057	1073	}
1058	1074
1059	1075	/* Reset the H/W. It will be in idle state after this returns */
1060		− hdev->asic_funcs->hw_fini(hdev, hard_reset);
	1076	+ hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset);
1061	1077
1062	1078	if (hard_reset) {
	1079	+ hdev->fw_loader.linux_loaded = false;
	1080	+
1063	1081	/* Release kernel context */
1064	1082	if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1)
1065	1083	hdev->kernel_ctx = NULL;
	1084	+
1066	1085	hl_vm_fini(hdev);
1067	1086	hl_mmu_fini(hdev);
1068	1087	hl_eq_reset(hdev, &hdev->event_queue);
@@ -1292,6 +1311,10 @@ int hl_device_init(struct hl_device hdev, struct class hclass) · 223 unmodified lines
1292	1311	if (rc)
1293	1312	goto user_interrupts_fini;
1294	1313
	1314	+
	1315	+ /* initialize completion structure for multi CS wait */
	1316	+ hl_multi_cs_completion_init(hdev);
	1317	+
1295	1318	/*
1296	1319	* Initialize the H/W queues. Must be done before hw_init, because
1297	1320	* there the addresses of the kernel queue are being written to the
@@ -1361,6 +1384,8 @@ int hl_device_init(struct hl_device hdev, struct class hclass) · 63 unmodified lines
1361	1384
1362	1385	hdev->compute_ctx = NULL;
1363	1386
	1387	+ hdev->asic_funcs->state_dump_init(hdev);
	1388	+
1364	1389	hl_debugfs_add_device(hdev);
1365	1390
1366	1391	/* debugfs nodes are created in hl_ctx_init so it must be called after
@@ -1567,31 +1592,13 @@ void hl_device_fini(struct hl_device *hdev) · 200 unmodified lines
1567	1592	/* Mark device as disabled */
1568	1593	hdev->disabled = true;
1569	1594
1570		− /* Flush anyone that is inside the critical section of enqueue
1571		− * jobs to the H/W
1572		− */
1573		− hdev->asic_funcs->hw_queues_lock(hdev);
1574		− hdev->asic_funcs->hw_queues_unlock(hdev);
1575		−
1576		− /* Flush anyone that is inside device open */
1577		− mutex_lock(&hdev->fpriv_list_lock);
1578		− mutex_unlock(&hdev->fpriv_list_lock);
	1595	+ take_release_locks(hdev);
1579	1596
1580	1597	hdev->hard_reset_pending = true;
1581	1598
1582	1599	hl_hwmon_fini(hdev);
1583	1600
1584		− device_late_fini(hdev);
1585		−
1586		− /*
1587		− * Halt the engines and disable interrupts so we won't get any more
1588		− * completions from H/W and we won't have any accesses from the
1589		− * H/W to the host machine
1590		− */
1591		− hdev->asic_funcs->halt_engines(hdev, true);
1592		−
1593		− /* Go over all the queues, release all CS and their jobs */
1594		− hl_cs_rollback_all(hdev);
	1601	+ cleanup_resources(hdev, true, false);
1595	1602
1596	1603	/* Kill processes here after CS rollback. This is because the process
1597	1604	* can't really exit until all its CSs are done, which is what we
@@ -1610,7 +1617,9 @@ void hl_device_fini(struct hl_device *hdev) · 12 unmodified lines
1610	1617	hl_cb_pool_fini(hdev);
1611	1618
1612	1619	/* Reset the H/W. It will be in idle state after this returns */
1613		− hdev->asic_funcs->hw_fini(hdev, true);
	1620	+ hdev->asic_funcs->hw_fini(hdev, true, false);
	1621	+
	1622	+ hdev->fw_loader.linux_loaded = false;
1614	1623
1615	1624	/* Release kernel context */
1616	1625	if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))

drivers/misc/habanalabs/common/firmware_if.c+32−24

@@ -1,7 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2
3	3	/*
4		− * Copyright 2016-2019 HabanaLabs, Ltd.
	4	+ * Copyright 2016-2021 HabanaLabs, Ltd.
5	5	* All Rights Reserved.
6	6	*/
7	7
@@ -240,11 +240,15 @@ int hl_fw_send_cpu_message(struct hl_device hdev, u32 hw_queue_id, u32 msg, · 232 unmodified lines
240	240	/* set fence to a non valid value */
241	241	pkt->fence = cpu_to_le32(UINT_MAX);
242	242
243		− rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, len, pkt_dma_addr);
244		− if (rc) {
245		− dev_err(hdev->dev, "Failed to send CB on CPU PQ (%d)\n", rc);
246		− goto out;
247		− }
	243	+ /*
	244	+ * The CPU queue is a synchronous queue with an effective depth of
	245	+ * a single entry (although it is allocated with room for multiple
	246	+ * entries). We lock on it using 'send_cpu_message_lock' which
	247	+ * serializes accesses to the CPU queue.
	248	+ * Which means that we don't need to lock the access to the entire H/W
	249	+ * queues module when submitting a JOB to the CPU queue.
	250	+ */
	251	+ hl_hw_queue_submit_bd(hdev, queue, 0, len, pkt_dma_addr);
248	252
249	253	if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
250	254	expected_ack_val = queue->pi;
@@ -663,17 +667,15 @@ int hl_fw_cpucp_info_get(struct hl_device *hdev, · 412 unmodified lines
663	667	hdev->event_queue.check_eqe_index = false;
664	668
665	669	/* Read FW application security bits again */
666		− if (hdev->asic_prop.fw_cpu_boot_dev_sts0_valid) {
667		− hdev->asic_prop.fw_app_cpu_boot_dev_sts0 =
668		− RREG32(sts_boot_dev_sts0_reg);
669		− if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
	670	+ if (prop->fw_cpu_boot_dev_sts0_valid) {
	671	+ prop->fw_app_cpu_boot_dev_sts0 = RREG32(sts_boot_dev_sts0_reg);
	672	+ if (prop->fw_app_cpu_boot_dev_sts0 &
670	673	CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
671	674	hdev->event_queue.check_eqe_index = true;
672	675	}
673	676
674		− if (hdev->asic_prop.fw_cpu_boot_dev_sts1_valid)
675		− hdev->asic_prop.fw_app_cpu_boot_dev_sts1 =
676		− RREG32(sts_boot_dev_sts1_reg);
	677	+ if (prop->fw_cpu_boot_dev_sts1_valid)
	678	+ prop->fw_app_cpu_boot_dev_sts1 = RREG32(sts_boot_dev_sts1_reg);
677	679
678	680	out:
679	681	hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
@@ -1008,6 +1010,11 @@ void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev) · 328 unmodified lines
1008	1010	} else {
1009	1011	WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
1010	1012	msleep(static_loader->cpu_reset_wait_msec);
	1013	+
	1014	+ /* Must clear this register in order to prevent preboot
	1015	+ * from reading WFE after reboot
	1016	+ */
	1017	+ WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_NA);
1011	1018	}
1012	1019
1013	1020	hdev->device_cpu_is_halted = true;
@@ -1055,6 +1062,10 @@ static void detect_cpu_boot_status(struct hl_device *hdev, u32 status) · 41 unmodified lines
1055	1062	dev_err(hdev->dev,
1056	1063	"Device boot progress - Thermal Sensor initialization failed\n");
1057	1064	break;
	1065	+ case CPU_BOOT_STATUS_SECURITY_READY:
	1066	+ dev_err(hdev->dev,
	1067	+ "Device boot progress - Stuck in preboot after security initialization\n");
	1068	+ break;
1058	1069	default:
1059	1070	dev_err(hdev->dev,
1060	1071	"Device boot progress - Invalid status code %d\n",
@@ -1238,11 +1249,6 @@ static void hl_fw_preboot_update_state(struct hl_device *hdev) · 177 unmodified lines
1238	1249	* b. Check whether hard reset is done by boot cpu
1239	1250	* 3. FW application - a. Fetch fw application security status
1240	1251	* b. Check whether hard reset is done by fw app
1241		− *
1242		− * Preboot:
1243		− * Check security status bit (CPU_BOOT_DEV_STS0_ENABLED). If set, then-
1244		− * check security enabled bit (CPU_BOOT_DEV_STS0_SECURITY_EN)
1245		− * If set, then mark GIC controller to be disabled.
1246	1252	*/
1247	1253	prop->hard_reset_done_by_fw =
1248	1254	!!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
@@ -1953,8 +1959,8 @@ static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev) · 704 unmodified lines
1953	1959	if (!hdev->asic_prop.gic_interrupts_enable &&
1954	1960	!(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
1955	1961	CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
1956		− dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_irq_ctrl;
1957		− dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_irq_ctrl;
	1962	+ dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_pi_upd_irq;
	1963	+ dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_pi_upd_irq;
1958	1964
1959	1965	dev_warn(hdev->dev,
1960	1966	"Using a single interrupt interface towards cpucp");
@@ -2122,8 +2128,7 @@ static void hl_fw_linux_update_state(struct hl_device *hdev, · 161 unmodified lines
2122	2128
2123	2129	/* Read FW application security bits */
2124	2130	if (prop->fw_cpu_boot_dev_sts0_valid) {
2125		− prop->fw_app_cpu_boot_dev_sts0 =
2126		− RREG32(cpu_boot_dev_sts0_reg);
	2131	+ prop->fw_app_cpu_boot_dev_sts0 = RREG32(cpu_boot_dev_sts0_reg);
2127	2132
2128	2133	if (prop->fw_app_cpu_boot_dev_sts0 &
2129	2134	CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
@@ -2143,8 +2148,7 @@ static void hl_fw_linux_update_state(struct hl_device *hdev, · 13 unmodified lines
2143	2148	}
2144	2149
2145	2150	if (prop->fw_cpu_boot_dev_sts1_valid) {
2146		− prop->fw_app_cpu_boot_dev_sts1 =
2147		− RREG32(cpu_boot_dev_sts1_reg);
	2151	+ prop->fw_app_cpu_boot_dev_sts1 = RREG32(cpu_boot_dev_sts1_reg);
2148	2152
2149	2153	dev_dbg(hdev->dev,
2150	2154	"Firmware application CPU status1 %#x\n",
@@ -2235,6 +2239,10 @@ static int hl_fw_dynamic_init_cpu(struct hl_device *hdev, · 84 unmodified lines
2235	2239	dev_info(hdev->dev,
2236	2240	"Loading firmware to device, may take some time...\n");
2237	2241
	2242	+ /*
	2243	+ * In this stage, "cpu_dyn_regs" contains only LKD's hard coded values!
	2244	+ * It will be updated from FW after hl_fw_dynamic_request_descriptor().
	2245	+ */
2238	2246	dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2239	2247
2240	2248	rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,

drivers/misc/habanalabs/common/habanalabs.h+375−46

@@ -20,6 +20,7 @@ · 19 unmodified lines
20	20	#include <linux/scatterlist.h>
21	21	#include <linux/hashtable.h>
22	22	#include <linux/debugfs.h>
	23	+#include <linux/rwsem.h>
23	24	#include <linux/bitfield.h>
24	25	#include <linux/genalloc.h>
25	26	#include <linux/sched/signal.h>
@@ -65,6 +66,11 @@ · 39 unmodified lines
65	66
66	67	#define HL_COMMON_USER_INTERRUPT_ID 0xFFF
67	68
	69	+#define HL_STATE_DUMP_HIST_LEN 5
	70	+
	71	+#define OBJ_NAMES_HASH_TABLE_BITS 7 /* 1 << 7 buckets */
	72	+#define SYNC_TO_ENGINE_HASH_TABLE_BITS 7 /* 1 << 7 buckets */
	73	+
68	74	/* Memory */
69	75	#define MEM_HASH_TABLE_BITS 7 /* 1 << 7 buckets */
70	76
@@ -122,12 +128,17 @@ enum hl_mmu_page_table_location { · 51 unmodified lines
122	128	*
123	129	* - HL_RESET_DEVICE_RELEASE
124	130	* Set if reset is due to device release
	131	+ *
	132	+ * - HL_RESET_FW
	133	+ * F/W will perform the reset. No need to ask it to reset the device. This is relevant
	134	+ * only when running with secured f/w
125	135	*/
126	136	#define HL_RESET_HARD (1 << 0)
127	137	#define HL_RESET_FROM_RESET_THREAD (1 << 1)
128	138	#define HL_RESET_HEARTBEAT (1 << 2)
129	139	#define HL_RESET_TDR (1 << 3)
130	140	#define HL_RESET_DEVICE_RELEASE (1 << 4)
	141	+#define HL_RESET_FW (1 << 5)
131	142
132	143	#define HL_MAX_SOBS_PER_MONITOR 8
133	144
@@ -236,7 +247,9 @@ enum hl_cs_type { · 102 unmodified lines
236	247	CS_TYPE_DEFAULT,
237	248	CS_TYPE_SIGNAL,
238	249	CS_TYPE_WAIT,
239		− CS_TYPE_COLLECTIVE_WAIT
	250	+ CS_TYPE_COLLECTIVE_WAIT,
	251	+ CS_RESERVE_SIGNALS,
	252	+ CS_UNRESERVE_SIGNALS
240	253	};
241	254
242	255	/*
@@ -281,13 +294,17 @@ enum queue_cb_alloc_flags { · 38 unmodified lines
281	294	* @hdev: habanalabs device structure.
282	295	* @kref: refcount of this SOB. The SOB will reset once the refcount is zero.
283	296	* @sob_id: id of this SOB.
	297	+ * @sob_addr: the sob offset from the base address.
284	298	* @q_idx: the H/W queue that uses this SOB.
	299	+ * @need_reset: reset indication set when switching to the other sob.
285	300	*/
286	301	struct hl_hw_sob {
287	302	struct hl_device *hdev;
288	303	struct kref kref;
289	304	u32 sob_id;
	305	+ u32 sob_addr;
290	306	u32 q_idx;
	307	+ bool need_reset;
291	308	};
292	309
293	310	enum hl_collective_mode {
@@ -317,11 +334,11 @@ struct hw_queue_properties { · 23 unmodified lines
317	334	};
318	335
319	336	/**
320		− * enum vm_type_t - virtual memory mapping request information.
	337	+ * enum vm_type - virtual memory mapping request information.
321	338	* @VM_TYPE_USERPTR: mapping of user memory to device virtual address.
322	339	* @VM_TYPE_PHYS_PACK: mapping of DRAM memory to device virtual address.
323	340	*/
324		−enum vm_type_t {
	341	+enum vm_type {
325	342	VM_TYPE_USERPTR = 0x1,
326	343	VM_TYPE_PHYS_PACK = 0x2
327	344	};
@@ -381,6 +398,16 @@ struct hl_mmu_properties { · 53 unmodified lines
381	398	u8 host_resident;
382	399	};
383	400
	401	+/**
	402	+ * struct hl_hints_range - hint addresses reserved va range.
	403	+ * @start_addr: start address of the va range.
	404	+ * @end_addr: end address of the va range.
	405	+ */
	406	+struct hl_hints_range {
	407	+ u64 start_addr;
	408	+ u64 end_addr;
	409	+};
	410	+
384	411	/**
385	412	* struct asic_fixed_properties - ASIC specific immutable properties.
386	413	* @hw_queues_props: H/W queues properties.
@@ -392,6 +419,10 @@ struct hl_mmu_properties { · 5 unmodified lines
392	419	* @pmmu: PCI (host) MMU address translation properties.
393	420	* @pmmu_huge: PCI (host) MMU address translation properties for memory
394	421	* allocated with huge pages.
	422	+ * @hints_dram_reserved_va_range: dram hint addresses reserved range.
	423	+ * @hints_host_reserved_va_range: host hint addresses reserved range.
	424	+ * @hints_host_hpage_reserved_va_range: host huge page hint addresses reserved
	425	+ * range.
395	426	* @sram_base_address: SRAM physical start address.
396	427	* @sram_end_address: SRAM physical end address.
397	428	* @sram_user_base_address - SRAM physical start address for user access.
@@ -412,6 +443,10 @@ struct hl_mmu_properties { · 14 unmodified lines
412	443	* to the device's MMU.
413	444	* @cb_va_end_addr: virtual end address of command buffers which are mapped to
414	445	* the device's MMU.
	446	+ * @dram_hints_align_mask: dram va hint addresses alignment mask which is used
	447	+ * for hints validity check.
	448	+ * device_dma_offset_for_host_access: the offset to add to host DMA addresses
	449	+ * to enable the device to access them.
415	450	* @mmu_pgt_size: MMU page tables total size.
416	451	* @mmu_pte_size: PTE size in MMU page tables.
417	452	* @mmu_hop_table_size: MMU hop table size.
@@ -459,6 +494,8 @@ struct hl_mmu_properties { · 41 unmodified lines
459	494	* reserved for the user
460	495	* @first_available_cq: first available CQ for the user.
461	496	* @user_interrupt_count: number of user interrupts.
	497	+ * @server_type: Server type that the ASIC is currently installed in.
	498	+ * The value is according to enum hl_server_type in uapi file.
462	499	* @tpc_enabled_mask: which TPCs are enabled.
463	500	* @completion_queues_count: number of completion queues.
464	501	* @fw_security_enabled: true if security measures are enabled in firmware,
@@ -470,6 +507,7 @@ struct hl_mmu_properties { · 5 unmodified lines
470	507	* @dram_supports_virtual_memory: is there an MMU towards the DRAM
471	508	* @hard_reset_done_by_fw: true if firmware is handling hard reset flow
472	509	* @num_functional_hbms: number of functional HBMs in each DCORE.
	510	+ * @hints_range_reservation: device support hint addresses range reservation.
473	511	* @iatu_done_by_fw: true if iATU configuration is being done by FW.
474	512	* @dynamic_fw_load: is dynamic FW load is supported.
475	513	* @gic_interrupts_enable: true if FW is not blocking GIC controller,
@@ -483,6 +521,9 @@ struct asic_fixed_properties { · 7 unmodified lines
483	521	struct hl_mmu_properties dmmu;
484	522	struct hl_mmu_properties pmmu;
485	523	struct hl_mmu_properties pmmu_huge;
	524	+ struct hl_hints_range hints_dram_reserved_va_range;
	525	+ struct hl_hints_range hints_host_reserved_va_range;
	526	+ struct hl_hints_range hints_host_hpage_reserved_va_range;
486	527	u64 sram_base_address;
487	528	u64 sram_end_address;
488	529	u64 sram_user_base_address;
@@ -500,6 +541,8 @@ struct asic_fixed_properties { · 11 unmodified lines
500	541	u64 mmu_dram_default_page_addr;
501	542	u64 cb_va_start_addr;
502	543	u64 cb_va_end_addr;
	544	+ u64 dram_hints_align_mask;
	545	+ u64 device_dma_offset_for_host_access;
503	546	u32 mmu_pgt_size;
504	547	u32 mmu_pte_size;
505	548	u32 mmu_hop_table_size;
@@ -534,6 +577,7 @@ struct asic_fixed_properties { · 28 unmodified lines
534	577	u16 first_available_user_msix_interrupt;
535	578	u16 first_available_cq[HL_MAX_DCORES];
536	579	u16 user_interrupt_count;
	580	+ u16 server_type;
537	581	u8 tpc_enabled_mask;
538	582	u8 completion_queues_count;
539	583	u8 fw_security_enabled;
@@ -542,6 +586,7 @@ struct asic_fixed_properties { · 2 unmodified lines
542	586	u8 dram_supports_virtual_memory;
543	587	u8 hard_reset_done_by_fw;
544	588	u8 num_functional_hbms;
	589	+ u8 hints_range_reservation;
545	590	u8 iatu_done_by_fw;
546	591	u8 dynamic_fw_load;
547	592	u8 gic_interrupts_enable;
@@ -552,40 +597,45 @@ struct asic_fixed_properties { · 4 unmodified lines
552	597	* @completion: fence is implemented using completion
553	598	* @refcount: refcount for this fence
554	599	* @cs_sequence: sequence of the corresponding command submission
	600	+ * @stream_master_qid_map: streams masters QID bitmap to represent all streams
	601	+ * masters QIDs that multi cs is waiting on
555	602	* @error: mark this fence with error
556	603	* @timestamp: timestamp upon completion
557		− *
558	604	*/
559	605	struct hl_fence {
560	606	struct completion completion;
561	607	struct kref refcount;
562	608	u64 cs_sequence;
	609	+ u32 stream_master_qid_map;
563	610	int error;
564	611	ktime_t timestamp;
565	612	};
566	613
567	614	/**
568	615	* struct hl_cs_compl - command submission completion object.
569		− * @sob_reset_work: workqueue object to run SOB reset flow.
570	616	* @base_fence: hl fence object.
571	617	* @lock: spinlock to protect fence.
572	618	* @hdev: habanalabs device structure.
573	619	* @hw_sob: the H/W SOB used in this signal/wait CS.
	620	+ * @encaps_sig_hdl: encaps signals hanlder.
574	621	* @cs_seq: command submission sequence number.
575	622	* @type: type of the CS - signal/wait.
576	623	* @sob_val: the SOB value that is used in this signal/wait CS.
577	624	* @sob_group: the SOB group that is used in this collective wait CS.
	625	+ * @encaps_signals: indication whether it's a completion object of cs with
	626	+ * encaps signals or not.
578	627	*/
579	628	struct hl_cs_compl {
580		− struct work_struct sob_reset_work;
581	629	struct hl_fence base_fence;
582	630	spinlock_t lock;
583	631	struct hl_device *hdev;
584	632	struct hl_hw_sob *hw_sob;
	633	+ struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
585	634	u64 cs_seq;
586	635	enum hl_cs_type type;
587	636	u16 sob_val;
588	637	u16 sob_group;
	638	+ bool encaps_signals;
589	639	};
590	640
591	641	/*
@@ -697,6 +747,17 @@ struct hl_sync_stream_properties { · 105 unmodified lines
697	747	u8 curr_sob_offset;
698	748	};
699	749
	750	+/**
	751	+ * struct hl_encaps_signals_mgr - describes sync stream encapsulated signals
	752	+ * handlers manager
	753	+ * @lock: protects handles.
	754	+ * @handles: an idr to hold all encapsulated signals handles.
	755	+ */
	756	+struct hl_encaps_signals_mgr {
	757	+ spinlock_t lock;
	758	+ struct idr handles;
	759	+};
	760	+
700	761	/**
701	762	* struct hl_hw_queue - describes a H/W transport queue.
702	763	* @shadow_queue: pointer to a shadow queue that holds pointers to jobs.
@@ -875,7 +936,7 @@ struct pci_mem_region { · 172 unmodified lines
875	936	u64 region_base;
876	937	u64 region_size;
877	938	u64 bar_size;
878		− u32 offset_in_bar;
	939	+ u64 offset_in_bar;
879	940	u8 bar_id;
880	941	u8 used;
881	942	};
@@ -996,7 +1057,7 @@ struct fw_load_mgr { · 114 unmodified lines
996	1057	* hw_fini and before CS rollback.
997	1058	* @suspend: handles IP specific H/W or SW changes for suspend.
998	1059	* @resume: handles IP specific H/W or SW changes for resume.
999		− * @cb_mmap: maps a CB.
	1060	+ * @mmap: maps a memory.
1000	1061	* @ring_doorbell: increment PI on a given QMAN.
1001	1062	* @pqe_write: Write the PQ entry to the PQ. This is ASIC-specific
1002	1063	* function because the PQs are located in different memory areas
@@ -1101,6 +1162,10 @@ struct fw_load_mgr { · 98 unmodified lines
1101	1162	* generic f/w compatible PLL Indexes
1102	1163	* @init_firmware_loader: initialize data for FW loader.
1103	1164	* @init_cpu_scrambler_dram: Enable CPU specific DRAM scrambling
	1165	+ * @state_dump_init: initialize constants required for state dump
	1166	+ * @get_sob_addr: get SOB base address offset.
	1167	+ * @set_pci_memory_regions: setting properties of PCI memory regions
	1168	+ * @get_stream_master_qid_arr: get pointer to stream masters QID array
1104	1169	*/
1105	1170	struct hl_asic_funcs {
1106	1171	int (early_init)(struct hl_device hdev);
@@ -1110,11 +1175,11 @@ struct hl_asic_funcs { · 3 unmodified lines
1110	1175	int (sw_init)(struct hl_device hdev);
1111	1176	int (sw_fini)(struct hl_device hdev);
1112	1177	int (hw_init)(struct hl_device hdev);
1113		− void (hw_fini)(struct hl_device hdev, bool hard_reset);
1114		− void (halt_engines)(struct hl_device hdev, bool hard_reset);
	1178	+ void (hw_fini)(struct hl_device hdev, bool hard_reset, bool fw_reset);
	1179	+ void (halt_engines)(struct hl_device hdev, bool hard_reset, bool fw_reset);
1115	1180	int (suspend)(struct hl_device hdev);
1116	1181	int (resume)(struct hl_device hdev);
1117		− int (cb_mmap)(struct hl_device hdev, struct vm_area_struct *vma,
	1182	+ int (mmap)(struct hl_device hdev, struct vm_area_struct *vma,
1118	1183	void *cpu_addr, dma_addr_t dma_addr, size_t size);
1119	1184	void (ring_doorbell)(struct hl_device hdev, u32 hw_queue_id, u32 pi);
1120	1185	void (pqe_write)(struct hl_device hdev, __le64 *pqe,
@@ -1210,10 +1275,11 @@ struct hl_asic_funcs { · 89 unmodified lines
1210	1275	void (reset_sob_group)(struct hl_device hdev, u16 sob_group);
1211	1276	void (set_dma_mask_from_fw)(struct hl_device hdev);
1212	1277	u64 (get_device_time)(struct hl_device hdev);
1213		− void (collective_wait_init_cs)(struct hl_cs cs);
	1278	+ int (collective_wait_init_cs)(struct hl_cs cs);
1214	1279	int (collective_wait_create_jobs)(struct hl_device hdev,
1215		− struct hl_ctx ctx, struct hl_cs cs, u32 wait_queue_id,
1216		− u32 collective_engine_id);
	1280	+ struct hl_ctx ctx, struct hl_cs cs,
	1281	+ u32 wait_queue_id, u32 collective_engine_id,
	1282	+ u32 encaps_signal_offset);
1217	1283	u64 (scramble_addr)(struct hl_device hdev, u64 addr);
1218	1284	u64 (descramble_addr)(struct hl_device hdev, u64 addr);
1219	1285	void (ack_protection_bits_errors)(struct hl_device hdev);
@@ -1226,6 +1292,10 @@ struct hl_asic_funcs { · 6 unmodified lines
1226	1292	int (*map_pll_idx_to_fw_idx)(u32 pll_idx);
1227	1293	void (init_firmware_loader)(struct hl_device hdev);
1228	1294	void (init_cpu_scrambler_dram)(struct hl_device hdev);
	1295	+ void (state_dump_init)(struct hl_device hdev);
	1296	+ u32 (get_sob_addr)(struct hl_device hdev, u32 sob_id);
	1297	+ void (set_pci_memory_regions)(struct hl_device hdev);
	1298	+ u32* (*get_stream_master_qid_arr)(void);
1229	1299	};
1230	1300
1231	1301
@@ -1282,20 +1352,6 @@ struct hl_cs_counters_atomic { · 50 unmodified lines
1282	1352	atomic64_t validation_drop_cnt;
1283	1353	};
1284	1354
1285		−/**
1286		− * struct hl_pending_cb - pending command buffer structure
1287		− * @cb_node: cb node in pending cb list
1288		− * @cb: command buffer to send in next submission
1289		− * @cb_size: command buffer size
1290		− * @hw_queue_id: destination queue id
1291		− */
1292		−struct hl_pending_cb {
1293		− struct list_head cb_node;
1294		− struct hl_cb *cb;
1295		− u32 cb_size;
1296		− u32 hw_queue_id;
1297		−};
1298		−
1299	1355	/**
1300	1356	* struct hl_ctx - user/kernel context.
1301	1357	* @mem_hash: holds mapping from virtual address to virtual memory area
@@ -1312,28 +1368,21 @@ struct hl_pending_cb { · 10 unmodified lines
1312	1368	* MMU hash or walking the PGT requires talking this lock.
1313	1369	* @hw_block_list_lock: protects the HW block memory list.
1314	1370	* @debugfs_list: node in debugfs list of contexts.
1315		− * pending_cb_list: list of pending command buffers waiting to be sent upon
1316		− * next user command submission context.
1317	1371	* @hw_block_mem_list: list of HW block virtual mapped addresses.
1318	1372	* @cs_counters: context command submission counters.
1319	1373	* @cb_va_pool: device VA pool for command buffers which are mapped to the
1320	1374	* device's MMU.
	1375	+ * @sig_mgr: encaps signals handle manager.
1321	1376	* @cs_sequence: sequence number for CS. Value is assigned to a CS and passed
1322	1377	* to user so user could inquire about CS. It is used as
1323	1378	* index to cs_pending array.
1324	1379	* @dram_default_hops: array that holds all hops addresses needed for default
1325	1380	* DRAM mapping.
1326		− * @pending_cb_lock: spinlock to protect pending cb list
1327	1381	* @cs_lock: spinlock to protect cs_sequence.
1328	1382	* @dram_phys_mem: amount of used physical DRAM memory by this context.
1329	1383	* @thread_ctx_switch_token: token to prevent multiple threads of the same
1330	1384	* context from running the context switch phase.
1331	1385	* Only a single thread should run it.
1332		− * @thread_pending_cb_token: token to prevent multiple threads from processing
1333		− * the pending CB list. Only a single thread should
1334		− * process the list since it is protected by a
1335		− * spinlock and we don't want to halt the entire
1336		− * command submission sequence.
1337	1386	* @thread_ctx_switch_wait_token: token to prevent the threads that didn't run
1338	1387	* the context switch phase from moving to their
1339	1388	* execution phase before the context switch phase
@@ -1353,17 +1402,15 @@ struct hl_ctx { · 13 unmodified lines
1353	1402	struct mutex mmu_lock;
1354	1403	struct mutex hw_block_list_lock;
1355	1404	struct list_head debugfs_list;
1356		− struct list_head pending_cb_list;
1357	1405	struct list_head hw_block_mem_list;
1358	1406	struct hl_cs_counters_atomic cs_counters;
1359	1407	struct gen_pool *cb_va_pool;
	1408	+ struct hl_encaps_signals_mgr sig_mgr;
1360	1409	u64 cs_sequence;
1361	1410	u64 *dram_default_hops;
1362		− spinlock_t pending_cb_lock;
1363	1411	spinlock_t cs_lock;
1364	1412	atomic64_t dram_phys_mem;
1365	1413	atomic_t thread_ctx_switch_token;
1366		− atomic_t thread_pending_cb_token;
1367	1414	u32 thread_ctx_switch_wait_token;
1368	1415	u32 asid;
1369	1416	u32 handle;
@@ -1394,20 +1441,22 @@ struct hl_ctx_mgr { · 24 unmodified lines
1394	1441	* @sgt: pointer to the scatter-gather table that holds the pages.
1395	1442	* @dir: for DMA unmapping, the direction must be supplied, so save it.
1396	1443	* @debugfs_list: node in debugfs list of command submissions.
	1444	+ * @pid: the pid of the user process owning the memory
1397	1445	* @addr: user-space virtual address of the start of the memory area.
1398	1446	* @size: size of the memory area to pin & map.
1399	1447	* @dma_mapped: true if the SG was mapped to DMA addresses, false otherwise.
1400	1448	*/
1401	1449	struct hl_userptr {
1402		− enum vm_type_t vm_type; /* must be first */
	1450	+ enum vm_type vm_type; /* must be first */
1403	1451	struct list_head job_node;
1404	1452	struct page **pages;
1405	1453	unsigned int npages;
1406	1454	struct sg_table *sgt;
1407	1455	enum dma_data_direction dir;
1408	1456	struct list_head debugfs_list;
	1457	+ pid_t pid;
1409	1458	u64 addr;
1410		− u32 size;
	1459	+ u64 size;
1411	1460	u8 dma_mapped;
1412	1461	};
1413	1462
@@ -1426,12 +1475,14 @@ struct hl_userptr { · 12 unmodified lines
1426	1475	* @mirror_node : node in device mirror list of command submissions.
1427	1476	* @staged_cs_node: node in the staged cs list.
1428	1477	* @debugfs_list: node in debugfs list of command submissions.
	1478	+ * @encaps_sig_hdl: holds the encaps signals handle.
1429	1479	* @sequence: the sequence number of this CS.
1430	1480	* @staged_sequence: the sequence of the staged submission this CS is part of,
1431	1481	* relevant only if staged_cs is set.
1432	1482	* @timeout_jiffies: cs timeout in jiffies.
1433	1483	* @submission_time_jiffies: submission time of the cs
1434	1484	* @type: CS_TYPE_*.
	1485	+ * @encaps_sig_hdl_id: encaps signals handle id, set for the first staged cs.
1435	1486	* @submitted: true if CS was submitted to H/W.
1436	1487	* @completed: true if CS was completed by device.
1437	1488	* @timedout : true if CS was timedout.
@@ -1445,6 +1496,7 @@ struct hl_userptr { · 7 unmodified lines
1445	1496	* @staged_cs: true if this CS is part of a staged submission.
1446	1497	* @skip_reset_on_timeout: true if we shall not reset the device in case
1447	1498	* timeout occurs (debug scenario).
	1499	+ * @encaps_signals: true if this CS has encaps reserved signals.
1448	1500	*/
1449	1501	struct hl_cs {
1450	1502	u16 *jobs_in_queue_cnt;
@@ -1459,11 +1511,13 @@ struct hl_cs { · 8 unmodified lines
1459	1511	struct list_head mirror_node;
1460	1512	struct list_head staged_cs_node;
1461	1513	struct list_head debugfs_list;
	1514	+ struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
1462	1515	u64 sequence;
1463	1516	u64 staged_sequence;
1464	1517	u64 timeout_jiffies;
1465	1518	u64 submission_time_jiffies;
1466	1519	enum hl_cs_type type;
	1520	+ u32 encaps_sig_hdl_id;
1467	1521	u8 submitted;
1468	1522	u8 completed;
1469	1523	u8 timedout;
@@ -1474,6 +1528,7 @@ struct hl_cs { · 4 unmodified lines
1474	1528	u8 staged_first;
1475	1529	u8 staged_cs;
1476	1530	u8 skip_reset_on_timeout;
	1531	+ u8 encaps_signals;
1477	1532	};
1478	1533
1479	1534	/**
@@ -1493,6 +1548,8 @@ struct hl_cs { · 13 unmodified lines
1493	1548	* @hw_queue_id: the id of the H/W queue this job is submitted to.
1494	1549	* @user_cb_size: the actual size of the CB we got from the user.
1495	1550	* @job_cb_size: the actual size of the CB that we put on the queue.
	1551	+ * @encaps_sig_wait_offset: encapsulated signals offset, which allow user
	1552	+ * to wait on part of the reserved signals.
1496	1553	* @is_kernel_allocated_cb: true if the CB handle we got from the user holds a
1497	1554	* handle to a kernel-allocated CB object, false
1498	1555	* otherwise (SRAM/DRAM/host address).
@@ -1517,6 +1574,7 @@ struct hl_cs_job { · 18 unmodified lines
1517	1574	u32 hw_queue_id;
1518	1575	u32 user_cb_size;
1519	1576	u32 job_cb_size;
	1577	+ u32 encaps_sig_wait_offset;
1520	1578	u8 is_kernel_allocated_cb;
1521	1579	u8 contains_dma_pkt;
1522	1580	};
@@ -1613,7 +1671,7 @@ struct hl_vm_hw_block_list_node { · 90 unmodified lines
1613	1671	* @created_from_userptr: is product of host virtual address.
1614	1672	*/
1615	1673	struct hl_vm_phys_pg_pack {
1616		− enum vm_type_t vm_type; /* must be first */
	1674	+ enum vm_type vm_type; /* must be first */
1617	1675	u64 *pages;
1618	1676	u64 npages;
1619	1677	u64 total_size;
@@ -1759,9 +1817,13 @@ struct hl_debugfs_entry { · 139 unmodified lines
1759	1817	* @ctx_mem_hash_list: list of available contexts with MMU mappings.
1760	1818	* @ctx_mem_hash_spinlock: protects cb_list.
1761	1819	* @blob_desc: descriptor of blob
	1820	+ * @state_dump: data of the system states in case of a bad cs.
	1821	+ * @state_dump_sem: protects state_dump.
1762	1822	* @addr: next address to read/write from/to in read/write32.
1763	1823	* @mmu_addr: next virtual address to translate to physical address in mmu_show.
	1824	+ * @userptr_lookup: the target user ptr to look up for on demand.
1764	1825	* @mmu_asid: ASID to use while translating in mmu_show.
	1826	+ * @state_dump_head: index of the latest state dump
1765	1827	* @i2c_bus: generic u8 debugfs file for bus value to use in i2c_data_read.
1766	1828	* @i2c_addr: generic u8 debugfs file for address value to use in i2c_data_read.
1767	1829	* @i2c_reg: generic u8 debugfs file for register value to use in i2c_data_read.
@@ -1783,14 +1845,149 @@ struct hl_dbg_device_entry { · 15 unmodified lines
1783	1845	struct list_head ctx_mem_hash_list;
1784	1846	spinlock_t ctx_mem_hash_spinlock;
1785	1847	struct debugfs_blob_wrapper blob_desc;
	1848	+ char *state_dump[HL_STATE_DUMP_HIST_LEN];
	1849	+ struct rw_semaphore state_dump_sem;
1786	1850	u64 addr;
1787	1851	u64 mmu_addr;
	1852	+ u64 userptr_lookup;
1788	1853	u32 mmu_asid;
	1854	+ u32 state_dump_head;
1789	1855	u8 i2c_bus;
1790	1856	u8 i2c_addr;
1791	1857	u8 i2c_reg;
1792	1858	};
1793	1859
	1860	+/**
	1861	+ * struct hl_hw_obj_name_entry - single hw object name, member of
	1862	+ * hl_state_dump_specs
	1863	+ * @node: link to the containing hash table
	1864	+ * @name: hw object name
	1865	+ * @id: object identifier
	1866	+ */
	1867	+struct hl_hw_obj_name_entry {
	1868	+ struct hlist_node node;
	1869	+ const char *name;
	1870	+ u32 id;
	1871	+};
	1872	+
	1873	+enum hl_state_dump_specs_props {
	1874	+ SP_SYNC_OBJ_BASE_ADDR,
	1875	+ SP_NEXT_SYNC_OBJ_ADDR,
	1876	+ SP_SYNC_OBJ_AMOUNT,
	1877	+ SP_MON_OBJ_WR_ADDR_LOW,
	1878	+ SP_MON_OBJ_WR_ADDR_HIGH,
	1879	+ SP_MON_OBJ_WR_DATA,
	1880	+ SP_MON_OBJ_ARM_DATA,
	1881	+ SP_MON_OBJ_STATUS,
	1882	+ SP_MONITORS_AMOUNT,
	1883	+ SP_TPC0_CMDQ,
	1884	+ SP_TPC0_CFG_SO,
	1885	+ SP_NEXT_TPC,
	1886	+ SP_MME_CMDQ,
	1887	+ SP_MME_CFG_SO,
	1888	+ SP_NEXT_MME,
	1889	+ SP_DMA_CMDQ,
	1890	+ SP_DMA_CFG_SO,
	1891	+ SP_DMA_QUEUES_OFFSET,
	1892	+ SP_NUM_OF_MME_ENGINES,
	1893	+ SP_SUB_MME_ENG_NUM,
	1894	+ SP_NUM_OF_DMA_ENGINES,
	1895	+ SP_NUM_OF_TPC_ENGINES,
	1896	+ SP_ENGINE_NUM_OF_QUEUES,
	1897	+ SP_ENGINE_NUM_OF_STREAMS,
	1898	+ SP_ENGINE_NUM_OF_FENCES,
	1899	+ SP_FENCE0_CNT_OFFSET,
	1900	+ SP_FENCE0_RDATA_OFFSET,
	1901	+ SP_CP_STS_OFFSET,
	1902	+ SP_NUM_CORES,
	1903	+
	1904	+ SP_MAX
	1905	+};
	1906	+
	1907	+enum hl_sync_engine_type {
	1908	+ ENGINE_TPC,
	1909	+ ENGINE_DMA,
	1910	+ ENGINE_MME,
	1911	+};
	1912	+
	1913	+/**
	1914	+ * struct hl_mon_state_dump - represents a state dump of a single monitor
	1915	+ * @id: monitor id
	1916	+ * @wr_addr_low: address monitor will write to, low bits
	1917	+ * @wr_addr_high: address monitor will write to, high bits
	1918	+ * @wr_data: data monitor will write
	1919	+ * @arm_data: register value containing monitor configuration
	1920	+ * @status: monitor status
	1921	+ */
	1922	+struct hl_mon_state_dump {
	1923	+ u32 id;
	1924	+ u32 wr_addr_low;
	1925	+ u32 wr_addr_high;
	1926	+ u32 wr_data;
	1927	+ u32 arm_data;
	1928	+ u32 status;
	1929	+};
	1930	+
	1931	+/**
	1932	+ * struct hl_sync_to_engine_map_entry - sync object id to engine mapping entry
	1933	+ * @engine_type: type of the engine
	1934	+ * @engine_id: id of the engine
	1935	+ * @sync_id: id of the sync object
	1936	+ */
	1937	+struct hl_sync_to_engine_map_entry {
	1938	+ struct hlist_node node;
	1939	+ enum hl_sync_engine_type engine_type;
	1940	+ u32 engine_id;
	1941	+ u32 sync_id;
	1942	+};
	1943	+
	1944	+/**
	1945	+ * struct hl_sync_to_engine_map - maps sync object id to associated engine id
	1946	+ * @tb: hash table containing the mapping, each element is of type
	1947	+ * struct hl_sync_to_engine_map_entry
	1948	+ */
	1949	+struct hl_sync_to_engine_map {
	1950	+ DECLARE_HASHTABLE(tb, SYNC_TO_ENGINE_HASH_TABLE_BITS);
	1951	+};
	1952	+
	1953	+/**
	1954	+ * struct hl_state_dump_specs_funcs - virtual functions used by the state dump
	1955	+ * @gen_sync_to_engine_map: generate a hash map from sync obj id to its engine
	1956	+ * @print_single_monitor: format monitor data as string
	1957	+ * @monitor_valid: return true if given monitor dump is valid
	1958	+ * @print_fences_single_engine: format fences data as string
	1959	+ */
	1960	+struct hl_state_dump_specs_funcs {
	1961	+ int (gen_sync_to_engine_map)(struct hl_device hdev,
	1962	+ struct hl_sync_to_engine_map *map);
	1963	+ int (print_single_monitor)(char buf, size_t size, size_t *offset,
	1964	+ struct hl_device *hdev,
	1965	+ struct hl_mon_state_dump *mon);
	1966	+ int (monitor_valid)(struct hl_mon_state_dump mon);
	1967	+ int (print_fences_single_engine)(struct hl_device hdev,
	1968	+ u64 base_offset,
	1969	+ u64 status_base_offset,
	1970	+ enum hl_sync_engine_type engine_type,
	1971	+ u32 engine_id, char **buf,
	1972	+ size_t size, size_t offset);
	1973	+};
	1974	+
	1975	+/**
	1976	+ * struct hl_state_dump_specs - defines ASIC known hw objects names
	1977	+ * @so_id_to_str_tb: sync objects names index table
	1978	+ * @monitor_id_to_str_tb: monitors names index table
	1979	+ * @funcs: virtual functions used for state dump
	1980	+ * @sync_namager_names: readable names for sync manager if available (ex: N_E)
	1981	+ * @props: pointer to a per asic const props array required for state dump
	1982	+ */
	1983	+struct hl_state_dump_specs {
	1984	+ DECLARE_HASHTABLE(so_id_to_str_tb, OBJ_NAMES_HASH_TABLE_BITS);
	1985	+ DECLARE_HASHTABLE(monitor_id_to_str_tb, OBJ_NAMES_HASH_TABLE_BITS);
	1986	+ struct hl_state_dump_specs_funcs funcs;
	1987	+ const char * const *sync_namager_names;
	1988	+ s64 *props;
	1989	+};
	1990	+
1794	1991
1795	1992	/*
1796	1993	* DEVICES
@@ -1798,7 +1995,7 @@ struct hl_dbg_device_entry { · 1 unmodified line
1798	1995
1799	1996	#define HL_STR_MAX 32
1800	1997
1801		−#define HL_DEV_STS_MAX (HL_DEVICE_STATUS_NEEDS_RESET + 1)
	1998	+#define HL_DEV_STS_MAX (HL_DEVICE_STATUS_LAST + 1)
1802	1999
1803	2000	/* Theoretical limit only. A single host can only contain up to 4 or 8 PCIe
1804	2001	* x16 cards. In extreme cases, there are hosts that can accommodate 16 cards.
@@ -1946,11 +2143,13 @@ struct hwmon_chip_info; · 141 unmodified lines
1946	2143	* @wq: work queue for device reset procedure.
1947	2144	* @reset_work: reset work to be done.
1948	2145	* @hdev: habanalabs device structure.
	2146	+ * @fw_reset: whether f/w will do the reset without us sending them a message to do it.
1949	2147	*/
1950	2148	struct hl_device_reset_work {
1951	2149	struct workqueue_struct *wq;
1952	2150	struct delayed_work reset_work;
1953	2151	struct hl_device *hdev;
	2152	+ bool fw_reset;
1954	2153	};
1955	2154
1956	2155	/**
@@ -2064,6 +2263,58 @@ struct hl_mmu_funcs { · 107 unmodified lines
2064	2263	u64 virt_addr, struct hl_mmu_hop_info *hops);
2065	2264	};
2066	2265
	2266	+/**
	2267	+ * number of user contexts allowed to call wait_for_multi_cs ioctl in
	2268	+ * parallel
	2269	+ */
	2270	+#define MULTI_CS_MAX_USER_CTX 2
	2271	+
	2272	+/**
	2273	+ * struct multi_cs_completion - multi CS wait completion.
	2274	+ * @completion: completion of any of the CS in the list
	2275	+ * @lock: spinlock for the completion structure
	2276	+ * @timestamp: timestamp for the multi-CS completion
	2277	+ * @stream_master_qid_map: bitmap of all stream masters on which the multi-CS
	2278	+ * is waiting
	2279	+ * @used: 1 if in use, otherwise 0
	2280	+ */
	2281	+struct multi_cs_completion {
	2282	+ struct completion completion;
	2283	+ spinlock_t lock;
	2284	+ s64 timestamp;
	2285	+ u32 stream_master_qid_map;
	2286	+ u8 used;
	2287	+};
	2288	+
	2289	+/**
	2290	+ * struct multi_cs_data - internal data for multi CS call
	2291	+ * @ctx: pointer to the context structure
	2292	+ * @fence_arr: array of fences of all CSs
	2293	+ * @seq_arr: array of CS sequence numbers
	2294	+ * @timeout_us: timeout in usec for waiting for CS to complete
	2295	+ * @timestamp: timestamp of first completed CS
	2296	+ * @wait_status: wait for CS status
	2297	+ * @completion_bitmap: bitmap of completed CSs (1- completed, otherwise 0)
	2298	+ * @stream_master_qid_map: bitmap of all stream master QIDs on which the
	2299	+ * multi-CS is waiting
	2300	+ * @arr_len: fence_arr and seq_arr array length
	2301	+ * @gone_cs: indication of gone CS (1- there was gone CS, otherwise 0)
	2302	+ * @update_ts: update timestamp. 1- update the timestamp, otherwise 0.
	2303	+ */
	2304	+struct multi_cs_data {
	2305	+ struct hl_ctx *ctx;
	2306	+ struct hl_fence **fence_arr;
	2307	+ u64 *seq_arr;
	2308	+ s64 timeout_us;
	2309	+ s64 timestamp;
	2310	+ long wait_status;
	2311	+ u32 completion_bitmap;
	2312	+ u32 stream_master_qid_map;
	2313	+ u8 arr_len;
	2314	+ u8 gone_cs;
	2315	+ u8 update_ts;
	2316	+};
	2317	+
2067	2318	/**
2068	2319	* struct hl_device - habanalabs device structure.
2069	2320	* @pdev: pointer to PCI device, can be NULL in case of simulator device.
@@ -2129,6 +2380,8 @@ struct hl_mmu_funcs { · 59 unmodified lines
2129	2380	* @mmu_func: device-related MMU functions.
2130	2381	* @fw_loader: FW loader manager.
2131	2382	* @pci_mem_region: array of memory regions in the PCI
	2383	+ * @state_dump_specs: constants and dictionaries needed to dump system state.
	2384	+ * @multi_cs_completion: array of multi-CS completion.
2132	2385	* @dram_used_mem: current DRAM memory consumption.
2133	2386	* @timeout_jiffies: device CS timeout value.
2134	2387	* @max_power: the max power of the device, as configured by the sysadmin. This
@@ -2205,6 +2458,7 @@ struct hl_mmu_funcs { · 70 unmodified lines
2205	2458	* halted. We can't halt it again because the COMMS
2206	2459	* protocol will throw an error. Relevant only for
2207	2460	* cases where Linux was not loaded to device CPU
	2461	+ * @supports_wait_for_multi_cs: true if wait for multi CS is supported
2208	2462	*/
2209	2463	struct hl_device {
2210	2464	struct pci_dev *pdev;
@@ -2273,6 +2527,11 @@ struct hl_device { · 62 unmodified lines
2273	2527
2274	2528	struct pci_mem_region pci_mem_region[PCI_REGION_NUMBER];
2275	2529
	2530	+ struct hl_state_dump_specs state_dump_specs;
	2531	+
	2532	+ struct multi_cs_completion multi_cs_completion[
	2533	+ MULTI_CS_MAX_USER_CTX];
	2534	+ u32 *stream_master_qid_arr;
2276	2535	atomic64_t dram_used_mem;
2277	2536	u64 timeout_jiffies;
2278	2537	u64 max_power;
@@ -2322,6 +2581,8 @@ struct hl_device { · 43 unmodified lines
2322	2581	u8 curr_reset_cause;
2323	2582	u8 skip_reset_on_timeout;
2324	2583	u8 device_cpu_is_halted;
	2584	+ u8 supports_wait_for_multi_cs;
	2585	+ u8 stream_master_qid_arr_size;
2325	2586
2326	2587	/* Parameters for bring-up */
2327	2588	u64 nic_ports_mask;
@@ -2343,6 +2604,29 @@ struct hl_device { · 15 unmodified lines
2343	2604	};
2344	2605
2345	2606
	2607	+/**
	2608	+ * struct hl_cs_encaps_sig_handle - encapsulated signals handle structure
	2609	+ * @refcount: refcount used to protect removing this id when several
	2610	+ * wait cs are used to wait of the reserved encaps signals.
	2611	+ * @hdev: pointer to habanalabs device structure.
	2612	+ * @hw_sob: pointer to H/W SOB used in the reservation.
	2613	+ * @cs_seq: staged cs sequence which contains encapsulated signals
	2614	+ * @id: idr handler id to be used to fetch the handler info
	2615	+ * @q_idx: stream queue index
	2616	+ * @pre_sob_val: current SOB value before reservation
	2617	+ * @count: signals number
	2618	+ */
	2619	+struct hl_cs_encaps_sig_handle {
	2620	+ struct kref refcount;
	2621	+ struct hl_device *hdev;
	2622	+ struct hl_hw_sob *hw_sob;
	2623	+ u64 cs_seq;
	2624	+ u32 id;
	2625	+ u32 q_idx;
	2626	+ u32 pre_sob_val;
	2627	+ u32 count;
	2628	+};
	2629	+
2346	2630	/*
2347	2631	* IOCTLs
2348	2632	*/
@@ -2372,6 +2656,23 @@ struct hl_ioctl_desc { · 23 unmodified lines
2372	2656	* Kernel module functions that can be accessed by entire module
2373	2657	*/
2374	2658
	2659	+/**
	2660	+ * hl_get_sg_info() - get number of pages and the DMA address from SG list.
	2661	+ * @sg: the SG list.
	2662	+ * @dma_addr: pointer to DMA address to return.
	2663	+ *
	2664	+ * Calculate the number of consecutive pages described by the SG list. Take the
	2665	+ * offset of the address in the first page, add to it the length and round it up
	2666	+ * to the number of needed pages.
	2667	+ */
	2668	+static inline u32 hl_get_sg_info(struct scatterlist sg, dma_addr_t dma_addr)
	2669	+{
	2670	+ *dma_addr = sg_dma_address(sg);
	2671	+
	2672	+ return ((((*dma_addr) & (PAGE_SIZE - 1)) + sg_dma_len(sg)) +
	2673	+ (PAGE_SIZE - 1)) >> PAGE_SHIFT;
	2674	+}
	2675	+
2375	2676	/**
2376	2677	* hl_mem_area_inside_range() - Checks whether address+size are inside a range.
2377	2678	* @address: The start address of the area we want to validate.
@@ -2436,7 +2737,9 @@ void destroy_hdev(struct hl_device *hdev); · 58 unmodified lines
2436	2737	int hl_hw_queues_create(struct hl_device *hdev);
2437	2738	void hl_hw_queues_destroy(struct hl_device *hdev);
2438	2739	int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
2439		− u32 cb_size, u64 cb_ptr);
	2740	+ u32 cb_size, u64 cb_ptr);
	2741	+void hl_hw_queue_submit_bd(struct hl_device hdev, struct hl_hw_queue q,
	2742	+ u32 ctl, u32 len, u64 ptr);
2440	2743	int hl_hw_queue_schedule_cs(struct hl_cs *cs);
2441	2744	u32 hl_hw_queue_add_ptr(u32 ptr, u16 val);
2442	2745	void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id);
@@ -2470,6 +2773,8 @@ void hl_ctx_do_release(struct kref *ref); · 27 unmodified lines
2470	2773	void hl_ctx_get(struct hl_device hdev, struct hl_ctx ctx);
2471	2774	int hl_ctx_put(struct hl_ctx *ctx);
2472	2775	struct hl_fence hl_ctx_get_fence(struct hl_ctx ctx, u64 seq);
	2776	+int hl_ctx_get_fences(struct hl_ctx ctx, u64 seq_arr,
	2777	+ struct hl_fence **fence, u32 arr_len);
2473	2778	void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr);
2474	2779	void hl_ctx_mgr_fini(struct hl_device hdev, struct hl_ctx_mgr mgr);
2475	2780
@@ -2511,18 +2816,19 @@ int hl_cb_va_pool_init(struct hl_ctx *ctx); · 35 unmodified lines
2511	2816	void hl_cb_va_pool_fini(struct hl_ctx *ctx);
2512	2817
2513	2818	void hl_cs_rollback_all(struct hl_device *hdev);
2514		−void hl_pending_cb_list_flush(struct hl_ctx *ctx);
2515	2819	struct hl_cs_job hl_cs_allocate_job(struct hl_device hdev,
2516	2820	enum hl_queue_type queue_type, bool is_kernel_allocated_cb);
2517	2821	void hl_sob_reset_error(struct kref *ref);
2518	2822	int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask);
2519	2823	void hl_fence_put(struct hl_fence *fence);
	2824	+void hl_fences_put(struct hl_fence **fence, int len);
2520	2825	void hl_fence_get(struct hl_fence *fence);
2521	2826	void cs_get(struct hl_cs *cs);
2522	2827	bool cs_needs_completion(struct hl_cs *cs);
2523	2828	bool cs_needs_timeout(struct hl_cs *cs);
2524	2829	bool is_staged_cs_last_exists(struct hl_device hdev, struct hl_cs cs);
2525	2830	struct hl_cs hl_staged_cs_find_first(struct hl_device hdev, u64 cs_seq);
	2831	+void hl_multi_cs_completion_init(struct hl_device *hdev);
2526	2832
2527	2833	void goya_set_asic_funcs(struct hl_device *hdev);
2528	2834	void gaudi_set_asic_funcs(struct hl_device *hdev);
@@ -2650,9 +2956,25 @@ int hl_set_voltage(struct hl_device *hdev, · 121 unmodified lines
2650	2956	int sensor_index, u32 attr, long value);
2651	2957	int hl_set_current(struct hl_device *hdev,
2652	2958	int sensor_index, u32 attr, long value);
	2959	+void hw_sob_get(struct hl_hw_sob *hw_sob);
	2960	+void hw_sob_put(struct hl_hw_sob *hw_sob);
	2961	+void hl_encaps_handle_do_release(struct kref *ref);
	2962	+void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev,
	2963	+ struct hl_cs cs, struct hl_cs_job job,
	2964	+ struct hl_cs_compl *cs_cmpl);
2653	2965	void hl_release_pending_user_interrupts(struct hl_device *hdev);
2654	2966	int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,
2655		− struct hl_hw_sob **hw_sob, u32 count);
	2967	+ struct hl_hw_sob **hw_sob, u32 count, bool encaps_sig);
	2968	+
	2969	+int hl_state_dump(struct hl_device *hdev);
	2970	+const char hl_state_dump_get_sync_name(struct hl_device hdev, u32 sync_id);
	2971	+const char hl_state_dump_get_monitor_name(struct hl_device hdev,
	2972	+ struct hl_mon_state_dump *mon);
	2973	+void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map);
	2974	+__printf(4, 5) int hl_snprintf_resize(char *buf, size_t size, size_t *offset,
	2975	+ const char *format, ...);
	2976	+char hl_format_as_binary(char buf, size_t buf_len, u32 n);
	2977	+const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type);
2656	2978
2657	2979	#ifdef CONFIG_DEBUG_FS
2658	2980
@@ -2673,6 +2995,8 @@ void hl_debugfs_remove_userptr(struct hl_device *hdev, · 14 unmodified lines
2673	2995	struct hl_userptr *userptr);
2674	2996	void hl_debugfs_add_ctx_mem_hash(struct hl_device hdev, struct hl_ctx ctx);
2675	2997	void hl_debugfs_remove_ctx_mem_hash(struct hl_device hdev, struct hl_ctx ctx);
	2998	+void hl_debugfs_set_state_dump(struct hl_device hdev, char data,
	2999	+ unsigned long length);
2676	3000
2677	3001	#else
2678	3002
@@ -2746,6 +3070,11 @@ static inline void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, · 67 unmodified lines
2746	3070	{
2747	3071	}
2748	3072
	3073	+static inline void hl_debugfs_set_state_dump(struct hl_device *hdev,
	3074	+ char *data, unsigned long length)
	3075	+{
	3076	+}
	3077	+
2749	3078	#endif
2750	3079
2751	3080	/* IOCTLs */

drivers/misc/habanalabs/common/habanalabs_drv.c+8−5

@@ -141,7 +141,7 @@ int hl_device_open(struct inode inode, struct file filp) · 140 unmodified lines
141	141	hl_cb_mgr_init(&hpriv->cb_mgr);
142	142	hl_ctx_mgr_init(&hpriv->ctx_mgr);
143	143
144		− hpriv->taskpid = find_get_pid(current->pid);
	144	+ hpriv->taskpid = get_task_pid(current, PIDTYPE_PID);
145	145
146	146	mutex_lock(&hdev->fpriv_list_lock);
147	147
@@ -194,7 +194,6 @@ int hl_device_open(struct inode inode, struct file filp) · 46 unmodified lines
194	194
195	195	out_err:
196	196	mutex_unlock(&hdev->fpriv_list_lock);
197		−
198	197	hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
199	198	hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
200	199	filp->private_data = NULL;
@@ -318,12 +317,16 @@ int create_hdev(struct hl_device *dev, struct pci_dev pdev, · 117 unmodified lines
318	317	hdev->asic_prop.fw_security_enabled = false;
319	318
320	319	/* Assign status description string */
321		− strncpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION],
322		− "disabled", HL_STR_MAX);
	320	+ strncpy(hdev->status[HL_DEVICE_STATUS_OPERATIONAL],
	321	+ "operational", HL_STR_MAX);
323	322	strncpy(hdev->status[HL_DEVICE_STATUS_IN_RESET],
324	323	"in reset", HL_STR_MAX);
	324	+ strncpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION],
	325	+ "disabled", HL_STR_MAX);
325	326	strncpy(hdev->status[HL_DEVICE_STATUS_NEEDS_RESET],
326	327	"needs reset", HL_STR_MAX);
	328	+ strncpy(hdev->status[HL_DEVICE_STATUS_IN_DEVICE_CREATION],
	329	+ "in device creation", HL_STR_MAX);
327	330
328	331	hdev->major = hl_major;
329	332	hdev->reset_on_lockup = reset_on_lockup;
@@ -532,7 +535,7 @@ hl_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t state) · 202 unmodified lines
532	535	result = PCI_ERS_RESULT_NONE;
533	536	}
534	537
535		− hdev->asic_funcs->halt_engines(hdev, true);
	538	+ hdev->asic_funcs->halt_engines(hdev, true, false);
536	539
537	540	return result;
538	541	}

drivers/misc/habanalabs/common/habanalabs_ioctl.c+2−0

@@ -94,6 +94,8 @@ static int hw_ip_info(struct hl_device hdev, struct hl_info_args args) · 93 unmodified lines
94	94
95	95	hw_ip.first_available_interrupt_id =
96	96	prop->first_available_user_msix_interrupt;
	97	+ hw_ip.server_type = prop->server_type;
	98	+
97	99	return copy_to_user(out, &hw_ip,
98	100	min((size_t) size, sizeof(hw_ip))) ? -EFAULT : 0;
99	101	}

drivers/misc/habanalabs/common/hw_queue.c+155−43

@@ -65,7 +65,7 @@ void hl_hw_queue_update_ci(struct hl_cs *cs) · 64 unmodified lines
65	65	}
66	66
67	67	/*
68		− * ext_and_hw_queue_submit_bd() - Submit a buffer descriptor to an external or a
	68	+ * hl_hw_queue_submit_bd() - Submit a buffer descriptor to an external or a
69	69	* H/W queue.
70	70	* @hdev: pointer to habanalabs device structure
71	71	* @q: pointer to habanalabs queue structure
@@ -80,8 +80,8 @@ void hl_hw_queue_update_ci(struct hl_cs *cs) · 8 unmodified lines
80	80	* This function must be called when the scheduler mutex is taken
81	81	*
82	82	*/
83		−static void ext_and_hw_queue_submit_bd(struct hl_device *hdev,
84		− struct hl_hw_queue *q, u32 ctl, u32 len, u64 ptr)
	83	+void hl_hw_queue_submit_bd(struct hl_device hdev, struct hl_hw_queue q,
	84	+ u32 ctl, u32 len, u64 ptr)
85	85	{
86	86	struct hl_bd *bd;
87	87
@@ -222,8 +222,8 @@ static int hw_queue_sanity_checks(struct hl_device hdev, struct hl_hw_queue q, · 134 unmodified lines
222	222	* @cb_size: size of CB
223	223	* @cb_ptr: pointer to CB location
224	224	*
225		− * This function sends a single CB, that must NOT generate a completion entry
226		− *
	225	+ * This function sends a single CB, that must NOT generate a completion entry.
	226	+ * Sending CPU messages can be done instead via 'hl_hw_queue_submit_bd()'
227	227	*/
228	228	int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
229	229	u32 cb_size, u64 cb_ptr)
@@ -231,16 +231,7 @@ int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id, · 1 unmodified line
231	231	struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];
232	232	int rc = 0;
233	233
234		− /*
235		− * The CPU queue is a synchronous queue with an effective depth of
236		− * a single entry (although it is allocated with room for multiple
237		− * entries). Therefore, there is a different lock, called
238		− * send_cpu_message_lock, that serializes accesses to the CPU queue.
239		− * As a result, we don't need to lock the access to the entire H/W
240		− * queues module when submitting a JOB to the CPU queue
241		− */
242		− if (q->queue_type != QUEUE_TYPE_CPU)
243		− hdev->asic_funcs->hw_queues_lock(hdev);
	234	+ hdev->asic_funcs->hw_queues_lock(hdev);
244	235
245	236	if (hdev->disabled) {
246	237	rc = -EPERM;
@@ -258,11 +249,10 @@ int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id, · 11 unmodified lines
258	249	goto out;
259	250	}
260	251
261		− ext_and_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);
	252	+ hl_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);
262	253
263	254	out:
264		− if (q->queue_type != QUEUE_TYPE_CPU)
265		− hdev->asic_funcs->hw_queues_unlock(hdev);
	255	+ hdev->asic_funcs->hw_queues_unlock(hdev);
266	256
267	257	return rc;
268	258	}
@@ -328,7 +318,7 @@ static void ext_queue_schedule_job(struct hl_cs_job *job) · 59 unmodified lines
328	318	cq->pi = hl_cq_inc_ptr(cq->pi);
329	319
330	320	submit_bd:
331		− ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
	321	+ hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
332	322	}
333	323
334	324	/*
@@ -407,7 +397,7 @@ static void hw_queue_schedule_job(struct hl_cs_job *job) · 72 unmodified lines
407	397	else
408	398	ptr = (u64) (uintptr_t) job->user_cb;
409	399
410		− ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
	400	+ hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
411	401	}
412	402
413	403	static int init_signal_cs(struct hl_device *hdev,
@@ -426,8 +416,9 @@ static int init_signal_cs(struct hl_device *hdev, · 12 unmodified lines
426	416	cs_cmpl->sob_val = prop->next_sob_val;
427	417
428	418	dev_dbg(hdev->dev,
429		− "generate signal CB, sob_id: %d, sob val: 0x%x, q_idx: %d\n",
430		− cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, q_idx);
	419	+ "generate signal CB, sob_id: %d, sob val: %u, q_idx: %d, seq: %llu\n",
	420	+ cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, q_idx,
	421	+ cs_cmpl->cs_seq);
431	422
432	423	/* we set an EB since we must make sure all oeprations are done
433	424	* when sending the signal
@@ -435,17 +426,37 @@ static int init_signal_cs(struct hl_device *hdev, · 1 unmodified line
435	426	hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb,
436	427	cs_cmpl->hw_sob->sob_id, 0, true);
437	428
438		− rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1);
	429	+ rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1,
	430	+ false);
439	431
440	432	return rc;
441	433	}
442	434
443		−static void init_wait_cs(struct hl_device hdev, struct hl_cs cs,
	435	+void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev,
	436	+ struct hl_cs cs, struct hl_cs_job job,
	437	+ struct hl_cs_compl *cs_cmpl)
	438	+{
	439	+ struct hl_cs_encaps_sig_handle *handle = cs->encaps_sig_hdl;
	440	+
	441	+ cs_cmpl->hw_sob = handle->hw_sob;
	442	+
	443	+ /* Note that encaps_sig_wait_offset was validated earlier in the flow
	444	+ * for offset value which exceeds the max reserved signal count.
	445	+ * always decrement 1 of the offset since when the user
	446	+ * set offset 1 for example he mean to wait only for the first
	447	+ * signal only, which will be pre_sob_val, and if he set offset 2
	448	+ * then the value required is (pre_sob_val + 1) and so on...
	449	+ */
	450	+ cs_cmpl->sob_val = handle->pre_sob_val +
	451	+ (job->encaps_sig_wait_offset - 1);
	452	+}
	453	+
	454	+static int init_wait_cs(struct hl_device hdev, struct hl_cs cs,
444	455	struct hl_cs_job job, struct hl_cs_compl cs_cmpl)
445	456	{
446		− struct hl_cs_compl *signal_cs_cmpl;
447		− struct hl_sync_stream_properties *prop;
448	457	struct hl_gen_wait_properties wait_prop;
	458	+ struct hl_sync_stream_properties *prop;
	459	+ struct hl_cs_compl *signal_cs_cmpl;
449	460	u32 q_idx;
450	461
451	462	q_idx = job->hw_queue_id;
@@ -455,14 +466,51 @@ static void init_wait_cs(struct hl_device hdev, struct hl_cs cs, · 3 unmodified lines
455	466	struct hl_cs_compl,
456	467	base_fence);
457	468
458		− /* copy the SOB id and value of the signal CS */
459		− cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;
460		− cs_cmpl->sob_val = signal_cs_cmpl->sob_val;
	469	+ if (cs->encaps_signals) {
	470	+ /* use the encaps signal handle stored earlier in the flow
	471	+ * and set the SOB information from the encaps
	472	+ * signals handle
	473	+ */
	474	+ hl_hw_queue_encaps_sig_set_sob_info(hdev, cs, job, cs_cmpl);
	475	+
	476	+ dev_dbg(hdev->dev, "Wait for encaps signals handle, qidx(%u), CS sequence(%llu), sob val: 0x%x, offset: %u\n",
	477	+ cs->encaps_sig_hdl->q_idx,
	478	+ cs->encaps_sig_hdl->cs_seq,
	479	+ cs_cmpl->sob_val,
	480	+ job->encaps_sig_wait_offset);
	481	+ } else {
	482	+ /* Copy the SOB id and value of the signal CS */
	483	+ cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;
	484	+ cs_cmpl->sob_val = signal_cs_cmpl->sob_val;
	485	+ }
	486	+
	487	+ /* check again if the signal cs already completed.
	488	+ * if yes then don't send any wait cs since the hw_sob
	489	+ * could be in reset already. if signal is not completed
	490	+ * then get refcount to hw_sob to prevent resetting the sob
	491	+ * while wait cs is not submitted.
	492	+ * note that this check is protected by two locks,
	493	+ * hw queue lock and completion object lock,
	494	+ * and the same completion object lock also protects
	495	+ * the hw_sob reset handler function.
	496	+ * The hw_queue lock prevent out of sync of hw_sob
	497	+ * refcount value, changed by signal/wait flows.
	498	+ */
	499	+ spin_lock(&signal_cs_cmpl->lock);
	500	+
	501	+ if (completion_done(&cs->signal_fence->completion)) {
	502	+ spin_unlock(&signal_cs_cmpl->lock);
	503	+ return -EINVAL;
	504	+ }
	505	+
	506	+ kref_get(&cs_cmpl->hw_sob->kref);
	507	+
	508	+ spin_unlock(&signal_cs_cmpl->lock);
461	509
462	510	dev_dbg(hdev->dev,
463		− "generate wait CB, sob_id: %d, sob_val: 0x%x, mon_id: %d, q_idx: %d\n",
	511	+ "generate wait CB, sob_id: %d, sob_val: 0x%x, mon_id: %d, q_idx: %d, seq: %llu\n",
464	512	cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val,
465		− prop->base_mon_id, q_idx);
	513	+ prop->base_mon_id, q_idx, cs->sequence);
466	514
467	515	wait_prop.data = (void *) job->patched_cb;
468	516	wait_prop.sob_base = cs_cmpl->hw_sob->sob_id;
@@ -471,17 +519,14 @@ static void init_wait_cs(struct hl_device hdev, struct hl_cs cs, · 2 unmodified lines
471	519	wait_prop.mon_id = prop->base_mon_id;
472	520	wait_prop.q_idx = q_idx;
473	521	wait_prop.size = 0;
	522	+
474	523	hdev->asic_funcs->gen_wait_cb(hdev, &wait_prop);
475	524
476		− kref_get(&cs_cmpl->hw_sob->kref);
477		− /*
478		− * Must put the signal fence after the SOB refcnt increment so
479		− * the SOB refcnt won't turn 0 and reset the SOB before the
480		− * wait CS was submitted.
481		− */
482	525	mb();
483	526	hl_fence_put(cs->signal_fence);
484	527	cs->signal_fence = NULL;
	528	+
	529	+ return 0;
485	530	}
486	531
487	532	/*
@@ -506,7 +551,60 @@ static int init_signal_wait_cs(struct hl_cs *cs) · 18 unmodified lines
506	551	if (cs->type & CS_TYPE_SIGNAL)
507	552	rc = init_signal_cs(hdev, job, cs_cmpl);
508	553	else if (cs->type & CS_TYPE_WAIT)
509		− init_wait_cs(hdev, cs, job, cs_cmpl);
	554	+ rc = init_wait_cs(hdev, cs, job, cs_cmpl);
	555	+
	556	+ return rc;
	557	+}
	558	+
	559	+static int encaps_sig_first_staged_cs_handler
	560	+ (struct hl_device hdev, struct hl_cs cs)
	561	+{
	562	+ struct hl_cs_compl *cs_cmpl =
	563	+ container_of(cs->fence,
	564	+ struct hl_cs_compl, base_fence);
	565	+ struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
	566	+ struct hl_encaps_signals_mgr *mgr;
	567	+ int rc = 0;
	568	+
	569	+ mgr = &hdev->compute_ctx->sig_mgr;
	570	+
	571	+ spin_lock(&mgr->lock);
	572	+ encaps_sig_hdl = idr_find(&mgr->handles, cs->encaps_sig_hdl_id);
	573	+ if (encaps_sig_hdl) {
	574	+ /*
	575	+ * Set handler CS sequence,
	576	+ * the CS which contains the encapsulated signals.
	577	+ */
	578	+ encaps_sig_hdl->cs_seq = cs->sequence;
	579	+ /* store the handle and set encaps signal indication,
	580	+ * to be used later in cs_do_release to put the last
	581	+ * reference to encaps signals handlers.
	582	+ */
	583	+ cs_cmpl->encaps_signals = true;
	584	+ cs_cmpl->encaps_sig_hdl = encaps_sig_hdl;
	585	+
	586	+ /* set hw_sob pointer in completion object
	587	+ * since it's used in cs_do_release flow to put
	588	+ * refcount to sob
	589	+ */
	590	+ cs_cmpl->hw_sob = encaps_sig_hdl->hw_sob;
	591	+ cs_cmpl->sob_val = encaps_sig_hdl->pre_sob_val +
	592	+ encaps_sig_hdl->count;
	593	+
	594	+ dev_dbg(hdev->dev, "CS seq (%llu) added to encaps signal handler id (%u), count(%u), qidx(%u), sob(%u), val(%u)\n",
	595	+ cs->sequence, encaps_sig_hdl->id,
	596	+ encaps_sig_hdl->count,
	597	+ encaps_sig_hdl->q_idx,
	598	+ cs_cmpl->hw_sob->sob_id,
	599	+ cs_cmpl->sob_val);
	600	+
	601	+ } else {
	602	+ dev_err(hdev->dev, "encaps handle id(%u) wasn't found!\n",
	603	+ cs->encaps_sig_hdl_id);
	604	+ rc = -EINVAL;
	605	+ }
	606	+
	607	+ spin_unlock(&mgr->lock);
510	608
511	609	return rc;
512	610	}
@@ -581,14 +679,21 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) · 68 unmodified lines
581	679
582	680	if ((cs->type == CS_TYPE_SIGNAL) \|\| (cs->type == CS_TYPE_WAIT)) {
583	681	rc = init_signal_wait_cs(cs);
584		− if (rc) {
585		− dev_err(hdev->dev, "Failed to submit signal cs\n");
	682	+ if (rc)
586	683	goto unroll_cq_resv;
587		− }
588		− } else if (cs->type == CS_TYPE_COLLECTIVE_WAIT)
589		− hdev->asic_funcs->collective_wait_init_cs(cs);
	684	+ } else if (cs->type == CS_TYPE_COLLECTIVE_WAIT) {
	685	+ rc = hdev->asic_funcs->collective_wait_init_cs(cs);
	686	+ if (rc)
	687	+ goto unroll_cq_resv;
	688	+ }
590	689
591	690
	691	+ if (cs->encaps_signals && cs->staged_first) {
	692	+ rc = encaps_sig_first_staged_cs_handler(hdev, cs);
	693	+ if (rc)
	694	+ goto unroll_cq_resv;
	695	+ }
	696	+
592	697	spin_lock(&hdev->cs_mirror_lock);
593	698
594	699	/* Verify staged CS exists and add to the staged list */
@@ -613,6 +718,11 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) · 18 unmodified lines
613	718	}
614	719
615	720	list_add_tail(&cs->staged_cs_node, &staged_cs->staged_cs_node);
	721	+
	722	+ /* update stream map of the first CS */
	723	+ if (hdev->supports_wait_for_multi_cs)
	724	+ staged_cs->fence->stream_master_qid_map \|=
	725	+ cs->fence->stream_master_qid_map;
616	726	}
617	727
618	728	list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list);
@@ -834,6 +944,8 @@ static void sync_stream_queue_init(struct hl_device *hdev, u32 q_idx) · 215 unmodified lines
834	944	hw_sob = &sync_stream_prop->hw_sob[sob];
835	945	hw_sob->hdev = hdev;
836	946	hw_sob->sob_id = sync_stream_prop->base_sob_id + sob;
	947	+ hw_sob->sob_addr =
	948	+ hdev->asic_funcs->get_sob_addr(hdev, hw_sob->sob_id);
837	949	hw_sob->q_idx = q_idx;
838	950	kref_init(&hw_sob->kref);
839	951	}

drivers/misc/habanalabs/common/memory.c+123−46

@@ -124,7 +124,7 @@ static int alloc_device_memory(struct hl_ctx ctx, struct hl_mem_in args, · 123 unmodified lines
124	124
125	125	spin_lock(&vm->idr_lock);
126	126	handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0,
127		− GFP_KERNEL);
	127	+ GFP_ATOMIC);
128	128	spin_unlock(&vm->idr_lock);
129	129
130	130	if (handle < 0) {
@@ -528,6 +528,33 @@ static inline int add_va_block(struct hl_device *hdev, · 397 unmodified lines
528	528	return rc;
529	529	}
530	530
	531	+/**
	532	+ * is_hint_crossing_range() - check if hint address crossing specified reserved
	533	+ * range.
	534	+ */
	535	+static inline bool is_hint_crossing_range(enum hl_va_range_type range_type,
	536	+ u64 start_addr, u32 size, struct asic_fixed_properties *prop) {
	537	+ bool range_cross;
	538	+
	539	+ if (range_type == HL_VA_RANGE_TYPE_DRAM)
	540	+ range_cross =
	541	+ hl_mem_area_crosses_range(start_addr, size,
	542	+ prop->hints_dram_reserved_va_range.start_addr,
	543	+ prop->hints_dram_reserved_va_range.end_addr);
	544	+ else if (range_type == HL_VA_RANGE_TYPE_HOST)
	545	+ range_cross =
	546	+ hl_mem_area_crosses_range(start_addr, size,
	547	+ prop->hints_host_reserved_va_range.start_addr,
	548	+ prop->hints_host_reserved_va_range.end_addr);
	549	+ else
	550	+ range_cross =
	551	+ hl_mem_area_crosses_range(start_addr, size,
	552	+ prop->hints_host_hpage_reserved_va_range.start_addr,
	553	+ prop->hints_host_hpage_reserved_va_range.end_addr);
	554	+
	555	+ return range_cross;
	556	+}
	557	+
531	558	/**
532	559	* get_va_block() - get a virtual block for the given size and alignment.
533	560	*
@@ -536,6 +563,8 @@ static inline int add_va_block(struct hl_device *hdev, · 2 unmodified lines
536	563	* @size: requested block size.
537	564	* @hint_addr: hint for requested address by the user.
538	565	* @va_block_align: required alignment of the virtual block start address.
	566	+ * @range_type: va range type (host, dram)
	567	+ * @flags: additional memory flags, currently only uses HL_MEM_FORCE_HINT
539	568	*
540	569	* This function does the following:
541	570	* - Iterate on the virtual block list to find a suitable virtual block for the
@@ -545,13 +574,19 @@ static inline int add_va_block(struct hl_device *hdev, · 3 unmodified lines
545	574	*/
546	575	static u64 get_va_block(struct hl_device *hdev,
547	576	struct hl_va_range *va_range,
548		− u64 size, u64 hint_addr, u32 va_block_align)
	577	+ u64 size, u64 hint_addr, u32 va_block_align,
	578	+ enum hl_va_range_type range_type,
	579	+ u32 flags)
549	580	{
550	581	struct hl_vm_va_block va_block, new_va_block = NULL;
	582	+ struct asic_fixed_properties *prop = &hdev->asic_prop;
551	583	u64 tmp_hint_addr, valid_start, valid_size, prev_start, prev_end,
552		− align_mask, reserved_valid_start = 0, reserved_valid_size = 0;
	584	+ align_mask, reserved_valid_start = 0, reserved_valid_size = 0,
	585	+ dram_hint_mask = prop->dram_hints_align_mask;
553	586	bool add_prev = false;
554	587	bool is_align_pow_2 = is_power_of_2(va_range->page_size);
	588	+ bool is_hint_dram_addr = hl_is_dram_va(hdev, hint_addr);
	589	+ bool force_hint = flags & HL_MEM_FORCE_HINT;
555	590
556	591	if (is_align_pow_2)
557	592	align_mask = ~((u64)va_block_align - 1);
@@ -564,12 +599,20 @@ static u64 get_va_block(struct hl_device *hdev, · 6 unmodified lines
564	599	size = DIV_ROUND_UP_ULL(size, va_range->page_size) *
565	600	va_range->page_size;
566	601
567		− tmp_hint_addr = hint_addr;
	602	+ tmp_hint_addr = hint_addr & ~dram_hint_mask;
568	603
569	604	/* Check if we need to ignore hint address */
570	605	if ((is_align_pow_2 && (hint_addr & (va_block_align - 1))) \|\|
571		− (!is_align_pow_2 &&
572		− do_div(tmp_hint_addr, va_range->page_size))) {
	606	+ (!is_align_pow_2 && is_hint_dram_addr &&
	607	+ do_div(tmp_hint_addr, va_range->page_size))) {
	608	+
	609	+ if (force_hint) {
	610	+ /* Hint must be respected, so here we just fail */
	611	+ dev_err(hdev->dev,
	612	+ "Hint address 0x%llx is not page aligned - cannot be respected\n",
	613	+ hint_addr);
	614	+ return 0;
	615	+ }
573	616
574	617	dev_dbg(hdev->dev,
575	618	"Hint address 0x%llx will be ignored because it is not aligned\n",
@@ -596,6 +639,16 @@ static u64 get_va_block(struct hl_device *hdev, · 20 unmodified lines
596	639	if (valid_size < size)
597	640	continue;
598	641
	642	+ /*
	643	+ * In case hint address is 0, and arc_hints_range_reservation
	644	+ * property enabled, then avoid allocating va blocks from the
	645	+ * range reserved for hint addresses
	646	+ */
	647	+ if (prop->hints_range_reservation && !hint_addr)
	648	+ if (is_hint_crossing_range(range_type, valid_start,
	649	+ size, prop))
	650	+ continue;
	651	+
599	652	/* Pick the minimal length block which has the required size */
600	653	if (!new_va_block \|\| (valid_size < reserved_valid_size)) {
601	654	new_va_block = va_block;
@@ -618,6 +671,17 @@ static u64 get_va_block(struct hl_device *hdev, · 16 unmodified lines
618	671	goto out;
619	672	}
620	673
	674	+ if (force_hint && reserved_valid_start != hint_addr) {
	675	+ /* Hint address must be respected. If we are here - this means
	676	+ * we could not respect it.
	677	+ */
	678	+ dev_err(hdev->dev,
	679	+ "Hint address 0x%llx could not be respected\n",
	680	+ hint_addr);
	681	+ reserved_valid_start = 0;
	682	+ goto out;
	683	+ }
	684	+
621	685	/*
622	686	* Check if there is some leftover range due to reserving the new
623	687	* va block, then return it to the main virtual addresses list.
@@ -670,7 +734,8 @@ u64 hl_reserve_va_block(struct hl_device hdev, struct hl_ctx ctx, · 46 unmodified lines
670	734	enum hl_va_range_type type, u32 size, u32 alignment)
671	735	{
672	736	return get_va_block(hdev, ctx->va_range[type], size, 0,
673		− max(alignment, ctx->va_range[type]->page_size));
	737	+ max(alignment, ctx->va_range[type]->page_size),
	738	+ type, 0);
674	739	}
675	740
676	741	/**
@@ -731,29 +796,16 @@ int hl_unreserve_va_block(struct hl_device hdev, struct hl_ctx ctx, · 54 unmodified lines
731	796	return rc;
732	797	}
733	798
734		−/**
735		− * get_sg_info() - get number of pages and the DMA address from SG list.
736		− * @sg: the SG list.
737		− * @dma_addr: pointer to DMA address to return.
738		− *
739		− * Calculate the number of consecutive pages described by the SG list. Take the
740		− * offset of the address in the first page, add to it the length and round it up
741		− * to the number of needed pages.
742		− */
743		−static u32 get_sg_info(struct scatterlist sg, dma_addr_t dma_addr)
744		−{
745		− *dma_addr = sg_dma_address(sg);
746		−
747		− return ((((*dma_addr) & (PAGE_SIZE - 1)) + sg_dma_len(sg)) +
748		− (PAGE_SIZE - 1)) >> PAGE_SHIFT;
749		−}
750		−
751	799	/**
752	800	* init_phys_pg_pack_from_userptr() - initialize physical page pack from host
753	801	* memory
754	802	* @ctx: pointer to the context structure.
755	803	* @userptr: userptr to initialize from.
756	804	* @pphys_pg_pack: result pointer.
	805	+ * @force_regular_page: tell the function to ignore huge page optimization,
	806	+ * even if possible. Needed for cases where the device VA
	807	+ * is allocated before we know the composition of the
	808	+ * physical pages
757	809	*
758	810	* This function does the following:
759	811	* - Pin the physical pages related to the given virtual block.
@@ -762,17 +814,18 @@ static u32 get_sg_info(struct scatterlist sg, dma_addr_t dma_addr) · 2 unmodified lines
762	814	*/
763	815	static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx,
764	816	struct hl_userptr *userptr,
765		− struct hl_vm_phys_pg_pack **pphys_pg_pack)
	817	+ struct hl_vm_phys_pg_pack **pphys_pg_pack,
	818	+ bool force_regular_page)
766	819	{
	820	+ u32 npages, page_size = PAGE_SIZE,
	821	+ huge_page_size = ctx->hdev->asic_prop.pmmu_huge.page_size;
	822	+ u32 pgs_in_huge_page = huge_page_size >> __ffs(page_size);
767	823	struct hl_vm_phys_pg_pack *phys_pg_pack;
	824	+ bool first = true, is_huge_page_opt;
	825	+ u64 page_mask, total_npages;
768	826	struct scatterlist *sg;
769	827	dma_addr_t dma_addr;
770		− u64 page_mask, total_npages;
771		− u32 npages, page_size = PAGE_SIZE,
772		− huge_page_size = ctx->hdev->asic_prop.pmmu_huge.page_size;
773		− bool first = true, is_huge_page_opt = true;
774	828	int rc, i, j;
775		− u32 pgs_in_huge_page = huge_page_size >> __ffs(page_size);
776	829
777	830	phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);
778	831	if (!phys_pg_pack)
@@ -783,6 +836,8 @@ static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx, · 4 unmodified lines
783	836	phys_pg_pack->asid = ctx->asid;
784	837	atomic_set(&phys_pg_pack->mapping_cnt, 1);
785	838
	839	+ is_huge_page_opt = (force_regular_page ? false : true);
	840	+
786	841	/* Only if all dma_addrs are aligned to 2MB and their
787	842	* sizes is at least 2MB, we can use huge page mapping.
788	843	* We limit the 2MB optimization to this condition,
@@ -791,7 +846,7 @@ static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx, · 2 unmodified lines
791	846	*/
792	847	total_npages = 0;
793	848	for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {
794		− npages = get_sg_info(sg, &dma_addr);
	849	+ npages = hl_get_sg_info(sg, &dma_addr);
795	850
796	851	total_npages += npages;
797	852
@@ -820,7 +875,7 @@ static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx, · 22 unmodified lines
820	875
821	876	j = 0;
822	877	for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {
823		− npages = get_sg_info(sg, &dma_addr);
	878	+ npages = hl_get_sg_info(sg, &dma_addr);
824	879
825	880	/* align down to physical page size and save the offset */
826	881	if (first) {
@@ -1001,11 +1056,12 @@ static int map_device_va(struct hl_ctx ctx, struct hl_mem_in args, · 174 unmodified lines
1001	1056	struct hl_userptr *userptr = NULL;
1002	1057	struct hl_vm_hash_node *hnode;
1003	1058	struct hl_va_range *va_range;
1004		− enum vm_type_t *vm_type;
	1059	+ enum vm_type *vm_type;
1005	1060	u64 ret_vaddr, hint_addr;
1006	1061	u32 handle = 0, va_block_align;
1007	1062	int rc;
1008	1063	bool is_userptr = args->flags & HL_MEM_USERPTR;
	1064	+ enum hl_va_range_type va_range_type = 0;
1009	1065
1010	1066	/* Assume failure */
1011	1067	*device_addr = 0;
@@ -1023,7 +1079,7 @@ static int map_device_va(struct hl_ctx ctx, struct hl_mem_in args, · 11 unmodified lines
1023	1079	}
1024	1080
1025	1081	rc = init_phys_pg_pack_from_userptr(ctx, userptr,
1026		− &phys_pg_pack);
	1082	+ &phys_pg_pack, false);
1027	1083	if (rc) {
1028	1084	dev_err(hdev->dev,
1029	1085	"unable to init page pack for vaddr 0x%llx\n",
@@ -1031,14 +1087,14 @@ static int map_device_va(struct hl_ctx ctx, struct hl_mem_in args, · 1 unmodified line
1031	1087	goto init_page_pack_err;
1032	1088	}
1033	1089
1034		− vm_type = (enum vm_type_t *) userptr;
	1090	+ vm_type = (enum vm_type *) userptr;
1035	1091	hint_addr = args->map_host.hint_addr;
1036	1092	handle = phys_pg_pack->handle;
1037	1093
1038	1094	/* get required alignment */
1039	1095	if (phys_pg_pack->page_size == page_size) {
1040	1096	va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];
1041		−
	1097	+ va_range_type = HL_VA_RANGE_TYPE_HOST;
1042	1098	/*
1043	1099	* huge page alignment may be needed in case of regular
1044	1100	* page mapping, depending on the host VA alignment
@@ -1053,6 +1109,7 @@ static int map_device_va(struct hl_ctx ctx, struct hl_mem_in args, · 8 unmodified lines
1053	1109	* mapping
1054	1110	*/
1055	1111	va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];
	1112	+ va_range_type = HL_VA_RANGE_TYPE_HOST_HUGE;
1056	1113	va_block_align = huge_page_size;
1057	1114	}
1058	1115	} else {
@@ -1072,12 +1129,13 @@ static int map_device_va(struct hl_ctx ctx, struct hl_mem_in args, · 13 unmodified lines
1072	1129
1073	1130	spin_unlock(&vm->idr_lock);
1074	1131
1075		− vm_type = (enum vm_type_t *) phys_pg_pack;
	1132	+ vm_type = (enum vm_type *) phys_pg_pack;
1076	1133
1077	1134	hint_addr = args->map_device.hint_addr;
1078	1135
1079	1136	/* DRAM VA alignment is the same as the MMU page size */
1080	1137	va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];
	1138	+ va_range_type = HL_VA_RANGE_TYPE_DRAM;
1081	1139	va_block_align = hdev->asic_prop.dmmu.page_size;
1082	1140	}
1083	1141
@@ -1100,8 +1158,23 @@ static int map_device_va(struct hl_ctx ctx, struct hl_mem_in args, · 16 unmodified lines
1100	1158	goto hnode_err;
1101	1159	}
1102	1160
	1161	+ if (hint_addr && phys_pg_pack->offset) {
	1162	+ if (args->flags & HL_MEM_FORCE_HINT) {
	1163	+ /* Fail if hint must be respected but it can't be */
	1164	+ dev_err(hdev->dev,
	1165	+ "Hint address 0x%llx cannot be respected because source memory is not aligned 0x%x\n",
	1166	+ hint_addr, phys_pg_pack->offset);
	1167	+ rc = -EINVAL;
	1168	+ goto va_block_err;
	1169	+ }
	1170	+ dev_dbg(hdev->dev,
	1171	+ "Hint address 0x%llx will be ignored because source memory is not aligned 0x%x\n",
	1172	+ hint_addr, phys_pg_pack->offset);
	1173	+ }
	1174	+
1103	1175	ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size,
1104		− hint_addr, va_block_align);
	1176	+ hint_addr, va_block_align,
	1177	+ va_range_type, args->flags);
1105	1178	if (!ret_vaddr) {
1106	1179	dev_err(hdev->dev, "no available va block for handle %u\n",
1107	1180	handle);
@@ -1181,17 +1254,19 @@ init_page_pack_err: · 73 unmodified lines
1181	1254	static int unmap_device_va(struct hl_ctx ctx, struct hl_mem_in args,
1182	1255	bool ctx_free)
1183	1256	{
1184		− struct hl_device *hdev = ctx->hdev;
1185		− struct asic_fixed_properties *prop = &hdev->asic_prop;
1186	1257	struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;
	1258	+ u64 vaddr = args->unmap.device_virt_addr;
1187	1259	struct hl_vm_hash_node *hnode = NULL;
	1260	+ struct asic_fixed_properties *prop;
	1261	+ struct hl_device *hdev = ctx->hdev;
1188	1262	struct hl_userptr *userptr = NULL;
1189	1263	struct hl_va_range *va_range;
1190		− u64 vaddr = args->unmap.device_virt_addr;
1191		− enum vm_type_t *vm_type;
	1264	+ enum vm_type *vm_type;
1192	1265	bool is_userptr;
1193	1266	int rc = 0;
1194	1267
	1268	+ prop = &hdev->asic_prop;
	1269	+
1195	1270	/* protect from double entrance */
1196	1271	mutex_lock(&ctx->mem_hash_lock);
1197	1272	hash_for_each_possible(ctx->mem_hash, hnode, node, (unsigned long)vaddr)
@@ -1214,8 +1289,9 @@ static int unmap_device_va(struct hl_ctx ctx, struct hl_mem_in args, · 16 unmodified lines
1214	1289	if (*vm_type == VM_TYPE_USERPTR) {
1215	1290	is_userptr = true;
1216	1291	userptr = hnode->ptr;
1217		− rc = init_phys_pg_pack_from_userptr(ctx, userptr,
1218		− &phys_pg_pack);
	1292	+
	1293	+ rc = init_phys_pg_pack_from_userptr(ctx, userptr, &phys_pg_pack,
	1294	+ false);
1219	1295	if (rc) {
1220	1296	dev_err(hdev->dev,
1221	1297	"unable to init page pack for vaddr 0x%llx\n",
@@ -1299,7 +1375,7 @@ static int unmap_device_va(struct hl_ctx ctx, struct hl_mem_in args, · 77 unmodified lines
1299	1375	kfree(hnode);
1300	1376
1301	1377	if (is_userptr) {
1302		− rc = free_phys_pg_pack(hdev, phys_pg_pack);
	1378	+ free_phys_pg_pack(hdev, phys_pg_pack);
1303	1379	dma_unmap_host_va(hdev, userptr);
1304	1380	}
1305	1381
@@ -1669,6 +1745,7 @@ int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size, · 363 unmodified lines
1669	1745	return -EINVAL;
1670	1746	}
1671	1747
	1748	+ userptr->pid = current->pid;
1672	1749	userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_KERNEL);
1673	1750	if (!userptr->sgt)
1674	1751	return -ENOMEM;
@@ -2033,7 +2110,7 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx) · 358 unmodified lines
2033	2110	* another side effect error
2034	2111	*/
2035	2112	if (!hdev->hard_reset_pending && !hash_empty(ctx->mem_hash))
2036		− dev_notice(hdev->dev,
	2113	+ dev_dbg(hdev->dev,
2037	2114	"user released device without removing its memory mappings\n");
2038	2115
2039	2116	hash_for_each_safe(ctx->mem_hash, i, tmp_node, hnode, node) {

drivers/misc/habanalabs/common/mmu/mmu_v1.c+6−6

@@ -470,13 +470,13 @@ static void hl_mmu_v1_fini(struct hl_device *hdev) · 469 unmodified lines
470	470	if (!ZERO_OR_NULL_PTR(hdev->mmu_priv.hr.mmu_shadow_hop0)) {
471	471	kvfree(hdev->mmu_priv.dr.mmu_shadow_hop0);
472	472	gen_pool_destroy(hdev->mmu_priv.dr.mmu_pgt_pool);
473		− }
474	473
475		− /* Make sure that if we arrive here again without init was called we
476		− * won't cause kernel panic. This can happen for example if we fail
477		− * during hard reset code at certain points
478		− */
479		− hdev->mmu_priv.dr.mmu_shadow_hop0 = NULL;
	474	+ /* Make sure that if we arrive here again without init was
	475	+ * called we won't cause kernel panic. This can happen for
	476	+ * example if we fail during hard reset code at certain points
	477	+ */
	478	+ hdev->mmu_priv.dr.mmu_shadow_hop0 = NULL;
	479	+ }
480	480	}
481	481
482	482	/**

drivers/misc/habanalabs/common/pci/pci.c+2−0

@@ -436,6 +436,8 @@ int hl_pci_init(struct hl_device *hdev) · 435 unmodified lines
436	436	goto unmap_pci_bars;
437	437	}
438	438
	439	+ dma_set_max_seg_size(&pdev->dev, U32_MAX);
	440	+
439	441	return 0;
440	442
441	443	unmap_pci_bars:

drivers/misc/habanalabs/common/state_dump.c+718−0

@@ -0,0 +1,718 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+/*
	4	+ * Copyright 2021 HabanaLabs, Ltd.
	5	+ * All Rights Reserved.
	6	+ */
	7	+
	8	+#include <linux/vmalloc.h>
	9	+#include <uapi/misc/habanalabs.h>
	10	+#include "habanalabs.h"
	11	+
	12	+/**
	13	+ * hl_format_as_binary - helper function, format an integer as binary
	14	+ * using supplied scratch buffer
	15	+ * @buf: the buffer to use
	16	+ * @buf_len: buffer capacity
	17	+ * @n: number to format
	18	+ *
	19	+ * Returns pointer to buffer
	20	+ */
	21	+char hl_format_as_binary(char buf, size_t buf_len, u32 n)
	22	+{
	23	+ int i;
	24	+ u32 bit;
	25	+ bool leading0 = true;
	26	+ char *wrptr = buf;
	27	+
	28	+ if (buf_len > 0 && buf_len < 3) {
	29	+ *wrptr = '\0';
	30	+ return buf;
	31	+ }
	32	+
	33	+ wrptr[0] = '0';
	34	+ wrptr[1] = 'b';
	35	+ wrptr += 2;
	36	+ /* Remove 3 characters from length for '0b' and '\0' termination */
	37	+ buf_len -= 3;
	38	+
	39	+ for (i = 0; i < sizeof(n) * BITS_PER_BYTE && buf_len; ++i, n <<= 1) {
	40	+ /* Writing bit calculation in one line would cause a false
	41	+ * positive static code analysis error, so splitting.
	42	+ */
	43	+ bit = n & (1 << (sizeof(n) * BITS_PER_BYTE - 1));
	44	+ bit = !!bit;
	45	+ leading0 &= !bit;
	46	+ if (!leading0) {
	47	+ *wrptr = '0' + bit;
	48	+ ++wrptr;
	49	+ }
	50	+ }
	51	+
	52	+ *wrptr = '\0';
	53	+
	54	+ return buf;
	55	+}
	56	+
	57	+/**
	58	+ * resize_to_fit - helper function, resize buffer to fit given amount of data
	59	+ * @buf: destination buffer double pointer
	60	+ * @size: pointer to the size container
	61	+ * @desired_size: size the buffer must contain
	62	+ *
	63	+ * Returns 0 on success or error code on failure.
	64	+ * On success, the size of buffer is at least desired_size. Buffer is allocated
	65	+ * via vmalloc and must be freed with vfree.
	66	+ */
	67	+static int resize_to_fit(char *buf, size_t size, size_t desired_size)
	68	+{
	69	+ char *resized_buf;
	70	+ size_t new_size;
	71	+
	72	+ if (*size >= desired_size)
	73	+ return 0;
	74	+
	75	+ /* Not enough space to print all, have to resize */
	76	+ new_size = max_t(size_t, PAGE_SIZE, round_up(desired_size, PAGE_SIZE));
	77	+ resized_buf = vmalloc(new_size);
	78	+ if (!resized_buf)
	79	+ return -ENOMEM;
	80	+ memcpy(resized_buf, buf, size);
	81	+ vfree(*buf);
	82	+ *buf = resized_buf;
	83	+ *size = new_size;
	84	+
	85	+ return 1;
	86	+}
	87	+
	88	+/**
	89	+ * hl_snprintf_resize() - print formatted data to buffer, resize as needed
	90	+ * @buf: buffer double pointer, to be written to and resized, must be either
	91	+ * NULL or allocated with vmalloc.
	92	+ * @size: current size of the buffer
	93	+ * @offset: current offset to write to
	94	+ * @format: format of the data
	95	+ *
	96	+ * This function will write formatted data into the buffer. If buffer is not
	97	+ * large enough, it will be resized using vmalloc. Size may be modified if the
	98	+ * buffer was resized, offset will be advanced by the number of bytes written
	99	+ * not including the terminating character
	100	+ *
	101	+ * Returns 0 on success or error code on failure
	102	+ *
	103	+ * Note that the buffer has to be manually released using vfree.
	104	+ */
	105	+int hl_snprintf_resize(char *buf, size_t size, size_t *offset,
	106	+ const char *format, ...)
	107	+{
	108	+ va_list args;
	109	+ size_t length;
	110	+ int rc;
	111	+
	112	+ if (buf == NULL && (size != 0 \|\| *offset != 0))
	113	+ return -EINVAL;
	114	+
	115	+ va_start(args, format);
	116	+ length = vsnprintf(buf + offset, size - offset, format, args);
	117	+ va_end(args);
	118	+
	119	+ rc = resize_to_fit(buf, size, *offset + length + 1);
	120	+ if (rc < 0)
	121	+ return rc;
	122	+ else if (rc > 0) {
	123	+ /* Resize was needed, write again */
	124	+ va_start(args, format);
	125	+ length = vsnprintf(buf + offset, size - offset, format,
	126	+ args);
	127	+ va_end(args);
	128	+ }
	129	+
	130	+ *offset += length;
	131	+
	132	+ return 0;
	133	+}
	134	+
	135	+/**
	136	+ * hl_sync_engine_to_string - convert engine type enum to string literal
	137	+ * @engine_type: engine type (TPC/MME/DMA)
	138	+ *
	139	+ * Return the resolved string literal
	140	+ */
	141	+const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type)
	142	+{
	143	+ switch (engine_type) {
	144	+ case ENGINE_DMA:
	145	+ return "DMA";
	146	+ case ENGINE_MME:
	147	+ return "MME";
	148	+ case ENGINE_TPC:
	149	+ return "TPC";
	150	+ }
	151	+ return "Invalid Engine Type";
	152	+}
	153	+
	154	+/**
	155	+ * hl_print_resize_sync_engine - helper function, format engine name and ID
	156	+ * using hl_snprintf_resize
	157	+ * @buf: destination buffer double pointer to be used with hl_snprintf_resize
	158	+ * @size: pointer to the size container
	159	+ * @offset: pointer to the offset container
	160	+ * @engine_type: engine type (TPC/MME/DMA)
	161	+ * @engine_id: engine numerical id
	162	+ *
	163	+ * Returns 0 on success or error code on failure
	164	+ */
	165	+static int hl_print_resize_sync_engine(char *buf, size_t size, size_t *offset,
	166	+ enum hl_sync_engine_type engine_type,
	167	+ u32 engine_id)
	168	+{
	169	+ return hl_snprintf_resize(buf, size, offset, "%s%u",
	170	+ hl_sync_engine_to_string(engine_type), engine_id);
	171	+}
	172	+
	173	+/**
	174	+ * hl_state_dump_get_sync_name - transform sync object id to name if available
	175	+ * @hdev: pointer to the device
	176	+ * @sync_id: sync object id
	177	+ *
	178	+ * Returns a name literal or NULL if not resolved.
	179	+ * Note: returning NULL shall not be considered as a failure, as not all
	180	+ * sync objects are named.
	181	+ */
	182	+const char hl_state_dump_get_sync_name(struct hl_device hdev, u32 sync_id)
	183	+{
	184	+ struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
	185	+ struct hl_hw_obj_name_entry *entry;
	186	+
	187	+ hash_for_each_possible(sds->so_id_to_str_tb, entry,
	188	+ node, sync_id)
	189	+ if (sync_id == entry->id)
	190	+ return entry->name;
	191	+
	192	+ return NULL;
	193	+}
	194	+
	195	+/**
	196	+ * hl_state_dump_get_monitor_name - transform monitor object dump to monitor
	197	+ * name if available
	198	+ * @hdev: pointer to the device
	199	+ * @mon: monitor state dump
	200	+ *
	201	+ * Returns a name literal or NULL if not resolved.
	202	+ * Note: returning NULL shall not be considered as a failure, as not all
	203	+ * monitors are named.
	204	+ */
	205	+const char hl_state_dump_get_monitor_name(struct hl_device hdev,
	206	+ struct hl_mon_state_dump *mon)
	207	+{
	208	+ struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
	209	+ struct hl_hw_obj_name_entry *entry;
	210	+
	211	+ hash_for_each_possible(sds->monitor_id_to_str_tb,
	212	+ entry, node, mon->id)
	213	+ if (mon->id == entry->id)
	214	+ return entry->name;
	215	+
	216	+ return NULL;
	217	+}
	218	+
	219	+/**
	220	+ * hl_state_dump_free_sync_to_engine_map - free sync object to engine map
	221	+ * @map: sync object to engine map
	222	+ *
	223	+ * Note: generic free implementation, the allocation is implemented per ASIC.
	224	+ */
	225	+void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map)
	226	+{
	227	+ struct hl_sync_to_engine_map_entry *entry;
	228	+ struct hlist_node *tmp_node;
	229	+ int i;
	230	+
	231	+ hash_for_each_safe(map->tb, i, tmp_node, entry, node) {
	232	+ hash_del(&entry->node);
	233	+ kfree(entry);
	234	+ }
	235	+}
	236	+
	237	+/**
	238	+ * hl_state_dump_get_sync_to_engine - transform sync_id to
	239	+ * hl_sync_to_engine_map_entry if available for current id
	240	+ * @map: sync object to engine map
	241	+ * @sync_id: sync object id
	242	+ *
	243	+ * Returns the translation entry if found or NULL if not.
	244	+ * Note, returned NULL shall not be considered as a failure as the map
	245	+ * does not cover all possible, it is a best effort sync ids.
	246	+ */
	247	+static struct hl_sync_to_engine_map_entry *
	248	+hl_state_dump_get_sync_to_engine(struct hl_sync_to_engine_map *map, u32 sync_id)
	249	+{
	250	+ struct hl_sync_to_engine_map_entry *entry;
	251	+
	252	+ hash_for_each_possible(map->tb, entry, node, sync_id)
	253	+ if (entry->sync_id == sync_id)
	254	+ return entry;
	255	+ return NULL;
	256	+}
	257	+
	258	+/**
	259	+ * hl_state_dump_read_sync_objects - read sync objects array
	260	+ * @hdev: pointer to the device
	261	+ * @index: sync manager block index starting with E_N
	262	+ *
	263	+ * Returns array of size SP_SYNC_OBJ_AMOUNT on success or NULL on failure
	264	+ */
	265	+static u32 hl_state_dump_read_sync_objects(struct hl_device hdev, u32 index)
	266	+{
	267	+ struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
	268	+ u32 *sync_objects;
	269	+ s64 base_addr; /* Base addr can be negative */
	270	+ int i;
	271	+
	272	+ base_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +
	273	+ sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;
	274	+
	275	+ sync_objects = vmalloc(sds->props[SP_SYNC_OBJ_AMOUNT] * sizeof(u32));
	276	+ if (!sync_objects)
	277	+ return NULL;
	278	+
	279	+ for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i)
	280	+ sync_objects[i] = RREG32(base_addr + i * sizeof(u32));
	281	+
	282	+ return sync_objects;
	283	+}
	284	+
	285	+/**
	286	+ * hl_state_dump_free_sync_objects - free sync objects array allocated by
	287	+ * hl_state_dump_read_sync_objects
	288	+ * @sync_objects: sync objects array
	289	+ */
	290	+static void hl_state_dump_free_sync_objects(u32 *sync_objects)
	291	+{
	292	+ vfree(sync_objects);
	293	+}
	294	+
	295	+
	296	+/**
	297	+ * hl_state_dump_print_syncs_single_block - print active sync objects on a
	298	+ * single block
	299	+ * @hdev: pointer to the device
	300	+ * @index: sync manager block index starting with E_N
	301	+ * @buf: destination buffer double pointer to be used with hl_snprintf_resize
	302	+ * @size: pointer to the size container
	303	+ * @offset: pointer to the offset container
	304	+ * @map: sync engines names map
	305	+ *
	306	+ * Returns 0 on success or error code on failure
	307	+ */
	308	+static int
	309	+hl_state_dump_print_syncs_single_block(struct hl_device *hdev, u32 index,
	310	+ char *buf, size_t size, size_t *offset,
	311	+ struct hl_sync_to_engine_map *map)
	312	+{
	313	+ struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
	314	+ const char *sync_name;
	315	+ u32 *sync_objects = NULL;
	316	+ int rc = 0, i;
	317	+
	318	+ if (sds->sync_namager_names) {
	319	+ rc = hl_snprintf_resize(
	320	+ buf, size, offset, "%s\n",
	321	+ sds->sync_namager_names[index]);
	322	+ if (rc)
	323	+ goto out;
	324	+ }
	325	+
	326	+ sync_objects = hl_state_dump_read_sync_objects(hdev, index);
	327	+ if (!sync_objects) {
	328	+ rc = -ENOMEM;
	329	+ goto out;
	330	+ }
	331	+
	332	+ for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i) {
	333	+ struct hl_sync_to_engine_map_entry *entry;
	334	+ u64 sync_object_addr;
	335	+
	336	+ if (!sync_objects[i])
	337	+ continue;
	338	+
	339	+ sync_object_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +
	340	+ sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index +
	341	+ i * sizeof(u32);
	342	+
	343	+ rc = hl_snprintf_resize(buf, size, offset, "sync id: %u", i);
	344	+ if (rc)
	345	+ goto free_sync_objects;
	346	+ sync_name = hl_state_dump_get_sync_name(hdev, i);
	347	+ if (sync_name) {
	348	+ rc = hl_snprintf_resize(buf, size, offset, " %s",
	349	+ sync_name);
	350	+ if (rc)
	351	+ goto free_sync_objects;
	352	+ }
	353	+ rc = hl_snprintf_resize(buf, size, offset, ", value: %u",
	354	+ sync_objects[i]);
	355	+ if (rc)
	356	+ goto free_sync_objects;
	357	+
	358	+ /* Append engine string */
	359	+ entry = hl_state_dump_get_sync_to_engine(map,
	360	+ (u32)sync_object_addr);
	361	+ if (entry) {
	362	+ rc = hl_snprintf_resize(buf, size, offset,
	363	+ ", Engine: ");
	364	+ if (rc)
	365	+ goto free_sync_objects;
	366	+ rc = hl_print_resize_sync_engine(buf, size, offset,
	367	+ entry->engine_type,
	368	+ entry->engine_id);
	369	+ if (rc)
	370	+ goto free_sync_objects;
	371	+ }
	372	+
	373	+ rc = hl_snprintf_resize(buf, size, offset, "\n");
	374	+ if (rc)
	375	+ goto free_sync_objects;
	376	+ }
	377	+
	378	+free_sync_objects:
	379	+ hl_state_dump_free_sync_objects(sync_objects);
	380	+out:
	381	+ return rc;
	382	+}
	383	+
	384	+/**
	385	+ * hl_state_dump_print_syncs - print active sync objects
	386	+ * @hdev: pointer to the device
	387	+ * @buf: destination buffer double pointer to be used with hl_snprintf_resize
	388	+ * @size: pointer to the size container
	389	+ * @offset: pointer to the offset container
	390	+ *
	391	+ * Returns 0 on success or error code on failure
	392	+ */
	393	+static int hl_state_dump_print_syncs(struct hl_device *hdev,
	394	+ char *buf, size_t size,
	395	+ size_t *offset)
	396	+
	397	+{
	398	+ struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
	399	+ struct hl_sync_to_engine_map *map;
	400	+ u32 index;
	401	+ int rc = 0;
	402	+
	403	+ map = kzalloc(sizeof(*map), GFP_KERNEL);
	404	+ if (!map)
	405	+ return -ENOMEM;
	406	+
	407	+ rc = sds->funcs.gen_sync_to_engine_map(hdev, map);
	408	+ if (rc)
	409	+ goto free_map_mem;
	410	+
	411	+ rc = hl_snprintf_resize(buf, size, offset, "Non zero sync objects:\n");
	412	+ if (rc)
	413	+ goto out;
	414	+
	415	+ if (sds->sync_namager_names) {
	416	+ for (index = 0; sds->sync_namager_names[index]; ++index) {
	417	+ rc = hl_state_dump_print_syncs_single_block(
	418	+ hdev, index, buf, size, offset, map);
	419	+ if (rc)
	420	+ goto out;
	421	+ }
	422	+ } else {
	423	+ for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {
	424	+ rc = hl_state_dump_print_syncs_single_block(
	425	+ hdev, index, buf, size, offset, map);
	426	+ if (rc)
	427	+ goto out;
	428	+ }
	429	+ }
	430	+
	431	+out:
	432	+ hl_state_dump_free_sync_to_engine_map(map);
	433	+free_map_mem:
	434	+ kfree(map);
	435	+
	436	+ return rc;
	437	+}
	438	+
	439	+/**
	440	+ * hl_state_dump_alloc_read_sm_block_monitors - read monitors for a specific
	441	+ * block
	442	+ * @hdev: pointer to the device
	443	+ * @index: sync manager block index starting with E_N
	444	+ *
	445	+ * Returns an array of monitor data of size SP_MONITORS_AMOUNT or NULL
	446	+ * on error
	447	+ */
	448	+static struct hl_mon_state_dump *
	449	+hl_state_dump_alloc_read_sm_block_monitors(struct hl_device *hdev, u32 index)
	450	+{
	451	+ struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
	452	+ struct hl_mon_state_dump *monitors;
	453	+ s64 base_addr; /* Base addr can be negative */
	454	+ int i;
	455	+
	456	+ monitors = vmalloc(sds->props[SP_MONITORS_AMOUNT] *
	457	+ sizeof(struct hl_mon_state_dump));
	458	+ if (!monitors)
	459	+ return NULL;
	460	+
	461	+ base_addr = sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;
	462	+
	463	+ for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {
	464	+ monitors[i].id = i;
	465	+ monitors[i].wr_addr_low =
	466	+ RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_LOW] +
	467	+ i * sizeof(u32));
	468	+
	469	+ monitors[i].wr_addr_high =
	470	+ RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_HIGH] +
	471	+ i * sizeof(u32));
	472	+
	473	+ monitors[i].wr_data =
	474	+ RREG32(base_addr + sds->props[SP_MON_OBJ_WR_DATA] +
	475	+ i * sizeof(u32));
	476	+
	477	+ monitors[i].arm_data =
	478	+ RREG32(base_addr + sds->props[SP_MON_OBJ_ARM_DATA] +
	479	+ i * sizeof(u32));
	480	+
	481	+ monitors[i].status =
	482	+ RREG32(base_addr + sds->props[SP_MON_OBJ_STATUS] +
	483	+ i * sizeof(u32));
	484	+ }
	485	+
	486	+ return monitors;
	487	+}
	488	+
	489	+/**
	490	+ * hl_state_dump_free_monitors - free the monitors structure
	491	+ * @monitors: monitors array created with
	492	+ * hl_state_dump_alloc_read_sm_block_monitors
	493	+ */
	494	+static void hl_state_dump_free_monitors(struct hl_mon_state_dump *monitors)
	495	+{
	496	+ vfree(monitors);
	497	+}
	498	+
	499	+/**
	500	+ * hl_state_dump_print_monitors_single_block - print active monitors on a
	501	+ * single block
	502	+ * @hdev: pointer to the device
	503	+ * @index: sync manager block index starting with E_N
	504	+ * @buf: destination buffer double pointer to be used with hl_snprintf_resize
	505	+ * @size: pointer to the size container
	506	+ * @offset: pointer to the offset container
	507	+ *
	508	+ * Returns 0 on success or error code on failure
	509	+ */
	510	+static int hl_state_dump_print_monitors_single_block(struct hl_device *hdev,
	511	+ u32 index,
	512	+ char *buf, size_t size,
	513	+ size_t *offset)
	514	+{
	515	+ struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
	516	+ struct hl_mon_state_dump *monitors = NULL;
	517	+ int rc = 0, i;
	518	+
	519	+ if (sds->sync_namager_names) {
	520	+ rc = hl_snprintf_resize(
	521	+ buf, size, offset, "%s\n",
	522	+ sds->sync_namager_names[index]);
	523	+ if (rc)
	524	+ goto out;
	525	+ }
	526	+
	527	+ monitors = hl_state_dump_alloc_read_sm_block_monitors(hdev, index);
	528	+ if (!monitors) {
	529	+ rc = -ENOMEM;
	530	+ goto out;
	531	+ }
	532	+
	533	+ for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {
	534	+ if (!(sds->funcs.monitor_valid(&monitors[i])))
	535	+ continue;
	536	+
	537	+ /* Monitor is valid, dump it */
	538	+ rc = sds->funcs.print_single_monitor(buf, size, offset, hdev,
	539	+ &monitors[i]);
	540	+ if (rc)
	541	+ goto free_monitors;
	542	+
	543	+ hl_snprintf_resize(buf, size, offset, "\n");
	544	+ }
	545	+
	546	+free_monitors:
	547	+ hl_state_dump_free_monitors(monitors);
	548	+out:
	549	+ return rc;
	550	+}
	551	+
	552	+/**
	553	+ * hl_state_dump_print_monitors - print active monitors
	554	+ * @hdev: pointer to the device
	555	+ * @buf: destination buffer double pointer to be used with hl_snprintf_resize
	556	+ * @size: pointer to the size container
	557	+ * @offset: pointer to the offset container
	558	+ *
	559	+ * Returns 0 on success or error code on failure
	560	+ */
	561	+static int hl_state_dump_print_monitors(struct hl_device *hdev,
	562	+ char *buf, size_t size,
	563	+ size_t *offset)
	564	+{
	565	+ struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
	566	+ u32 index;
	567	+ int rc = 0;
	568	+
	569	+ rc = hl_snprintf_resize(buf, size, offset,
	570	+ "Valid (armed) monitor objects:\n");
	571	+ if (rc)
	572	+ goto out;
	573	+
	574	+ if (sds->sync_namager_names) {
	575	+ for (index = 0; sds->sync_namager_names[index]; ++index) {
	576	+ rc = hl_state_dump_print_monitors_single_block(
	577	+ hdev, index, buf, size, offset);
	578	+ if (rc)
	579	+ goto out;
	580	+ }
	581	+ } else {
	582	+ for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {
	583	+ rc = hl_state_dump_print_monitors_single_block(
	584	+ hdev, index, buf, size, offset);
	585	+ if (rc)
	586	+ goto out;
	587	+ }
	588	+ }
	589	+
	590	+out:
	591	+ return rc;
	592	+}
	593	+
	594	+/**
	595	+ * hl_state_dump_print_engine_fences - print active fences for a specific
	596	+ * engine
	597	+ * @hdev: pointer to the device
	598	+ * @engine_type: engine type to use
	599	+ * @buf: destination buffer double pointer to be used with hl_snprintf_resize
	600	+ * @size: pointer to the size container
	601	+ * @offset: pointer to the offset container
	602	+ */
	603	+static int
	604	+hl_state_dump_print_engine_fences(struct hl_device *hdev,
	605	+ enum hl_sync_engine_type engine_type,
	606	+ char *buf, size_t size, size_t *offset)
	607	+{
	608	+ struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
	609	+ int rc = 0, i, n_fences;
	610	+ u64 base_addr, next_fence;
	611	+
	612	+ switch (engine_type) {
	613	+ case ENGINE_TPC:
	614	+ n_fences = sds->props[SP_NUM_OF_TPC_ENGINES];
	615	+ base_addr = sds->props[SP_TPC0_CMDQ];
	616	+ next_fence = sds->props[SP_NEXT_TPC];
	617	+ break;
	618	+ case ENGINE_MME:
	619	+ n_fences = sds->props[SP_NUM_OF_MME_ENGINES];
	620	+ base_addr = sds->props[SP_MME_CMDQ];
	621	+ next_fence = sds->props[SP_NEXT_MME];
	622	+ break;
	623	+ case ENGINE_DMA:
	624	+ n_fences = sds->props[SP_NUM_OF_DMA_ENGINES];
	625	+ base_addr = sds->props[SP_DMA_CMDQ];
	626	+ next_fence = sds->props[SP_DMA_QUEUES_OFFSET];
	627	+ break;
	628	+ default:
	629	+ return -EINVAL;
	630	+ }
	631	+ for (i = 0; i < n_fences; ++i) {
	632	+ rc = sds->funcs.print_fences_single_engine(
	633	+ hdev,
	634	+ base_addr + next_fence * i +
	635	+ sds->props[SP_FENCE0_CNT_OFFSET],
	636	+ base_addr + next_fence * i +
	637	+ sds->props[SP_CP_STS_OFFSET],
	638	+ engine_type, i, buf, size, offset);
	639	+ if (rc)
	640	+ goto out;
	641	+ }
	642	+out:
	643	+ return rc;
	644	+}
	645	+
	646	+/**
	647	+ * hl_state_dump_print_fences - print active fences
	648	+ * @hdev: pointer to the device
	649	+ * @buf: destination buffer double pointer to be used with hl_snprintf_resize
	650	+ * @size: pointer to the size container
	651	+ * @offset: pointer to the offset container
	652	+ */
	653	+static int hl_state_dump_print_fences(struct hl_device hdev, char *buf,
	654	+ size_t size, size_t offset)
	655	+{
	656	+ int rc = 0;
	657	+
	658	+ rc = hl_snprintf_resize(buf, size, offset, "Valid (armed) fences:\n");
	659	+ if (rc)
	660	+ goto out;
	661	+
	662	+ rc = hl_state_dump_print_engine_fences(hdev, ENGINE_TPC, buf, size, offset);
	663	+ if (rc)
	664	+ goto out;
	665	+
	666	+ rc = hl_state_dump_print_engine_fences(hdev, ENGINE_MME, buf, size, offset);
	667	+ if (rc)
	668	+ goto out;
	669	+
	670	+ rc = hl_state_dump_print_engine_fences(hdev, ENGINE_DMA, buf, size, offset);
	671	+ if (rc)
	672	+ goto out;
	673	+
	674	+out:
	675	+ return rc;
	676	+}
	677	+
	678	+/**
	679	+ * hl_state_dump() - dump system state
	680	+ * @hdev: pointer to device structure
	681	+ */
	682	+int hl_state_dump(struct hl_device *hdev)
	683	+{
	684	+ char *buf = NULL;
	685	+ size_t offset = 0, size = 0;
	686	+ int rc;
	687	+
	688	+ rc = hl_snprintf_resize(&buf, &size, &offset,
	689	+ "Timestamp taken on: %llu\n\n",
	690	+ ktime_to_ns(ktime_get()));
	691	+ if (rc)
	692	+ goto err;
	693	+
	694	+ rc = hl_state_dump_print_syncs(hdev, &buf, &size, &offset);
	695	+ if (rc)
	696	+ goto err;
	697	+
	698	+ hl_snprintf_resize(&buf, &size, &offset, "\n");
	699	+
	700	+ rc = hl_state_dump_print_monitors(hdev, &buf, &size, &offset);
	701	+ if (rc)
	702	+ goto err;
	703	+
	704	+ hl_snprintf_resize(&buf, &size, &offset, "\n");
	705	+
	706	+ rc = hl_state_dump_print_fences(hdev, &buf, &size, &offset);
	707	+ if (rc)
	708	+ goto err;
	709	+
	710	+ hl_snprintf_resize(&buf, &size, &offset, "\n");
	711	+
	712	+ hl_debugfs_set_state_dump(hdev, buf, size);
	713	+
	714	+ return 0;
	715	+err:
	716	+ vfree(buf);
	717	+ return rc;
	718	+}

drivers/misc/habanalabs/common/sysfs.c+7−13

@@ -9,8 +9,7 @@ · 8 unmodified lines
9	9
10	10	#include <linux/pci.h>
11	11
12		−long hl_get_frequency(struct hl_device *hdev, u32 pll_index,
13		− bool curr)
	12	+long hl_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr)
14	13	{
15	14	struct cpucp_packet pkt;
16	15	u32 used_pll_idx;
@@ -44,8 +43,7 @@ long hl_get_frequency(struct hl_device *hdev, u32 pll_index, · 27 unmodified lines
44	43	return (long) result;
45	44	}
46	45
47		−void hl_set_frequency(struct hl_device *hdev, u32 pll_index,
48		− u64 freq)
	46	+void hl_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq)
49	47	{
50	48	struct cpucp_packet pkt;
51	49	u32 used_pll_idx;
@@ -285,16 +283,12 @@ static ssize_t status_show(struct device dev, struct device_attribute attr, · 233 unmodified lines
285	283	char *buf)
286	284	{
287	285	struct hl_device *hdev = dev_get_drvdata(dev);
288		− char *str;
	286	+ char str[HL_STR_MAX];
289	287
290		− if (atomic_read(&hdev->in_reset))
291		− str = "In reset";
292		− else if (hdev->disabled)
293		− str = "Malfunction";
294		− else if (hdev->needs_reset)
295		− str = "Needs Reset";
296		− else
297		− str = "Operational";
	288	+ strscpy(str, hdev->status[hl_device_status(hdev)], HL_STR_MAX);
	289	+
	290	+ /* use uppercase for backward compatibility */
	291	+ str[0] = 'A' + (str[0] - 'a');
298	292
299	293	return sprintf(buf, "%s\n", str);
300	294	}

drivers/misc/habanalabs/gaudi/gaudi.c+571−145

@@ -76,7 +76,7 @@ · 75 unmodified lines
76	76	#define GAUDI_PLDM_MMU_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 100)
77	77	#define GAUDI_PLDM_QMAN0_TIMEOUT_USEC (HL_DEVICE_TIMEOUT_USEC * 30)
78	78	#define GAUDI_PLDM_TPC_KERNEL_WAIT_USEC (HL_DEVICE_TIMEOUT_USEC * 30)
79		−#define GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC 1000000 /* 1s */
	79	+#define GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC 4000000 /* 4s */
80	80	#define GAUDI_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */
81	81	#define GAUDI_WAIT_FOR_BL_TIMEOUT_USEC 15000000 /* 15s */
82	82
@@ -106,6 +106,21 @@ · 23 unmodified lines
106	106
107	107	#define GAUDI_PLL_MAX 10
108	108
	109	+#define BIN_REG_STRING_SIZE sizeof("0b10101010101010101010101010101010")
	110	+
	111	+#define MONITOR_SOB_STRING_SIZE 256
	112	+
	113	+static u32 gaudi_stream_master[GAUDI_STREAM_MASTER_ARR_SIZE] = {
	114	+ GAUDI_QUEUE_ID_DMA_0_0,
	115	+ GAUDI_QUEUE_ID_DMA_0_1,
	116	+ GAUDI_QUEUE_ID_DMA_0_2,
	117	+ GAUDI_QUEUE_ID_DMA_0_3,
	118	+ GAUDI_QUEUE_ID_DMA_1_0,
	119	+ GAUDI_QUEUE_ID_DMA_1_1,
	120	+ GAUDI_QUEUE_ID_DMA_1_2,
	121	+ GAUDI_QUEUE_ID_DMA_1_3
	122	+};
	123	+
109	124	static const char gaudi_irq_name[GAUDI_MSI_ENTRIES][GAUDI_MAX_STRING_LEN] = {
110	125	"gaudi cq 0_0", "gaudi cq 0_1", "gaudi cq 0_2", "gaudi cq 0_3",
111	126	"gaudi cq 1_0", "gaudi cq 1_1", "gaudi cq 1_2", "gaudi cq 1_3",
@@ -348,6 +363,97 @@ static enum hl_queue_type gaudi_queue_type[GAUDI_QUEUE_ID_SIZE] = { · 236 unmodified lines
348	363	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_3 */
349	364	};
350	365
	366	+static struct hl_hw_obj_name_entry gaudi_so_id_to_str[] = {
	367	+ { .id = 0, .name = "SYNC_OBJ_DMA_DOWN_FEEDBACK" },
	368	+ { .id = 1, .name = "SYNC_OBJ_DMA_UP_FEEDBACK" },
	369	+ { .id = 2, .name = "SYNC_OBJ_DMA_STATIC_DRAM_SRAM_FEEDBACK" },
	370	+ { .id = 3, .name = "SYNC_OBJ_DMA_SRAM_DRAM_FEEDBACK" },
	371	+ { .id = 4, .name = "SYNC_OBJ_FIRST_COMPUTE_FINISH" },
	372	+ { .id = 5, .name = "SYNC_OBJ_HOST_DRAM_DONE" },
	373	+ { .id = 6, .name = "SYNC_OBJ_DBG_CTR_DEPRECATED" },
	374	+ { .id = 7, .name = "SYNC_OBJ_DMA_ACTIVATIONS_DRAM_SRAM_FEEDBACK" },
	375	+ { .id = 8, .name = "SYNC_OBJ_ENGINE_SEM_MME_0" },
	376	+ { .id = 9, .name = "SYNC_OBJ_ENGINE_SEM_MME_1" },
	377	+ { .id = 10, .name = "SYNC_OBJ_ENGINE_SEM_TPC_0" },
	378	+ { .id = 11, .name = "SYNC_OBJ_ENGINE_SEM_TPC_1" },
	379	+ { .id = 12, .name = "SYNC_OBJ_ENGINE_SEM_TPC_2" },
	380	+ { .id = 13, .name = "SYNC_OBJ_ENGINE_SEM_TPC_3" },
	381	+ { .id = 14, .name = "SYNC_OBJ_ENGINE_SEM_TPC_4" },
	382	+ { .id = 15, .name = "SYNC_OBJ_ENGINE_SEM_TPC_5" },
	383	+ { .id = 16, .name = "SYNC_OBJ_ENGINE_SEM_TPC_6" },
	384	+ { .id = 17, .name = "SYNC_OBJ_ENGINE_SEM_TPC_7" },
	385	+ { .id = 18, .name = "SYNC_OBJ_ENGINE_SEM_DMA_1" },
	386	+ { .id = 19, .name = "SYNC_OBJ_ENGINE_SEM_DMA_2" },
	387	+ { .id = 20, .name = "SYNC_OBJ_ENGINE_SEM_DMA_3" },
	388	+ { .id = 21, .name = "SYNC_OBJ_ENGINE_SEM_DMA_4" },
	389	+ { .id = 22, .name = "SYNC_OBJ_ENGINE_SEM_DMA_5" },
	390	+ { .id = 23, .name = "SYNC_OBJ_ENGINE_SEM_DMA_6" },
	391	+ { .id = 24, .name = "SYNC_OBJ_ENGINE_SEM_DMA_7" },
	392	+ { .id = 25, .name = "SYNC_OBJ_DBG_CTR_0" },
	393	+ { .id = 26, .name = "SYNC_OBJ_DBG_CTR_1" },
	394	+};
	395	+
	396	+static struct hl_hw_obj_name_entry gaudi_monitor_id_to_str[] = {
	397	+ { .id = 200, .name = "MON_OBJ_DMA_DOWN_FEEDBACK_RESET" },
	398	+ { .id = 201, .name = "MON_OBJ_DMA_UP_FEADBACK_RESET" },
	399	+ { .id = 203, .name = "MON_OBJ_DRAM_TO_SRAM_QUEUE_FENCE" },
	400	+ { .id = 204, .name = "MON_OBJ_TPC_0_CLK_GATE" },
	401	+ { .id = 205, .name = "MON_OBJ_TPC_1_CLK_GATE" },
	402	+ { .id = 206, .name = "MON_OBJ_TPC_2_CLK_GATE" },
	403	+ { .id = 207, .name = "MON_OBJ_TPC_3_CLK_GATE" },
	404	+ { .id = 208, .name = "MON_OBJ_TPC_4_CLK_GATE" },
	405	+ { .id = 209, .name = "MON_OBJ_TPC_5_CLK_GATE" },
	406	+ { .id = 210, .name = "MON_OBJ_TPC_6_CLK_GATE" },
	407	+ { .id = 211, .name = "MON_OBJ_TPC_7_CLK_GATE" },
	408	+};
	409	+
	410	+static s64 gaudi_state_dump_specs_props[] = {
	411	+ [SP_SYNC_OBJ_BASE_ADDR] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0,
	412	+ [SP_NEXT_SYNC_OBJ_ADDR] = NEXT_SYNC_OBJ_ADDR_INTERVAL,
	413	+ [SP_SYNC_OBJ_AMOUNT] = NUM_OF_SOB_IN_BLOCK,
	414	+ [SP_MON_OBJ_WR_ADDR_LOW] =
	415	+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0,
	416	+ [SP_MON_OBJ_WR_ADDR_HIGH] =
	417	+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0,
	418	+ [SP_MON_OBJ_WR_DATA] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_DATA_0,
	419	+ [SP_MON_OBJ_ARM_DATA] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_ARM_0,
	420	+ [SP_MON_OBJ_STATUS] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0,
	421	+ [SP_MONITORS_AMOUNT] = NUM_OF_MONITORS_IN_BLOCK,
	422	+ [SP_TPC0_CMDQ] = mmTPC0_QM_GLBL_CFG0,
	423	+ [SP_TPC0_CFG_SO] = mmTPC0_CFG_QM_SYNC_OBJECT_ADDR,
	424	+ [SP_NEXT_TPC] = mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0,
	425	+ [SP_MME_CMDQ] = mmMME0_QM_GLBL_CFG0,
	426	+ [SP_MME_CFG_SO] = mmMME0_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL,
	427	+ [SP_NEXT_MME] = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0,
	428	+ [SP_DMA_CMDQ] = mmDMA0_QM_GLBL_CFG0,
	429	+ [SP_DMA_CFG_SO] = mmDMA0_CORE_WR_COMP_ADDR_LO,
	430	+ [SP_DMA_QUEUES_OFFSET] = mmDMA1_QM_GLBL_CFG0 - mmDMA0_QM_GLBL_CFG0,
	431	+ [SP_NUM_OF_MME_ENGINES] = NUM_OF_MME_ENGINES,
	432	+ [SP_SUB_MME_ENG_NUM] = NUM_OF_MME_SUB_ENGINES,
	433	+ [SP_NUM_OF_DMA_ENGINES] = NUM_OF_DMA_ENGINES,
	434	+ [SP_NUM_OF_TPC_ENGINES] = NUM_OF_TPC_ENGINES,
	435	+ [SP_ENGINE_NUM_OF_QUEUES] = NUM_OF_QUEUES,
	436	+ [SP_ENGINE_NUM_OF_STREAMS] = NUM_OF_STREAMS,
	437	+ [SP_ENGINE_NUM_OF_FENCES] = NUM_OF_FENCES,
	438	+ [SP_FENCE0_CNT_OFFSET] =
	439	+ mmDMA0_QM_CP_FENCE0_CNT_0 - mmDMA0_QM_GLBL_CFG0,
	440	+ [SP_FENCE0_RDATA_OFFSET] =
	441	+ mmDMA0_QM_CP_FENCE0_RDATA_0 - mmDMA0_QM_GLBL_CFG0,
	442	+ [SP_CP_STS_OFFSET] = mmDMA0_QM_CP_STS_0 - mmDMA0_QM_GLBL_CFG0,
	443	+ [SP_NUM_CORES] = 1,
	444	+};
	445	+
	446	+/* The order here is opposite to the order of the indexing in the h/w.
	447	+ * i.e. SYNC_MGR_W_S is actually 0, SYNC_MGR_E_S is 1, etc.
	448	+ */
	449	+static const char * const gaudi_sync_manager_names[] = {
	450	+ "SYNC_MGR_E_N",
	451	+ "SYNC_MGR_W_N",
	452	+ "SYNC_MGR_E_S",
	453	+ "SYNC_MGR_W_S",
	454	+ NULL
	455	+};
	456	+
351	457	struct ecc_info_extract_params {
352	458	u64 block_address;
353	459	u32 num_memories;
@@ -363,8 +469,6 @@ static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr, · 9 unmodified lines
363	469	u32 size, u64 val);
364	470	static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base,
365	471	u32 num_regs, u32 val);
366		−static int gaudi_schedule_register_memset(struct hl_device *hdev,
367		− u32 hw_queue_id, u64 reg_base, u32 num_regs, u32 val);
368	472	static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel,
369	473	u32 tpc_id);
370	474	static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev);
@@ -375,7 +479,6 @@ static u32 gaudi_gen_signal_cb(struct hl_device hdev, void data, u16 sob_id, · 4 unmodified lines
375	479	u32 size, bool eb);
376	480	static u32 gaudi_gen_wait_cb(struct hl_device *hdev,
377	481	struct hl_gen_wait_properties *prop);
378		−
379	482	static inline enum hl_collective_mode
380	483	get_collective_mode(struct hl_device *hdev, u32 queue_id)
381	484	{
@@ -403,7 +506,11 @@ static inline void set_default_power_values(struct hl_device *hdev) · 21 unmodified lines
403	506
404	507	if (hdev->card_type == cpucp_card_type_pmc) {
405	508	prop->max_power_default = MAX_POWER_DEFAULT_PMC;
406		− prop->dc_power_default = DC_POWER_DEFAULT_PMC;
	509	+
	510	+ if (prop->fw_security_enabled)
	511	+ prop->dc_power_default = DC_POWER_DEFAULT_PMC_SEC;
	512	+ else
	513	+ prop->dc_power_default = DC_POWER_DEFAULT_PMC;
407	514	} else {
408	515	prop->max_power_default = MAX_POWER_DEFAULT_PCI;
409	516	prop->dc_power_default = DC_POWER_DEFAULT_PCI;
@@ -450,6 +557,7 @@ static int gaudi_set_fixed_properties(struct hl_device *hdev) · 40 unmodified lines
450	557	get_collective_mode(hdev, i);
451	558	}
452	559
	560	+ prop->device_dma_offset_for_host_access = HOST_PHYS_BASE;
453	561	prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;
454	562	prop->collective_first_sob = 0;
455	563	prop->collective_first_mon = 0;
@@ -551,6 +659,8 @@ static int gaudi_set_fixed_properties(struct hl_device *hdev) · 95 unmodified lines
551	659	prop->hard_reset_done_by_fw = false;
552	660	prop->gic_interrupts_enable = true;
553	661
	662	+ prop->server_type = HL_SERVER_TYPE_UNKNOWN;
	663	+
554	664	return 0;
555	665	}
556	666
@@ -723,14 +833,14 @@ pci_init: · 166 unmodified lines
723	833	GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC);
724	834	if (rc) {
725	835	if (hdev->reset_on_preboot_fail)
726		− hdev->asic_funcs->hw_fini(hdev, true);
	836	+ hdev->asic_funcs->hw_fini(hdev, true, false);
727	837	goto pci_fini;
728	838	}
729	839
730	840	if (gaudi_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
731	841	dev_info(hdev->dev,
732	842	"H/W state is dirty, must reset before initializing\n");
733		− hdev->asic_funcs->hw_fini(hdev, true);
	843	+ hdev->asic_funcs->hw_fini(hdev, true, false);
734	844	}
735	845
736	846	return 0;
@@ -974,17 +1084,11 @@ static void gaudi_sob_group_hw_reset(struct kref *ref) · 237 unmodified lines
974	1084	struct gaudi_hw_sob_group *hw_sob_group =
975	1085	container_of(ref, struct gaudi_hw_sob_group, kref);
976	1086	struct hl_device *hdev = hw_sob_group->hdev;
977		− u64 base_addr;
978		− int rc;
	1087	+ int i;
979	1088
980		− base_addr = CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 +
981		− hw_sob_group->base_sob_id * 4;
982		− rc = gaudi_schedule_register_memset(hdev, hw_sob_group->queue_id,
983		− base_addr, NUMBER_OF_SOBS_IN_GRP, 0);
984		− if (rc)
985		− dev_err(hdev->dev,
986		− "failed resetting sob group - sob base %u, count %u",
987		− hw_sob_group->base_sob_id, NUMBER_OF_SOBS_IN_GRP);
	1089	+ for (i = 0 ; i < NUMBER_OF_SOBS_IN_GRP ; i++)
	1090	+ WREG32((mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 +
	1091	+ (hw_sob_group->base_sob_id * 4) + (i * 4)), 0);
988	1092
989	1093	kref_init(&hw_sob_group->kref);
990	1094	}
@@ -1121,6 +1225,20 @@ static void gaudi_collective_slave_init_job(struct hl_device *hdev, · 130 unmodified lines
1121	1225	queue_id = job->hw_queue_id;
1122	1226	prop = &hdev->kernel_queues[queue_id].sync_stream_prop;
1123	1227
	1228	+ if (job->cs->encaps_signals) {
	1229	+ /* use the encaps signal handle store earlier in the flow
	1230	+ * and set the SOB information from the encaps
	1231	+ * signals handle
	1232	+ */
	1233	+ hl_hw_queue_encaps_sig_set_sob_info(hdev, job->cs, job,
	1234	+ cs_cmpl);
	1235	+
	1236	+ dev_dbg(hdev->dev, "collective wait: Sequence %llu found, sob_id: %u, wait for sob_val: %u\n",
	1237	+ job->cs->sequence,
	1238	+ cs_cmpl->hw_sob->sob_id,
	1239	+ cs_cmpl->sob_val);
	1240	+ }
	1241	+
1124	1242	/* Add to wait CBs using slave monitor */
1125	1243	wait_prop.data = (void *) job->user_cb;
1126	1244	wait_prop.sob_base = cs_cmpl->hw_sob->sob_id;
@@ -1131,7 +1249,7 @@ static void gaudi_collective_slave_init_job(struct hl_device *hdev, · 4 unmodified lines
1131	1249	wait_prop.size = cb_size;
1132	1250
1133	1251	dev_dbg(hdev->dev,
1134		− "Generate slave wait CB, sob %d, val:0x%x, mon %d, q %d\n",
	1252	+ "Generate slave wait CB, sob %d, val:%x, mon %d, q %d\n",
1135	1253	cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val,
1136	1254	prop->collective_slave_mon_id, queue_id);
1137	1255
@@ -1145,7 +1263,7 @@ static void gaudi_collective_slave_init_job(struct hl_device *hdev, · 7 unmodified lines
1145	1263	prop->collective_sob_id, cb_size, false);
1146	1264	}
1147	1265
1148		−static void gaudi_collective_wait_init_cs(struct hl_cs *cs)
	1266	+static int gaudi_collective_wait_init_cs(struct hl_cs *cs)
1149	1267	{
1150	1268	struct hl_cs_compl *signal_cs_cmpl =
1151	1269	container_of(cs->signal_fence, struct hl_cs_compl, base_fence);
@@ -1163,9 +1281,37 @@ static void gaudi_collective_wait_init_cs(struct hl_cs *cs) · 11 unmodified lines
1163	1281	gaudi = hdev->asic_specific;
1164	1282	cprop = &gaudi->collective_props;
1165	1283
1166		− /* copy the SOB id and value of the signal CS */
1167		− cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;
1168		− cs_cmpl->sob_val = signal_cs_cmpl->sob_val;
	1284	+ /* In encaps signals case the SOB info will be retrieved from
	1285	+ * the handle in gaudi_collective_slave_init_job.
	1286	+ */
	1287	+ if (!cs->encaps_signals) {
	1288	+ /* copy the SOB id and value of the signal CS */
	1289	+ cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;
	1290	+ cs_cmpl->sob_val = signal_cs_cmpl->sob_val;
	1291	+ }
	1292	+
	1293	+ /* check again if the signal cs already completed.
	1294	+ * if yes then don't send any wait cs since the hw_sob
	1295	+ * could be in reset already. if signal is not completed
	1296	+ * then get refcount to hw_sob to prevent resetting the sob
	1297	+ * while wait cs is not submitted.
	1298	+ * note that this check is protected by two locks,
	1299	+ * hw queue lock and completion object lock,
	1300	+ * and the same completion object lock also protects
	1301	+ * the hw_sob reset handler function.
	1302	+ * The hw_queue lock prevent out of sync of hw_sob
	1303	+ * refcount value, changed by signal/wait flows.
	1304	+ */
	1305	+ spin_lock(&signal_cs_cmpl->lock);
	1306	+
	1307	+ if (completion_done(&cs->signal_fence->completion)) {
	1308	+ spin_unlock(&signal_cs_cmpl->lock);
	1309	+ return -EINVAL;
	1310	+ }
	1311	+ /* Increment kref since all slave queues are now waiting on it */
	1312	+ kref_get(&cs_cmpl->hw_sob->kref);
	1313	+
	1314	+ spin_unlock(&signal_cs_cmpl->lock);
1169	1315
1170	1316	/* Calculate the stream from collective master queue (1st job) */
1171	1317	job = list_first_entry(&cs->job_list, struct hl_cs_job, cs_node);
@@ -1210,21 +1356,17 @@ static void gaudi_collective_wait_init_cs(struct hl_cs *cs) · 38 unmodified lines
1210	1356	cprop->curr_sob_group_idx[stream], stream);
1211	1357	}
1212	1358
1213		− /* Increment kref since all slave queues are now waiting on it */
1214		− kref_get(&cs_cmpl->hw_sob->kref);
1215		− /*
1216		− * Must put the signal fence after the SOB refcnt increment so
1217		− * the SOB refcnt won't turn 0 and reset the SOB before the
1218		− * wait CS was submitted.
1219		− */
1220	1359	mb();
1221	1360	hl_fence_put(cs->signal_fence);
1222	1361	cs->signal_fence = NULL;
	1362	+
	1363	+ return 0;
1223	1364	}
1224	1365
1225	1366	static int gaudi_collective_wait_create_job(struct hl_device *hdev,
1226	1367	struct hl_ctx ctx, struct hl_cs cs,
1227		− enum hl_collective_mode mode, u32 queue_id, u32 wait_queue_id)
	1368	+ enum hl_collective_mode mode, u32 queue_id, u32 wait_queue_id,
	1369	+ u32 encaps_signal_offset)
1228	1370	{
1229	1371	struct hw_queue_properties *hw_queue_prop;
1230	1372	struct hl_cs_counters_atomic *cntr;
@@ -1284,6 +1426,13 @@ static int gaudi_collective_wait_create_job(struct hl_device *hdev, · 53 unmodified lines
1284	1426	job->user_cb_size = cb_size;
1285	1427	job->hw_queue_id = queue_id;
1286	1428
	1429	+ /* since its guaranteed to have only one chunk in the collective wait
	1430	+ * cs, we can use this chunk to set the encapsulated signal offset
	1431	+ * in the jobs.
	1432	+ */
	1433	+ if (cs->encaps_signals)
	1434	+ job->encaps_sig_wait_offset = encaps_signal_offset;
	1435	+
1287	1436	/*
1288	1437	* No need in parsing, user CB is the patched CB.
1289	1438	* We call hl_cb_destroy() out of two reasons - we don't need
@@ -1312,8 +1461,9 @@ static int gaudi_collective_wait_create_job(struct hl_device *hdev, · 22 unmodified lines
1312	1461	}
1313	1462
1314	1463	static int gaudi_collective_wait_create_jobs(struct hl_device *hdev,
1315		− struct hl_ctx ctx, struct hl_cs cs, u32 wait_queue_id,
1316		− u32 collective_engine_id)
	1464	+ struct hl_ctx ctx, struct hl_cs cs,
	1465	+ u32 wait_queue_id, u32 collective_engine_id,
	1466	+ u32 encaps_signal_offset)
1317	1467	{
1318	1468	struct gaudi_device *gaudi = hdev->asic_specific;
1319	1469	struct hw_queue_properties *hw_queue_prop;
@@ -1363,7 +1513,8 @@ static int gaudi_collective_wait_create_jobs(struct hl_device *hdev, · 43 unmodified lines
1363	1513	if (i == 0) {
1364	1514	queue_id = wait_queue_id;
1365	1515	rc = gaudi_collective_wait_create_job(hdev, ctx, cs,
1366		− HL_COLLECTIVE_MASTER, queue_id, wait_queue_id);
	1516	+ HL_COLLECTIVE_MASTER, queue_id,
	1517	+ wait_queue_id, encaps_signal_offset);
1367	1518	} else {
1368	1519	if (nic_idx < NIC_NUMBER_OF_ENGINES) {
1369	1520	if (gaudi->hw_cap_initialized &
@@ -1383,7 +1534,8 @@ static int gaudi_collective_wait_create_jobs(struct hl_device *hdev, · 13 unmodified lines
1383	1534	}
1384	1535
1385	1536	rc = gaudi_collective_wait_create_job(hdev, ctx, cs,
1386		− HL_COLLECTIVE_SLAVE, queue_id, wait_queue_id);
	1537	+ HL_COLLECTIVE_SLAVE, queue_id,
	1538	+ wait_queue_id, encaps_signal_offset);
1387	1539	}
1388	1540
1389	1541	if (rc)
@@ -1431,6 +1583,11 @@ static int gaudi_late_init(struct hl_device *hdev) · 41 unmodified lines
1431	1583	return rc;
1432	1584	}
1433	1585
	1586	+ /* Scrub both SRAM and DRAM */
	1587	+ rc = hdev->asic_funcs->scrub_device_mem(hdev, 0, 0);
	1588	+ if (rc)
	1589	+ goto disable_pci_access;
	1590	+
1434	1591	rc = gaudi_fetch_psoc_frequency(hdev);
1435	1592	if (rc) {
1436	1593	dev_err(hdev->dev, "Failed to fetch psoc frequency\n");
@@ -1455,6 +1612,11 @@ static int gaudi_late_init(struct hl_device *hdev) · 18 unmodified lines
1455	1612	goto disable_pci_access;
1456	1613	}
1457	1614
	1615	+ /* We only support a single ASID for the user, so for the sake of optimization, just
	1616	+ * initialize the ASID one time during device initialization with the fixed value of 1
	1617	+ */
	1618	+ gaudi_mmu_prepare(hdev, 1);
	1619	+
1458	1620	return 0;
1459	1621
1460	1622	disable_pci_access:
@@ -1720,8 +1882,12 @@ static int gaudi_sw_init(struct hl_device *hdev) · 259 unmodified lines
1720	1882	hdev->supports_sync_stream = true;
1721	1883	hdev->supports_coresight = true;
1722	1884	hdev->supports_staged_submission = true;
	1885	+ hdev->supports_wait_for_multi_cs = true;
1723	1886
1724		− gaudi_set_pci_memory_regions(hdev);
	1887	+ hdev->asic_funcs->set_pci_memory_regions(hdev);
	1888	+ hdev->stream_master_qid_arr =
	1889	+ hdev->asic_funcs->get_stream_master_qid_arr();
	1890	+ hdev->stream_master_qid_arr_size = GAUDI_STREAM_MASTER_ARR_SIZE;
1725	1891
1726	1892	return 0;
1727	1893
@@ -2523,7 +2689,7 @@ static void gaudi_init_golden_registers(struct hl_device *hdev) · 795 unmodified lines
2523	2689	tpc_id < TPC_NUMBER_OF_ENGINES;
2524	2690	tpc_id++, tpc_offset += TPC_CFG_OFFSET) {
2525	2691	/* Mask all arithmetic interrupts from TPC */
2526		− WREG32(mmTPC0_CFG_TPC_INTR_MASK + tpc_offset, 0x8FFF);
	2692	+ WREG32(mmTPC0_CFG_TPC_INTR_MASK + tpc_offset, 0x8FFE);
2527	2693	/* Set 16 cache lines */
2528	2694	WREG32_FIELD(TPC0_CFG_MSS_CONFIG, tpc_offset,
2529	2695	ICACHE_FETCH_LINE_NUM, 2);
@@ -3670,7 +3836,7 @@ static void gaudi_disable_timestamp(struct hl_device *hdev) · 1140 unmodified lines
3670	3836	WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
3671	3837	}
3672	3838
3673		−static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset)
	3839	+static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset)
3674	3840	{
3675	3841	u32 wait_timeout_ms;
3676	3842
@@ -3682,6 +3848,9 @@ static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset) · 5 unmodified lines
3682	3848	else
3683	3849	wait_timeout_ms = GAUDI_RESET_WAIT_MSEC;
3684	3850
	3851	+ if (fw_reset)
	3852	+ goto skip_engines;
	3853	+
3685	3854	gaudi_stop_nic_qmans(hdev);
3686	3855	gaudi_stop_mme_qmans(hdev);
3687	3856	gaudi_stop_tpc_qmans(hdev);
@@ -3707,6 +3876,7 @@ static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset) · 19 unmodified lines
3707	3876
3708	3877	gaudi_disable_timestamp(hdev);
3709	3878
	3879	+skip_engines:
3710	3880	gaudi_disable_msi(hdev);
3711	3881	}
3712	3882
@@ -3739,6 +3909,9 @@ static int gaudi_mmu_init(struct hl_device *hdev) · 26 unmodified lines
3739	3909	WREG32(mmSTLB_CACHE_INV_BASE_39_8, MMU_CACHE_MNG_ADDR >> 8);
3740	3910	WREG32(mmSTLB_CACHE_INV_BASE_49_40, MMU_CACHE_MNG_ADDR >> 40);
3741	3911
	3912	+ /* mem cache invalidation */
	3913	+ WREG32(mmSTLB_MEM_CACHE_INVALIDATION, 1);
	3914	+
3742	3915	hdev->asic_funcs->mmu_invalidate_cache(hdev, true, 0);
3743	3916
3744	3917	WREG32(mmMMU_UP_MMU_ENABLE, 1);
@@ -4071,7 +4244,7 @@ disable_queues: · 326 unmodified lines
4071	4244	return rc;
4072	4245	}
4073	4246
4074		−static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset)
	4247	+static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
4075	4248	{
4076	4249	struct cpu_dyn_regs *dyn_regs =
4077	4250	&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
@@ -4092,6 +4265,14 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset) · 14 unmodified lines
4092	4265	cpu_timeout_ms = GAUDI_CPU_RESET_WAIT_MSEC;
4093	4266	}
4094	4267
	4268	+ if (fw_reset) {
	4269	+ dev_info(hdev->dev,
	4270	+ "Firmware performs HARD reset, going to wait %dms\n",
	4271	+ reset_timeout_ms);
	4272	+
	4273	+ goto skip_reset;
	4274	+ }
	4275	+
4095	4276	driver_performs_reset = !!(!hdev->asic_prop.fw_security_enabled &&
4096	4277	!hdev->asic_prop.hard_reset_done_by_fw);
4097	4278
@@ -4168,6 +4349,7 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset) · 70 unmodified lines
4168	4349	reset_timeout_ms);
4169	4350	}
4170	4351
	4352	+skip_reset:
4171	4353	/*
4172	4354	* After hard reset, we can't poll the BTM_FSM register because the PSOC
4173	4355	* itself is in reset. Need to wait until the reset is deasserted
@@ -4212,7 +4394,7 @@ static int gaudi_resume(struct hl_device *hdev) · 38 unmodified lines
4212	4394	return gaudi_init_iatu(hdev);
4213	4395	}
4214	4396
4215		−static int gaudi_cb_mmap(struct hl_device hdev, struct vm_area_struct vma,
	4397	+static int gaudi_mmap(struct hl_device hdev, struct vm_area_struct vma,
4216	4398	void *cpu_addr, dma_addr_t dma_addr, size_t size)
4217	4399	{
4218	4400	int rc;
@@ -4621,8 +4803,8 @@ static int gaudi_hbm_scrubbing(struct hl_device *hdev) · 402 unmodified lines
4621	4803	"Doing HBM scrubbing for 0x%09llx - 0x%09llx\n",
4622	4804	cur_addr, cur_addr + chunk_size);
4623	4805
4624		− WREG32(mmDMA0_CORE_SRC_BASE_LO + dma_offset, 0);
4625		− WREG32(mmDMA0_CORE_SRC_BASE_HI + dma_offset, 0);
	4806	+ WREG32(mmDMA0_CORE_SRC_BASE_LO + dma_offset, 0xdeadbeaf);
	4807	+ WREG32(mmDMA0_CORE_SRC_BASE_HI + dma_offset, 0xdeadbeaf);
4626	4808	WREG32(mmDMA0_CORE_DST_BASE_LO + dma_offset,
4627	4809	lower_32_bits(cur_addr));
4628	4810	WREG32(mmDMA0_CORE_DST_BASE_HI + dma_offset,
@@ -5796,78 +5978,6 @@ release_cb: · 1167 unmodified lines
5796	5978	return rc;
5797	5979	}
5798	5980
5799		−static int gaudi_schedule_register_memset(struct hl_device *hdev,
5800		− u32 hw_queue_id, u64 reg_base, u32 num_regs, u32 val)
5801		−{
5802		− struct hl_ctx *ctx;
5803		− struct hl_pending_cb *pending_cb;
5804		− struct packet_msg_long *pkt;
5805		− u32 cb_size, ctl;
5806		− struct hl_cb *cb;
5807		− int i, rc;
5808		−
5809		− mutex_lock(&hdev->fpriv_list_lock);
5810		− ctx = hdev->compute_ctx;
5811		−
5812		− /* If no compute context available or context is going down
5813		− * memset registers directly
5814		− */
5815		− if (!ctx \|\| kref_read(&ctx->refcount) == 0) {
5816		− rc = gaudi_memset_registers(hdev, reg_base, num_regs, val);
5817		− mutex_unlock(&hdev->fpriv_list_lock);
5818		− return rc;
5819		− }
5820		−
5821		− mutex_unlock(&hdev->fpriv_list_lock);
5822		−
5823		− cb_size = (sizeof(pkt) num_regs) +
5824		− sizeof(struct packet_msg_prot) * 2;
5825		−
5826		− if (cb_size > SZ_2M) {
5827		− dev_err(hdev->dev, "CB size must be smaller than %uMB", SZ_2M);
5828		− return -ENOMEM;
5829		− }
5830		−
5831		− pending_cb = kzalloc(sizeof(*pending_cb), GFP_KERNEL);
5832		− if (!pending_cb)
5833		− return -ENOMEM;
5834		−
5835		− cb = hl_cb_kernel_create(hdev, cb_size, false);
5836		− if (!cb) {
5837		− kfree(pending_cb);
5838		− return -EFAULT;
5839		− }
5840		−
5841		− pkt = cb->kernel_address;
5842		−
5843		− ctl = FIELD_PREP(GAUDI_PKT_LONG_CTL_OP_MASK, 0); /* write the value */
5844		− ctl \|= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_LONG);
5845		− ctl \|= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1);
5846		− ctl \|= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1);
5847		− ctl \|= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);
5848		−
5849		− for (i = 0; i < num_regs ; i++, pkt++) {
5850		− pkt->ctl = cpu_to_le32(ctl);
5851		− pkt->value = cpu_to_le32(val);
5852		− pkt->addr = cpu_to_le64(reg_base + (i * 4));
5853		− }
5854		−
5855		− hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
5856		−
5857		− pending_cb->cb = cb;
5858		− pending_cb->cb_size = cb_size;
5859		− /* The queue ID MUST be an external queue ID. Otherwise, we will
5860		− * have undefined behavior
5861		− */
5862		− pending_cb->hw_queue_id = hw_queue_id;
5863		−
5864		− spin_lock(&ctx->pending_cb_lock);
5865		− list_add_tail(&pending_cb->cb_node, &ctx->pending_cb_list);
5866		− spin_unlock(&ctx->pending_cb_lock);
5867		−
5868		− return 0;
5869		−}
5870		−
5871	5981	static int gaudi_restore_sm_registers(struct hl_device *hdev)
5872	5982	{
5873	5983	u64 base_addr;
@@ -6013,7 +6123,7 @@ static int gaudi_restore_user_registers(struct hl_device *hdev) · 139 unmodified lines
6013	6123
6014	6124	static int gaudi_context_switch(struct hl_device *hdev, u32 asid)
6015	6125	{
6016		− return gaudi_restore_user_registers(hdev);
	6126	+ return 0;
6017	6127	}
6018	6128
6019	6129	static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev)
@@ -6723,6 +6833,9 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) · 703 unmodified lines
6723	6833	asid);
6724	6834	}
6725	6835
	6836	+ gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_ARUSER, asid);
	6837	+ gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_AWUSER, asid);
	6838	+
6726	6839	hdev->asic_funcs->set_clock_gating(hdev);
6727	6840
6728	6841	mutex_unlock(&gaudi->clk_gate_mutex);
@@ -6772,7 +6885,8 @@ static int gaudi_send_job_on_qman0(struct hl_device *hdev, · 43 unmodified lines
6772	6885
6773	6886	dma_offset = gaudi_dma_assignment[GAUDI_PCI_DMA_1] * DMA_CORE_OFFSET;
6774	6887
6775		− WREG32_OR(mmDMA0_CORE_PROT + dma_offset, BIT(DMA0_CORE_PROT_VAL_SHIFT));
	6888	+ WREG32(mmDMA0_CORE_PROT + dma_offset,
	6889	+ BIT(DMA0_CORE_PROT_ERR_VAL_SHIFT) \| BIT(DMA0_CORE_PROT_VAL_SHIFT));
6776	6890
6777	6891	rc = hl_hw_queue_send_cb_no_cmpl(hdev, GAUDI_QUEUE_ID_DMA_0_0,
6778	6892	job->job_cb_size, cb->bus_address);
@@ -6793,8 +6907,7 @@ static int gaudi_send_job_on_qman0(struct hl_device *hdev, · 14 unmodified lines
6793	6907	}
6794	6908
6795	6909	free_fence_ptr:
6796		− WREG32_AND(mmDMA0_CORE_PROT + dma_offset,
6797		− ~BIT(DMA0_CORE_PROT_VAL_SHIFT));
	6910	+ WREG32(mmDMA0_CORE_PROT + dma_offset, BIT(DMA0_CORE_PROT_ERR_VAL_SHIFT));
6798	6911
6799	6912	hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
6800	6913	fence_dma_addr);
@@ -7168,7 +7281,7 @@ static void gaudi_print_sw_config_stream_data(struct hl_device *hdev, u32 stream · 367 unmodified lines
7168	7281
7169	7282	cq_ptr = (((u64) RREG32(cq_ptr_hi)) << 32) \| RREG32(cq_ptr_lo);
7170	7283	size = RREG32(cq_tsize);
7171		− dev_info(hdev->dev, "stop on err: stream: %u, addr: %#llx, size: %x\n",
	7284	+ dev_info(hdev->dev, "stop on err: stream: %u, addr: %#llx, size: %u\n",
7172	7285	stream, cq_ptr, size);
7173	7286	}
7174	7287
@@ -7224,7 +7337,7 @@ static void gaudi_print_last_pqes_on_err(struct hl_device *hdev, u32 qid_base, · 49 unmodified lines
7224	7337
7225	7338	addr = le64_to_cpu(bd->ptr);
7226	7339
7227		− dev_info(hdev->dev, "stop on err PQE(stream %u): ci: %u, addr: %#llx, size: %x\n",
	7340	+ dev_info(hdev->dev, "stop on err PQE(stream %u): ci: %u, addr: %#llx, size: %u\n",
7228	7341	stream, ci, addr, len);
7229	7342
7230	7343	/* get previous ci, wrap if needed */
@@ -7326,24 +7439,30 @@ static void gaudi_print_sm_sei_info(struct hl_device *hdev, u16 event_type, · 95 unmodified lines
7326	7439	{
7327	7440	u32 index = event_type - GAUDI_EVENT_DMA_IF_SEI_0;
7328	7441
	7442	+ /* Flip the bits as the enum is ordered in the opposite way */
	7443	+ index = (index ^ 0x3) & 0x3;
	7444	+
7329	7445	switch (sei_data->sei_cause) {
7330	7446	case SM_SEI_SO_OVERFLOW:
7331		− dev_err(hdev->dev,
7332		− "SM %u SEI Error: SO %u overflow/underflow",
7333		− index, le32_to_cpu(sei_data->sei_log));
	7447	+ dev_err_ratelimited(hdev->dev,
	7448	+ "%s SEI Error: SOB Group %u overflow/underflow",
	7449	+ gaudi_sync_manager_names[index],
	7450	+ le32_to_cpu(sei_data->sei_log));
7334	7451	break;
7335	7452	case SM_SEI_LBW_4B_UNALIGNED:
7336		− dev_err(hdev->dev,
7337		− "SM %u SEI Error: Unaligned 4B LBW access, monitor agent address low - %#x",
7338		− index, le32_to_cpu(sei_data->sei_log));
	7453	+ dev_err_ratelimited(hdev->dev,
	7454	+ "%s SEI Error: Unaligned 4B LBW access, monitor agent address low - %#x",
	7455	+ gaudi_sync_manager_names[index],
	7456	+ le32_to_cpu(sei_data->sei_log));
7339	7457	break;
7340	7458	case SM_SEI_AXI_RESPONSE_ERR:
7341		− dev_err(hdev->dev,
7342		− "SM %u SEI Error: AXI ID %u response error",
7343		− index, le32_to_cpu(sei_data->sei_log));
	7459	+ dev_err_ratelimited(hdev->dev,
	7460	+ "%s SEI Error: AXI ID %u response error",
	7461	+ gaudi_sync_manager_names[index],
	7462	+ le32_to_cpu(sei_data->sei_log));
7344	7463	break;
7345	7464	default:
7346		− dev_err(hdev->dev, "Unknown SM SEI cause %u",
	7465	+ dev_err_ratelimited(hdev->dev, "Unknown SM SEI cause %u",
7347	7466	le32_to_cpu(sei_data->sei_log));
7348	7467	break;
7349	7468	}
@@ -7358,6 +7477,11 @@ static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type, · 8 unmodified lines
7358	7477	bool extract_info_from_fw;
7359	7478	int rc;
7360	7479
	7480	+ if (hdev->asic_prop.fw_security_enabled) {
	7481	+ extract_info_from_fw = true;
	7482	+ goto extract_ecc_info;
	7483	+ }
	7484	+
7361	7485	switch (event_type) {
7362	7486	case GAUDI_EVENT_PCIE_CORE_SERR ... GAUDI_EVENT_PCIE_PHY_DERR:
7363	7487	case GAUDI_EVENT_DMA0_SERR_ECC ... GAUDI_EVENT_MMU_DERR:
@@ -7430,6 +7554,7 @@ static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type, · 66 unmodified lines
7430	7554	return;
7431	7555	}
7432	7556
	7557	+extract_ecc_info:
7433	7558	if (extract_info_from_fw) {
7434	7559	ecc_address = le64_to_cpu(ecc_data->ecc_address);
7435	7560	ecc_syndrom = le64_to_cpu(ecc_data->ecc_syndrom);
@@ -7806,8 +7931,15 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 370 unmodified lines
7806	7931	u32 ctl = le32_to_cpu(eq_entry->hdr.ctl);
7807	7932	u16 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK)
7808	7933	>> EQ_CTL_EVENT_TYPE_SHIFT);
7809		− u8 cause;
7810	7934	bool reset_required;
	7935	+ u8 cause;
	7936	+ int rc;
	7937	+
	7938	+ if (event_type >= GAUDI_EVENT_SIZE) {
	7939	+ dev_err(hdev->dev, "Event type %u exceeds maximum of %u",
	7940	+ event_type, GAUDI_EVENT_SIZE - 1);
	7941	+ return;
	7942	+ }
7811	7943
7812	7944	gaudi->events_stat[event_type]++;
7813	7945	gaudi->events_stat_aggregate[event_type]++;
@@ -7880,10 +8012,10 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 66 unmodified lines
7880	8012	tpc_dec_event_to_tpc_id(event_type),
7881	8013	"AXI_SLV_DEC_Error");
7882	8014	if (reset_required) {
7883		− dev_err(hdev->dev, "hard reset required due to %s\n",
	8015	+ dev_err(hdev->dev, "reset required due to %s\n",
7884	8016	gaudi_irq_map_table[event_type].name);
7885	8017
7886		− goto reset_device;
	8018	+ hl_device_reset(hdev, 0);
7887	8019	} else {
7888	8020	hl_fw_unmask_irq(hdev, event_type);
7889	8021	}
@@ -7902,10 +8034,10 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 12 unmodified lines
7902	8034	tpc_krn_event_to_tpc_id(event_type),
7903	8035	"KRN_ERR");
7904	8036	if (reset_required) {
7905		− dev_err(hdev->dev, "hard reset required due to %s\n",
	8037	+ dev_err(hdev->dev, "reset required due to %s\n",
7906	8038	gaudi_irq_map_table[event_type].name);
7907	8039
7908		− goto reset_device;
	8040	+ hl_device_reset(hdev, 0);
7909	8041	} else {
7910	8042	hl_fw_unmask_irq(hdev, event_type);
7911	8043	}
@@ -7993,6 +8125,10 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 81 unmodified lines
7993	8125	gaudi_print_irq_info(hdev, event_type, false);
7994	8126	gaudi_print_sm_sei_info(hdev, event_type,
7995	8127	&eq_entry->sm_sei_data);
	8128	+ rc = hl_state_dump(hdev);
	8129	+ if (rc)
	8130	+ dev_err(hdev->dev,
	8131	+ "Error during system state dump %d\n", rc);
7996	8132	hl_fw_unmask_irq(hdev, event_type);
7997	8133	break;
7998	8134
@@ -8031,7 +8167,9 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 32 unmodified lines
8031	8167	return;
8032	8168
8033	8169	reset_device:
8034		− if (hdev->hard_reset_on_fw_events)
	8170	+ if (hdev->asic_prop.fw_security_enabled)
	8171	+ hl_device_reset(hdev, HL_RESET_HARD \| HL_RESET_FW);
	8172	+ else if (hdev->hard_reset_on_fw_events)
8035	8173	hl_device_reset(hdev, HL_RESET_HARD);
8036	8174	else
8037	8175	hl_fw_unmask_irq(hdev, event_type);
@@ -8563,11 +8701,20 @@ static void gaudi_internal_cb_pool_fini(struct hl_device *hdev, · 525 unmodified lines
8563	8701
8564	8702	static int gaudi_ctx_init(struct hl_ctx *ctx)
8565	8703	{
	8704	+ int rc;
	8705	+
8566	8706	if (ctx->asid == HL_KERNEL_ASID_ID)
8567	8707	return 0;
8568	8708
8569		− gaudi_mmu_prepare(ctx->hdev, ctx->asid);
8570		− return gaudi_internal_cb_pool_init(ctx->hdev, ctx);
	8709	+ rc = gaudi_internal_cb_pool_init(ctx->hdev, ctx);
	8710	+ if (rc)
	8711	+ return rc;
	8712	+
	8713	+ rc = gaudi_restore_user_registers(ctx->hdev);
	8714	+ if (rc)
	8715	+ gaudi_internal_cb_pool_fini(ctx->hdev, ctx);
	8716	+
	8717	+ return rc;
8571	8718	}
8572	8719
8573	8720	static void gaudi_ctx_fini(struct hl_ctx *ctx)
@@ -8596,6 +8743,11 @@ static u32 gaudi_get_wait_cb_size(struct hl_device *hdev) · 22 unmodified lines
8596	8743	sizeof(struct packet_msg_prot) * 2;
8597	8744	}
8598	8745
	8746	+static u32 gaudi_get_sob_addr(struct hl_device *hdev, u32 sob_id)
	8747	+{
	8748	+ return mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + (sob_id * 4);
	8749	+}
	8750	+
8599	8751	static u32 gaudi_gen_signal_cb(struct hl_device hdev, void data, u16 sob_id,
8600	8752	u32 size, bool eb)
8601	8753	{
@@ -8902,16 +9054,12 @@ static u32 gaudi_gen_wait_cb(struct hl_device *hdev, · 300 unmodified lines
8902	9054	static void gaudi_reset_sob(struct hl_device hdev, void data)
8903	9055	{
8904	9056	struct hl_hw_sob hw_sob = (struct hl_hw_sob ) data;
8905		− int rc;
8906	9057
8907	9058	dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx,
8908	9059	hw_sob->sob_id);
8909	9060
8910		− rc = gaudi_schedule_register_memset(hdev, hw_sob->q_idx,
8911		− CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 +
8912		− hw_sob->sob_id * 4, 1, 0);
8913		− if (rc)
8914		− dev_err(hdev->dev, "failed resetting sob %u", hw_sob->sob_id);
	9061	+ WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 +
	9062	+ hw_sob->sob_id * 4, 0);
8915	9063
8916	9064	kref_init(&hw_sob->kref);
8917	9065	}
@@ -8977,6 +9125,280 @@ static int gaudi_map_pll_idx_to_fw_idx(u32 pll_idx) · 59 unmodified lines
8977	9125	}
8978	9126	}
8979	9127
	9128	+static int gaudi_add_sync_to_engine_map_entry(
	9129	+ struct hl_sync_to_engine_map *map, u32 reg_value,
	9130	+ enum hl_sync_engine_type engine_type, u32 engine_id)
	9131	+{
	9132	+ struct hl_sync_to_engine_map_entry *entry;
	9133	+
	9134	+ /* Reg value represents a partial address of sync object,
	9135	+ * it is used as unique identifier. For this we need to
	9136	+ * clear the cutoff cfg base bits from the value.
	9137	+ */
	9138	+ if (reg_value == 0 \|\| reg_value == 0xffffffff)
	9139	+ return 0;
	9140	+ reg_value -= (u32)CFG_BASE;
	9141	+
	9142	+ /* create a new hash entry */
	9143	+ entry = kzalloc(sizeof(*entry), GFP_KERNEL);
	9144	+ if (!entry)
	9145	+ return -ENOMEM;
	9146	+ entry->engine_type = engine_type;
	9147	+ entry->engine_id = engine_id;
	9148	+ entry->sync_id = reg_value;
	9149	+ hash_add(map->tb, &entry->node, reg_value);
	9150	+
	9151	+ return 0;
	9152	+}
	9153	+
	9154	+static int gaudi_gen_sync_to_engine_map(struct hl_device *hdev,
	9155	+ struct hl_sync_to_engine_map *map)
	9156	+{
	9157	+ struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
	9158	+ struct gaudi_device *gaudi = hdev->asic_specific;
	9159	+ int i, j, rc;
	9160	+ u32 reg_value;
	9161	+
	9162	+ /* Iterate over TPC engines */
	9163	+ for (i = 0; i < sds->props[SP_NUM_OF_TPC_ENGINES]; ++i) {
	9164	+ /* TPC registered must be accessed with clock gating disabled */
	9165	+ mutex_lock(&gaudi->clk_gate_mutex);
	9166	+ hdev->asic_funcs->disable_clock_gating(hdev);
	9167	+
	9168	+ reg_value = RREG32(sds->props[SP_TPC0_CFG_SO] +
	9169	+ sds->props[SP_NEXT_TPC] * i);
	9170	+
	9171	+ /* We can reenable clock_gating */
	9172	+ hdev->asic_funcs->set_clock_gating(hdev);
	9173	+ mutex_unlock(&gaudi->clk_gate_mutex);
	9174	+
	9175	+ rc = gaudi_add_sync_to_engine_map_entry(map, reg_value,
	9176	+ ENGINE_TPC, i);
	9177	+ if (rc)
	9178	+ goto free_sync_to_engine_map;
	9179	+ }
	9180	+
	9181	+ /* Iterate over MME engines */
	9182	+ for (i = 0; i < sds->props[SP_NUM_OF_MME_ENGINES]; ++i) {
	9183	+ for (j = 0; j < sds->props[SP_SUB_MME_ENG_NUM]; ++j) {
	9184	+ /* MME registered must be accessed with clock gating
	9185	+ * disabled
	9186	+ */
	9187	+ mutex_lock(&gaudi->clk_gate_mutex);
	9188	+ hdev->asic_funcs->disable_clock_gating(hdev);
	9189	+
	9190	+ reg_value = RREG32(sds->props[SP_MME_CFG_SO] +
	9191	+ sds->props[SP_NEXT_MME] * i +
	9192	+ j * sizeof(u32));
	9193	+
	9194	+ /* We can reenable clock_gating */
	9195	+ hdev->asic_funcs->set_clock_gating(hdev);
	9196	+ mutex_unlock(&gaudi->clk_gate_mutex);
	9197	+
	9198	+ rc = gaudi_add_sync_to_engine_map_entry(
	9199	+ map, reg_value, ENGINE_MME,
	9200	+ i * sds->props[SP_SUB_MME_ENG_NUM] + j);
	9201	+ if (rc)
	9202	+ goto free_sync_to_engine_map;
	9203	+ }
	9204	+ }
	9205	+
	9206	+ /* Iterate over DMA engines */
	9207	+ for (i = 0; i < sds->props[SP_NUM_OF_DMA_ENGINES]; ++i) {
	9208	+ reg_value = RREG32(sds->props[SP_DMA_CFG_SO] +
	9209	+ sds->props[SP_DMA_QUEUES_OFFSET] * i);
	9210	+ rc = gaudi_add_sync_to_engine_map_entry(map, reg_value,
	9211	+ ENGINE_DMA, i);
	9212	+ if (rc)
	9213	+ goto free_sync_to_engine_map;
	9214	+ }
	9215	+
	9216	+ return 0;
	9217	+
	9218	+free_sync_to_engine_map:
	9219	+ hl_state_dump_free_sync_to_engine_map(map);
	9220	+
	9221	+ return rc;
	9222	+}
	9223	+
	9224	+static int gaudi_monitor_valid(struct hl_mon_state_dump *mon)
	9225	+{
	9226	+ return FIELD_GET(
	9227	+ SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_VALID_MASK,
	9228	+ mon->status);
	9229	+}
	9230	+
	9231	+static void gaudi_fill_sobs_from_mon(char sobs, struct hl_mon_state_dump mon)
	9232	+{
	9233	+ const size_t max_write = 10;
	9234	+ u32 gid, mask, sob;
	9235	+ int i, offset;
	9236	+
	9237	+ /* Sync object ID is calculated as follows:
	9238	+ * (8 * group_id + cleared bits in mask)
	9239	+ */
	9240	+ gid = FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SID_MASK,
	9241	+ mon->arm_data);
	9242	+ mask = FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_MASK_MASK,
	9243	+ mon->arm_data);
	9244	+
	9245	+ for (i = 0, offset = 0; mask && offset < MONITOR_SOB_STRING_SIZE -
	9246	+ max_write; mask >>= 1, i++) {
	9247	+ if (!(mask & 1)) {
	9248	+ sob = gid * MONITOR_MAX_SOBS + i;
	9249	+
	9250	+ if (offset > 0)
	9251	+ offset += snprintf(sobs + offset, max_write,
	9252	+ ", ");
	9253	+
	9254	+ offset += snprintf(sobs + offset, max_write, "%u", sob);
	9255	+ }
	9256	+ }
	9257	+}
	9258	+
	9259	+static int gaudi_print_single_monitor(char *buf, size_t size, size_t *offset,
	9260	+ struct hl_device *hdev,
	9261	+ struct hl_mon_state_dump *mon)
	9262	+{
	9263	+ const char *name;
	9264	+ char scratch_buf1[BIN_REG_STRING_SIZE],
	9265	+ scratch_buf2[BIN_REG_STRING_SIZE];
	9266	+ char monitored_sobs[MONITOR_SOB_STRING_SIZE] = {0};
	9267	+
	9268	+ name = hl_state_dump_get_monitor_name(hdev, mon);
	9269	+ if (!name)
	9270	+ name = "";
	9271	+
	9272	+ gaudi_fill_sobs_from_mon(monitored_sobs, mon);
	9273	+
	9274	+ return hl_snprintf_resize(
	9275	+ buf, size, offset,
	9276	+ "Mon id: %u%s, wait for group id: %u mask %s to reach val: %u and write %u to address 0x%llx. Pending: %s. Means sync objects [%s] are being monitored.",
	9277	+ mon->id, name,
	9278	+ FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SID_MASK,
	9279	+ mon->arm_data),
	9280	+ hl_format_as_binary(
	9281	+ scratch_buf1, sizeof(scratch_buf1),
	9282	+ FIELD_GET(
	9283	+ SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_MASK_MASK,
	9284	+ mon->arm_data)),
	9285	+ FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SOD_MASK,
	9286	+ mon->arm_data),
	9287	+ mon->wr_data,
	9288	+ (((u64)mon->wr_addr_high) << 32) \| mon->wr_addr_low,
	9289	+ hl_format_as_binary(
	9290	+ scratch_buf2, sizeof(scratch_buf2),
	9291	+ FIELD_GET(
	9292	+ SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_PENDING_MASK,
	9293	+ mon->status)),
	9294	+ monitored_sobs);
	9295	+}
	9296	+
	9297	+
	9298	+static int gaudi_print_fences_single_engine(
	9299	+ struct hl_device *hdev, u64 base_offset, u64 status_base_offset,
	9300	+ enum hl_sync_engine_type engine_type, u32 engine_id, char **buf,
	9301	+ size_t size, size_t offset)
	9302	+{
	9303	+ struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
	9304	+ int rc = -ENOMEM, i;
	9305	+ u32 statuses, fences;
	9306	+
	9307	+ statuses = kcalloc(sds->props[SP_ENGINE_NUM_OF_QUEUES],
	9308	+ sizeof(*statuses), GFP_KERNEL);
	9309	+ if (!statuses)
	9310	+ goto out;
	9311	+
	9312	+ fences = kcalloc(sds->props[SP_ENGINE_NUM_OF_FENCES] *
	9313	+ sds->props[SP_ENGINE_NUM_OF_QUEUES],
	9314	+ sizeof(*fences), GFP_KERNEL);
	9315	+ if (!fences)
	9316	+ goto free_status;
	9317	+
	9318	+ for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_FENCES]; ++i)
	9319	+ statuses[i] = RREG32(status_base_offset + i * sizeof(u32));
	9320	+
	9321	+ for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_FENCES] *
	9322	+ sds->props[SP_ENGINE_NUM_OF_QUEUES]; ++i)
	9323	+ fences[i] = RREG32(base_offset + i * sizeof(u32));
	9324	+
	9325	+ /* The actual print */
	9326	+ for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_QUEUES]; ++i) {
	9327	+ u32 fence_id;
	9328	+ u64 fence_cnt, fence_rdata;
	9329	+ const char *engine_name;
	9330	+
	9331	+ if (!FIELD_GET(TPC0_QM_CP_STS_0_FENCE_IN_PROGRESS_MASK,
	9332	+ statuses[i]))
	9333	+ continue;
	9334	+
	9335	+ fence_id =
	9336	+ FIELD_GET(TPC0_QM_CP_STS_0_FENCE_ID_MASK, statuses[i]);
	9337	+ fence_cnt = base_offset + CFG_BASE +
	9338	+ sizeof(u32) *
	9339	+ (i + fence_id * sds->props[SP_ENGINE_NUM_OF_QUEUES]);
	9340	+ fence_rdata = fence_cnt - sds->props[SP_FENCE0_CNT_OFFSET] +
	9341	+ sds->props[SP_FENCE0_RDATA_OFFSET];
	9342	+ engine_name = hl_sync_engine_to_string(engine_type);
	9343	+
	9344	+ rc = hl_snprintf_resize(
	9345	+ buf, size, offset,
	9346	+ "%s%u, stream %u: fence id %u cnt = 0x%llx (%s%u_QM.CP_FENCE%u_CNT_%u) rdata = 0x%llx (%s%u_QM.CP_FENCE%u_RDATA_%u) value = %u, cp_status = %u\n",
	9347	+ engine_name, engine_id,
	9348	+ i, fence_id,
	9349	+ fence_cnt, engine_name, engine_id, fence_id, i,
	9350	+ fence_rdata, engine_name, engine_id, fence_id, i,
	9351	+ fences[fence_id],
	9352	+ statuses[i]);
	9353	+ if (rc)
	9354	+ goto free_fences;
	9355	+ }
	9356	+
	9357	+ rc = 0;
	9358	+
	9359	+free_fences:
	9360	+ kfree(fences);
	9361	+free_status:
	9362	+ kfree(statuses);
	9363	+out:
	9364	+ return rc;
	9365	+}
	9366	+
	9367	+
	9368	+static struct hl_state_dump_specs_funcs gaudi_state_dump_funcs = {
	9369	+ .monitor_valid = gaudi_monitor_valid,
	9370	+ .print_single_monitor = gaudi_print_single_monitor,
	9371	+ .gen_sync_to_engine_map = gaudi_gen_sync_to_engine_map,
	9372	+ .print_fences_single_engine = gaudi_print_fences_single_engine,
	9373	+};
	9374	+
	9375	+static void gaudi_state_dump_init(struct hl_device *hdev)
	9376	+{
	9377	+ struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
	9378	+ int i;
	9379	+
	9380	+ for (i = 0; i < ARRAY_SIZE(gaudi_so_id_to_str); ++i)
	9381	+ hash_add(sds->so_id_to_str_tb,
	9382	+ &gaudi_so_id_to_str[i].node,
	9383	+ gaudi_so_id_to_str[i].id);
	9384	+
	9385	+ for (i = 0; i < ARRAY_SIZE(gaudi_monitor_id_to_str); ++i)
	9386	+ hash_add(sds->monitor_id_to_str_tb,
	9387	+ &gaudi_monitor_id_to_str[i].node,
	9388	+ gaudi_monitor_id_to_str[i].id);
	9389	+
	9390	+ sds->props = gaudi_state_dump_specs_props;
	9391	+
	9392	+ sds->sync_namager_names = gaudi_sync_manager_names;
	9393	+
	9394	+ sds->funcs = gaudi_state_dump_funcs;
	9395	+}
	9396	+
	9397	+static u32 *gaudi_get_stream_master_qid_arr(void)
	9398	+{
	9399	+ return gaudi_stream_master;
	9400	+}
	9401	+
8980	9402	static const struct hl_asic_funcs gaudi_funcs = {
8981	9403	.early_init = gaudi_early_init,
8982	9404	.early_fini = gaudi_early_fini,
@@ -8989,7 +9411,7 @@ static const struct hl_asic_funcs gaudi_funcs = { · 6 unmodified lines
8989	9411	.halt_engines = gaudi_halt_engines,
8990	9412	.suspend = gaudi_suspend,
8991	9413	.resume = gaudi_resume,
8992		− .cb_mmap = gaudi_cb_mmap,
	9414	+ .mmap = gaudi_mmap,
8993	9415	.ring_doorbell = gaudi_ring_doorbell,
8994	9416	.pqe_write = gaudi_pqe_write,
8995	9417	.asic_dma_alloc_coherent = gaudi_dma_alloc_coherent,
@@ -9062,7 +9484,11 @@ static const struct hl_asic_funcs gaudi_funcs = { · 66 unmodified lines
9062	9484	.enable_events_from_fw = gaudi_enable_events_from_fw,
9063	9485	.map_pll_idx_to_fw_idx = gaudi_map_pll_idx_to_fw_idx,
9064	9486	.init_firmware_loader = gaudi_init_firmware_loader,
9065		− .init_cpu_scrambler_dram = gaudi_init_scrambler_hbm
	9487	+ .init_cpu_scrambler_dram = gaudi_init_scrambler_hbm,
	9488	+ .state_dump_init = gaudi_state_dump_init,
	9489	+ .get_sob_addr = gaudi_get_sob_addr,
	9490	+ .set_pci_memory_regions = gaudi_set_pci_memory_regions,
	9491	+ .get_stream_master_qid_arr = gaudi_get_stream_master_qid_arr
9066	9492	};
9067	9493
9068	9494	/**

drivers/misc/habanalabs/gaudi/gaudiP.h+18−1

@@ -36,6 +36,8 @@ · 35 unmodified lines
36	36	#define NUMBER_OF_INTERRUPTS (NUMBER_OF_CMPLT_QUEUES + \
37	37	NUMBER_OF_CPU_HW_QUEUES)
38	38
	39	+#define GAUDI_STREAM_MASTER_ARR_SIZE 8
	40	+
39	41	#if (NUMBER_OF_INTERRUPTS > GAUDI_MSI_ENTRIES)
40	42	#error "Number of MSI interrupts must be smaller or equal to GAUDI_MSI_ENTRIES"
41	43	#endif
@@ -50,6 +52,8 @@ · 8 unmodified lines
50	52	#define DC_POWER_DEFAULT_PCI 60000 /* 60W */
51	53	#define DC_POWER_DEFAULT_PMC 60000 /* 60W */
52	54
	55	+#define DC_POWER_DEFAULT_PMC_SEC 97000 /* 97W */
	56	+
53	57	#define GAUDI_CPU_TIMEOUT_USEC 30000000 /* 30s */
54	58
55	59	#define TPC_ENABLED_MASK 0xFF
@@ -62,7 +66,7 @@ · 6 unmodified lines
62	66
63	67	#define DMA_MAX_TRANSFER_SIZE U32_MAX
64	68
65		−#define GAUDI_DEFAULT_CARD_NAME "HL2000"
	69	+#define GAUDI_DEFAULT_CARD_NAME "HL205"
66	70
67	71	#define GAUDI_MAX_PENDING_CS SZ_16K
68	72
@@ -117,6 +121,7 @@ · 48 unmodified lines
117	121	(((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_511 - \
118	122	mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0) + 4) >> 2)
119	123
	124	+#define MONITOR_MAX_SOBS 8
120	125
121	126	/* DRAM Memory Map */
122	127
@@ -200,6 +205,18 @@ · 77 unmodified lines
200	205	#define HW_CAP_TPC_MASK GENMASK(31, 24)
201	206	#define HW_CAP_TPC_SHIFT 24
202	207
	208	+#define NEXT_SYNC_OBJ_ADDR_INTERVAL \
	209	+ (mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0 - \
	210	+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0)
	211	+#define NUM_OF_MME_ENGINES 2
	212	+#define NUM_OF_MME_SUB_ENGINES 2
	213	+#define NUM_OF_TPC_ENGINES 8
	214	+#define NUM_OF_DMA_ENGINES 8
	215	+#define NUM_OF_QUEUES 5
	216	+#define NUM_OF_STREAMS 4
	217	+#define NUM_OF_FENCES 4
	218	+
	219	+
203	220	#define GAUDI_CPU_PCI_MSB_ADDR(addr) (((addr) & GENMASK_ULL(49, 39)) >> 39)
204	221	#define GAUDI_PCI_TO_CPU_ADDR(addr) \
205	222	do { \

drivers/misc/habanalabs/gaudi/gaudi_coresight.c+0−5

@@ -622,11 +622,6 @@ static int gaudi_config_etr(struct hl_device *hdev, · 621 unmodified lines
622	622	return -EINVAL;
623	623	}
624	624
625		− gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_ARUSER,
626		− hdev->compute_ctx->asid);
627		− gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_AWUSER,
628		− hdev->compute_ctx->asid);
629		−
630	625	msb = upper_32_bits(input->buffer_address) >> 8;
631	626	msb &= PSOC_GLOBAL_CONF_TRACE_ADDR_MSB_MASK;
632	627	WREG32(mmPSOC_GLOBAL_CONF_TRACE_ADDR, msb);

drivers/misc/habanalabs/gaudi/gaudi_security.c+8−0

@@ -9559,6 +9559,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 9558 unmodified lines
9559	9559	mask \|= 1U << ((mmTPC0_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
9560	9560	mask \|= 1U << ((mmTPC0_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
9561	9561	mask \|= 1U << ((mmTPC0_CFG_TPC_STALL & 0x7F) >> 2);
	9562	+ mask \|= 1U << ((mmTPC0_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
9562	9563	mask \|= 1U << ((mmTPC0_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
9563	9564	mask \|= 1U << ((mmTPC0_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
9564	9565	mask \|= 1U << ((mmTPC0_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -10013,6 +10014,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 448 unmodified lines
10013	10014	mask \|= 1U << ((mmTPC1_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
10014	10015	mask \|= 1U << ((mmTPC1_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
10015	10016	mask \|= 1U << ((mmTPC1_CFG_TPC_STALL & 0x7F) >> 2);
	10017	+ mask \|= 1U << ((mmTPC1_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
10016	10018	mask \|= 1U << ((mmTPC1_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
10017	10019	mask \|= 1U << ((mmTPC1_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
10018	10020	mask \|= 1U << ((mmTPC1_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -10466,6 +10468,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 447 unmodified lines
10466	10468	mask \|= 1U << ((mmTPC2_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
10467	10469	mask \|= 1U << ((mmTPC2_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
10468	10470	mask \|= 1U << ((mmTPC2_CFG_TPC_STALL & 0x7F) >> 2);
	10471	+ mask \|= 1U << ((mmTPC2_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
10469	10472	mask \|= 1U << ((mmTPC2_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
10470	10473	mask \|= 1U << ((mmTPC2_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
10471	10474	mask \|= 1U << ((mmTPC2_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -10919,6 +10922,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 447 unmodified lines
10919	10922	mask \|= 1U << ((mmTPC3_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
10920	10923	mask \|= 1U << ((mmTPC3_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
10921	10924	mask \|= 1U << ((mmTPC3_CFG_TPC_STALL & 0x7F) >> 2);
	10925	+ mask \|= 1U << ((mmTPC3_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
10922	10926	mask \|= 1U << ((mmTPC3_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
10923	10927	mask \|= 1U << ((mmTPC3_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
10924	10928	mask \|= 1U << ((mmTPC3_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -11372,6 +11376,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 447 unmodified lines
11372	11376	mask \|= 1U << ((mmTPC4_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
11373	11377	mask \|= 1U << ((mmTPC4_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
11374	11378	mask \|= 1U << ((mmTPC4_CFG_TPC_STALL & 0x7F) >> 2);
	11379	+ mask \|= 1U << ((mmTPC4_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
11375	11380	mask \|= 1U << ((mmTPC4_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
11376	11381	mask \|= 1U << ((mmTPC4_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
11377	11382	mask \|= 1U << ((mmTPC4_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -11825,6 +11830,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 447 unmodified lines
11825	11830	mask \|= 1U << ((mmTPC5_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
11826	11831	mask \|= 1U << ((mmTPC5_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
11827	11832	mask \|= 1U << ((mmTPC5_CFG_TPC_STALL & 0x7F) >> 2);
	11833	+ mask \|= 1U << ((mmTPC5_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
11828	11834	mask \|= 1U << ((mmTPC5_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
11829	11835	mask \|= 1U << ((mmTPC5_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
11830	11836	mask \|= 1U << ((mmTPC5_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -12280,6 +12286,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 449 unmodified lines
12280	12286	mask \|= 1U << ((mmTPC6_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
12281	12287	mask \|= 1U << ((mmTPC6_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
12282	12288	mask \|= 1U << ((mmTPC6_CFG_TPC_STALL & 0x7F) >> 2);
	12289	+ mask \|= 1U << ((mmTPC6_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
12283	12290	mask \|= 1U << ((mmTPC6_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
12284	12291	mask \|= 1U << ((mmTPC6_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
12285	12292	mask \|= 1U << ((mmTPC6_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -12735,6 +12742,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 449 unmodified lines
12735	12742	mask \|= 1U << ((mmTPC7_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
12736	12743	mask \|= 1U << ((mmTPC7_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
12737	12744	mask \|= 1U << ((mmTPC7_CFG_TPC_STALL & 0x7F) >> 2);
	12745	+ mask \|= 1U << ((mmTPC7_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
12738	12746	mask \|= 1U << ((mmTPC7_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
12739	12747	mask \|= 1U << ((mmTPC7_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
12740	12748	mask \|= 1U << ((mmTPC7_CFG_MSS_CONFIG & 0x7F) >> 2);

drivers/misc/habanalabs/goya/goya.c+84−18

@@ -350,6 +350,8 @@ static u32 goya_all_events[] = { · 349 unmodified lines
350	350	GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E
351	351	};
352	352
	353	+static s64 goya_state_dump_specs_props[SP_MAX] = {0};
	354	+
353	355	static int goya_mmu_clear_pgt_range(struct hl_device *hdev);
354	356	static int goya_mmu_set_dram_default_page(struct hl_device *hdev);
355	357	static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev);
@@ -387,6 +389,7 @@ int goya_set_fixed_properties(struct hl_device *hdev) · 31 unmodified lines
387	389	prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_USER;
388	390	}
389	391
	392	+ prop->device_dma_offset_for_host_access = HOST_PHYS_BASE;
390	393	prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;
391	394
392	395	prop->dram_base_address = DRAM_PHYS_BASE;
@@ -466,6 +469,8 @@ int goya_set_fixed_properties(struct hl_device *hdev) · 73 unmodified lines
466	469	prop->hard_reset_done_by_fw = false;
467	470	prop->gic_interrupts_enable = true;
468	471
	472	+ prop->server_type = HL_SERVER_TYPE_UNKNOWN;
	473	+
469	474	return 0;
470	475	}
471	476
@@ -649,14 +654,14 @@ pci_init: · 177 unmodified lines
649	654	GOYA_BOOT_FIT_REQ_TIMEOUT_USEC);
650	655	if (rc) {
651	656	if (hdev->reset_on_preboot_fail)
652		− hdev->asic_funcs->hw_fini(hdev, true);
	657	+ hdev->asic_funcs->hw_fini(hdev, true, false);
653	658	goto pci_fini;
654	659	}
655	660
656	661	if (goya_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
657	662	dev_info(hdev->dev,
658	663	"H/W state is dirty, must reset before initializing\n");
659		− hdev->asic_funcs->hw_fini(hdev, true);
	664	+ hdev->asic_funcs->hw_fini(hdev, true, false);
660	665	}
661	666
662	667	if (!hdev->pldm) {
@@ -955,8 +960,9 @@ static int goya_sw_init(struct hl_device *hdev) · 292 unmodified lines
955	960	hdev->supports_coresight = true;
956	961	hdev->supports_soft_reset = true;
957	962	hdev->allow_external_soft_reset = true;
	963	+ hdev->supports_wait_for_multi_cs = false;
958	964
959		− goya_set_pci_memory_regions(hdev);
	965	+ hdev->asic_funcs->set_pci_memory_regions(hdev);
960	966
961	967	return 0;
962	968
@@ -2374,7 +2380,7 @@ static void goya_disable_timestamp(struct hl_device *hdev) · 1411 unmodified lines
2374	2380	WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
2375	2381	}
2376	2382
2377		−static void goya_halt_engines(struct hl_device *hdev, bool hard_reset)
	2383	+static void goya_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset)
2378	2384	{
2379	2385	u32 wait_timeout_ms;
2380	2386
@@ -2493,6 +2499,7 @@ static void goya_init_firmware_loader(struct hl_device *hdev) · 112 unmodified lines
2493	2499	struct fw_load_mgr *fw_loader = &hdev->fw_loader;
2494	2500
2495	2501	/* fill common fields */
	2502	+ fw_loader->linux_loaded = false;
2496	2503	fw_loader->boot_fit_img.image_name = GOYA_BOOT_FIT_FILE;
2497	2504	fw_loader->linux_img.image_name = GOYA_LINUX_FW_FILE;
2498	2505	fw_loader->cpu_timeout = GOYA_CPU_TIMEOUT_USEC;
@@ -2696,14 +2703,7 @@ disable_queues: · 197 unmodified lines
2696	2703	return rc;
2697	2704	}
2698	2705
2699		−/*
2700		− * goya_hw_fini - Goya hardware tear-down code
2701		− *
2702		− * @hdev: pointer to hl_device structure
2703		− * @hard_reset: should we do hard reset to all engines or just reset the
2704		− * compute/dma engines
2705		− */
2706		−static void goya_hw_fini(struct hl_device *hdev, bool hard_reset)
	2706	+static void goya_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
2707	2707	{
2708	2708	struct goya_device *goya = hdev->asic_specific;
2709	2709	u32 reset_timeout_ms, cpu_timeout_ms, status;
@@ -2796,7 +2796,7 @@ int goya_resume(struct hl_device *hdev) · 86 unmodified lines
2796	2796	return goya_init_iatu(hdev);
2797	2797	}
2798	2798
2799		−static int goya_cb_mmap(struct hl_device hdev, struct vm_area_struct vma,
	2799	+static int goya_mmap(struct hl_device hdev, struct vm_area_struct vma,
2800	2800	void *cpu_addr, dma_addr_t dma_addr, size_t size)
2801	2801	{
2802	2802	int rc;
@@ -4797,6 +4797,12 @@ void goya_handle_eqe(struct hl_device hdev, struct hl_eq_entry eq_entry) · 1994 unmodified lines
4797	4797	>> EQ_CTL_EVENT_TYPE_SHIFT);
4798	4798	struct goya_device *goya = hdev->asic_specific;
4799	4799
	4800	+ if (event_type >= GOYA_ASYNC_EVENT_ID_SIZE) {
	4801	+ dev_err(hdev->dev, "Event type %u exceeds maximum of %u",
	4802	+ event_type, GOYA_ASYNC_EVENT_ID_SIZE - 1);
	4803	+ return;
	4804	+ }
	4805	+
4800	4806	goya->events_stat[event_type]++;
4801	4807	goya->events_stat_aggregate[event_type]++;
4802	4808
@@ -5475,14 +5481,14 @@ u64 goya_get_device_time(struct hl_device *hdev) · 672 unmodified lines
5475	5481	return device_time \| RREG32(mmPSOC_TIMESTAMP_CNTCVL);
5476	5482	}
5477	5483
5478		−static void goya_collective_wait_init_cs(struct hl_cs *cs)
	5484	+static int goya_collective_wait_init_cs(struct hl_cs *cs)
5479	5485	{
5480		−
	5486	+ return 0;
5481	5487	}
5482	5488
5483	5489	static int goya_collective_wait_create_jobs(struct hl_device *hdev,
5484	5490	struct hl_ctx ctx, struct hl_cs cs, u32 wait_queue_id,
5485		− u32 collective_engine_id)
	5491	+ u32 collective_engine_id, u32 encaps_signal_offset)
5486	5492	{
5487	5493	return -EINVAL;
5488	5494	}
@@ -5524,6 +5530,62 @@ static int goya_map_pll_idx_to_fw_idx(u32 pll_idx) · 35 unmodified lines
5524	5530	}
5525	5531	}
5526	5532
	5533	+static int goya_gen_sync_to_engine_map(struct hl_device *hdev,
	5534	+ struct hl_sync_to_engine_map *map)
	5535	+{
	5536	+ /* Not implemented */
	5537	+ return 0;
	5538	+}
	5539	+
	5540	+static int goya_monitor_valid(struct hl_mon_state_dump *mon)
	5541	+{
	5542	+ /* Not implemented */
	5543	+ return 0;
	5544	+}
	5545	+
	5546	+static int goya_print_single_monitor(char *buf, size_t size, size_t *offset,
	5547	+ struct hl_device *hdev,
	5548	+ struct hl_mon_state_dump *mon)
	5549	+{
	5550	+ /* Not implemented */
	5551	+ return 0;
	5552	+}
	5553	+
	5554	+
	5555	+static int goya_print_fences_single_engine(
	5556	+ struct hl_device *hdev, u64 base_offset, u64 status_base_offset,
	5557	+ enum hl_sync_engine_type engine_type, u32 engine_id, char **buf,
	5558	+ size_t size, size_t offset)
	5559	+{
	5560	+ /* Not implemented */
	5561	+ return 0;
	5562	+}
	5563	+
	5564	+
	5565	+static struct hl_state_dump_specs_funcs goya_state_dump_funcs = {
	5566	+ .monitor_valid = goya_monitor_valid,
	5567	+ .print_single_monitor = goya_print_single_monitor,
	5568	+ .gen_sync_to_engine_map = goya_gen_sync_to_engine_map,
	5569	+ .print_fences_single_engine = goya_print_fences_single_engine,
	5570	+};
	5571	+
	5572	+static void goya_state_dump_init(struct hl_device *hdev)
	5573	+{
	5574	+ /* Not implemented */
	5575	+ hdev->state_dump_specs.props = goya_state_dump_specs_props;
	5576	+ hdev->state_dump_specs.funcs = goya_state_dump_funcs;
	5577	+}
	5578	+
	5579	+static u32 goya_get_sob_addr(struct hl_device *hdev, u32 sob_id)
	5580	+{
	5581	+ return 0;
	5582	+}
	5583	+
	5584	+static u32 *goya_get_stream_master_qid_arr(void)
	5585	+{
	5586	+ return NULL;
	5587	+}
	5588	+
5527	5589	static const struct hl_asic_funcs goya_funcs = {
5528	5590	.early_init = goya_early_init,
5529	5591	.early_fini = goya_early_fini,
@@ -5536,7 +5598,7 @@ static const struct hl_asic_funcs goya_funcs = { · 6 unmodified lines
5536	5598	.halt_engines = goya_halt_engines,
5537	5599	.suspend = goya_suspend,
5538	5600	.resume = goya_resume,
5539		− .cb_mmap = goya_cb_mmap,
	5601	+ .mmap = goya_mmap,
5540	5602	.ring_doorbell = goya_ring_doorbell,
5541	5603	.pqe_write = goya_pqe_write,
5542	5604	.asic_dma_alloc_coherent = goya_dma_alloc_coherent,
@@ -5609,7 +5671,11 @@ static const struct hl_asic_funcs goya_funcs = { · 66 unmodified lines
5609	5671	.enable_events_from_fw = goya_enable_events_from_fw,
5610	5672	.map_pll_idx_to_fw_idx = goya_map_pll_idx_to_fw_idx,
5611	5673	.init_firmware_loader = goya_init_firmware_loader,
5612		− .init_cpu_scrambler_dram = goya_cpu_init_scrambler_dram
	5674	+ .init_cpu_scrambler_dram = goya_cpu_init_scrambler_dram,
	5675	+ .state_dump_init = goya_state_dump_init,
	5676	+ .get_sob_addr = &goya_get_sob_addr,
	5677	+ .set_pci_memory_regions = goya_set_pci_memory_regions,
	5678	+ .get_stream_master_qid_arr = goya_get_stream_master_qid_arr,
5613	5679	};
5614	5680
5615	5681	/*

drivers/misc/habanalabs/include/common/cpucp_if.h+109−6

@@ -98,6 +98,18 @@ struct hl_eq_fw_alive { · 97 unmodified lines
98	98	__u8 pad[7];
99	99	};
100	100
	101	+enum hl_pcie_addr_dec_cause {
	102	+ PCIE_ADDR_DEC_HBW_ERR_RESP,
	103	+ PCIE_ADDR_DEC_LBW_ERR_RESP,
	104	+ PCIE_ADDR_DEC_TLP_BLOCKED_BY_RR
	105	+};
	106	+
	107	+struct hl_eq_pcie_addr_dec_data {
	108	+ /* enum hl_pcie_addr_dec_cause */
	109	+ __u8 addr_dec_cause;
	110	+ __u8 pad[7];
	111	+};
	112	+
101	113	struct hl_eq_entry {
102	114	struct hl_eq_header hdr;
103	115	union {
@@ -106,6 +118,7 @@ struct hl_eq_entry { · 2 unmodified lines
106	118	struct hl_eq_sm_sei_data sm_sei_data;
107	119	struct cpucp_pkt_sync_err pkt_sync_err;
108	120	struct hl_eq_fw_alive fw_alive;
	121	+ struct hl_eq_pcie_addr_dec_data pcie_addr_dec_data;
109	122	__le64 data[7];
110	123	};
111	124	};
@@ -116,7 +129,7 @@ struct hl_eq_entry { · 4 unmodified lines
116	129	#define EQ_CTL_READY_MASK 0x80000000
117	130
118	131	#define EQ_CTL_EVENT_TYPE_SHIFT 16
119		−#define EQ_CTL_EVENT_TYPE_MASK 0x03FF0000
	132	+#define EQ_CTL_EVENT_TYPE_MASK 0x0FFF0000
120	133
121	134	#define EQ_CTL_INDEX_SHIFT 0
122	135	#define EQ_CTL_INDEX_MASK 0x0000FFFF
@@ -300,7 +313,7 @@ enum pq_init_status { · 177 unmodified lines
300	313	* The packet's arguments specify the desired sensor and the field to
301	314	* set.
302	315	*
303		− * CPUCP_PACKET_PCIE_THROUGHPUT_GET
	316	+ * CPUCP_PACKET_PCIE_THROUGHPUT_GET -
304	317	* Get throughput of PCIe.
305	318	* The packet's arguments specify the transaction direction (TX/RX).
306	319	* The window measurement is 10[msec], and the return value is in KB/sec.
@@ -309,19 +322,19 @@ enum pq_init_status { · 2 unmodified lines
309	322	* Replay count measures number of "replay" events, which is basicly
310	323	* number of retries done by PCIe.
311	324	*
312		− * CPUCP_PACKET_TOTAL_ENERGY_GET
	325	+ * CPUCP_PACKET_TOTAL_ENERGY_GET -
313	326	* Total Energy is measurement of energy from the time FW Linux
314	327	* is loaded. It is calculated by multiplying the average power
315	328	* by time (passed from armcp start). The units are in MilliJouls.
316	329	*
317		− * CPUCP_PACKET_PLL_INFO_GET
	330	+ * CPUCP_PACKET_PLL_INFO_GET -
318	331	* Fetch frequencies of PLL from the required PLL IP.
319	332	* The packet's arguments specify the device PLL type
320	333	* Pll type is the PLL from device pll_index enum.
321	334	* The result is composed of 4 outputs, each is 16-bit
322	335	* frequency in MHz.
323	336	*
324		− * CPUCP_PACKET_POWER_GET
	337	+ * CPUCP_PACKET_POWER_GET -
325	338	* Fetch the present power consumption of the device (Current * Voltage).
326	339	*
327	340	* CPUCP_PACKET_NIC_PFC_SET -
@@ -345,6 +358,24 @@ enum pq_init_status { · 17 unmodified lines
345	358	* CPUCP_PACKET_MSI_INFO_SET -
346	359	* set the index number for each supported msi type going from
347	360	* host to device
	361	+ *
	362	+ * CPUCP_PACKET_NIC_XPCS91_REGS_GET -
	363	+ * Fetch the un/correctable counters values from the NIC MAC.
	364	+ *
	365	+ * CPUCP_PACKET_NIC_STAT_REGS_GET -
	366	+ * Fetch various NIC MAC counters from the NIC STAT.
	367	+ *
	368	+ * CPUCP_PACKET_NIC_STAT_REGS_CLR -
	369	+ * Clear the various NIC MAC counters in the NIC STAT.
	370	+ *
	371	+ * CPUCP_PACKET_NIC_STAT_REGS_ALL_GET -
	372	+ * Fetch all NIC MAC counters from the NIC STAT.
	373	+ *
	374	+ * CPUCP_PACKET_IS_IDLE_CHECK -
	375	+ * Check if the device is IDLE in regard to the DMA/compute engines
	376	+ * and QMANs. The f/w will return a bitmask where each bit represents
	377	+ * a different engine or QMAN according to enum cpucp_idle_mask.
	378	+ * The bit will be 1 if the engine is NOT idle.
348	379	*/
349	380
350	381	enum cpucp_packet_id {
@@ -385,6 +416,11 @@ enum cpucp_packet_id { · 34 unmodified lines
385	416	CPUCP_PACKET_NIC_LPBK_SET, /* internal */
386	417	CPUCP_PACKET_NIC_MAC_CFG, /* internal */
387	418	CPUCP_PACKET_MSI_INFO_SET, /* internal */
	419	+ CPUCP_PACKET_NIC_XPCS91_REGS_GET, /* internal */
	420	+ CPUCP_PACKET_NIC_STAT_REGS_GET, /* internal */
	421	+ CPUCP_PACKET_NIC_STAT_REGS_CLR, /* internal */
	422	+ CPUCP_PACKET_NIC_STAT_REGS_ALL_GET, /* internal */
	423	+ CPUCP_PACKET_IS_IDLE_CHECK, /* internal */
388	424	};
389	425
390	426	#define CPUCP_PACKET_FENCE_VAL 0xFE8CE7A5
@@ -414,6 +450,11 @@ enum cpucp_packet_id { · 23 unmodified lines
414	450	#define CPUCP_PKT_VAL_LPBK_IN2_SHIFT 1
415	451	#define CPUCP_PKT_VAL_LPBK_IN2_MASK 0x000000000000001Eull
416	452
	453	+#define CPUCP_PKT_VAL_MAC_CNT_IN1_SHIFT 0
	454	+#define CPUCP_PKT_VAL_MAC_CNT_IN1_MASK 0x0000000000000001ull
	455	+#define CPUCP_PKT_VAL_MAC_CNT_IN2_SHIFT 1
	456	+#define CPUCP_PKT_VAL_MAC_CNT_IN2_MASK 0x00000000FFFFFFFEull
	457	+
417	458	/* heartbeat status bits */
418	459	#define CPUCP_PKT_HB_STATUS_EQ_FAULT_SHIFT 0
419	460	#define CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK 0x00000001
@@ -467,7 +508,8 @@ struct cpucp_packet { · 47 unmodified lines
467	508	__le32 status_mask;
468	509	};
469	510
470		− __le32 reserved;
	511	+ /* For NIC requests */
	512	+ __le32 port_index;
471	513	};
472	514
473	515	struct cpucp_unmask_irq_arr_packet {
@@ -476,6 +518,12 @@ struct cpucp_unmask_irq_arr_packet { · 2 unmodified lines
476	518	__le32 irqs[0];
477	519	};
478	520
	521	+struct cpucp_nic_status_packet {
	522	+ struct cpucp_packet cpucp_pkt;
	523	+ __le32 length;
	524	+ __le32 data[0];
	525	+};
	526	+
479	527	struct cpucp_array_data_packet {
480	528	struct cpucp_packet cpucp_pkt;
481	529	__le32 length;
@@ -595,6 +643,18 @@ enum pll_index { · 113 unmodified lines
595	643	PLL_MAX
596	644	};
597	645
	646	+enum rl_index {
	647	+ TPC_RL = 0,
	648	+ MME_RL,
	649	+};
	650	+
	651	+enum pvt_index {
	652	+ PVT_SW,
	653	+ PVT_SE,
	654	+ PVT_NW,
	655	+ PVT_NE
	656	+};
	657	+
598	658	/* Event Queue Packets */
599	659
600	660	struct eq_generic_event {
@@ -700,6 +760,15 @@ struct cpucp_mac_addr { · 99 unmodified lines
700	760	__u8 mac_addr[ETH_ALEN];
701	761	};
702	762
	763	+enum cpucp_serdes_type {
	764	+ TYPE_1_SERDES_TYPE,
	765	+ TYPE_2_SERDES_TYPE,
	766	+ HLS1_SERDES_TYPE,
	767	+ HLS1H_SERDES_TYPE,
	768	+ UNKNOWN_SERDES_TYPE,
	769	+ MAX_NUM_SERDES_TYPE = UNKNOWN_SERDES_TYPE
	770	+};
	771	+
703	772	struct cpucp_nic_info {
704	773	struct cpucp_mac_addr mac_addrs[CPUCP_MAX_NICS];
705	774	__le64 link_mask[CPUCP_NIC_MASK_ARR_LEN];
@@ -708,6 +777,40 @@ struct cpucp_nic_info { · 2 unmodified lines
708	777	__le64 link_ext_mask[CPUCP_NIC_MASK_ARR_LEN];
709	778	__u8 qsfp_eeprom[CPUCP_NIC_QSFP_EEPROM_MAX_LEN];
710	779	__le64 auto_neg_mask[CPUCP_NIC_MASK_ARR_LEN];
	780	+ __le16 serdes_type; /* enum cpucp_serdes_type */
	781	+ __u8 reserved[6];
	782	+};
	783	+
	784	+/*
	785	+ * struct cpucp_nic_status - describes the status of a NIC port.
	786	+ * @port: NIC port index.
	787	+ * @bad_format_cnt: e.g. CRC.
	788	+ * @responder_out_of_sequence_psn_cnt: e.g NAK.
	789	+ * @high_ber_reinit_cnt: link reinit due to high BER.
	790	+ * @correctable_err_cnt: e.g. bit-flip.
	791	+ * @uncorrectable_err_cnt: e.g. MAC errors.
	792	+ * @retraining_cnt: re-training counter.
	793	+ * @up: is port up.
	794	+ * @pcs_link: has PCS link.
	795	+ * @phy_ready: is PHY ready.
	796	+ * @auto_neg: is Autoneg enabled.
	797	+ * @timeout_retransmission_cnt: timeout retransmission events
	798	+ * @high_ber_cnt: high ber events
	799	+ */
	800	+struct cpucp_nic_status {
	801	+ __le32 port;
	802	+ __le32 bad_format_cnt;
	803	+ __le32 responder_out_of_sequence_psn_cnt;
	804	+ __le32 high_ber_reinit;
	805	+ __le32 correctable_err_cnt;
	806	+ __le32 uncorrectable_err_cnt;
	807	+ __le32 retraining_cnt;
	808	+ __u8 up;
	809	+ __u8 pcs_link;
	810	+ __u8 phy_ready;
	811	+ __u8 auto_neg;
	812	+ __le32 timeout_retransmission_cnt;
	813	+ __le32 high_ber_cnt;
711	814	};
712	815
713	816	#endif /* CPUCP_IF_H */

drivers/misc/habanalabs/include/common/hl_boot_if.h+57−5

@@ -78,6 +78,26 @@ · 77 unmodified lines
78	78	* CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL Device is unusable and customer support
79	79	* should be contacted.
80	80	*
	81	+ * CPU_BOOT_ERR0_ARC0_HALT_ACK_NOT_RCVD HALT ACK from ARC0 is not received
	82	+ * within specified retries after issuing
	83	+ * HALT request. ARC0 appears to be in bad
	84	+ * reset.
	85	+ *
	86	+ * CPU_BOOT_ERR0_ARC1_HALT_ACK_NOT_RCVD HALT ACK from ARC1 is not received
	87	+ * within specified retries after issuing
	88	+ * HALT request. ARC1 appears to be in bad
	89	+ * reset.
	90	+ *
	91	+ * CPU_BOOT_ERR0_ARC0_RUN_ACK_NOT_RCVD RUN ACK from ARC0 is not received
	92	+ * within specified timeout after issuing
	93	+ * RUN request. ARC0 appears to be in bad
	94	+ * reset.
	95	+ *
	96	+ * CPU_BOOT_ERR0_ARC1_RUN_ACK_NOT_RCVD RUN ACK from ARC1 is not received
	97	+ * within specified timeout after issuing
	98	+ * RUN request. ARC1 appears to be in bad
	99	+ * reset.
	100	+ *
81	101	* CPU_BOOT_ERR0_ENABLED Error registers enabled.
82	102	* This is a main indication that the
83	103	* running FW populates the error
@@ -98,6 +118,10 @@ · 14 unmodified lines
98	118	#define CPU_BOOT_ERR0_SEC_IMG_VER_FAIL (1 << 11)
99	119	#define CPU_BOOT_ERR0_PLL_FAIL (1 << 12)
100	120	#define CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL (1 << 13)
	121	+#define CPU_BOOT_ERR0_ARC0_HALT_ACK_NOT_RCVD (1 << 14)
	122	+#define CPU_BOOT_ERR0_ARC1_HALT_ACK_NOT_RCVD (1 << 15)
	123	+#define CPU_BOOT_ERR0_ARC0_RUN_ACK_NOT_RCVD (1 << 16)
	124	+#define CPU_BOOT_ERR0_ARC1_RUN_ACK_NOT_RCVD (1 << 17)
101	125	#define CPU_BOOT_ERR0_ENABLED (1 << 31)
102	126	#define CPU_BOOT_ERR1_ENABLED (1 << 31)
103	127
@@ -186,6 +210,10 @@ · 82 unmodified lines
186	210	* configured and is ready for use.
187	211	* Initialized in: ppboot
188	212	*
	213	+ * CPU_BOOT_DEV_STS0_FW_NIC_MAC_EN NIC MAC channels init is done by FW and
	214	+ * any access to them is done via the FW.
	215	+ * Initialized in: linux
	216	+ *
189	217	* CPU_BOOT_DEV_STS0_DYN_PLL_EN Dynamic PLL configuration is enabled.
190	218	* FW sends to host a bitmap of supported
191	219	* PLLs.
@@ -209,6 +237,21 @@ · 17 unmodified lines
209	237	* prevent IRQs overriding each other.
210	238	* Initialized in: linux
211	239	*
	240	+ * CPU_BOOT_DEV_STS0_FW_NIC_STAT_XPCS91_EN
	241	+ * NIC STAT and XPCS91 access is restricted
	242	+ * and is done via FW only.
	243	+ * Initialized in: linux
	244	+ *
	245	+ * CPU_BOOT_DEV_STS0_FW_NIC_STAT_EXT_EN
	246	+ * NIC STAT get all is supported.
	247	+ * Initialized in: linux
	248	+ *
	249	+ * CPU_BOOT_DEV_STS0_IS_IDLE_CHECK_EN
	250	+ * F/W checks if the device is idle by reading defined set
	251	+ * of registers. It returns a bitmask of all the engines,
	252	+ * where a bit is set if the engine is not idle.
	253	+ * Initialized in: linux
	254	+ *
212	255	* CPU_BOOT_DEV_STS0_ENABLED Device status register enabled.
213	256	* This is a main indication that the
214	257	* running FW populates the device status
@@ -236,10 +279,14 @@ · 21 unmodified lines
236	279	#define CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN (1 << 15)
237	280	#define CPU_BOOT_DEV_STS0_FW_LD_COM_EN (1 << 16)
238	281	#define CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN (1 << 17)
	282	+#define CPU_BOOT_DEV_STS0_FW_NIC_MAC_EN (1 << 18)
239	283	#define CPU_BOOT_DEV_STS0_DYN_PLL_EN (1 << 19)
240	284	#define CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN (1 << 20)
241	285	#define CPU_BOOT_DEV_STS0_EQ_INDEX_EN (1 << 21)
242	286	#define CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN (1 << 22)
	287	+#define CPU_BOOT_DEV_STS0_FW_NIC_STAT_XPCS91_EN (1 << 23)
	288	+#define CPU_BOOT_DEV_STS0_FW_NIC_STAT_EXT_EN (1 << 24)
	289	+#define CPU_BOOT_DEV_STS0_IS_IDLE_CHECK_EN (1 << 25)
243	290	#define CPU_BOOT_DEV_STS0_ENABLED (1 << 31)
244	291	#define CPU_BOOT_DEV_STS1_ENABLED (1 << 31)
245	292
@@ -313,10 +360,7 @@ struct cpu_dyn_regs { · 67 unmodified lines
313	360	__le32 hw_state;
314	361	__le32 kmd_msg_to_cpu;
315	362	__le32 cpu_cmd_status_to_host;
316		− union {
317		− __le32 gic_host_irq_ctrl;
318		− __le32 gic_host_pi_upd_irq;
319		− };
	363	+ __le32 gic_host_pi_upd_irq;
320	364	__le32 gic_tpc_qm_irq_ctrl;
321	365	__le32 gic_mme_qm_irq_ctrl;
322	366	__le32 gic_dma_qm_irq_ctrl;
@@ -324,7 +368,9 @@ struct cpu_dyn_regs { · 1 unmodified line
324	368	__le32 gic_dma_core_irq_ctrl;
325	369	__le32 gic_host_halt_irq;
326	370	__le32 gic_host_ints_irq;
327		− __le32 reserved1[24]; /* reserve for future use */
	371	+ __le32 gic_host_soft_rst_irq;
	372	+ __le32 gic_rot_qm_irq_ctrl;
	373	+ __le32 reserved1[22]; /* reserve for future use */
328	374	};
329	375
330	376	/* TODO: remove the desc magic after the code is updated to use message */
@@ -462,6 +508,11 @@ struct lkd_fw_comms_msg { · 131 unmodified lines
462	508	* Do not wait for BMC response.
463	509	*
464	510	* COMMS_LOW_PLL_OPP Initialize PLLs for low OPP.
	511	+ *
	512	+ * COMMS_PREP_DESC_ELBI Same as COMMS_PREP_DESC only that the memory
	513	+ * space is allocated in a ELBI access only
	514	+ * address range.
	515	+ *
465	516	*/
466	517	enum comms_cmd {
467	518	COMMS_NOOP = 0,
@@ -474,6 +525,7 @@ enum comms_cmd { · 6 unmodified lines
474	525	COMMS_GOTO_WFE = 7,
475	526	COMMS_SKIP_BMC = 8,
476	527	COMMS_LOW_PLL_OPP = 9,
	528	+ COMMS_PREP_DESC_ELBI = 10,
477	529	COMMS_INVLD_LAST
478	530	};
479	531

drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_regs.h+3−0

@@ -126,6 +126,9 @@ · 125 unmodified lines
126	126	#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_1 0x4F2004
127	127	#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_2047 0x4F3FFC
128	128	#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 0x4F4000
	129	+#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 0x4F4800
	130	+#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_DATA_0 0x4F5000
	131	+#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_ARM_0 0x4F5800
129	132	#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0 0x4F6000
130	133	#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_511 0x4F67FC
131	134

drivers/misc/habanalabs/include/gaudi/gaudi_masks.h+17−0

@@ -449,4 +449,21 @@ enum axi_id { · 448 unmodified lines
449	449	#define PCIE_AUX_FLR_CTRL_HW_CTRL_MASK 0x1
450	450	#define PCIE_AUX_FLR_CTRL_INT_MASK_MASK 0x2
451	451
	452	+#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_VALID_SHIFT 0
	453	+#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_VALID_MASK 0x1
	454	+#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_PENDING_SHIFT 1
	455	+#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_PENDING_MASK 0x1FE
	456	+#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SID_SHIFT 0
	457	+#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SID_MASK 0xFF
	458	+#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_MASK_SHIFT 8
	459	+#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_MASK_MASK 0xFF00
	460	+#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SOP_SHIFT 16
	461	+#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SOP_MASK 0x10000
	462	+#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SOD_SHIFT 17
	463	+#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SOD_MASK 0xFFFE0000
	464	+#define TPC0_QM_CP_STS_0_FENCE_ID_SHIFT 20
	465	+#define TPC0_QM_CP_STS_0_FENCE_ID_MASK 0x300000
	466	+#define TPC0_QM_CP_STS_0_FENCE_IN_PROGRESS_SHIFT 22
	467	+#define TPC0_QM_CP_STS_0_FENCE_IN_PROGRESS_MASK 0x400000
	468	+
452	469	#endif /* GAUDI_MASKS_H_ */

drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h+0−2

@@ -12,8 +12,6 @@ · 11 unmodified lines
12	12	* PSOC scratch-pad registers
13	13	*/
14	14	#define mmHW_STATE mmPSOC_GLOBAL_CONF_SCRATCHPAD_0
15		−/* TODO: remove mmGIC_HOST_IRQ_CTRL_POLL_REG */
16		−#define mmGIC_HOST_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_1
17	15	#define mmGIC_HOST_PI_UPD_IRQ_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_1
18	16	#define mmGIC_TPC_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_2
19	17	#define mmGIC_MME_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_3

include/uapi/misc/habanalabs.h+166−20

@@ -276,7 +276,17 @@ enum hl_device_status { · 275 unmodified lines
276	276	HL_DEVICE_STATUS_OPERATIONAL,
277	277	HL_DEVICE_STATUS_IN_RESET,
278	278	HL_DEVICE_STATUS_MALFUNCTION,
279		− HL_DEVICE_STATUS_NEEDS_RESET
	279	+ HL_DEVICE_STATUS_NEEDS_RESET,
	280	+ HL_DEVICE_STATUS_IN_DEVICE_CREATION,
	281	+ HL_DEVICE_STATUS_LAST = HL_DEVICE_STATUS_IN_DEVICE_CREATION
	282	+};
	283	+
	284	+enum hl_server_type {
	285	+ HL_SERVER_TYPE_UNKNOWN = 0,
	286	+ HL_SERVER_GAUDI_HLS1 = 1,
	287	+ HL_SERVER_GAUDI_HLS1H = 2,
	288	+ HL_SERVER_GAUDI_TYPE1 = 3,
	289	+ HL_SERVER_GAUDI_TYPE2 = 4
280	290	};
281	291
282	292	/* Opcode for management ioctl
@@ -337,17 +347,49 @@ enum hl_device_status { · 54 unmodified lines
337	347	#define HL_INFO_VERSION_MAX_LEN 128
338	348	#define HL_INFO_CARD_NAME_MAX_LEN 16
339	349
	350	+/**
	351	+ * struct hl_info_hw_ip_info - hardware information on various IPs in the ASIC
	352	+ * @sram_base_address: The first SRAM physical base address that is free to be
	353	+ * used by the user.
	354	+ * @dram_base_address: The first DRAM virtual or physical base address that is
	355	+ * free to be used by the user.
	356	+ * @dram_size: The DRAM size that is available to the user.
	357	+ * @sram_size: The SRAM size that is available to the user.
	358	+ * @num_of_events: The number of events that can be received from the f/w. This
	359	+ * is needed so the user can what is the size of the h/w events
	360	+ * array he needs to pass to the kernel when he wants to fetch
	361	+ * the event counters.
	362	+ * @device_id: PCI device ID of the ASIC.
	363	+ * @module_id: Module ID of the ASIC for mezzanine cards in servers
	364	+ * (From OCP spec).
	365	+ * @first_available_interrupt_id: The first available interrupt ID for the user
	366	+ * to be used when it works with user interrupts.
	367	+ * @server_type: Server type that the Gaudi ASIC is currently installed in.
	368	+ * The value is according to enum hl_server_type
	369	+ * @cpld_version: CPLD version on the board.
	370	+ * @psoc_pci_pll_nr: PCI PLL NR value. Needed by the profiler in some ASICs.
	371	+ * @psoc_pci_pll_nf: PCI PLL NF value. Needed by the profiler in some ASICs.
	372	+ * @psoc_pci_pll_od: PCI PLL OD value. Needed by the profiler in some ASICs.
	373	+ * @psoc_pci_pll_div_factor: PCI PLL DIV factor value. Needed by the profiler
	374	+ * in some ASICs.
	375	+ * @tpc_enabled_mask: Bit-mask that represents which TPCs are enabled. Relevant
	376	+ * for Goya/Gaudi only.
	377	+ * @dram_enabled: Whether the DRAM is enabled.
	378	+ * @cpucp_version: The CPUCP f/w version.
	379	+ * @card_name: The card name as passed by the f/w.
	380	+ * @dram_page_size: The DRAM physical page size.
	381	+ */
340	382	struct hl_info_hw_ip_info {
341	383	__u64 sram_base_address;
342	384	__u64 dram_base_address;
343	385	__u64 dram_size;
344	386	__u32 sram_size;
345	387	__u32 num_of_events;
346		− __u32 device_id; /* PCI Device ID */
347		− __u32 module_id; /* For mezzanine cards in servers (From OCP spec.) */
	388	+ __u32 device_id;
	389	+ __u32 module_id;
348	390	__u32 reserved;
349	391	__u16 first_available_interrupt_id;
350		− __u16 reserved2;
	392	+ __u16 server_type;
351	393	__u32 cpld_version;
352	394	__u32 psoc_pci_pll_nr;
353	395	__u32 psoc_pci_pll_nf;
@@ -358,7 +400,7 @@ struct hl_info_hw_ip_info { · 4 unmodified lines
358	400	__u8 pad[2];
359	401	__u8 cpucp_version[HL_INFO_VERSION_MAX_LEN];
360	402	__u8 card_name[HL_INFO_CARD_NAME_MAX_LEN];
361		− __u64 reserved3;
	403	+ __u64 reserved2;
362	404	__u64 dram_page_size;
363	405	};
364	406
@@ -628,12 +670,21 @@ struct hl_cs_chunk { · 263 unmodified lines
628	670	__u64 cb_handle;
629	671
630	672	/* Relevant only when HL_CS_FLAGS_WAIT or
631		− * HL_CS_FLAGS_COLLECTIVE_WAIT is set.
	673	+ * HL_CS_FLAGS_COLLECTIVE_WAIT is set
632	674	* This holds address of array of u64 values that contain
633		− * signal CS sequence numbers. The wait described by this job
634		− * will listen on all those signals (wait event per signal)
	675	+ * signal CS sequence numbers. The wait described by
	676	+ * this job will listen on all those signals
	677	+ * (wait event per signal)
635	678	*/
636	679	__u64 signal_seq_arr;
	680	+
	681	+ /*
	682	+ * Relevant only when HL_CS_FLAGS_WAIT or
	683	+ * HL_CS_FLAGS_COLLECTIVE_WAIT is set
	684	+ * along with HL_CS_FLAGS_ENCAP_SIGNALS.
	685	+ * This is the CS sequence which has the encapsulated signals.
	686	+ */
	687	+ __u64 encaps_signal_seq;
637	688	};
638	689
639	690	/* Index of queue to put the CB on */
@@ -651,6 +702,17 @@ struct hl_cs_chunk { · 11 unmodified lines
651	702	* Number of entries in signal_seq_arr
652	703	*/
653	704	__u32 num_signal_seq_arr;
	705	+
	706	+ /* Relevant only when HL_CS_FLAGS_WAIT or
	707	+ * HL_CS_FLAGS_COLLECTIVE_WAIT is set along
	708	+ * with HL_CS_FLAGS_ENCAP_SIGNALS
	709	+ * This set the signals range that the user want to wait for
	710	+ * out of the whole reserved signals range.
	711	+ * e.g if the signals range is 20, and user don't want
	712	+ * to wait for signal 8, so he set this offset to 7, then
	713	+ * he call the API again with 9 and so on till 20.
	714	+ */
	715	+ __u32 encaps_signal_offset;
654	716	};
655	717
656	718	/* HL_CS_CHUNK_FLAGS_* */
@@ -678,6 +740,28 @@ struct hl_cs_chunk { · 21 unmodified lines
678	740	#define HL_CS_FLAGS_CUSTOM_TIMEOUT 0x200
679	741	#define HL_CS_FLAGS_SKIP_RESET_ON_TIMEOUT 0x400
680	742
	743	+/*
	744	+ * The encapsulated signals CS is merged into the existing CS ioctls.
	745	+ * In order to use this feature need to follow the below procedure:
	746	+ * 1. Reserve signals, set the CS type to HL_CS_FLAGS_RESERVE_SIGNALS_ONLY
	747	+ * the output of this API will be the SOB offset from CFG_BASE.
	748	+ * this address will be used to patch CB cmds to do the signaling for this
	749	+ * SOB by incrementing it's value.
	750	+ * for reverting the reservation use HL_CS_FLAGS_UNRESERVE_SIGNALS_ONLY
	751	+ * CS type, note that this might fail if out-of-sync happened to the SOB
	752	+ * value, in case other signaling request to the same SOB occurred between
	753	+ * reserve-unreserve calls.
	754	+ * 2. Use the staged CS to do the encapsulated signaling jobs.
	755	+ * use HL_CS_FLAGS_STAGED_SUBMISSION and HL_CS_FLAGS_STAGED_SUBMISSION_FIRST
	756	+ * along with HL_CS_FLAGS_ENCAP_SIGNALS flag, and set encaps_signal_offset
	757	+ * field. This offset allows app to wait on part of the reserved signals.
	758	+ * 3. Use WAIT/COLLECTIVE WAIT CS along with HL_CS_FLAGS_ENCAP_SIGNALS flag
	759	+ * to wait for the encapsulated signals.
	760	+ */
	761	+#define HL_CS_FLAGS_ENCAP_SIGNALS 0x800
	762	+#define HL_CS_FLAGS_RESERVE_SIGNALS_ONLY 0x1000
	763	+#define HL_CS_FLAGS_UNRESERVE_SIGNALS_ONLY 0x2000
	764	+
681	765	#define HL_CS_STATUS_SUCCESS 0
682	766
683	767	#define HL_MAX_JOBS_PER_CS 512
@@ -690,10 +774,35 @@ struct hl_cs_in { · 6 unmodified lines
690	774	/* holds address of array of hl_cs_chunk for execution phase */
691	775	__u64 chunks_execute;
692	776
693		− /* Sequence number of a staged submission CS
694		− * valid only if HL_CS_FLAGS_STAGED_SUBMISSION is set
695		− */
696		− __u64 seq;
	777	+ union {
	778	+ /*
	779	+ * Sequence number of a staged submission CS
	780	+ * valid only if HL_CS_FLAGS_STAGED_SUBMISSION is set and
	781	+ * HL_CS_FLAGS_STAGED_SUBMISSION_FIRST is unset.
	782	+ */
	783	+ __u64 seq;
	784	+
	785	+ /*
	786	+ * Encapsulated signals handle id
	787	+ * Valid for two flows:
	788	+ * 1. CS with encapsulated signals:
	789	+ * when HL_CS_FLAGS_STAGED_SUBMISSION and
	790	+ * HL_CS_FLAGS_STAGED_SUBMISSION_FIRST
	791	+ * and HL_CS_FLAGS_ENCAP_SIGNALS are set.
	792	+ * 2. unreserve signals:
	793	+ * valid when HL_CS_FLAGS_UNRESERVE_SIGNALS_ONLY is set.
	794	+ */
	795	+ __u32 encaps_sig_handle_id;
	796	+
	797	+ /* Valid only when HL_CS_FLAGS_RESERVE_SIGNALS_ONLY is set */
	798	+ struct {
	799	+ /* Encapsulated signals number */
	800	+ __u32 encaps_signals_count;
	801	+
	802	+ /* Encapsulated signals queue index (stream) */
	803	+ __u32 encaps_signals_q_idx;
	804	+ };
	805	+ };
697	806
698	807	/* Number of chunks in restore phase array. Maximum number is
699	808	* HL_MAX_JOBS_PER_CS
@@ -718,14 +827,31 @@ struct hl_cs_in { · 18 unmodified lines
718	827	};
719	828
720	829	struct hl_cs_out {
	830	+ union {
	831	+ /*
	832	+ * seq holds the sequence number of the CS to pass to wait
	833	+ * ioctl. All values are valid except for 0 and ULLONG_MAX
	834	+ */
	835	+ __u64 seq;
	836	+
	837	+ /* Valid only when HL_CS_FLAGS_RESERVE_SIGNALS_ONLY is set */
	838	+ struct {
	839	+ /* This is the resereved signal handle id */
	840	+ __u32 handle_id;
	841	+
	842	+ /* This is the signals count */
	843	+ __u32 count;
	844	+ };
	845	+ };
	846	+
	847	+ /* HL_CS_STATUS */
	848	+ __u32 status;
	849	+
721	850	/*
722		− * seq holds the sequence number of the CS to pass to wait ioctl. All
723		− * values are valid except for 0 and ULLONG_MAX
	851	+ * SOB base address offset
	852	+ * Valid only when HL_CS_FLAGS_RESERVE_SIGNALS_ONLY is set
724	853	*/
725		− __u64 seq;
726		− /* HL_CS_STATUS_* */
727		− __u32 status;
728		− __u32 pad;
	854	+ __u32 sob_base_addr_offset;
729	855	};
730	856
731	857	union hl_cs_args {
@@ -735,11 +861,18 @@ union hl_cs_args { · 3 unmodified lines
735	861
736	862	#define HL_WAIT_CS_FLAGS_INTERRUPT 0x2
737	863	#define HL_WAIT_CS_FLAGS_INTERRUPT_MASK 0xFFF00000
	864	+#define HL_WAIT_CS_FLAGS_MULTI_CS 0x4
	865	+
	866	+#define HL_WAIT_MULTI_CS_LIST_MAX_LEN 32
738	867
739	868	struct hl_wait_cs_in {
740	869	union {
741	870	struct {
742		− /* Command submission sequence number */
	871	+ /*
	872	+ * In case of wait_cs holds the CS sequence number.
	873	+ * In case of wait for multi CS hold a user pointer to
	874	+ * an array of CS sequence numbers
	875	+ */
743	876	__u64 seq;
744	877	/* Absolute timeout to wait for command submission
745	878	* in microseconds
@@ -767,12 +900,17 @@ struct hl_wait_cs_in { · 21 unmodified lines
767	900
768	901	/* Context ID - Currently not in use */
769	902	__u32 ctx_id;
	903	+
770	904	/* HL_WAIT_CS_FLAGS_*
771	905	* If HL_WAIT_CS_FLAGS_INTERRUPT is set, this field should include
772	906	* interrupt id according to HL_WAIT_CS_FLAGS_INTERRUPT_MASK, in order
773	907	* not to specify an interrupt id ,set mask to all 1s.
774	908	*/
775	909	__u32 flags;
	910	+
	911	+ /* Multi CS API info- valid entries in multi-CS array */
	912	+ __u8 seq_arr_len;
	913	+ __u8 pad[7];
776	914	};
777	915
778	916	#define HL_WAIT_CS_STATUS_COMPLETED 0
@@ -789,8 +927,15 @@ struct hl_wait_cs_out { · 10 unmodified lines
789	927	__u32 status;
790	928	/* HL_WAIT_CS_STATUS_FLAG* */
791	929	__u32 flags;
792		− /* valid only if HL_WAIT_CS_STATUS_FLAG_TIMESTAMP_VLD is set */
	930	+ /*
	931	+ * valid only if HL_WAIT_CS_STATUS_FLAG_TIMESTAMP_VLD is set
	932	+ * for wait_cs: timestamp of CS completion
	933	+ * for wait_multi_cs: timestamp of FIRST CS completion
	934	+ */
793	935	__s64 timestamp_nsec;
	936	+ /* multi CS completion bitmap */
	937	+ __u32 cs_completion_map;
	938	+ __u32 pad;
794	939	};
795	940
796	941	union hl_wait_cs_args {
@@ -813,6 +958,7 @@ union hl_wait_cs_args { · 16 unmodified lines
813	958	#define HL_MEM_CONTIGUOUS 0x1
814	959	#define HL_MEM_SHARED 0x2
815	960	#define HL_MEM_USERPTR 0x4
	961	+#define HL_MEM_FORCE_HINT 0x8
816	962
817	963	struct hl_mem_in {
818	964	union {