History

Merge tag 'char-misc-5.15-rc1-2' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc

Pull habanalabs updates from Greg KH: "Here is another round of misc driver patches for 5.15-rc1. In here is only updates for the Habanalabs driver. This request is late because the previously-objected-to dma-buf patches are all removed and some fixes that you and others found are now included in here as well. All of these have been in linux-next for well over a week with no reports of problems, and they are all self-contained to only this one driver. Full details are in the shortlog" * tag 'char-misc-5.15-rc1-2' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc: (61 commits) habanalabs/gaudi: hwmon default card name habanalabs: add support for f/w reset habanalabs/gaudi: block ICACHE_BASE_ADDERESS_HIGH in TPC habanalabs: cannot sleep while holding spinlock habanalabs: never copy_from_user inside spinlock habanalabs: remove unnecessary device status check habanalabs: disable IRQ in user interrupts spinlock habanalabs: add "in device creation" status habanalabs/gaudi: invalidate PMMU mem cache on init habanalabs/gaudi: size should be printed in decimal habanalabs/gaudi: define DC POWER for secured PMC habanalabs/gaudi: unmask out of bounds SLM access interrupt habanalabs: add userptr_lookup node in debugfs habanalabs/gaudi: fetch TPC/MME ECC errors from F/W habanalabs: modify multi-CS to wait on stream masters habanalabs/gaudi: add monitored SOBs to state dump habanalabs/gaudi: restore user registers when context opens habanalabs/gaudi: increase boot fit timeout habanalabs: update to latest firmware headers habanalabs/gaudi: minimize number of register reads ...

Linus Torvalds committed 5 years ago

Commit 5ffc06e2 parents a668acb 4cd67ad28 files · +3918 −834Browse files

Unified · Split

28 files
  1. Documentation/ABI/testing/debugfs-driver-habanalabs +19 −0
  2. drivers/misc/habanalabs/common/Makefile +2 −1
  3. drivers/misc/habanalabs/common/command_buffer.c +1 −3
  4. drivers/misc/habanalabs/common/command_submission.c +1049 −338
  5. drivers/misc/habanalabs/common/context.c +126 −20
  6. drivers/misc/habanalabs/common/debugfs.c +174 −10
  7. drivers/misc/habanalabs/common/device.c +86 −77
  8. drivers/misc/habanalabs/common/firmware_if.c +32 −24
  9. drivers/misc/habanalabs/common/habanalabs.h +375 −46
  10. drivers/misc/habanalabs/common/habanalabs_drv.c +8 −5
  11. drivers/misc/habanalabs/common/habanalabs_ioctl.c +2 −0
  12. drivers/misc/habanalabs/common/hw_queue.c +155 −43
  13. drivers/misc/habanalabs/common/memory.c +123 −46
  14. drivers/misc/habanalabs/common/mmu/mmu_v1.c +6 −6
  15. drivers/misc/habanalabs/common/pci/pci.c +2 −0
  16. drivers/misc/habanalabs/common/state_dump.c +718 −0
  17. drivers/misc/habanalabs/common/sysfs.c +7 −13
  18. drivers/misc/habanalabs/gaudi/gaudi.c +571 −145
  19. drivers/misc/habanalabs/gaudi/gaudiP.h +18 −1
  20. drivers/misc/habanalabs/gaudi/gaudi_coresight.c +0 −5
  21. drivers/misc/habanalabs/gaudi/gaudi_security.c +8 −0
  22. drivers/misc/habanalabs/goya/goya.c +84 −18
  23. drivers/misc/habanalabs/include/common/cpucp_if.h +109 −6
  24. drivers/misc/habanalabs/include/common/hl_boot_if.h +57 −5
  25. drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_regs.h +3 −0
  26. drivers/misc/habanalabs/include/gaudi/gaudi_masks.h +17 −0
  27. drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h +0 −2
  28. include/uapi/misc/habanalabs.h +166 −20
Documentation/ABI/testing/debugfs-driver-habanalabs+19−0
@@ -215,6 +215,17 @@ Description: Sets the skip reset on timeout option for the device. Value of · 214 unmodified lines
215 "0" means device will be reset in case some CS has timed out,215 "0" means device will be reset in case some CS has timed out,
216 otherwise it will not be reset.216 otherwise it will not be reset.
217 217
218What: /sys/kernel/debug/habanalabs/hl<n>/state_dump
219Date: Oct 2021
220KernelVersion: 5.15
221Contact: ynudelman@habana.ai
222Description: Gets the state dump occurring on a CS timeout or failure.
223 State dump is used for debug and is created each time in case of
224 a problem in a CS execution, before reset.
225 Reading from the node returns the newest state dump available.
226 Writing an integer X discards X state dumps, so that the
227 next read would return X+1-st newest state dump.
228
218What: /sys/kernel/debug/habanalabs/hl<n>/stop_on_err229What: /sys/kernel/debug/habanalabs/hl<n>/stop_on_err
219Date: Mar 2020230Date: Mar 2020
220KernelVersion: 5.6231KernelVersion: 5.6
@@ -230,6 +241,14 @@ Description: Displays a list with information about the currently user · 9 unmodified lines
230 pointers (user virtual addresses) that are pinned and mapped241 pointers (user virtual addresses) that are pinned and mapped
231 to DMA addresses242 to DMA addresses
232 243
244What: /sys/kernel/debug/habanalabs/hl<n>/userptr_lookup
245Date: Aug 2021
246KernelVersion: 5.15
247Contact: ogabbay@kernel.org
248Description: Allows to search for specific user pointers (user virtual
249 addresses) that are pinned and mapped to DMA addresses, and see
250 their resolution to the specific dma address.
251
233What: /sys/kernel/debug/habanalabs/hl<n>/vm252What: /sys/kernel/debug/habanalabs/hl<n>/vm
234Date: Jan 2019253Date: Jan 2019
235KernelVersion: 5.1254KernelVersion: 5.1
drivers/misc/habanalabs/common/Makefile+2−1
@@ -10,4 +10,5 @@ HL_COMMON_FILES := common/habanalabs_drv.o common/device.o common/context.o \ · 9 unmodified lines
10 common/asid.o common/habanalabs_ioctl.o \10 common/asid.o common/habanalabs_ioctl.o \
11 common/command_buffer.o common/hw_queue.o common/irq.o \11 common/command_buffer.o common/hw_queue.o common/irq.o \
12 common/sysfs.o common/hwmon.o common/memory.o \12 common/sysfs.o common/hwmon.o common/memory.o \
13 common/command_submission.o common/firmware_if.o13 common/command_submission.o common/firmware_if.o \
14 common/state_dump.o
drivers/misc/habanalabs/common/command_buffer.c+1−3
@@ -314,8 +314,6 @@ int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr, · 313 unmodified lines
314 314
315 spin_lock(&mgr->cb_lock);315 spin_lock(&mgr->cb_lock);
316 rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);316 rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
317 if (rc < 0)
318 rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_KERNEL);
319 spin_unlock(&mgr->cb_lock);317 spin_unlock(&mgr->cb_lock);
320 318
321 if (rc < 0) {319 if (rc < 0) {
@@ -552,7 +550,7 @@ int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma) · 230 unmodified lines
552 550
553 vma->vm_private_data = cb;551 vma->vm_private_data = cb;
554 552
555 rc = hdev->asic_funcs->cb_mmap(hdev, vma, cb->kernel_address,553 rc = hdev->asic_funcs->mmap(hdev, vma, cb->kernel_address,
556 cb->bus_address, cb->size);554 cb->bus_address, cb->size);
557 if (rc) {555 if (rc) {
558 spin_lock(&cb->lock);556 spin_lock(&cb->lock);
drivers/misc/habanalabs/common/command_submission.c+1049−338
@@ -38,7 +38,11 @@ static void hl_sob_reset(struct kref *ref) · 37 unmodified lines
38 kref);38 kref);
39 struct hl_device *hdev = hw_sob->hdev;39 struct hl_device *hdev = hw_sob->hdev;
40 40
41 dev_dbg(hdev->dev, "reset sob id %u\n", hw_sob->sob_id);
42
41 hdev->asic_funcs->reset_sob(hdev, hw_sob);43 hdev->asic_funcs->reset_sob(hdev, hw_sob);
44
45 hw_sob->need_reset = false;
42}46}
43 47
44void hl_sob_reset_error(struct kref *ref)48void hl_sob_reset_error(struct kref *ref)
@@ -52,6 +56,24 @@ void hl_sob_reset_error(struct kref *ref) · 7 unmodified lines
52 hw_sob->q_idx, hw_sob->sob_id);56 hw_sob->q_idx, hw_sob->sob_id);
53}57}
54 58
59void hw_sob_put(struct hl_hw_sob *hw_sob)
60{
61 if (hw_sob)
62 kref_put(&hw_sob->kref, hl_sob_reset);
63}
64
65static void hw_sob_put_err(struct hl_hw_sob *hw_sob)
66{
67 if (hw_sob)
68 kref_put(&hw_sob->kref, hl_sob_reset_error);
69}
70
71void hw_sob_get(struct hl_hw_sob *hw_sob)
72{
73 if (hw_sob)
74 kref_get(&hw_sob->kref);
75}
76
55/**77/**
56 * hl_gen_sob_mask() - Generates a sob mask to be used in a monitor arm packet78 * hl_gen_sob_mask() - Generates a sob mask to be used in a monitor arm packet
57 * @sob_base: sob base id79 * @sob_base: sob base id
@@ -84,76 +106,29 @@ int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask) · 26 unmodified lines
84 return 0;106 return 0;
85}107}
86 108
87static void sob_reset_work(struct work_struct *work)
88{
89 struct hl_cs_compl *hl_cs_cmpl =
90 container_of(work, struct hl_cs_compl, sob_reset_work);
91 struct hl_device *hdev = hl_cs_cmpl->hdev;
92
93 /*
94 * A signal CS can get completion while the corresponding wait
95 * for signal CS is on its way to the PQ. The wait for signal CS
96 * will get stuck if the signal CS incremented the SOB to its
97 * max value and there are no pending (submitted) waits on this
98 * SOB.
99 * We do the following to void this situation:
100 * 1. The wait for signal CS must get a ref for the signal CS as
101 * soon as possible in cs_ioctl_signal_wait() and put it
102 * before being submitted to the PQ but after it incremented
103 * the SOB refcnt in init_signal_wait_cs().
104 * 2. Signal/Wait for signal CS will decrement the SOB refcnt
105 * here.
106 * These two measures guarantee that the wait for signal CS will
107 * reset the SOB upon completion rather than the signal CS and
108 * hence the above scenario is avoided.
109 */
110 kref_put(&hl_cs_cmpl->hw_sob->kref, hl_sob_reset);
111
112 if (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT)
113 hdev->asic_funcs->reset_sob_group(hdev,
114 hl_cs_cmpl->sob_group);
115
116 kfree(hl_cs_cmpl);
117}
118
119static void hl_fence_release(struct kref *kref)109static void hl_fence_release(struct kref *kref)
120{110{
121 struct hl_fence *fence =111 struct hl_fence *fence =
122 container_of(kref, struct hl_fence, refcount);112 container_of(kref, struct hl_fence, refcount);
123 struct hl_cs_compl *hl_cs_cmpl =113 struct hl_cs_compl *hl_cs_cmpl =
124 container_of(fence, struct hl_cs_compl, base_fence);114 container_of(fence, struct hl_cs_compl, base_fence);
125 struct hl_device *hdev = hl_cs_cmpl->hdev;
126
127 /* EBUSY means the CS was never submitted and hence we don't have
128 * an attached hw_sob object that we should handle here
129 */
130 if (fence->error == -EBUSY)
131 goto free;
132
133 if ((hl_cs_cmpl->type == CS_TYPE_SIGNAL) ||
134 (hl_cs_cmpl->type == CS_TYPE_WAIT) ||
135 (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT)) {
136
137 dev_dbg(hdev->dev,
138 "CS 0x%llx type %d finished, sob_id: %d, sob_val: 0x%x\n",
139 hl_cs_cmpl->cs_seq,
140 hl_cs_cmpl->type,
141 hl_cs_cmpl->hw_sob->sob_id,
142 hl_cs_cmpl->sob_val);
143
144 queue_work(hdev->sob_reset_wq, &hl_cs_cmpl->sob_reset_work);
145 115
146 return;
147 }
148
149free:
150 kfree(hl_cs_cmpl);116 kfree(hl_cs_cmpl);
151}117}
152 118
153void hl_fence_put(struct hl_fence *fence)119void hl_fence_put(struct hl_fence *fence)
154{120{
155 if (fence)121 if (IS_ERR_OR_NULL(fence))
156 kref_put(&fence->refcount, hl_fence_release);122 return;
123 kref_put(&fence->refcount, hl_fence_release);
124}
125
126void hl_fences_put(struct hl_fence **fence, int len)
127{
128 int i;
129
130 for (i = 0; i < len; i++, fence++)
131 hl_fence_put(*fence);
157}132}
158 133
159void hl_fence_get(struct hl_fence *fence)134void hl_fence_get(struct hl_fence *fence)
@@ -473,11 +448,139 @@ static void cs_handle_tdr(struct hl_device *hdev, struct hl_cs *cs) · 313 unmodified lines
473 spin_unlock(&hdev->cs_mirror_lock);448 spin_unlock(&hdev->cs_mirror_lock);
474}449}
475 450
451/*
452 * force_complete_multi_cs - complete all contexts that wait on multi-CS
453 *
454 * @hdev: pointer to habanalabs device structure
455 */
456static void force_complete_multi_cs(struct hl_device *hdev)
457{
458 int i;
459
460 for (i = 0; i < MULTI_CS_MAX_USER_CTX; i++) {
461 struct multi_cs_completion *mcs_compl;
462
463 mcs_compl = &hdev->multi_cs_completion[i];
464
465 spin_lock(&mcs_compl->lock);
466
467 if (!mcs_compl->used) {
468 spin_unlock(&mcs_compl->lock);
469 continue;
470 }
471
472 /* when calling force complete no context should be waiting on
473 * multi-cS.
474 * We are calling the function as a protection for such case
475 * to free any pending context and print error message
476 */
477 dev_err(hdev->dev,
478 "multi-CS completion context %d still waiting when calling force completion\n",
479 i);
480 complete_all(&mcs_compl->completion);
481 spin_unlock(&mcs_compl->lock);
482 }
483}
484
485/*
486 * complete_multi_cs - complete all waiting entities on multi-CS
487 *
488 * @hdev: pointer to habanalabs device structure
489 * @cs: CS structure
490 * The function signals a waiting entity that has an overlapping stream masters
491 * with the completed CS.
492 * For example:
493 * - a completed CS worked on stream master QID 4, multi CS completion
494 * is actively waiting on stream master QIDs 3, 5. don't send signal as no
495 * common stream master QID
496 * - a completed CS worked on stream master QID 4, multi CS completion
497 * is actively waiting on stream master QIDs 3, 4. send signal as stream
498 * master QID 4 is common
499 */
500static void complete_multi_cs(struct hl_device *hdev, struct hl_cs *cs)
501{
502 struct hl_fence *fence = cs->fence;
503 int i;
504
505 /* in case of multi CS check for completion only for the first CS */
506 if (cs->staged_cs && !cs->staged_first)
507 return;
508
509 for (i = 0; i < MULTI_CS_MAX_USER_CTX; i++) {
510 struct multi_cs_completion *mcs_compl;
511
512 mcs_compl = &hdev->multi_cs_completion[i];
513 if (!mcs_compl->used)
514 continue;
515
516 spin_lock(&mcs_compl->lock);
517
518 /*
519 * complete if:
520 * 1. still waiting for completion
521 * 2. the completed CS has at least one overlapping stream
522 * master with the stream masters in the completion
523 */
524 if (mcs_compl->used &&
525 (fence->stream_master_qid_map &
526 mcs_compl->stream_master_qid_map)) {
527 /* extract the timestamp only of first completed CS */
528 if (!mcs_compl->timestamp)
529 mcs_compl->timestamp =
530 ktime_to_ns(fence->timestamp);
531 complete_all(&mcs_compl->completion);
532 }
533
534 spin_unlock(&mcs_compl->lock);
535 }
536}
537
538static inline void cs_release_sob_reset_handler(struct hl_device *hdev,
539 struct hl_cs *cs,
540 struct hl_cs_compl *hl_cs_cmpl)
541{
542 /* Skip this handler if the cs wasn't submitted, to avoid putting
543 * the hw_sob twice, since this case already handled at this point,
544 * also skip if the hw_sob pointer wasn't set.
545 */
546 if (!hl_cs_cmpl->hw_sob || !cs->submitted)
547 return;
548
549 spin_lock(&hl_cs_cmpl->lock);
550
551 /*
552 * we get refcount upon reservation of signals or signal/wait cs for the
553 * hw_sob object, and need to put it when the first staged cs
554 * (which cotains the encaps signals) or cs signal/wait is completed.
555 */
556 if ((hl_cs_cmpl->type == CS_TYPE_SIGNAL) ||
557 (hl_cs_cmpl->type == CS_TYPE_WAIT) ||
558 (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT) ||
559 (!!hl_cs_cmpl->encaps_signals)) {
560 dev_dbg(hdev->dev,
561 "CS 0x%llx type %d finished, sob_id: %d, sob_val: %u\n",
562 hl_cs_cmpl->cs_seq,
563 hl_cs_cmpl->type,
564 hl_cs_cmpl->hw_sob->sob_id,
565 hl_cs_cmpl->sob_val);
566
567 hw_sob_put(hl_cs_cmpl->hw_sob);
568
569 if (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT)
570 hdev->asic_funcs->reset_sob_group(hdev,
571 hl_cs_cmpl->sob_group);
572 }
573
574 spin_unlock(&hl_cs_cmpl->lock);
575}
576
476static void cs_do_release(struct kref *ref)577static void cs_do_release(struct kref *ref)
477{578{
478 struct hl_cs *cs = container_of(ref, struct hl_cs, refcount);579 struct hl_cs *cs = container_of(ref, struct hl_cs, refcount);
479 struct hl_device *hdev = cs->ctx->hdev;580 struct hl_device *hdev = cs->ctx->hdev;
480 struct hl_cs_job *job, *tmp;581 struct hl_cs_job *job, *tmp;
582 struct hl_cs_compl *hl_cs_cmpl =
583 container_of(cs->fence, struct hl_cs_compl, base_fence);
481 584
482 cs->completed = true;585 cs->completed = true;
483 586
@@ -493,8 +596,9 @@ static void cs_do_release(struct kref *ref) · 9 unmodified lines
493 complete_job(hdev, job);596 complete_job(hdev, job);
494 597
495 if (!cs->submitted) {598 if (!cs->submitted) {
496 /* In case the wait for signal CS was submitted, the put occurs599 /*
497 * in init_signal_wait_cs() or collective_wait_init_cs()600 * In case the wait for signal CS was submitted, the fence put
601 * occurs in init_signal_wait_cs() or collective_wait_init_cs()
498 * right before hanging on the PQ.602 * right before hanging on the PQ.
499 */603 */
500 if (cs->type == CS_TYPE_WAIT ||604 if (cs->type == CS_TYPE_WAIT ||
@@ -535,8 +639,20 @@ static void cs_do_release(struct kref *ref) · 34 unmodified lines
535 list_del(&cs->staged_cs_node);639 list_del(&cs->staged_cs_node);
536 spin_unlock(&hdev->cs_mirror_lock);640 spin_unlock(&hdev->cs_mirror_lock);
537 }641 }
642
643 /* decrement refcount to handle when first staged cs
644 * with encaps signals is completed.
645 */
646 if (hl_cs_cmpl->encaps_signals)
647 kref_put(&hl_cs_cmpl->encaps_sig_hdl->refcount,
648 hl_encaps_handle_do_release);
538 }649 }
539 650
651 if ((cs->type == CS_TYPE_WAIT || cs->type == CS_TYPE_COLLECTIVE_WAIT)
652 && cs->encaps_signals)
653 kref_put(&cs->encaps_sig_hdl->refcount,
654 hl_encaps_handle_do_release);
655
540out:656out:
541 /* Must be called before hl_ctx_put because inside we use ctx to get657 /* Must be called before hl_ctx_put because inside we use ctx to get
542 * the device658 * the device
@@ -566,6 +682,10 @@ out: · 23 unmodified lines
566 if (cs->timestamp)682 if (cs->timestamp)
567 cs->fence->timestamp = ktime_get();683 cs->fence->timestamp = ktime_get();
568 complete_all(&cs->fence->completion);684 complete_all(&cs->fence->completion);
685 complete_multi_cs(hdev, cs);
686
687 cs_release_sob_reset_handler(hdev, cs, hl_cs_cmpl);
688
569 hl_fence_put(cs->fence);689 hl_fence_put(cs->fence);
570 690
571 kfree(cs->jobs_in_queue_cnt);691 kfree(cs->jobs_in_queue_cnt);
@@ -621,6 +741,10 @@ static void cs_timedout(struct work_struct *work) · 49 unmodified lines
621 break;741 break;
622 }742 }
623 743
744 rc = hl_state_dump(hdev);
745 if (rc)
746 dev_err(hdev->dev, "Error during system state dump %d\n", rc);
747
624 cs_put(cs);748 cs_put(cs);
625 749
626 if (likely(!skip_reset_on_timeout)) {750 if (likely(!skip_reset_on_timeout)) {
@@ -661,6 +785,7 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, · 34 unmodified lines
661 cs->completed = false;785 cs->completed = false;
662 cs->type = cs_type;786 cs->type = cs_type;
663 cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP);787 cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP);
788 cs->encaps_signals = !!(flags & HL_CS_FLAGS_ENCAP_SIGNALS);
664 cs->timeout_jiffies = timeout;789 cs->timeout_jiffies = timeout;
665 cs->skip_reset_on_timeout =790 cs->skip_reset_on_timeout =
666 hdev->skip_reset_on_timeout ||791 hdev->skip_reset_on_timeout ||
@@ -671,9 +796,9 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, · 4 unmodified lines
671 kref_init(&cs->refcount);796 kref_init(&cs->refcount);
672 spin_lock_init(&cs->job_lock);797 spin_lock_init(&cs->job_lock);
673 798
674 cs_cmpl = kmalloc(sizeof(*cs_cmpl), GFP_ATOMIC);799 cs_cmpl = kzalloc(sizeof(*cs_cmpl), GFP_ATOMIC);
675 if (!cs_cmpl)800 if (!cs_cmpl)
676 cs_cmpl = kmalloc(sizeof(*cs_cmpl), GFP_KERNEL);801 cs_cmpl = kzalloc(sizeof(*cs_cmpl), GFP_KERNEL);
677 802
678 if (!cs_cmpl) {803 if (!cs_cmpl) {
679 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);804 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
@@ -698,7 +823,6 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, · 18 unmodified lines
698 cs_cmpl->hdev = hdev;823 cs_cmpl->hdev = hdev;
699 cs_cmpl->type = cs->type;824 cs_cmpl->type = cs->type;
700 spin_lock_init(&cs_cmpl->lock);825 spin_lock_init(&cs_cmpl->lock);
701 INIT_WORK(&cs_cmpl->sob_reset_work, sob_reset_work);
702 cs->fence = &cs_cmpl->base_fence;826 cs->fence = &cs_cmpl->base_fence;
703 827
704 spin_lock(&ctx->cs_lock);828 spin_lock(&ctx->cs_lock);
@@ -791,31 +915,22 @@ void hl_cs_rollback_all(struct hl_device *hdev) · 86 unmodified lines
791 cs_rollback(hdev, cs);915 cs_rollback(hdev, cs);
792 cs_put(cs);916 cs_put(cs);
793 }917 }
794}
795
796void hl_pending_cb_list_flush(struct hl_ctx *ctx)
797{
798 struct hl_pending_cb *pending_cb, *tmp;
799 918
800 list_for_each_entry_safe(pending_cb, tmp,919 force_complete_multi_cs(hdev);
801 &ctx->pending_cb_list, cb_node) {
802 list_del(&pending_cb->cb_node);
803 hl_cb_put(pending_cb->cb);
804 kfree(pending_cb);
805 }
806}920}
807 921
808static void922static void
809wake_pending_user_interrupt_threads(struct hl_user_interrupt *interrupt)923wake_pending_user_interrupt_threads(struct hl_user_interrupt *interrupt)
810{924{
811 struct hl_user_pending_interrupt *pend;925 struct hl_user_pending_interrupt *pend;
926 unsigned long flags;
812 927
813 spin_lock(&interrupt->wait_list_lock);928 spin_lock_irqsave(&interrupt->wait_list_lock, flags);
814 list_for_each_entry(pend, &interrupt->wait_list_head, wait_list_node) {929 list_for_each_entry(pend, &interrupt->wait_list_head, wait_list_node) {
815 pend->fence.error = -EIO;930 pend->fence.error = -EIO;
816 complete_all(&pend->fence.completion);931 complete_all(&pend->fence.completion);
817 }932 }
818 spin_unlock(&interrupt->wait_list_lock);933 spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
819}934}
820 935
821void hl_release_pending_user_interrupts(struct hl_device *hdev)936void hl_release_pending_user_interrupts(struct hl_device *hdev)
@@ -981,6 +1096,10 @@ static enum hl_cs_type hl_cs_get_cs_type(u32 cs_type_flags) · 159 unmodified lines
981 return CS_TYPE_WAIT;1096 return CS_TYPE_WAIT;
982 else if (cs_type_flags & HL_CS_FLAGS_COLLECTIVE_WAIT)1097 else if (cs_type_flags & HL_CS_FLAGS_COLLECTIVE_WAIT)
983 return CS_TYPE_COLLECTIVE_WAIT;1098 return CS_TYPE_COLLECTIVE_WAIT;
1099 else if (cs_type_flags & HL_CS_FLAGS_RESERVE_SIGNALS_ONLY)
1100 return CS_RESERVE_SIGNALS;
1101 else if (cs_type_flags & HL_CS_FLAGS_UNRESERVE_SIGNALS_ONLY)
1102 return CS_UNRESERVE_SIGNALS;
984 else1103 else
985 return CS_TYPE_DEFAULT;1104 return CS_TYPE_DEFAULT;
986}1105}
@@ -1081,7 +1200,8 @@ static int hl_cs_copy_chunk_array(struct hl_device *hdev, · 94 unmodified lines
1081}1200}
1082 1201
1083static int cs_staged_submission(struct hl_device *hdev, struct hl_cs *cs,1202static int cs_staged_submission(struct hl_device *hdev, struct hl_cs *cs,
1084 u64 sequence, u32 flags)1203 u64 sequence, u32 flags,
1204 u32 encaps_signal_handle)
1085{1205{
1086 if (!(flags & HL_CS_FLAGS_STAGED_SUBMISSION))1206 if (!(flags & HL_CS_FLAGS_STAGED_SUBMISSION))
1087 return 0;1207 return 0;
@@ -1093,6 +1213,9 @@ static int cs_staged_submission(struct hl_device *hdev, struct hl_cs *cs, · 5 unmodified lines
1093 /* Staged CS sequence is the first CS sequence */1213 /* Staged CS sequence is the first CS sequence */
1094 INIT_LIST_HEAD(&cs->staged_cs_node);1214 INIT_LIST_HEAD(&cs->staged_cs_node);
1095 cs->staged_sequence = cs->sequence;1215 cs->staged_sequence = cs->sequence;
1216
1217 if (cs->encaps_signals)
1218 cs->encaps_sig_hdl_id = encaps_signal_handle;
1096 } else {1219 } else {
1097 /* User sequence will be validated in 'hl_hw_queue_schedule_cs'1220 /* User sequence will be validated in 'hl_hw_queue_schedule_cs'
1098 * under the cs_mirror_lock1221 * under the cs_mirror_lock
@@ -1108,9 +1231,20 @@ static int cs_staged_submission(struct hl_device *hdev, struct hl_cs *cs, · 9 unmodified lines
1108 return 0;1231 return 0;
1109}1232}
1110 1233
1234static u32 get_stream_master_qid_mask(struct hl_device *hdev, u32 qid)
1235{
1236 int i;
1237
1238 for (i = 0; i < hdev->stream_master_qid_arr_size; i++)
1239 if (qid == hdev->stream_master_qid_arr[i])
1240 return BIT(i);
1241
1242 return 0;
1243}
1244
1111static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks,1245static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks,
1112 u32 num_chunks, u64 *cs_seq, u32 flags,1246 u32 num_chunks, u64 *cs_seq, u32 flags,
1113 u32 timeout)1247 u32 encaps_signals_handle, u32 timeout)
1114{1248{
1115 bool staged_mid, int_queues_only = true;1249 bool staged_mid, int_queues_only = true;
1116 struct hl_device *hdev = hpriv->hdev;1250 struct hl_device *hdev = hpriv->hdev;
@@ -1121,6 +1255,7 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, · 4 unmodified lines
1121 struct hl_cs *cs;1255 struct hl_cs *cs;
1122 struct hl_cb *cb;1256 struct hl_cb *cb;
1123 u64 user_sequence;1257 u64 user_sequence;
1258 u8 stream_master_qid_map = 0;
1124 int rc, i;1259 int rc, i;
1125 1260
1126 cntr = &hdev->aggregated_cs_counters;1261 cntr = &hdev->aggregated_cs_counters;
@@ -1148,7 +1283,8 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, · 21 unmodified lines
1148 1283
1149 hl_debugfs_add_cs(cs);1284 hl_debugfs_add_cs(cs);
1150 1285
1151 rc = cs_staged_submission(hdev, cs, user_sequence, flags);1286 rc = cs_staged_submission(hdev, cs, user_sequence, flags,
1287 encaps_signals_handle);
1152 if (rc)1288 if (rc)
1153 goto free_cs_object;1289 goto free_cs_object;
1154 1290
@@ -1179,9 +1315,20 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, · 24 unmodified lines
1179 cb = (struct hl_cb *) (uintptr_t) chunk->cb_handle;1315 cb = (struct hl_cb *) (uintptr_t) chunk->cb_handle;
1180 }1316 }
1181 1317
1182 if (queue_type == QUEUE_TYPE_EXT || queue_type == QUEUE_TYPE_HW)1318 if (queue_type == QUEUE_TYPE_EXT ||
1319 queue_type == QUEUE_TYPE_HW) {
1183 int_queues_only = false;1320 int_queues_only = false;
1184 1321
1322 /*
1323 * store which stream are being used for external/HW
1324 * queues of this CS
1325 */
1326 if (hdev->supports_wait_for_multi_cs)
1327 stream_master_qid_map |=
1328 get_stream_master_qid_mask(hdev,
1329 chunk->queue_index);
1330 }
1331
1185 job = hl_cs_allocate_job(hdev, queue_type,1332 job = hl_cs_allocate_job(hdev, queue_type,
1186 is_kernel_allocated_cb);1333 is_kernel_allocated_cb);
1187 if (!job) {1334 if (!job) {
@@ -1242,6 +1389,13 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, · 54 unmodified lines
1242 goto free_cs_object;1389 goto free_cs_object;
1243 }1390 }
1244 1391
1392 /*
1393 * store the (external/HW queues) streams used by the CS in the
1394 * fence object for multi-CS completion
1395 */
1396 if (hdev->supports_wait_for_multi_cs)
1397 cs->fence->stream_master_qid_map = stream_master_qid_map;
1398
1245 rc = hl_hw_queue_schedule_cs(cs);1399 rc = hl_hw_queue_schedule_cs(cs);
1246 if (rc) {1400 if (rc) {
1247 if (rc != -EAGAIN)1401 if (rc != -EAGAIN)
@@ -1270,130 +1424,6 @@ out: · 22 unmodified lines
1270 return rc;1424 return rc;
1271}1425}
1272 1426
1273static int pending_cb_create_job(struct hl_device *hdev, struct hl_ctx *ctx,
1274 struct hl_cs *cs, struct hl_cb *cb, u32 size, u32 hw_queue_id)
1275{
1276 struct hw_queue_properties *hw_queue_prop;
1277 struct hl_cs_counters_atomic *cntr;
1278 struct hl_cs_job *job;
1279
1280 hw_queue_prop = &hdev->asic_prop.hw_queues_props[hw_queue_id];
1281 cntr = &hdev->aggregated_cs_counters;
1282
1283 job = hl_cs_allocate_job(hdev, hw_queue_prop->type, true);
1284 if (!job) {
1285 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
1286 atomic64_inc(&cntr->out_of_mem_drop_cnt);
1287 dev_err(hdev->dev, "Failed to allocate a new job\n");
1288 return -ENOMEM;
1289 }
1290
1291 job->id = 0;
1292 job->cs = cs;
1293 job->user_cb = cb;
1294 atomic_inc(&job->user_cb->cs_cnt);
1295 job->user_cb_size = size;
1296 job->hw_queue_id = hw_queue_id;
1297 job->patched_cb = job->user_cb;
1298 job->job_cb_size = job->user_cb_size;
1299
1300 /* increment refcount as for external queues we get completion */
1301 cs_get(cs);
1302
1303 cs->jobs_in_queue_cnt[job->hw_queue_id]++;
1304
1305 list_add_tail(&job->cs_node, &cs->job_list);
1306
1307 hl_debugfs_add_job(hdev, job);
1308
1309 return 0;
1310}
1311
1312static int hl_submit_pending_cb(struct hl_fpriv *hpriv)
1313{
1314 struct hl_device *hdev = hpriv->hdev;
1315 struct hl_ctx *ctx = hpriv->ctx;
1316 struct hl_pending_cb *pending_cb, *tmp;
1317 struct list_head local_cb_list;
1318 struct hl_cs *cs;
1319 struct hl_cb *cb;
1320 u32 hw_queue_id;
1321 u32 cb_size;
1322 int process_list, rc = 0;
1323
1324 if (list_empty(&ctx->pending_cb_list))
1325 return 0;
1326
1327 process_list = atomic_cmpxchg(&ctx->thread_pending_cb_token, 1, 0);
1328
1329 /* Only a single thread is allowed to process the list */
1330 if (!process_list)
1331 return 0;
1332
1333 if (list_empty(&ctx->pending_cb_list))
1334 goto free_pending_cb_token;
1335
1336 /* move all list elements to a local list */
1337 INIT_LIST_HEAD(&local_cb_list);
1338 spin_lock(&ctx->pending_cb_lock);
1339 list_for_each_entry_safe(pending_cb, tmp, &ctx->pending_cb_list,
1340 cb_node)
1341 list_move_tail(&pending_cb->cb_node, &local_cb_list);
1342 spin_unlock(&ctx->pending_cb_lock);
1343
1344 rc = allocate_cs(hdev, ctx, CS_TYPE_DEFAULT, ULLONG_MAX, &cs, 0,
1345 hdev->timeout_jiffies);
1346 if (rc)
1347 goto add_list_elements;
1348
1349 hl_debugfs_add_cs(cs);
1350
1351 /* Iterate through pending cb list, create jobs and add to CS */
1352 list_for_each_entry(pending_cb, &local_cb_list, cb_node) {
1353 cb = pending_cb->cb;
1354 cb_size = pending_cb->cb_size;
1355 hw_queue_id = pending_cb->hw_queue_id;
1356
1357 rc = pending_cb_create_job(hdev, ctx, cs, cb, cb_size,
1358 hw_queue_id);
1359 if (rc)
1360 goto free_cs_object;
1361 }
1362
1363 rc = hl_hw_queue_schedule_cs(cs);
1364 if (rc) {
1365 if (rc != -EAGAIN)
1366 dev_err(hdev->dev,
1367 "Failed to submit CS %d.%llu (%d)\n",
1368 ctx->asid, cs->sequence, rc);
1369 goto free_cs_object;
1370 }
1371
1372 /* pending cb was scheduled successfully */
1373 list_for_each_entry_safe(pending_cb, tmp, &local_cb_list, cb_node) {
1374 list_del(&pending_cb->cb_node);
1375 kfree(pending_cb);
1376 }
1377
1378 cs_put(cs);
1379
1380 goto free_pending_cb_token;
1381
1382free_cs_object:
1383 cs_rollback(hdev, cs);
1384 cs_put(cs);
1385add_list_elements:
1386 spin_lock(&ctx->pending_cb_lock);
1387 list_for_each_entry_safe_reverse(pending_cb, tmp, &local_cb_list,
1388 cb_node)
1389 list_move(&pending_cb->cb_node, &ctx->pending_cb_list);
1390 spin_unlock(&ctx->pending_cb_lock);
1391free_pending_cb_token:
1392 atomic_set(&ctx->thread_pending_cb_token, 1);
1393
1394 return rc;
1395}
1396
1397static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args,1427static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args,
1398 u64 *cs_seq)1428 u64 *cs_seq)
1399{1429{
@@ -1443,7 +1473,7 @@ static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args, · 43 unmodified lines
1443 rc = 0;1473 rc = 0;
1444 } else {1474 } else {
1445 rc = cs_ioctl_default(hpriv, chunks, num_chunks,1475 rc = cs_ioctl_default(hpriv, chunks, num_chunks,
1446 cs_seq, 0, hdev->timeout_jiffies);1476 cs_seq, 0, 0, hdev->timeout_jiffies);
1447 }1477 }
1448 1478
1449 mutex_unlock(&hpriv->restore_phase_mutex);1479 mutex_unlock(&hpriv->restore_phase_mutex);
@@ -1501,10 +1531,17 @@ out: · 51 unmodified lines
1501 * hl_cs_signal_sob_wraparound_handler: handle SOB value wrapaound case.1531 * hl_cs_signal_sob_wraparound_handler: handle SOB value wrapaound case.
1502 * if the SOB value reaches the max value move to the other SOB reserved1532 * if the SOB value reaches the max value move to the other SOB reserved
1503 * to the queue.1533 * to the queue.
1534 * @hdev: pointer to device structure
1535 * @q_idx: stream queue index
1536 * @hw_sob: the H/W SOB used in this signal CS.
1537 * @count: signals count
1538 * @encaps_sig: tells whether it's reservation for encaps signals or not.
1539 *
1504 * Note that this function must be called while hw_queues_lock is taken.1540 * Note that this function must be called while hw_queues_lock is taken.
1505 */1541 */
1506int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,1542int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,
1507 struct hl_hw_sob **hw_sob, u32 count)1543 struct hl_hw_sob **hw_sob, u32 count, bool encaps_sig)
1544
1508{1545{
1509 struct hl_sync_stream_properties *prop;1546 struct hl_sync_stream_properties *prop;
1510 struct hl_hw_sob *sob = *hw_sob, *other_sob;1547 struct hl_hw_sob *sob = *hw_sob, *other_sob;
@@ -1512,7 +1549,7 @@ int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, · 1 unmodified line
1512 1549
1513 prop = &hdev->kernel_queues[q_idx].sync_stream_prop;1550 prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
1514 1551
1515 kref_get(&sob->kref);1552 hw_sob_get(sob);
1516 1553
1517 /* check for wraparound */1554 /* check for wraparound */
1518 if (prop->next_sob_val + count >= HL_MAX_SOB_VAL) {1555 if (prop->next_sob_val + count >= HL_MAX_SOB_VAL) {
@@ -1522,7 +1559,7 @@ int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, · 3 unmodified lines
1522 * just incremented the refcount right before calling this1559 * just incremented the refcount right before calling this
1523 * function.1560 * function.
1524 */1561 */
1525 kref_put(&sob->kref, hl_sob_reset_error);1562 hw_sob_put_err(sob);
1526 1563
1527 /*1564 /*
1528 * check the other sob value, if it still in use then fail1565 * check the other sob value, if it still in use then fail
@@ -1537,12 +1574,42 @@ int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, · 8 unmodified lines
1537 return -EINVAL;1574 return -EINVAL;
1538 }1575 }
1539 1576
1540 prop->next_sob_val = 1;1577 /*
1578 * next_sob_val always points to the next available signal
1579 * in the sob, so in encaps signals it will be the next one
1580 * after reserving the required amount.
1581 */
1582 if (encaps_sig)
1583 prop->next_sob_val = count + 1;
1584 else
1585 prop->next_sob_val = count;
1541 1586
1542 /* only two SOBs are currently in use */1587 /* only two SOBs are currently in use */
1543 prop->curr_sob_offset = other_sob_offset;1588 prop->curr_sob_offset = other_sob_offset;
1544 *hw_sob = other_sob;1589 *hw_sob = other_sob;
1545 1590
1591 /*
1592 * check if other_sob needs reset, then do it before using it
1593 * for the reservation or the next signal cs.
1594 * we do it here, and for both encaps and regular signal cs
1595 * cases in order to avoid possible races of two kref_put
1596 * of the sob which can occur at the same time if we move the
1597 * sob reset(kref_put) to cs_do_release function.
1598 * in addition, if we have combination of cs signal and
1599 * encaps, and at the point we need to reset the sob there was
1600 * no more reservations and only signal cs keep coming,
1601 * in such case we need signal_cs to put the refcount and
1602 * reset the sob.
1603 */
1604 if (other_sob->need_reset)
1605 hw_sob_put(other_sob);
1606
1607 if (encaps_sig) {
1608 /* set reset indication for the sob */
1609 sob->need_reset = true;
1610 hw_sob_get(other_sob);
1611 }
1612
1546 dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n",1613 dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n",
1547 prop->curr_sob_offset, q_idx);1614 prop->curr_sob_offset, q_idx);
1548 } else {1615 } else {
@@ -1553,12 +1620,18 @@ int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, · 4 unmodified lines
1553}1620}
1554 1621
1555static int cs_ioctl_extract_signal_seq(struct hl_device *hdev,1622static int cs_ioctl_extract_signal_seq(struct hl_device *hdev,
1556 struct hl_cs_chunk *chunk, u64 *signal_seq, struct hl_ctx *ctx)1623 struct hl_cs_chunk *chunk, u64 *signal_seq, struct hl_ctx *ctx,
1624 bool encaps_signals)
1557{1625{
1558 u64 *signal_seq_arr = NULL;1626 u64 *signal_seq_arr = NULL;
1559 u32 size_to_copy, signal_seq_arr_len;1627 u32 size_to_copy, signal_seq_arr_len;
1560 int rc = 0;1628 int rc = 0;
1561 1629
1630 if (encaps_signals) {
1631 *signal_seq = chunk->encaps_signal_seq;
1632 return 0;
1633 }
1634
1562 signal_seq_arr_len = chunk->num_signal_seq_arr;1635 signal_seq_arr_len = chunk->num_signal_seq_arr;
1563 1636
1564 /* currently only one signal seq is supported */1637 /* currently only one signal seq is supported */
@@ -1583,7 +1656,7 @@ static int cs_ioctl_extract_signal_seq(struct hl_device *hdev, · 18 unmodified lines
1583 return -ENOMEM;1656 return -ENOMEM;
1584 }1657 }
1585 1658
1586 size_to_copy = chunk->num_signal_seq_arr * sizeof(*signal_seq_arr);1659 size_to_copy = signal_seq_arr_len * sizeof(*signal_seq_arr);
1587 if (copy_from_user(signal_seq_arr,1660 if (copy_from_user(signal_seq_arr,
1588 u64_to_user_ptr(chunk->signal_seq_arr),1661 u64_to_user_ptr(chunk->signal_seq_arr),
1589 size_to_copy)) {1662 size_to_copy)) {
@@ -1605,8 +1678,8 @@ out: · 15 unmodified lines
1605}1678}
1606 1679
1607static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev,1680static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev,
1608 struct hl_ctx *ctx, struct hl_cs *cs, enum hl_queue_type q_type,1681 struct hl_ctx *ctx, struct hl_cs *cs,
1609 u32 q_idx)1682 enum hl_queue_type q_type, u32 q_idx, u32 encaps_signal_offset)
1610{1683{
1611 struct hl_cs_counters_atomic *cntr;1684 struct hl_cs_counters_atomic *cntr;
1612 struct hl_cs_job *job;1685 struct hl_cs_job *job;
@@ -1644,6 +1717,9 @@ static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev, · 31 unmodified lines
1644 job->user_cb_size = cb_size;1717 job->user_cb_size = cb_size;
1645 job->hw_queue_id = q_idx;1718 job->hw_queue_id = q_idx;
1646 1719
1720 if ((cs->type == CS_TYPE_WAIT || cs->type == CS_TYPE_COLLECTIVE_WAIT)
1721 && cs->encaps_signals)
1722 job->encaps_sig_wait_offset = encaps_signal_offset;
1647 /*1723 /*
1648 * No need in parsing, user CB is the patched CB.1724 * No need in parsing, user CB is the patched CB.
1649 * We call hl_cb_destroy() out of two reasons - we don't need the CB in1725 * We call hl_cb_destroy() out of two reasons - we don't need the CB in
@@ -1666,75 +1742,307 @@ static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev, · 16 unmodified lines
1666 return 0;1742 return 0;
1667}1743}
1668 1744
1669static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,1745static int cs_ioctl_reserve_signals(struct hl_fpriv *hpriv,
1670 void __user *chunks, u32 num_chunks,1746 u32 q_idx, u32 count,
1671 u64 *cs_seq, u32 flags, u32 timeout)1747 u32 *handle_id, u32 *sob_addr,
1748 u32 *signals_count)
1672{1749{
1673 struct hl_cs_chunk *cs_chunk_array, *chunk;
1674 struct hw_queue_properties *hw_queue_prop;1750 struct hw_queue_properties *hw_queue_prop;
1751 struct hl_sync_stream_properties *prop;
1675 struct hl_device *hdev = hpriv->hdev;1752 struct hl_device *hdev = hpriv->hdev;
1676 struct hl_cs_compl *sig_waitcs_cmpl;1753 struct hl_cs_encaps_sig_handle *handle;
1677 u32 q_idx, collective_engine_id = 0;1754 struct hl_encaps_signals_mgr *mgr;
1678 struct hl_cs_counters_atomic *cntr;1755 struct hl_hw_sob *hw_sob;
1679 struct hl_fence *sig_fence = NULL;1756 int hdl_id;
1680 struct hl_ctx *ctx = hpriv->ctx;1757 int rc = 0;
1681 enum hl_queue_type q_type;
1682 struct hl_cs *cs;
1683 u64 signal_seq;
1684 int rc;
1685
1686 cntr = &hdev->aggregated_cs_counters;
1687 *cs_seq = ULLONG_MAX;
1688 1758
1689 rc = hl_cs_copy_chunk_array(hdev, &cs_chunk_array, chunks, num_chunks,1759 if (count >= HL_MAX_SOB_VAL) {
1690 ctx);1760 dev_err(hdev->dev, "signals count(%u) exceeds the max SOB value\n",
1691 if (rc)1761 count);
1762 rc = -EINVAL;
1692 goto out;1763 goto out;
1764 }
1693 1765
1694 /* currently it is guaranteed to have only one chunk */1766 if (q_idx >= hdev->asic_prop.max_queues) {
1695 chunk = &cs_chunk_array[0];
1696
1697 if (chunk->queue_index >= hdev->asic_prop.max_queues) {
1698 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1699 atomic64_inc(&cntr->validation_drop_cnt);
1700 dev_err(hdev->dev, "Queue index %d is invalid\n",1767 dev_err(hdev->dev, "Queue index %d is invalid\n",
1701 chunk->queue_index);1768 q_idx);
1702 rc = -EINVAL;1769 rc = -EINVAL;
1703 goto free_cs_chunk_array;1770 goto out;
1704 }1771 }
1705 1772
1706 q_idx = chunk->queue_index;
1707 hw_queue_prop = &hdev->asic_prop.hw_queues_props[q_idx];1773 hw_queue_prop = &hdev->asic_prop.hw_queues_props[q_idx];
1708 q_type = hw_queue_prop->type;
1709 1774
1710 if (!hw_queue_prop->supports_sync_stream) {1775 if (!hw_queue_prop->supports_sync_stream) {
1711 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1712 atomic64_inc(&cntr->validation_drop_cnt);
1713 dev_err(hdev->dev,1776 dev_err(hdev->dev,
1714 "Queue index %d does not support sync stream operations\n",1777 "Queue index %d does not support sync stream operations\n",
1715 q_idx);1778 q_idx);
1716 rc = -EINVAL;1779 rc = -EINVAL;
1717 goto free_cs_chunk_array;1780 goto out;
1718 }1781 }
1719 1782
1720 if (cs_type == CS_TYPE_COLLECTIVE_WAIT) {1783 prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
1721 if (!(hw_queue_prop->collective_mode == HL_COLLECTIVE_MASTER)) {
1722 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1723 atomic64_inc(&cntr->validation_drop_cnt);
1724 dev_err(hdev->dev,
1725 "Queue index %d is invalid\n", q_idx);
1726 rc = -EINVAL;
1727 goto free_cs_chunk_array;
1728 }
1729 1784
1730 collective_engine_id = chunk->collective_engine_id;1785 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
1731 }1786 if (!handle) {
1787 rc = -ENOMEM;
1788 goto out;
1789 }
1790
1791 handle->count = count;
1792 mgr = &hpriv->ctx->sig_mgr;
1793
1794 spin_lock(&mgr->lock);
1795 hdl_id = idr_alloc(&mgr->handles, handle, 1, 0, GFP_ATOMIC);
1796 spin_unlock(&mgr->lock);
1797
1798 if (hdl_id < 0) {
1799 dev_err(hdev->dev, "Failed to allocate IDR for a new signal reservation\n");
1800 rc = -EINVAL;
1801 goto out;
1802 }
1803
1804 handle->id = hdl_id;
1805 handle->q_idx = q_idx;
1806 handle->hdev = hdev;
1807 kref_init(&handle->refcount);
1808
1809 hdev->asic_funcs->hw_queues_lock(hdev);
1810
1811 hw_sob = &prop->hw_sob[prop->curr_sob_offset];
1812
1813 /*
1814 * Increment the SOB value by count by user request
1815 * to reserve those signals
1816 * check if the signals amount to reserve is not exceeding the max sob
1817 * value, if yes then switch sob.
1818 */
1819 rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, count,
1820 true);
1821 if (rc) {
1822 dev_err(hdev->dev, "Failed to switch SOB\n");
1823 hdev->asic_funcs->hw_queues_unlock(hdev);
1824 rc = -EINVAL;
1825 goto remove_idr;
1826 }
1827 /* set the hw_sob to the handle after calling the sob wraparound handler
1828 * since sob could have changed.
1829 */
1830 handle->hw_sob = hw_sob;
1831
1832 /* store the current sob value for unreserve validity check, and
1833 * signal offset support
1834 */
1835 handle->pre_sob_val = prop->next_sob_val - handle->count;
1836
1837 *signals_count = prop->next_sob_val;
1838 hdev->asic_funcs->hw_queues_unlock(hdev);
1839
1840 *sob_addr = handle->hw_sob->sob_addr;
1841 *handle_id = hdl_id;
1842
1843 dev_dbg(hdev->dev,
1844 "Signals reserved, sob_id: %d, sob addr: 0x%x, last sob_val: %u, q_idx: %d, hdl_id: %d\n",
1845 hw_sob->sob_id, handle->hw_sob->sob_addr,
1846 prop->next_sob_val - 1, q_idx, hdl_id);
1847 goto out;
1848
1849remove_idr:
1850 spin_lock(&mgr->lock);
1851 idr_remove(&mgr->handles, hdl_id);
1852 spin_unlock(&mgr->lock);
1853
1854 kfree(handle);
1855out:
1856 return rc;
1857}
1858
1859static int cs_ioctl_unreserve_signals(struct hl_fpriv *hpriv, u32 handle_id)
1860{
1861 struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
1862 struct hl_sync_stream_properties *prop;
1863 struct hl_device *hdev = hpriv->hdev;
1864 struct hl_encaps_signals_mgr *mgr;
1865 struct hl_hw_sob *hw_sob;
1866 u32 q_idx, sob_addr;
1867 int rc = 0;
1868
1869 mgr = &hpriv->ctx->sig_mgr;
1870
1871 spin_lock(&mgr->lock);
1872 encaps_sig_hdl = idr_find(&mgr->handles, handle_id);
1873 if (encaps_sig_hdl) {
1874 dev_dbg(hdev->dev, "unreserve signals, handle: %u, SOB:0x%x, count: %u\n",
1875 handle_id, encaps_sig_hdl->hw_sob->sob_addr,
1876 encaps_sig_hdl->count);
1877
1878 hdev->asic_funcs->hw_queues_lock(hdev);
1879
1880 q_idx = encaps_sig_hdl->q_idx;
1881 prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
1882 hw_sob = &prop->hw_sob[prop->curr_sob_offset];
1883 sob_addr = hdev->asic_funcs->get_sob_addr(hdev, hw_sob->sob_id);
1884
1885 /* Check if sob_val got out of sync due to other
1886 * signal submission requests which were handled
1887 * between the reserve-unreserve calls or SOB switch
1888 * upon reaching SOB max value.
1889 */
1890 if (encaps_sig_hdl->pre_sob_val + encaps_sig_hdl->count
1891 != prop->next_sob_val ||
1892 sob_addr != encaps_sig_hdl->hw_sob->sob_addr) {
1893 dev_err(hdev->dev, "Cannot unreserve signals, SOB val ran out of sync, expected: %u, actual val: %u\n",
1894 encaps_sig_hdl->pre_sob_val,
1895 (prop->next_sob_val - encaps_sig_hdl->count));
1896
1897 hdev->asic_funcs->hw_queues_unlock(hdev);
1898 rc = -EINVAL;
1899 goto out;
1900 }
1901
1902 /*
1903 * Decrement the SOB value by count by user request
1904 * to unreserve those signals
1905 */
1906 prop->next_sob_val -= encaps_sig_hdl->count;
1732 1907
1733 if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_COLLECTIVE_WAIT) {1908 hdev->asic_funcs->hw_queues_unlock(hdev);
1734 rc = cs_ioctl_extract_signal_seq(hdev, chunk, &signal_seq, ctx);1909
1910 hw_sob_put(hw_sob);
1911
1912 /* Release the id and free allocated memory of the handle */
1913 idr_remove(&mgr->handles, handle_id);
1914 kfree(encaps_sig_hdl);
1915 } else {
1916 rc = -EINVAL;
1917 dev_err(hdev->dev, "failed to unreserve signals, cannot find handler\n");
1918 }
1919out:
1920 spin_unlock(&mgr->lock);
1921
1922 return rc;
1923}
1924
1925static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
1926 void __user *chunks, u32 num_chunks,
1927 u64 *cs_seq, u32 flags, u32 timeout)
1928{
1929 struct hl_cs_encaps_sig_handle *encaps_sig_hdl = NULL;
1930 bool handle_found = false, is_wait_cs = false,
1931 wait_cs_submitted = false,
1932 cs_encaps_signals = false;
1933 struct hl_cs_chunk *cs_chunk_array, *chunk;
1934 bool staged_cs_with_encaps_signals = false;
1935 struct hw_queue_properties *hw_queue_prop;
1936 struct hl_device *hdev = hpriv->hdev;
1937 struct hl_cs_compl *sig_waitcs_cmpl;
1938 u32 q_idx, collective_engine_id = 0;
1939 struct hl_cs_counters_atomic *cntr;
1940 struct hl_fence *sig_fence = NULL;
1941 struct hl_ctx *ctx = hpriv->ctx;
1942 enum hl_queue_type q_type;
1943 struct hl_cs *cs;
1944 u64 signal_seq;
1945 int rc;
1946
1947 cntr = &hdev->aggregated_cs_counters;
1948 *cs_seq = ULLONG_MAX;
1949
1950 rc = hl_cs_copy_chunk_array(hdev, &cs_chunk_array, chunks, num_chunks,
1951 ctx);
1952 if (rc)
1953 goto out;
1954
1955 /* currently it is guaranteed to have only one chunk */
1956 chunk = &cs_chunk_array[0];
1957
1958 if (chunk->queue_index >= hdev->asic_prop.max_queues) {
1959 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1960 atomic64_inc(&cntr->validation_drop_cnt);
1961 dev_err(hdev->dev, "Queue index %d is invalid\n",
1962 chunk->queue_index);
1963 rc = -EINVAL;
1964 goto free_cs_chunk_array;
1965 }
1966
1967 q_idx = chunk->queue_index;
1968 hw_queue_prop = &hdev->asic_prop.hw_queues_props[q_idx];
1969 q_type = hw_queue_prop->type;
1970
1971 if (!hw_queue_prop->supports_sync_stream) {
1972 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1973 atomic64_inc(&cntr->validation_drop_cnt);
1974 dev_err(hdev->dev,
1975 "Queue index %d does not support sync stream operations\n",
1976 q_idx);
1977 rc = -EINVAL;
1978 goto free_cs_chunk_array;
1979 }
1980
1981 if (cs_type == CS_TYPE_COLLECTIVE_WAIT) {
1982 if (!(hw_queue_prop->collective_mode == HL_COLLECTIVE_MASTER)) {
1983 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1984 atomic64_inc(&cntr->validation_drop_cnt);
1985 dev_err(hdev->dev,
1986 "Queue index %d is invalid\n", q_idx);
1987 rc = -EINVAL;
1988 goto free_cs_chunk_array;
1989 }
1990
1991 collective_engine_id = chunk->collective_engine_id;
1992 }
1993
1994 is_wait_cs = !!(cs_type == CS_TYPE_WAIT ||
1995 cs_type == CS_TYPE_COLLECTIVE_WAIT);
1996
1997 cs_encaps_signals = !!(flags & HL_CS_FLAGS_ENCAP_SIGNALS);
1998
1999 if (is_wait_cs) {
2000 rc = cs_ioctl_extract_signal_seq(hdev, chunk, &signal_seq,
2001 ctx, cs_encaps_signals);
1735 if (rc)2002 if (rc)
1736 goto free_cs_chunk_array;2003 goto free_cs_chunk_array;
1737 2004
2005 if (cs_encaps_signals) {
2006 /* check if cs sequence has encapsulated
2007 * signals handle
2008 */
2009 struct idr *idp;
2010 u32 id;
2011
2012 spin_lock(&ctx->sig_mgr.lock);
2013 idp = &ctx->sig_mgr.handles;
2014 idr_for_each_entry(idp, encaps_sig_hdl, id) {
2015 if (encaps_sig_hdl->cs_seq == signal_seq) {
2016 handle_found = true;
2017 /* get refcount to protect removing
2018 * this handle from idr, needed when
2019 * multiple wait cs are used with offset
2020 * to wait on reserved encaps signals.
2021 */
2022 kref_get(&encaps_sig_hdl->refcount);
2023 break;
2024 }
2025 }
2026 spin_unlock(&ctx->sig_mgr.lock);
2027
2028 if (!handle_found) {
2029 dev_err(hdev->dev, "Cannot find encapsulated signals handle for seq 0x%llx\n",
2030 signal_seq);
2031 rc = -EINVAL;
2032 goto free_cs_chunk_array;
2033 }
2034
2035 /* validate also the signal offset value */
2036 if (chunk->encaps_signal_offset >
2037 encaps_sig_hdl->count) {
2038 dev_err(hdev->dev, "offset(%u) value exceed max reserved signals count(%u)!\n",
2039 chunk->encaps_signal_offset,
2040 encaps_sig_hdl->count);
2041 rc = -EINVAL;
2042 goto free_cs_chunk_array;
2043 }
2044 }
2045
1738 sig_fence = hl_ctx_get_fence(ctx, signal_seq);2046 sig_fence = hl_ctx_get_fence(ctx, signal_seq);
1739 if (IS_ERR(sig_fence)) {2047 if (IS_ERR(sig_fence)) {
1740 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);2048 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
@@ -1755,11 +2063,16 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, · 14 unmodified lines
1755 sig_waitcs_cmpl =2063 sig_waitcs_cmpl =
1756 container_of(sig_fence, struct hl_cs_compl, base_fence);2064 container_of(sig_fence, struct hl_cs_compl, base_fence);
1757 2065
1758 if (sig_waitcs_cmpl->type != CS_TYPE_SIGNAL) {2066 staged_cs_with_encaps_signals = !!
2067 (sig_waitcs_cmpl->type == CS_TYPE_DEFAULT &&
2068 (flags & HL_CS_FLAGS_ENCAP_SIGNALS));
2069
2070 if (sig_waitcs_cmpl->type != CS_TYPE_SIGNAL &&
2071 !staged_cs_with_encaps_signals) {
1759 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);2072 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1760 atomic64_inc(&cntr->validation_drop_cnt);2073 atomic64_inc(&cntr->validation_drop_cnt);
1761 dev_err(hdev->dev,2074 dev_err(hdev->dev,
1762 "CS seq 0x%llx is not of a signal CS\n",2075 "CS seq 0x%llx is not of a signal/encaps-signal CS\n",
1763 signal_seq);2076 signal_seq);
1764 hl_fence_put(sig_fence);2077 hl_fence_put(sig_fence);
1765 rc = -EINVAL;2078 rc = -EINVAL;
@@ -1776,18 +2089,27 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, · 10 unmodified lines
1776 2089
1777 rc = allocate_cs(hdev, ctx, cs_type, ULLONG_MAX, &cs, flags, timeout);2090 rc = allocate_cs(hdev, ctx, cs_type, ULLONG_MAX, &cs, flags, timeout);
1778 if (rc) {2091 if (rc) {
1779 if (cs_type == CS_TYPE_WAIT ||2092 if (is_wait_cs)
1780 cs_type == CS_TYPE_COLLECTIVE_WAIT)
1781 hl_fence_put(sig_fence);2093 hl_fence_put(sig_fence);
2094
1782 goto free_cs_chunk_array;2095 goto free_cs_chunk_array;
1783 }2096 }
1784 2097
1785 /*2098 /*
1786 * Save the signal CS fence for later initialization right before2099 * Save the signal CS fence for later initialization right before
1787 * hanging the wait CS on the queue.2100 * hanging the wait CS on the queue.
2101 * for encaps signals case, we save the cs sequence and handle pointer
2102 * for later initialization.
1788 */2103 */
1789 if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_COLLECTIVE_WAIT)2104 if (is_wait_cs) {
1790 cs->signal_fence = sig_fence;2105 cs->signal_fence = sig_fence;
2106 /* store the handle pointer, so we don't have to
2107 * look for it again, later on the flow
2108 * when we need to set SOB info in hw_queue.
2109 */
2110 if (cs->encaps_signals)
2111 cs->encaps_sig_hdl = encaps_sig_hdl;
2112 }
1791 2113
1792 hl_debugfs_add_cs(cs);2114 hl_debugfs_add_cs(cs);
1793 2115
@@ -1795,10 +2117,11 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, · 1 unmodified line
1795 2117
1796 if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_SIGNAL)2118 if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_SIGNAL)
1797 rc = cs_ioctl_signal_wait_create_jobs(hdev, ctx, cs, q_type,2119 rc = cs_ioctl_signal_wait_create_jobs(hdev, ctx, cs, q_type,
1798 q_idx);2120 q_idx, chunk->encaps_signal_offset);
1799 else if (cs_type == CS_TYPE_COLLECTIVE_WAIT)2121 else if (cs_type == CS_TYPE_COLLECTIVE_WAIT)
1800 rc = hdev->asic_funcs->collective_wait_create_jobs(hdev, ctx,2122 rc = hdev->asic_funcs->collective_wait_create_jobs(hdev, ctx,
1801 cs, q_idx, collective_engine_id);2123 cs, q_idx, collective_engine_id,
2124 chunk->encaps_signal_offset);
1802 else {2125 else {
1803 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);2126 atomic64_inc(&ctx->cs_counters.validation_drop_cnt);
1804 atomic64_inc(&cntr->validation_drop_cnt);2127 atomic64_inc(&cntr->validation_drop_cnt);
@@ -1810,7 +2133,13 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, · 5 unmodified lines
1810 2133
1811 rc = hl_hw_queue_schedule_cs(cs);2134 rc = hl_hw_queue_schedule_cs(cs);
1812 if (rc) {2135 if (rc) {
1813 if (rc != -EAGAIN)2136 /* In case wait cs failed here, it means the signal cs
2137 * already completed. we want to free all it's related objects
2138 * but we don't want to fail the ioctl.
2139 */
2140 if (is_wait_cs)
2141 rc = 0;
2142 else if (rc != -EAGAIN)
1814 dev_err(hdev->dev,2143 dev_err(hdev->dev,
1815 "Failed to submit CS %d.%llu to H/W queues, error %d\n",2144 "Failed to submit CS %d.%llu to H/W queues, error %d\n",
1816 ctx->asid, cs->sequence, rc);2145 ctx->asid, cs->sequence, rc);
@@ -1818,6 +2147,8 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, · 1 unmodified line
1818 }2147 }
1819 2148
1820 rc = HL_CS_STATUS_SUCCESS;2149 rc = HL_CS_STATUS_SUCCESS;
2150 if (is_wait_cs)
2151 wait_cs_submitted = true;
1821 goto put_cs;2152 goto put_cs;
1822 2153
1823free_cs_object:2154free_cs_object:
@@ -1828,6 +2159,10 @@ put_cs: · 4 unmodified lines
1828 /* We finished with the CS in this function, so put the ref */2159 /* We finished with the CS in this function, so put the ref */
1829 cs_put(cs);2160 cs_put(cs);
1830free_cs_chunk_array:2161free_cs_chunk_array:
2162 if (!wait_cs_submitted && cs_encaps_signals && handle_found &&
2163 is_wait_cs)
2164 kref_put(&encaps_sig_hdl->refcount,
2165 hl_encaps_handle_do_release);
1831 kfree(cs_chunk_array);2166 kfree(cs_chunk_array);
1832out:2167out:
1833 return rc;2168 return rc;
@@ -1836,10 +2171,11 @@ out: · 2 unmodified lines
1836int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)2171int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
1837{2172{
1838 union hl_cs_args *args = data;2173 union hl_cs_args *args = data;
1839 enum hl_cs_type cs_type;2174 enum hl_cs_type cs_type = 0;
1840 u64 cs_seq = ULONG_MAX;2175 u64 cs_seq = ULONG_MAX;
1841 void __user *chunks;2176 void __user *chunks;
1842 u32 num_chunks, flags, timeout;2177 u32 num_chunks, flags, timeout,
2178 signals_count = 0, sob_addr = 0, handle_id = 0;
1843 int rc;2179 int rc;
1844 2180
1845 rc = hl_cs_sanity_checks(hpriv, args);2181 rc = hl_cs_sanity_checks(hpriv, args);
@@ -1850,10 +2186,6 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) · 4 unmodified lines
1850 if (rc)2186 if (rc)
1851 goto out;2187 goto out;
1852 2188
1853 rc = hl_submit_pending_cb(hpriv);
1854 if (rc)
1855 goto out;
1856
1857 cs_type = hl_cs_get_cs_type(args->in.cs_flags &2189 cs_type = hl_cs_get_cs_type(args->in.cs_flags &
1858 ~HL_CS_FLAGS_FORCE_RESTORE);2190 ~HL_CS_FLAGS_FORCE_RESTORE);
1859 chunks = (void __user *) (uintptr_t) args->in.chunks_execute;2191 chunks = (void __user *) (uintptr_t) args->in.chunks_execute;
@@ -1876,80 +2208,448 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) · 16 unmodified lines
1876 rc = cs_ioctl_signal_wait(hpriv, cs_type, chunks, num_chunks,2208 rc = cs_ioctl_signal_wait(hpriv, cs_type, chunks, num_chunks,
1877 &cs_seq, args->in.cs_flags, timeout);2209 &cs_seq, args->in.cs_flags, timeout);
1878 break;2210 break;
2211 case CS_RESERVE_SIGNALS:
2212 rc = cs_ioctl_reserve_signals(hpriv,
2213 args->in.encaps_signals_q_idx,
2214 args->in.encaps_signals_count,
2215 &handle_id, &sob_addr, &signals_count);
2216 break;
2217 case CS_UNRESERVE_SIGNALS:
2218 rc = cs_ioctl_unreserve_signals(hpriv,
2219 args->in.encaps_sig_handle_id);
2220 break;
1879 default:2221 default:
1880 rc = cs_ioctl_default(hpriv, chunks, num_chunks, &cs_seq,2222 rc = cs_ioctl_default(hpriv, chunks, num_chunks, &cs_seq,
1881 args->in.cs_flags, timeout);2223 args->in.cs_flags,
2224 args->in.encaps_sig_handle_id,
2225 timeout);
1882 break;2226 break;
1883 }2227 }
1884
1885out:2228out:
1886 if (rc != -EAGAIN) {2229 if (rc != -EAGAIN) {
1887 memset(args, 0, sizeof(*args));2230 memset(args, 0, sizeof(*args));
2231
2232 if (cs_type == CS_RESERVE_SIGNALS) {
2233 args->out.handle_id = handle_id;
2234 args->out.sob_base_addr_offset = sob_addr;
2235 args->out.count = signals_count;
2236 } else {
2237 args->out.seq = cs_seq;
2238 }
1888 args->out.status = rc;2239 args->out.status = rc;
1889 args->out.seq = cs_seq;
1890 }2240 }
1891 2241
1892 return rc;2242 return rc;
1893}2243}
1894 2244
2245static int hl_wait_for_fence(struct hl_ctx *ctx, u64 seq, struct hl_fence *fence,
2246 enum hl_cs_wait_status *status, u64 timeout_us,
2247 s64 *timestamp)
2248{
2249 struct hl_device *hdev = ctx->hdev;
2250 long completion_rc;
2251 int rc = 0;
2252
2253 if (IS_ERR(fence)) {
2254 rc = PTR_ERR(fence);
2255 if (rc == -EINVAL)
2256 dev_notice_ratelimited(hdev->dev,
2257 "Can't wait on CS %llu because current CS is at seq %llu\n",
2258 seq, ctx->cs_sequence);
2259 return rc;
2260 }
2261
2262 if (!fence) {
2263 dev_dbg(hdev->dev,
2264 "Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n",
2265 seq, ctx->cs_sequence);
2266
2267 *status = CS_WAIT_STATUS_GONE;
2268 return 0;
2269 }
2270
2271 if (!timeout_us) {
2272 completion_rc = completion_done(&fence->completion);
2273 } else {
2274 unsigned long timeout;
2275
2276 timeout = (timeout_us == MAX_SCHEDULE_TIMEOUT) ?
2277 timeout_us : usecs_to_jiffies(timeout_us);
2278 completion_rc =
2279 wait_for_completion_interruptible_timeout(
2280 &fence->completion, timeout);
2281 }
2282
2283 if (completion_rc > 0) {
2284 *status = CS_WAIT_STATUS_COMPLETED;
2285 if (timestamp)
2286 *timestamp = ktime_to_ns(fence->timestamp);
2287 } else {
2288 *status = CS_WAIT_STATUS_BUSY;
2289 }
2290
2291 if (fence->error == -ETIMEDOUT)
2292 rc = -ETIMEDOUT;
2293 else if (fence->error == -EIO)
2294 rc = -EIO;
2295
2296 return rc;
2297}
2298
2299/*
2300 * hl_cs_poll_fences - iterate CS fences to check for CS completion
2301 *
2302 * @mcs_data: multi-CS internal data
2303 *
2304 * @return 0 on success, otherwise non 0 error code
2305 *
2306 * The function iterates on all CS sequence in the list and set bit in
2307 * completion_bitmap for each completed CS.
2308 * while iterating, the function can extracts the stream map to be later
2309 * used by the waiting function.
2310 * this function shall be called after taking context ref
2311 */
2312static int hl_cs_poll_fences(struct multi_cs_data *mcs_data)
2313{
2314 struct hl_fence **fence_ptr = mcs_data->fence_arr;
2315 struct hl_device *hdev = mcs_data->ctx->hdev;
2316 int i, rc, arr_len = mcs_data->arr_len;
2317 u64 *seq_arr = mcs_data->seq_arr;
2318 ktime_t max_ktime, first_cs_time;
2319 enum hl_cs_wait_status status;
2320
2321 memset(fence_ptr, 0, arr_len * sizeof(*fence_ptr));
2322
2323 /* get all fences under the same lock */
2324 rc = hl_ctx_get_fences(mcs_data->ctx, seq_arr, fence_ptr, arr_len);
2325 if (rc)
2326 return rc;
2327
2328 /*
2329 * set to maximum time to verify timestamp is valid: if at the end
2330 * this value is maintained- no timestamp was updated
2331 */
2332 max_ktime = ktime_set(KTIME_SEC_MAX, 0);
2333 first_cs_time = max_ktime;
2334
2335 for (i = 0; i < arr_len; i++, fence_ptr++) {
2336 struct hl_fence *fence = *fence_ptr;
2337
2338 /*
2339 * function won't sleep as it is called with timeout 0 (i.e.
2340 * poll the fence)
2341 */
2342 rc = hl_wait_for_fence(mcs_data->ctx, seq_arr[i], fence,
2343 &status, 0, NULL);
2344 if (rc) {
2345 dev_err(hdev->dev,
2346 "wait_for_fence error :%d for CS seq %llu\n",
2347 rc, seq_arr[i]);
2348 break;
2349 }
2350
2351 mcs_data->stream_master_qid_map |= fence->stream_master_qid_map;
2352
2353 if (status == CS_WAIT_STATUS_BUSY)
2354 continue;
2355
2356 mcs_data->completion_bitmap |= BIT(i);
2357
2358 /*
2359 * best effort to extract timestamp. few notes:
2360 * - if even single fence is gone we cannot extract timestamp
2361 * (as fence not exist anymore)
2362 * - for all completed CSs we take the earliest timestamp.
2363 * for this we have to validate that:
2364 * 1. given timestamp was indeed set
2365 * 2. the timestamp is earliest of all timestamps so far
2366 */
2367
2368 if (status == CS_WAIT_STATUS_GONE) {
2369 mcs_data->update_ts = false;
2370 mcs_data->gone_cs = true;
2371 } else if (mcs_data->update_ts &&
2372 (ktime_compare(fence->timestamp,
2373 ktime_set(0, 0)) > 0) &&
2374 (ktime_compare(fence->timestamp, first_cs_time) < 0)) {
2375 first_cs_time = fence->timestamp;
2376 }
2377 }
2378
2379 hl_fences_put(mcs_data->fence_arr, arr_len);
2380
2381 if (mcs_data->update_ts &&
2382 (ktime_compare(first_cs_time, max_ktime) != 0))
2383 mcs_data->timestamp = ktime_to_ns(first_cs_time);
2384
2385 return rc;
2386}
2387
1895static int _hl_cs_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,2388static int _hl_cs_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
1896 u64 timeout_us, u64 seq,2389 u64 timeout_us, u64 seq,
1897 enum hl_cs_wait_status *status, s64 *timestamp)2390 enum hl_cs_wait_status *status, s64 *timestamp)
1898{2391{
1899 struct hl_fence *fence;2392 struct hl_fence *fence;
1900 unsigned long timeout;
1901 int rc = 0;2393 int rc = 0;
1902 long completion_rc;
1903 2394
1904 if (timestamp)2395 if (timestamp)
1905 *timestamp = 0;2396 *timestamp = 0;
1906 2397
1907 if (timeout_us == MAX_SCHEDULE_TIMEOUT)
1908 timeout = timeout_us;
1909 else
1910 timeout = usecs_to_jiffies(timeout_us);
1911
1912 hl_ctx_get(hdev, ctx);2398 hl_ctx_get(hdev, ctx);
1913 2399
1914 fence = hl_ctx_get_fence(ctx, seq);2400 fence = hl_ctx_get_fence(ctx, seq);
1915 if (IS_ERR(fence)) {
1916 rc = PTR_ERR(fence);
1917 if (rc == -EINVAL)
1918 dev_notice_ratelimited(hdev->dev,
1919 "Can't wait on CS %llu because current CS is at seq %llu\n",
1920 seq, ctx->cs_sequence);
1921 } else if (fence) {
1922 if (!timeout_us)
1923 completion_rc = completion_done(&fence->completion);
1924 else
1925 completion_rc =
1926 wait_for_completion_interruptible_timeout(
1927 &fence->completion, timeout);
1928 2401
1929 if (completion_rc > 0) {2402 rc = hl_wait_for_fence(ctx, seq, fence, status, timeout_us, timestamp);
1930 *status = CS_WAIT_STATUS_COMPLETED;2403 hl_fence_put(fence);
1931 if (timestamp)2404 hl_ctx_put(ctx);
1932 *timestamp = ktime_to_ns(fence->timestamp);2405
1933 } else {2406 return rc;
1934 *status = CS_WAIT_STATUS_BUSY;2407}
2408
2409/*
2410 * hl_wait_multi_cs_completion_init - init completion structure
2411 *
2412 * @hdev: pointer to habanalabs device structure
2413 * @stream_master_bitmap: stream master QIDs map, set bit indicates stream
2414 * master QID to wait on
2415 *
2416 * @return valid completion struct pointer on success, otherwise error pointer
2417 *
2418 * up to MULTI_CS_MAX_USER_CTX calls can be done concurrently to the driver.
2419 * the function gets the first available completion (by marking it "used")
2420 * and initialize its values.
2421 */
2422static struct multi_cs_completion *hl_wait_multi_cs_completion_init(
2423 struct hl_device *hdev,
2424 u8 stream_master_bitmap)
2425{
2426 struct multi_cs_completion *mcs_compl;
2427 int i;
2428
2429 /* find free multi_cs completion structure */
2430 for (i = 0; i < MULTI_CS_MAX_USER_CTX; i++) {
2431 mcs_compl = &hdev->multi_cs_completion[i];
2432 spin_lock(&mcs_compl->lock);
2433 if (!mcs_compl->used) {
2434 mcs_compl->used = 1;
2435 mcs_compl->timestamp = 0;
2436 mcs_compl->stream_master_qid_map = stream_master_bitmap;
2437 reinit_completion(&mcs_compl->completion);
2438 spin_unlock(&mcs_compl->lock);
2439 break;
1935 }2440 }
2441 spin_unlock(&mcs_compl->lock);
2442 }
1936 2443
1937 if (fence->error == -ETIMEDOUT)2444 if (i == MULTI_CS_MAX_USER_CTX) {
1938 rc = -ETIMEDOUT;2445 dev_err(hdev->dev,
1939 else if (fence->error == -EIO)2446 "no available multi-CS completion structure\n");
1940 rc = -EIO;2447 return ERR_PTR(-ENOMEM);
2448 }
2449 return mcs_compl;
2450}
1941 2451
1942 hl_fence_put(fence);2452/*
1943 } else {2453 * hl_wait_multi_cs_completion_fini - return completion structure and set as
1944 dev_dbg(hdev->dev,2454 * unused
1945 "Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n",2455 *
1946 seq, ctx->cs_sequence);2456 * @mcs_compl: pointer to the completion structure
1947 *status = CS_WAIT_STATUS_GONE;2457 */
2458static void hl_wait_multi_cs_completion_fini(
2459 struct multi_cs_completion *mcs_compl)
2460{
2461 /*
2462 * free completion structure, do it under lock to be in-sync with the
2463 * thread that signals completion
2464 */
2465 spin_lock(&mcs_compl->lock);
2466 mcs_compl->used = 0;
2467 spin_unlock(&mcs_compl->lock);
2468}
2469
2470/*
2471 * hl_wait_multi_cs_completion - wait for first CS to complete
2472 *
2473 * @mcs_data: multi-CS internal data
2474 *
2475 * @return 0 on success, otherwise non 0 error code
2476 */
2477static int hl_wait_multi_cs_completion(struct multi_cs_data *mcs_data)
2478{
2479 struct hl_device *hdev = mcs_data->ctx->hdev;
2480 struct multi_cs_completion *mcs_compl;
2481 long completion_rc;
2482
2483 mcs_compl = hl_wait_multi_cs_completion_init(hdev,
2484 mcs_data->stream_master_qid_map);
2485 if (IS_ERR(mcs_compl))
2486 return PTR_ERR(mcs_compl);
2487
2488 completion_rc = wait_for_completion_interruptible_timeout(
2489 &mcs_compl->completion,
2490 usecs_to_jiffies(mcs_data->timeout_us));
2491
2492 /* update timestamp */
2493 if (completion_rc > 0)
2494 mcs_data->timestamp = mcs_compl->timestamp;
2495
2496 hl_wait_multi_cs_completion_fini(mcs_compl);
2497
2498 mcs_data->wait_status = completion_rc;
2499
2500 return 0;
2501}
2502
2503/*
2504 * hl_multi_cs_completion_init - init array of multi-CS completion structures
2505 *
2506 * @hdev: pointer to habanalabs device structure
2507 */
2508void hl_multi_cs_completion_init(struct hl_device *hdev)
2509{
2510 struct multi_cs_completion *mcs_cmpl;
2511 int i;
2512
2513 for (i = 0; i < MULTI_CS_MAX_USER_CTX; i++) {
2514 mcs_cmpl = &hdev->multi_cs_completion[i];
2515 mcs_cmpl->used = 0;
2516 spin_lock_init(&mcs_cmpl->lock);
2517 init_completion(&mcs_cmpl->completion);
2518 }
2519}
2520
2521/*
2522 * hl_multi_cs_wait_ioctl - implementation of the multi-CS wait ioctl
2523 *
2524 * @hpriv: pointer to the private data of the fd
2525 * @data: pointer to multi-CS wait ioctl in/out args
2526 *
2527 */
2528static int hl_multi_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data)
2529{
2530 struct hl_device *hdev = hpriv->hdev;
2531 struct multi_cs_data mcs_data = {0};
2532 union hl_wait_cs_args *args = data;
2533 struct hl_ctx *ctx = hpriv->ctx;
2534 struct hl_fence **fence_arr;
2535 void __user *seq_arr;
2536 u32 size_to_copy;
2537 u64 *cs_seq_arr;
2538 u8 seq_arr_len;
2539 int rc;
2540
2541 if (!hdev->supports_wait_for_multi_cs) {
2542 dev_err(hdev->dev, "Wait for multi CS is not supported\n");
2543 return -EPERM;
2544 }
2545
2546 seq_arr_len = args->in.seq_arr_len;
2547
2548 if (seq_arr_len > HL_WAIT_MULTI_CS_LIST_MAX_LEN) {
2549 dev_err(hdev->dev, "Can wait only up to %d CSs, input sequence is of length %u\n",
2550 HL_WAIT_MULTI_CS_LIST_MAX_LEN, seq_arr_len);
2551 return -EINVAL;
2552 }
2553
2554 /* allocate memory for sequence array */
2555 cs_seq_arr =
2556 kmalloc_array(seq_arr_len, sizeof(*cs_seq_arr), GFP_KERNEL);
2557 if (!cs_seq_arr)
2558 return -ENOMEM;
2559
2560 /* copy CS sequence array from user */
2561 seq_arr = (void __user *) (uintptr_t) args->in.seq;
2562 size_to_copy = seq_arr_len * sizeof(*cs_seq_arr);
2563 if (copy_from_user(cs_seq_arr, seq_arr, size_to_copy)) {
2564 dev_err(hdev->dev, "Failed to copy multi-cs sequence array from user\n");
2565 rc = -EFAULT;
2566 goto free_seq_arr;
2567 }
2568
2569 /* allocate array for the fences */
2570 fence_arr = kmalloc_array(seq_arr_len, sizeof(*fence_arr), GFP_KERNEL);
2571 if (!fence_arr) {
2572 rc = -ENOMEM;
2573 goto free_seq_arr;
2574 }
2575
2576 /* initialize the multi-CS internal data */
2577 mcs_data.ctx = ctx;
2578 mcs_data.seq_arr = cs_seq_arr;
2579 mcs_data.fence_arr = fence_arr;
2580 mcs_data.arr_len = seq_arr_len;
2581
2582 hl_ctx_get(hdev, ctx);
2583
2584 /* poll all CS fences, extract timestamp */
2585 mcs_data.update_ts = true;
2586 rc = hl_cs_poll_fences(&mcs_data);
2587 /*
2588 * skip wait for CS completion when one of the below is true:
2589 * - an error on the poll function
2590 * - one or more CS in the list completed
2591 * - the user called ioctl with timeout 0
2592 */
2593 if (rc || mcs_data.completion_bitmap || !args->in.timeout_us)
2594 goto put_ctx;
2595
2596 /* wait (with timeout) for the first CS to be completed */
2597 mcs_data.timeout_us = args->in.timeout_us;
2598 rc = hl_wait_multi_cs_completion(&mcs_data);
2599 if (rc)
2600 goto put_ctx;
2601
2602 if (mcs_data.wait_status > 0) {
2603 /*
2604 * poll fences once again to update the CS map.
2605 * no timestamp should be updated this time.
2606 */
2607 mcs_data.update_ts = false;
2608 rc = hl_cs_poll_fences(&mcs_data);
2609
2610 /*
2611 * if hl_wait_multi_cs_completion returned before timeout (i.e.
2612 * it got a completion) we expect to see at least one CS
2613 * completed after the poll function.
2614 */
2615 if (!mcs_data.completion_bitmap) {
2616 dev_err(hdev->dev, "Multi-CS got completion on wait but no CS completed\n");
2617 rc = -EFAULT;
2618 }
1948 }2619 }
1949 2620
2621put_ctx:
1950 hl_ctx_put(ctx);2622 hl_ctx_put(ctx);
2623 kfree(fence_arr);
1951 2624
1952 return rc;2625free_seq_arr:
2626 kfree(cs_seq_arr);
2627
2628 /* update output args */
2629 memset(args, 0, sizeof(*args));
2630 if (rc)
2631 return rc;
2632
2633 if (mcs_data.completion_bitmap) {
2634 args->out.status = HL_WAIT_CS_STATUS_COMPLETED;
2635 args->out.cs_completion_map = mcs_data.completion_bitmap;
2636
2637 /* if timestamp not 0- it's valid */
2638 if (mcs_data.timestamp) {
2639 args->out.timestamp_nsec = mcs_data.timestamp;
2640 args->out.flags |= HL_WAIT_CS_STATUS_FLAG_TIMESTAMP_VLD;
2641 }
2642
2643 /* update if some CS was gone */
2644 if (mcs_data.timestamp)
2645 args->out.flags |= HL_WAIT_CS_STATUS_FLAG_GONE;
2646 } else if (mcs_data.wait_status == -ERESTARTSYS) {
2647 args->out.status = HL_WAIT_CS_STATUS_INTERRUPTED;
2648 } else {
2649 args->out.status = HL_WAIT_CS_STATUS_BUSY;
2650 }
2651
2652 return 0;
1953}2653}
1954 2654
1955static int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data)2655static int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data)
@@ -2015,9 +2715,9 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, · 59 unmodified lines
2015{2715{
2016 struct hl_user_pending_interrupt *pend;2716 struct hl_user_pending_interrupt *pend;
2017 struct hl_user_interrupt *interrupt;2717 struct hl_user_interrupt *interrupt;
2018 unsigned long timeout;2718 unsigned long timeout, flags;
2019 long completion_rc;
2020 u32 completion_value;2719 u32 completion_value;
2720 long completion_rc;
2021 int rc = 0;2721 int rc = 0;
2022 2722
2023 if (timeout_us == U32_MAX)2723 if (timeout_us == U32_MAX)
@@ -2040,17 +2740,10 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, · 16 unmodified lines
2040 else2740 else
2041 interrupt = &hdev->user_interrupt[interrupt_offset];2741 interrupt = &hdev->user_interrupt[interrupt_offset];
2042 2742
2043 spin_lock(&interrupt->wait_list_lock);
2044 if (!hl_device_operational(hdev, NULL)) {
2045 rc = -EPERM;
2046 goto unlock_and_free_fence;
2047 }
2048
2049 if (copy_from_user(&completion_value, u64_to_user_ptr(user_address), 4)) {2743 if (copy_from_user(&completion_value, u64_to_user_ptr(user_address), 4)) {
2050 dev_err(hdev->dev,2744 dev_err(hdev->dev, "Failed to copy completion value from user\n");
2051 "Failed to copy completion value from user\n");
2052 rc = -EFAULT;2745 rc = -EFAULT;
2053 goto unlock_and_free_fence;2746 goto free_fence;
2054 }2747 }
2055 2748
2056 if (completion_value >= target_value)2749 if (completion_value >= target_value)
@@ -2059,48 +2752,57 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, · 2 unmodified lines
2059 *status = CS_WAIT_STATUS_BUSY;2752 *status = CS_WAIT_STATUS_BUSY;
2060 2753
2061 if (!timeout_us || (*status == CS_WAIT_STATUS_COMPLETED))2754 if (!timeout_us || (*status == CS_WAIT_STATUS_COMPLETED))
2062 goto unlock_and_free_fence;2755 goto free_fence;
2063 2756
2064 /* Add pending user interrupt to relevant list for the interrupt2757 /* Add pending user interrupt to relevant list for the interrupt
2065 * handler to monitor2758 * handler to monitor
2066 */2759 */
2760 spin_lock_irqsave(&interrupt->wait_list_lock, flags);
2067 list_add_tail(&pend->wait_list_node, &interrupt->wait_list_head);2761 list_add_tail(&pend->wait_list_node, &interrupt->wait_list_head);
2068 spin_unlock(&interrupt->wait_list_lock);2762 spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
2069 2763
2070wait_again:2764wait_again:
2071 /* Wait for interrupt handler to signal completion */2765 /* Wait for interrupt handler to signal completion */
2072 completion_rc =2766 completion_rc = wait_for_completion_interruptible_timeout(&pend->fence.completion,
2073 wait_for_completion_interruptible_timeout(2767 timeout);
2074 &pend->fence.completion, timeout);
2075 2768
2076 /* If timeout did not expire we need to perform the comparison.2769 /* If timeout did not expire we need to perform the comparison.
2077 * If comparison fails, keep waiting until timeout expires2770 * If comparison fails, keep waiting until timeout expires
2078 */2771 */
2079 if (completion_rc > 0) {2772 if (completion_rc > 0) {
2080 if (copy_from_user(&completion_value,2773 if (copy_from_user(&completion_value, u64_to_user_ptr(user_address), 4)) {
2081 u64_to_user_ptr(user_address), 4)) {2774 dev_err(hdev->dev, "Failed to copy completion value from user\n");
2082 dev_err(hdev->dev,
2083 "Failed to copy completion value from user\n");
2084 rc = -EFAULT;2775 rc = -EFAULT;
2776
2085 goto remove_pending_user_interrupt;2777 goto remove_pending_user_interrupt;
2086 }2778 }
2087 2779
2088 if (completion_value >= target_value) {2780 if (completion_value >= target_value) {
2089 *status = CS_WAIT_STATUS_COMPLETED;2781 *status = CS_WAIT_STATUS_COMPLETED;
2090 } else {2782 } else {
2783 spin_lock_irqsave(&interrupt->wait_list_lock, flags);
2784 reinit_completion(&pend->fence.completion);
2091 timeout = completion_rc;2785 timeout = completion_rc;
2786
2787 spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
2092 goto wait_again;2788 goto wait_again;
2093 }2789 }
2790 } else if (completion_rc == -ERESTARTSYS) {
2791 dev_err_ratelimited(hdev->dev,
2792 "user process got signal while waiting for interrupt ID %d\n",
2793 interrupt->interrupt_id);
2794 *status = HL_WAIT_CS_STATUS_INTERRUPTED;
2795 rc = -EINTR;
2094 } else {2796 } else {
2095 *status = CS_WAIT_STATUS_BUSY;2797 *status = CS_WAIT_STATUS_BUSY;
2096 }2798 }
2097 2799
2098remove_pending_user_interrupt:2800remove_pending_user_interrupt:
2099 spin_lock(&interrupt->wait_list_lock);2801 spin_lock_irqsave(&interrupt->wait_list_lock, flags);
2100 list_del(&pend->wait_list_node);2802 list_del(&pend->wait_list_node);
2803 spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
2101 2804
2102unlock_and_free_fence:2805free_fence:
2103 spin_unlock(&interrupt->wait_list_lock);
2104 kfree(pend);2806 kfree(pend);
2105 hl_ctx_put(ctx);2807 hl_ctx_put(ctx);
2106 2808
@@ -2148,8 +2850,9 @@ static int hl_interrupt_wait_ioctl(struct hl_fpriv *hpriv, void *data) · 41 unmodified lines
2148 memset(args, 0, sizeof(*args));2850 memset(args, 0, sizeof(*args));
2149 2851
2150 if (rc) {2852 if (rc) {
2151 dev_err_ratelimited(hdev->dev,2853 if (rc != -EINTR)
2152 "interrupt_wait_ioctl failed (%d)\n", rc);2854 dev_err_ratelimited(hdev->dev,
2855 "interrupt_wait_ioctl failed (%d)\n", rc);
2153 2856
2154 return rc;2857 return rc;
2155 }2858 }
@@ -2173,8 +2876,16 @@ int hl_wait_ioctl(struct hl_fpriv *hpriv, void *data) · 17 unmodified lines
2173 u32 flags = args->in.flags;2876 u32 flags = args->in.flags;
2174 int rc;2877 int rc;
2175 2878
2879 /* If the device is not operational, no point in waiting for any command submission or
2880 * user interrupt
2881 */
2882 if (!hl_device_operational(hpriv->hdev, NULL))
2883 return -EPERM;
2884
2176 if (flags & HL_WAIT_CS_FLAGS_INTERRUPT)2885 if (flags & HL_WAIT_CS_FLAGS_INTERRUPT)
2177 rc = hl_interrupt_wait_ioctl(hpriv, data);2886 rc = hl_interrupt_wait_ioctl(hpriv, data);
2887 else if (flags & HL_WAIT_CS_FLAGS_MULTI_CS)
2888 rc = hl_multi_cs_wait_ioctl(hpriv, data);
2178 else2889 else
2179 rc = hl_cs_wait_ioctl(hpriv, data);2890 rc = hl_cs_wait_ioctl(hpriv, data);
2180 2891
drivers/misc/habanalabs/common/context.c+126−20
@@ -9,16 +9,70 @@ · 8 unmodified lines
9 9
10#include <linux/slab.h>10#include <linux/slab.h>
11 11
12void hl_encaps_handle_do_release(struct kref *ref)
13{
14 struct hl_cs_encaps_sig_handle *handle =
15 container_of(ref, struct hl_cs_encaps_sig_handle, refcount);
16 struct hl_ctx *ctx = handle->hdev->compute_ctx;
17 struct hl_encaps_signals_mgr *mgr = &ctx->sig_mgr;
18
19 spin_lock(&mgr->lock);
20 idr_remove(&mgr->handles, handle->id);
21 spin_unlock(&mgr->lock);
22
23 kfree(handle);
24}
25
26static void hl_encaps_handle_do_release_sob(struct kref *ref)
27{
28 struct hl_cs_encaps_sig_handle *handle =
29 container_of(ref, struct hl_cs_encaps_sig_handle, refcount);
30 struct hl_ctx *ctx = handle->hdev->compute_ctx;
31 struct hl_encaps_signals_mgr *mgr = &ctx->sig_mgr;
32
33 /* if we're here, then there was a signals reservation but cs with
34 * encaps signals wasn't submitted, so need to put refcount
35 * to hw_sob taken at the reservation.
36 */
37 hw_sob_put(handle->hw_sob);
38
39 spin_lock(&mgr->lock);
40 idr_remove(&mgr->handles, handle->id);
41 spin_unlock(&mgr->lock);
42
43 kfree(handle);
44}
45
46static void hl_encaps_sig_mgr_init(struct hl_encaps_signals_mgr *mgr)
47{
48 spin_lock_init(&mgr->lock);
49 idr_init(&mgr->handles);
50}
51
52static void hl_encaps_sig_mgr_fini(struct hl_device *hdev,
53 struct hl_encaps_signals_mgr *mgr)
54{
55 struct hl_cs_encaps_sig_handle *handle;
56 struct idr *idp;
57 u32 id;
58
59 idp = &mgr->handles;
60
61 if (!idr_is_empty(idp)) {
62 dev_warn(hdev->dev, "device released while some encaps signals handles are still allocated\n");
63 idr_for_each_entry(idp, handle, id)
64 kref_put(&handle->refcount,
65 hl_encaps_handle_do_release_sob);
66 }
67
68 idr_destroy(&mgr->handles);
69}
70
12static void hl_ctx_fini(struct hl_ctx *ctx)71static void hl_ctx_fini(struct hl_ctx *ctx)
13{72{
14 struct hl_device *hdev = ctx->hdev;73 struct hl_device *hdev = ctx->hdev;
15 int i;74 int i;
16 75
17 /* Release all allocated pending cb's, those cb's were never
18 * scheduled so it is safe to release them here
19 */
20 hl_pending_cb_list_flush(ctx);
21
22 /* Release all allocated HW block mapped list entries and destroy76 /* Release all allocated HW block mapped list entries and destroy
23 * the mutex.77 * the mutex.
24 */78 */
@@ -53,6 +107,7 @@ static void hl_ctx_fini(struct hl_ctx *ctx) · 28 unmodified lines
53 hl_cb_va_pool_fini(ctx);107 hl_cb_va_pool_fini(ctx);
54 hl_vm_ctx_fini(ctx);108 hl_vm_ctx_fini(ctx);
55 hl_asid_free(hdev, ctx->asid);109 hl_asid_free(hdev, ctx->asid);
110 hl_encaps_sig_mgr_fini(hdev, &ctx->sig_mgr);
56 111
57 /* Scrub both SRAM and DRAM */112 /* Scrub both SRAM and DRAM */
58 hdev->asic_funcs->scrub_device_mem(hdev, 0, 0);113 hdev->asic_funcs->scrub_device_mem(hdev, 0, 0);
@@ -130,9 +185,6 @@ void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx) · 71 unmodified lines
130{185{
131 if (kref_put(&ctx->refcount, hl_ctx_do_release) == 1)186 if (kref_put(&ctx->refcount, hl_ctx_do_release) == 1)
132 return;187 return;
133
134 dev_warn(hdev->dev,
135 "user process released device but its command submissions are still executing\n");
136}188}
137 189
138int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)190int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
@@ -144,11 +196,8 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx) · 5 unmodified lines
144 kref_init(&ctx->refcount);196 kref_init(&ctx->refcount);
145 197
146 ctx->cs_sequence = 1;198 ctx->cs_sequence = 1;
147 INIT_LIST_HEAD(&ctx->pending_cb_list);
148 spin_lock_init(&ctx->pending_cb_lock);
149 spin_lock_init(&ctx->cs_lock);199 spin_lock_init(&ctx->cs_lock);
150 atomic_set(&ctx->thread_ctx_switch_token, 1);200 atomic_set(&ctx->thread_ctx_switch_token, 1);
151 atomic_set(&ctx->thread_pending_cb_token, 1);
152 ctx->thread_ctx_switch_wait_token = 0;201 ctx->thread_ctx_switch_wait_token = 0;
153 ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs,202 ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs,
154 sizeof(struct hl_fence *),203 sizeof(struct hl_fence *),
@@ -200,6 +249,8 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx) · 45 unmodified lines
200 goto err_cb_va_pool_fini;249 goto err_cb_va_pool_fini;
201 }250 }
202 251
252 hl_encaps_sig_mgr_init(&ctx->sig_mgr);
253
203 dev_dbg(hdev->dev, "create user context %d\n", ctx->asid);254 dev_dbg(hdev->dev, "create user context %d\n", ctx->asid);
204 }255 }
205 256
@@ -229,31 +280,86 @@ int hl_ctx_put(struct hl_ctx *ctx) · 23 unmodified lines
229 return kref_put(&ctx->refcount, hl_ctx_do_release);280 return kref_put(&ctx->refcount, hl_ctx_do_release);
230}281}
231 282
232struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)283/*
284 * hl_ctx_get_fence_locked - get CS fence under CS lock
285 *
286 * @ctx: pointer to the context structure.
287 * @seq: CS sequences number
288 *
289 * @return valid fence pointer on success, NULL if fence is gone, otherwise
290 * error pointer.
291 *
292 * NOTE: this function shall be called with cs_lock locked
293 */
294static struct hl_fence *hl_ctx_get_fence_locked(struct hl_ctx *ctx, u64 seq)
233{295{
234 struct asic_fixed_properties *asic_prop = &ctx->hdev->asic_prop;296 struct asic_fixed_properties *asic_prop = &ctx->hdev->asic_prop;
235 struct hl_fence *fence;297 struct hl_fence *fence;
236 298
237 spin_lock(&ctx->cs_lock);299 if (seq >= ctx->cs_sequence)
238
239 if (seq >= ctx->cs_sequence) {
240 spin_unlock(&ctx->cs_lock);
241 return ERR_PTR(-EINVAL);300 return ERR_PTR(-EINVAL);
242 }
243 301
244 if (seq + asic_prop->max_pending_cs < ctx->cs_sequence) {302 if (seq + asic_prop->max_pending_cs < ctx->cs_sequence)
245 spin_unlock(&ctx->cs_lock);
246 return NULL;303 return NULL;
247 }
248 304
249 fence = ctx->cs_pending[seq & (asic_prop->max_pending_cs - 1)];305 fence = ctx->cs_pending[seq & (asic_prop->max_pending_cs - 1)];
250 hl_fence_get(fence);306 hl_fence_get(fence);
307 return fence;
308}
309
310struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
311{
312 struct hl_fence *fence;
313
314 spin_lock(&ctx->cs_lock);
315
316 fence = hl_ctx_get_fence_locked(ctx, seq);
251 317
252 spin_unlock(&ctx->cs_lock);318 spin_unlock(&ctx->cs_lock);
253 319
254 return fence;320 return fence;
255}321}
256 322
323/*
324 * hl_ctx_get_fences - get multiple CS fences under the same CS lock
325 *
326 * @ctx: pointer to the context structure.
327 * @seq_arr: array of CS sequences to wait for
328 * @fence: fence array to store the CS fences
329 * @arr_len: length of seq_arr and fence_arr
330 *
331 * @return 0 on success, otherwise non 0 error code
332 */
333int hl_ctx_get_fences(struct hl_ctx *ctx, u64 *seq_arr,
334 struct hl_fence **fence, u32 arr_len)
335{
336 struct hl_fence **fence_arr_base = fence;
337 int i, rc = 0;
338
339 spin_lock(&ctx->cs_lock);
340
341 for (i = 0; i < arr_len; i++, fence++) {
342 u64 seq = seq_arr[i];
343
344 *fence = hl_ctx_get_fence_locked(ctx, seq);
345
346 if (IS_ERR(*fence)) {
347 dev_err(ctx->hdev->dev,
348 "Failed to get fence for CS with seq 0x%llx\n",
349 seq);
350 rc = PTR_ERR(*fence);
351 break;
352 }
353 }
354
355 spin_unlock(&ctx->cs_lock);
356
357 if (rc)
358 hl_fences_put(fence_arr_base, i);
359
360 return rc;
361}
362
257/*363/*
258 * hl_ctx_mgr_init - initialize the context manager364 * hl_ctx_mgr_init - initialize the context manager
259 *365 *
drivers/misc/habanalabs/common/debugfs.c+174−10
@@ -209,12 +209,12 @@ static int userptr_show(struct seq_file *s, void *data) · 208 unmodified lines
209 if (first) {209 if (first) {
210 first = false;210 first = false;
211 seq_puts(s, "\n");211 seq_puts(s, "\n");
212 seq_puts(s, " user virtual address size dma dir\n");212 seq_puts(s, " pid user virtual address size dma dir\n");
213 seq_puts(s, "----------------------------------------------------------\n");213 seq_puts(s, "----------------------------------------------------------\n");
214 }214 }
215 seq_printf(s,215 seq_printf(s, " %-7d 0x%-14llx %-10llu %-30s\n",
216 " 0x%-14llx %-10u %-30s\n",216 userptr->pid, userptr->addr, userptr->size,
217 userptr->addr, userptr->size, dma_dir[userptr->dir]);217 dma_dir[userptr->dir]);
218 }218 }
219 219
220 spin_unlock(&dev_entry->userptr_spinlock);220 spin_unlock(&dev_entry->userptr_spinlock);
@@ -235,7 +235,7 @@ static int vm_show(struct seq_file *s, void *data) · 14 unmodified lines
235 struct hl_vm_hash_node *hnode;235 struct hl_vm_hash_node *hnode;
236 struct hl_userptr *userptr;236 struct hl_userptr *userptr;
237 struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;237 struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;
238 enum vm_type_t *vm_type;238 enum vm_type *vm_type;
239 bool once = true;239 bool once = true;
240 u64 j;240 u64 j;
241 int i;241 int i;
@@ -261,7 +261,7 @@ static int vm_show(struct seq_file *s, void *data) · 19 unmodified lines
261 if (*vm_type == VM_TYPE_USERPTR) {261 if (*vm_type == VM_TYPE_USERPTR) {
262 userptr = hnode->ptr;262 userptr = hnode->ptr;
263 seq_printf(s,263 seq_printf(s,
264 " 0x%-14llx %-10u\n",264 " 0x%-14llx %-10llu\n",
265 hnode->vaddr, userptr->size);265 hnode->vaddr, userptr->size);
266 } else {266 } else {
267 phys_pg_pack = hnode->ptr;267 phys_pg_pack = hnode->ptr;
@@ -320,6 +320,77 @@ static int vm_show(struct seq_file *s, void *data) · 52 unmodified lines
320 return 0;320 return 0;
321}321}
322 322
323static int userptr_lookup_show(struct seq_file *s, void *data)
324{
325 struct hl_debugfs_entry *entry = s->private;
326 struct hl_dbg_device_entry *dev_entry = entry->dev_entry;
327 struct scatterlist *sg;
328 struct hl_userptr *userptr;
329 bool first = true;
330 u64 total_npages, npages, sg_start, sg_end;
331 dma_addr_t dma_addr;
332 int i;
333
334 spin_lock(&dev_entry->userptr_spinlock);
335
336 list_for_each_entry(userptr, &dev_entry->userptr_list, debugfs_list) {
337 if (dev_entry->userptr_lookup >= userptr->addr &&
338 dev_entry->userptr_lookup < userptr->addr + userptr->size) {
339 total_npages = 0;
340 for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents,
341 i) {
342 npages = hl_get_sg_info(sg, &dma_addr);
343 sg_start = userptr->addr +
344 total_npages * PAGE_SIZE;
345 sg_end = userptr->addr +
346 (total_npages + npages) * PAGE_SIZE;
347
348 if (dev_entry->userptr_lookup >= sg_start &&
349 dev_entry->userptr_lookup < sg_end) {
350 dma_addr += (dev_entry->userptr_lookup -
351 sg_start);
352 if (first) {
353 first = false;
354 seq_puts(s, "\n");
355 seq_puts(s, " user virtual address dma address pid region start region size\n");
356 seq_puts(s, "---------------------------------------------------------------------------------------\n");
357 }
358 seq_printf(s, " 0x%-18llx 0x%-16llx %-8u 0x%-16llx %-12llu\n",
359 dev_entry->userptr_lookup,
360 (u64)dma_addr, userptr->pid,
361 userptr->addr, userptr->size);
362 }
363 total_npages += npages;
364 }
365 }
366 }
367
368 spin_unlock(&dev_entry->userptr_spinlock);
369
370 if (!first)
371 seq_puts(s, "\n");
372
373 return 0;
374}
375
376static ssize_t userptr_lookup_write(struct file *file, const char __user *buf,
377 size_t count, loff_t *f_pos)
378{
379 struct seq_file *s = file->private_data;
380 struct hl_debugfs_entry *entry = s->private;
381 struct hl_dbg_device_entry *dev_entry = entry->dev_entry;
382 ssize_t rc;
383 u64 value;
384
385 rc = kstrtoull_from_user(buf, count, 16, &value);
386 if (rc)
387 return rc;
388
389 dev_entry->userptr_lookup = value;
390
391 return count;
392}
393
323static int mmu_show(struct seq_file *s, void *data)394static int mmu_show(struct seq_file *s, void *data)
324{395{
325 struct hl_debugfs_entry *entry = s->private;396 struct hl_debugfs_entry *entry = s->private;
@@ -349,7 +420,7 @@ static int mmu_show(struct seq_file *s, void *data) · 23 unmodified lines
349 return 0;420 return 0;
350 }421 }
351 422
352 phys_addr = hops_info.hop_info[hops_info.used_hops - 1].hop_pte_val;423 hl_mmu_va_to_pa(ctx, virt_addr, &phys_addr);
353 424
354 if (hops_info.scrambled_vaddr &&425 if (hops_info.scrambled_vaddr &&
355 (dev_entry->mmu_addr != hops_info.scrambled_vaddr))426 (dev_entry->mmu_addr != hops_info.scrambled_vaddr))
@@ -491,11 +562,10 @@ static int device_va_to_pa(struct hl_device *hdev, u64 virt_addr, u32 size, · 135 unmodified lines
491 struct hl_vm_phys_pg_pack *phys_pg_pack;562 struct hl_vm_phys_pg_pack *phys_pg_pack;
492 struct hl_ctx *ctx = hdev->compute_ctx;563 struct hl_ctx *ctx = hdev->compute_ctx;
493 struct hl_vm_hash_node *hnode;564 struct hl_vm_hash_node *hnode;
565 u64 end_address, range_size;
494 struct hl_userptr *userptr;566 struct hl_userptr *userptr;
495 enum vm_type_t *vm_type;567 enum vm_type *vm_type;
496 bool valid = false;568 bool valid = false;
497 u64 end_address;
498 u32 range_size;
499 int i, rc = 0;569 int i, rc = 0;
500 570
501 if (!ctx) {571 if (!ctx) {
@@ -1043,6 +1113,60 @@ static ssize_t hl_security_violations_read(struct file *f, char __user *buf, · 541 unmodified lines
1043 return 0;1113 return 0;
1044}1114}
1045 1115
1116static ssize_t hl_state_dump_read(struct file *f, char __user *buf,
1117 size_t count, loff_t *ppos)
1118{
1119 struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
1120 ssize_t rc;
1121
1122 down_read(&entry->state_dump_sem);
1123 if (!entry->state_dump[entry->state_dump_head])
1124 rc = 0;
1125 else
1126 rc = simple_read_from_buffer(
1127 buf, count, ppos,
1128 entry->state_dump[entry->state_dump_head],
1129 strlen(entry->state_dump[entry->state_dump_head]));
1130 up_read(&entry->state_dump_sem);
1131
1132 return rc;
1133}
1134
1135static ssize_t hl_state_dump_write(struct file *f, const char __user *buf,
1136 size_t count, loff_t *ppos)
1137{
1138 struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
1139 struct hl_device *hdev = entry->hdev;
1140 ssize_t rc;
1141 u32 size;
1142 int i;
1143
1144 rc = kstrtouint_from_user(buf, count, 10, &size);
1145 if (rc)
1146 return rc;
1147
1148 if (size <= 0 || size >= ARRAY_SIZE(entry->state_dump)) {
1149 dev_err(hdev->dev, "Invalid number of dumps to skip\n");
1150 return -EINVAL;
1151 }
1152
1153 if (entry->state_dump[entry->state_dump_head]) {
1154 down_write(&entry->state_dump_sem);
1155 for (i = 0; i < size; ++i) {
1156 vfree(entry->state_dump[entry->state_dump_head]);
1157 entry->state_dump[entry->state_dump_head] = NULL;
1158 if (entry->state_dump_head > 0)
1159 entry->state_dump_head--;
1160 else
1161 entry->state_dump_head =
1162 ARRAY_SIZE(entry->state_dump) - 1;
1163 }
1164 up_write(&entry->state_dump_sem);
1165 }
1166
1167 return count;
1168}
1169
1046static const struct file_operations hl_data32b_fops = {1170static const struct file_operations hl_data32b_fops = {
1047 .owner = THIS_MODULE,1171 .owner = THIS_MODULE,
1048 .read = hl_data_read32,1172 .read = hl_data_read32,
@@ -1110,12 +1234,19 @@ static const struct file_operations hl_security_violations_fops = { · 61 unmodified lines
1110 .read = hl_security_violations_read1234 .read = hl_security_violations_read
1111};1235};
1112 1236
1237static const struct file_operations hl_state_dump_fops = {
1238 .owner = THIS_MODULE,
1239 .read = hl_state_dump_read,
1240 .write = hl_state_dump_write
1241};
1242
1113static const struct hl_info_list hl_debugfs_list[] = {1243static const struct hl_info_list hl_debugfs_list[] = {
1114 {"command_buffers", command_buffers_show, NULL},1244 {"command_buffers", command_buffers_show, NULL},
1115 {"command_submission", command_submission_show, NULL},1245 {"command_submission", command_submission_show, NULL},
1116 {"command_submission_jobs", command_submission_jobs_show, NULL},1246 {"command_submission_jobs", command_submission_jobs_show, NULL},
1117 {"userptr", userptr_show, NULL},1247 {"userptr", userptr_show, NULL},
1118 {"vm", vm_show, NULL},1248 {"vm", vm_show, NULL},
1249 {"userptr_lookup", userptr_lookup_show, userptr_lookup_write},
1119 {"mmu", mmu_show, mmu_asid_va_write},1250 {"mmu", mmu_show, mmu_asid_va_write},
1120 {"engines", engines_show, NULL}1251 {"engines", engines_show, NULL}
1121};1252};
@@ -1172,6 +1303,7 @@ void hl_debugfs_add_device(struct hl_device *hdev) · 50 unmodified lines
1172 INIT_LIST_HEAD(&dev_entry->userptr_list);1303 INIT_LIST_HEAD(&dev_entry->userptr_list);
1173 INIT_LIST_HEAD(&dev_entry->ctx_mem_hash_list);1304 INIT_LIST_HEAD(&dev_entry->ctx_mem_hash_list);
1174 mutex_init(&dev_entry->file_mutex);1305 mutex_init(&dev_entry->file_mutex);
1306 init_rwsem(&dev_entry->state_dump_sem);
1175 spin_lock_init(&dev_entry->cb_spinlock);1307 spin_lock_init(&dev_entry->cb_spinlock);
1176 spin_lock_init(&dev_entry->cs_spinlock);1308 spin_lock_init(&dev_entry->cs_spinlock);
1177 spin_lock_init(&dev_entry->cs_job_spinlock);1309 spin_lock_init(&dev_entry->cs_job_spinlock);
@@ -1283,6 +1415,12 @@ void hl_debugfs_add_device(struct hl_device *hdev) · 105 unmodified lines
1283 dev_entry->root,1415 dev_entry->root,
1284 &hdev->skip_reset_on_timeout);1416 &hdev->skip_reset_on_timeout);
1285 1417
1418 debugfs_create_file("state_dump",
1419 0600,
1420 dev_entry->root,
1421 dev_entry,
1422 &hl_state_dump_fops);
1423
1286 for (i = 0, entry = dev_entry->entry_arr ; i < count ; i++, entry++) {1424 for (i = 0, entry = dev_entry->entry_arr ; i < count ; i++, entry++) {
1287 debugfs_create_file(hl_debugfs_list[i].name,1425 debugfs_create_file(hl_debugfs_list[i].name,
1288 0444,1426 0444,
@@ -1297,6 +1435,7 @@ void hl_debugfs_add_device(struct hl_device *hdev) · 8 unmodified lines
1297void hl_debugfs_remove_device(struct hl_device *hdev)1435void hl_debugfs_remove_device(struct hl_device *hdev)
1298{1436{
1299 struct hl_dbg_device_entry *entry = &hdev->hl_debugfs;1437 struct hl_dbg_device_entry *entry = &hdev->hl_debugfs;
1438 int i;
1300 1439
1301 debugfs_remove_recursive(entry->root);1440 debugfs_remove_recursive(entry->root);
1302 1441
@@ -1304,6 +1443,9 @@ void hl_debugfs_remove_device(struct hl_device *hdev) · 1 unmodified line
1304 1443
1305 vfree(entry->blob_desc.data);1444 vfree(entry->blob_desc.data);
1306 1445
1446 for (i = 0; i < ARRAY_SIZE(entry->state_dump); ++i)
1447 vfree(entry->state_dump[i]);
1448
1307 kfree(entry->entry_arr);1449 kfree(entry->entry_arr);
1308}1450}
1309 1451
@@ -1416,6 +1558,28 @@ void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx) · 106 unmodified lines
1416 spin_unlock(&dev_entry->ctx_mem_hash_spinlock);1558 spin_unlock(&dev_entry->ctx_mem_hash_spinlock);
1417}1559}
1418 1560
1561/**
1562 * hl_debugfs_set_state_dump - register state dump making it accessible via
1563 * debugfs
1564 * @hdev: pointer to the device structure
1565 * @data: the actual dump data
1566 * @length: the length of the data
1567 */
1568void hl_debugfs_set_state_dump(struct hl_device *hdev, char *data,
1569 unsigned long length)
1570{
1571 struct hl_dbg_device_entry *dev_entry = &hdev->hl_debugfs;
1572
1573 down_write(&dev_entry->state_dump_sem);
1574
1575 dev_entry->state_dump_head = (dev_entry->state_dump_head + 1) %
1576 ARRAY_SIZE(dev_entry->state_dump);
1577 vfree(dev_entry->state_dump[dev_entry->state_dump_head]);
1578 dev_entry->state_dump[dev_entry->state_dump_head] = data;
1579
1580 up_write(&dev_entry->state_dump_sem);
1581}
1582
1419void __init hl_debugfs_init(void)1583void __init hl_debugfs_init(void)
1420{1584{
1421 hl_debug_root = debugfs_create_dir("habanalabs", NULL);1585 hl_debug_root = debugfs_create_dir("habanalabs", NULL);
drivers/misc/habanalabs/common/device.c+86−77
@@ -7,11 +7,11 @@ · 6 unmodified lines
7 7
8#define pr_fmt(fmt) "habanalabs: " fmt8#define pr_fmt(fmt) "habanalabs: " fmt
9 9
10#include <uapi/misc/habanalabs.h>
10#include "habanalabs.h"11#include "habanalabs.h"
11 12
12#include <linux/pci.h>13#include <linux/pci.h>
13#include <linux/hwmon.h>14#include <linux/hwmon.h>
14#include <uapi/misc/habanalabs.h>
15 15
16enum hl_device_status hl_device_status(struct hl_device *hdev)16enum hl_device_status hl_device_status(struct hl_device *hdev)
17{17{
@@ -23,6 +23,8 @@ enum hl_device_status hl_device_status(struct hl_device *hdev) · 5 unmodified lines
23 status = HL_DEVICE_STATUS_NEEDS_RESET;23 status = HL_DEVICE_STATUS_NEEDS_RESET;
24 else if (hdev->disabled)24 else if (hdev->disabled)
25 status = HL_DEVICE_STATUS_MALFUNCTION;25 status = HL_DEVICE_STATUS_MALFUNCTION;
26 else if (!hdev->init_done)
27 status = HL_DEVICE_STATUS_IN_DEVICE_CREATION;
26 else28 else
27 status = HL_DEVICE_STATUS_OPERATIONAL;29 status = HL_DEVICE_STATUS_OPERATIONAL;
28 30
@@ -44,6 +46,7 @@ bool hl_device_operational(struct hl_device *hdev, · 15 unmodified lines
44 case HL_DEVICE_STATUS_NEEDS_RESET:46 case HL_DEVICE_STATUS_NEEDS_RESET:
45 return false;47 return false;
46 case HL_DEVICE_STATUS_OPERATIONAL:48 case HL_DEVICE_STATUS_OPERATIONAL:
49 case HL_DEVICE_STATUS_IN_DEVICE_CREATION:
47 default:50 default:
48 return true;51 return true;
49 }52 }
@@ -129,8 +132,8 @@ static int hl_device_release(struct inode *inode, struct file *filp) · 79 unmodified lines
129 hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);132 hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);
130 133
131 if (!hl_hpriv_put(hpriv))134 if (!hl_hpriv_put(hpriv))
132 dev_warn(hdev->dev,135 dev_notice(hdev->dev,
133 "Device is still in use because there are live CS and/or memory mappings\n");136 "User process closed FD but device still in use\n");
134 137
135 hdev->last_open_session_duration_jif =138 hdev->last_open_session_duration_jif =
136 jiffies - hdev->last_successful_open_jif;139 jiffies - hdev->last_successful_open_jif;
@@ -308,9 +311,15 @@ static void device_hard_reset_pending(struct work_struct *work) · 171 unmodified lines
308 container_of(work, struct hl_device_reset_work,311 container_of(work, struct hl_device_reset_work,
309 reset_work.work);312 reset_work.work);
310 struct hl_device *hdev = device_reset_work->hdev;313 struct hl_device *hdev = device_reset_work->hdev;
314 u32 flags;
311 int rc;315 int rc;
312 316
313 rc = hl_device_reset(hdev, HL_RESET_HARD | HL_RESET_FROM_RESET_THREAD);317 flags = HL_RESET_HARD | HL_RESET_FROM_RESET_THREAD;
318
319 if (device_reset_work->fw_reset)
320 flags |= HL_RESET_FW;
321
322 rc = hl_device_reset(hdev, flags);
314 if ((rc == -EBUSY) && !hdev->device_fini_pending) {323 if ((rc == -EBUSY) && !hdev->device_fini_pending) {
315 dev_info(hdev->dev,324 dev_info(hdev->dev,
316 "Could not reset device. will try again in %u seconds",325 "Could not reset device. will try again in %u seconds",
@@ -682,6 +691,44 @@ out: · 365 unmodified lines
682 return rc;691 return rc;
683}692}
684 693
694static void take_release_locks(struct hl_device *hdev)
695{
696 /* Flush anyone that is inside the critical section of enqueue
697 * jobs to the H/W
698 */
699 hdev->asic_funcs->hw_queues_lock(hdev);
700 hdev->asic_funcs->hw_queues_unlock(hdev);
701
702 /* Flush processes that are sending message to CPU */
703 mutex_lock(&hdev->send_cpu_message_lock);
704 mutex_unlock(&hdev->send_cpu_message_lock);
705
706 /* Flush anyone that is inside device open */
707 mutex_lock(&hdev->fpriv_list_lock);
708 mutex_unlock(&hdev->fpriv_list_lock);
709}
710
711static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_reset)
712{
713 if (hard_reset)
714 device_late_fini(hdev);
715
716 /*
717 * Halt the engines and disable interrupts so we won't get any more
718 * completions from H/W and we won't have any accesses from the
719 * H/W to the host machine
720 */
721 hdev->asic_funcs->halt_engines(hdev, hard_reset, fw_reset);
722
723 /* Go over all the queues, release all CS and their jobs */
724 hl_cs_rollback_all(hdev);
725
726 /* Release all pending user interrupts, each pending user interrupt
727 * holds a reference to user context
728 */
729 hl_release_pending_user_interrupts(hdev);
730}
731
685/*732/*
686 * hl_device_suspend - initiate device suspend733 * hl_device_suspend - initiate device suspend
687 *734 *
@@ -707,16 +754,7 @@ int hl_device_suspend(struct hl_device *hdev) · 19 unmodified lines
707 /* This blocks all other stuff that is not blocked by in_reset */754 /* This blocks all other stuff that is not blocked by in_reset */
708 hdev->disabled = true;755 hdev->disabled = true;
709 756
710 /*757 take_release_locks(hdev);
711 * Flush anyone that is inside the critical section of enqueue
712 * jobs to the H/W
713 */
714 hdev->asic_funcs->hw_queues_lock(hdev);
715 hdev->asic_funcs->hw_queues_unlock(hdev);
716
717 /* Flush processes that are sending message to CPU */
718 mutex_lock(&hdev->send_cpu_message_lock);
719 mutex_unlock(&hdev->send_cpu_message_lock);
720 758
721 rc = hdev->asic_funcs->suspend(hdev);759 rc = hdev->asic_funcs->suspend(hdev);
722 if (rc)760 if (rc)
@@ -819,6 +857,11 @@ static int device_kill_open_processes(struct hl_device *hdev, u32 timeout) · 96 unmodified lines
819 usleep_range(1000, 10000);857 usleep_range(1000, 10000);
820 858
821 put_task_struct(task);859 put_task_struct(task);
860 } else {
861 dev_warn(hdev->dev,
862 "Can't get task struct for PID so giving up on killing process\n");
863 mutex_unlock(&hdev->fpriv_list_lock);
864 return -ETIME;
822 }865 }
823 }866 }
824 867
@@ -885,7 +928,7 @@ static void device_disable_open_processes(struct hl_device *hdev) · 60 unmodified lines
885int hl_device_reset(struct hl_device *hdev, u32 flags)928int hl_device_reset(struct hl_device *hdev, u32 flags)
886{929{
887 u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};930 u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
888 bool hard_reset, from_hard_reset_thread, hard_instead_soft = false;931 bool hard_reset, from_hard_reset_thread, fw_reset, hard_instead_soft = false;
889 int i, rc;932 int i, rc;
890 933
891 if (!hdev->init_done) {934 if (!hdev->init_done) {
@@ -894,8 +937,9 @@ int hl_device_reset(struct hl_device *hdev, u32 flags) · 2 unmodified lines
894 return 0;937 return 0;
895 }938 }
896 939
897 hard_reset = (flags & HL_RESET_HARD) != 0;940 hard_reset = !!(flags & HL_RESET_HARD);
898 from_hard_reset_thread = (flags & HL_RESET_FROM_RESET_THREAD) != 0;941 from_hard_reset_thread = !!(flags & HL_RESET_FROM_RESET_THREAD);
942 fw_reset = !!(flags & HL_RESET_FW);
899 943
900 if (!hard_reset && !hdev->supports_soft_reset) {944 if (!hard_reset && !hdev->supports_soft_reset) {
901 hard_instead_soft = true;945 hard_instead_soft = true;
@@ -947,11 +991,13 @@ do_reset: · 45 unmodified lines
947 else991 else
948 hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;992 hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
949 993
950 /*994 /* If reset is due to heartbeat, device CPU is no responsive in
951 * if reset is due to heartbeat, device CPU is no responsive in995 * which case no point sending PCI disable message to it.
952 * which case no point sending PCI disable message to it996 *
997 * If F/W is performing the reset, no need to send it a message to disable
998 * PCI access
953 */999 */
954 if (hard_reset && !(flags & HL_RESET_HEARTBEAT)) {1000 if (hard_reset && !(flags & (HL_RESET_HEARTBEAT | HL_RESET_FW))) {
955 /* Disable PCI access from device F/W so he won't send1001 /* Disable PCI access from device F/W so he won't send
956 * us additional interrupts. We disable MSI/MSI-X at1002 * us additional interrupts. We disable MSI/MSI-X at
957 * the halt_engines function and we can't have the F/W1003 * the halt_engines function and we can't have the F/W
@@ -970,15 +1016,7 @@ do_reset: · 12 unmodified lines
970 /* This also blocks future CS/VM/JOB completion operations */1016 /* This also blocks future CS/VM/JOB completion operations */
971 hdev->disabled = true;1017 hdev->disabled = true;
972 1018
973 /* Flush anyone that is inside the critical section of enqueue1019 take_release_locks(hdev);
974 * jobs to the H/W
975 */
976 hdev->asic_funcs->hw_queues_lock(hdev);
977 hdev->asic_funcs->hw_queues_unlock(hdev);
978
979 /* Flush anyone that is inside device open */
980 mutex_lock(&hdev->fpriv_list_lock);
981 mutex_unlock(&hdev->fpriv_list_lock);
982 1020
983 dev_err(hdev->dev, "Going to RESET device!\n");1021 dev_err(hdev->dev, "Going to RESET device!\n");
984 }1022 }
@@ -989,6 +1027,8 @@ again: · 4 unmodified lines
989 1027
990 hdev->process_kill_trial_cnt = 0;1028 hdev->process_kill_trial_cnt = 0;
991 1029
1030 hdev->device_reset_work.fw_reset = fw_reset;
1031
992 /*1032 /*
993 * Because the reset function can't run from heartbeat work,1033 * Because the reset function can't run from heartbeat work,
994 * we need to call the reset function from a dedicated work.1034 * we need to call the reset function from a dedicated work.
@@ -999,31 +1039,7 @@ again: · 4 unmodified lines
999 return 0;1039 return 0;
1000 }1040 }
1001 1041
1002 if (hard_reset) {1042 cleanup_resources(hdev, hard_reset, fw_reset);
1003 device_late_fini(hdev);
1004
1005 /*
1006 * Now that the heartbeat thread is closed, flush processes
1007 * which are sending messages to CPU
1008 */
1009 mutex_lock(&hdev->send_cpu_message_lock);
1010 mutex_unlock(&hdev->send_cpu_message_lock);
1011 }
1012
1013 /*
1014 * Halt the engines and disable interrupts so we won't get any more
1015 * completions from H/W and we won't have any accesses from the
1016 * H/W to the host machine
1017 */
1018 hdev->asic_funcs->halt_engines(hdev, hard_reset);
1019
1020 /* Go over all the queues, release all CS and their jobs */
1021 hl_cs_rollback_all(hdev);
1022
1023 /* Release all pending user interrupts, each pending user interrupt
1024 * holds a reference to user context
1025 */
1026 hl_release_pending_user_interrupts(hdev);
1027 1043
1028kill_processes:1044kill_processes:
1029 if (hard_reset) {1045 if (hard_reset) {
@@ -1057,12 +1073,15 @@ kill_processes: · 27 unmodified lines
1057 }1073 }
1058 1074
1059 /* Reset the H/W. It will be in idle state after this returns */1075 /* Reset the H/W. It will be in idle state after this returns */
1060 hdev->asic_funcs->hw_fini(hdev, hard_reset);1076 hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset);
1061 1077
1062 if (hard_reset) {1078 if (hard_reset) {
1079 hdev->fw_loader.linux_loaded = false;
1080
1063 /* Release kernel context */1081 /* Release kernel context */
1064 if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1)1082 if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1)
1065 hdev->kernel_ctx = NULL;1083 hdev->kernel_ctx = NULL;
1084
1066 hl_vm_fini(hdev);1085 hl_vm_fini(hdev);
1067 hl_mmu_fini(hdev);1086 hl_mmu_fini(hdev);
1068 hl_eq_reset(hdev, &hdev->event_queue);1087 hl_eq_reset(hdev, &hdev->event_queue);
@@ -1292,6 +1311,10 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) · 223 unmodified lines
1292 if (rc)1311 if (rc)
1293 goto user_interrupts_fini;1312 goto user_interrupts_fini;
1294 1313
1314
1315 /* initialize completion structure for multi CS wait */
1316 hl_multi_cs_completion_init(hdev);
1317
1295 /*1318 /*
1296 * Initialize the H/W queues. Must be done before hw_init, because1319 * Initialize the H/W queues. Must be done before hw_init, because
1297 * there the addresses of the kernel queue are being written to the1320 * there the addresses of the kernel queue are being written to the
@@ -1361,6 +1384,8 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) · 63 unmodified lines
1361 1384
1362 hdev->compute_ctx = NULL;1385 hdev->compute_ctx = NULL;
1363 1386
1387 hdev->asic_funcs->state_dump_init(hdev);
1388
1364 hl_debugfs_add_device(hdev);1389 hl_debugfs_add_device(hdev);
1365 1390
1366 /* debugfs nodes are created in hl_ctx_init so it must be called after1391 /* debugfs nodes are created in hl_ctx_init so it must be called after
@@ -1567,31 +1592,13 @@ void hl_device_fini(struct hl_device *hdev) · 200 unmodified lines
1567 /* Mark device as disabled */1592 /* Mark device as disabled */
1568 hdev->disabled = true;1593 hdev->disabled = true;
1569 1594
1570 /* Flush anyone that is inside the critical section of enqueue1595 take_release_locks(hdev);
1571 * jobs to the H/W
1572 */
1573 hdev->asic_funcs->hw_queues_lock(hdev);
1574 hdev->asic_funcs->hw_queues_unlock(hdev);
1575
1576 /* Flush anyone that is inside device open */
1577 mutex_lock(&hdev->fpriv_list_lock);
1578 mutex_unlock(&hdev->fpriv_list_lock);
1579 1596
1580 hdev->hard_reset_pending = true;1597 hdev->hard_reset_pending = true;
1581 1598
1582 hl_hwmon_fini(hdev);1599 hl_hwmon_fini(hdev);
1583 1600
1584 device_late_fini(hdev);1601 cleanup_resources(hdev, true, false);
1585
1586 /*
1587 * Halt the engines and disable interrupts so we won't get any more
1588 * completions from H/W and we won't have any accesses from the
1589 * H/W to the host machine
1590 */
1591 hdev->asic_funcs->halt_engines(hdev, true);
1592
1593 /* Go over all the queues, release all CS and their jobs */
1594 hl_cs_rollback_all(hdev);
1595 1602
1596 /* Kill processes here after CS rollback. This is because the process1603 /* Kill processes here after CS rollback. This is because the process
1597 * can't really exit until all its CSs are done, which is what we1604 * can't really exit until all its CSs are done, which is what we
@@ -1610,7 +1617,9 @@ void hl_device_fini(struct hl_device *hdev) · 12 unmodified lines
1610 hl_cb_pool_fini(hdev);1617 hl_cb_pool_fini(hdev);
1611 1618
1612 /* Reset the H/W. It will be in idle state after this returns */1619 /* Reset the H/W. It will be in idle state after this returns */
1613 hdev->asic_funcs->hw_fini(hdev, true);1620 hdev->asic_funcs->hw_fini(hdev, true, false);
1621
1622 hdev->fw_loader.linux_loaded = false;
1614 1623
1615 /* Release kernel context */1624 /* Release kernel context */
1616 if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))1625 if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
drivers/misc/habanalabs/common/firmware_if.c+32−24
@@ -1,7 +1,7 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2 2
3/*3/*
4 * Copyright 2016-2019 HabanaLabs, Ltd.4 * Copyright 2016-2021 HabanaLabs, Ltd.
5 * All Rights Reserved.5 * All Rights Reserved.
6 */6 */
7 7
@@ -240,11 +240,15 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, · 232 unmodified lines
240 /* set fence to a non valid value */240 /* set fence to a non valid value */
241 pkt->fence = cpu_to_le32(UINT_MAX);241 pkt->fence = cpu_to_le32(UINT_MAX);
242 242
243 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, len, pkt_dma_addr);243 /*
244 if (rc) {244 * The CPU queue is a synchronous queue with an effective depth of
245 dev_err(hdev->dev, "Failed to send CB on CPU PQ (%d)\n", rc);245 * a single entry (although it is allocated with room for multiple
246 goto out;246 * entries). We lock on it using 'send_cpu_message_lock' which
247 }247 * serializes accesses to the CPU queue.
248 * Which means that we don't need to lock the access to the entire H/W
249 * queues module when submitting a JOB to the CPU queue.
250 */
251 hl_hw_queue_submit_bd(hdev, queue, 0, len, pkt_dma_addr);
248 252
249 if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)253 if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
250 expected_ack_val = queue->pi;254 expected_ack_val = queue->pi;
@@ -663,17 +667,15 @@ int hl_fw_cpucp_info_get(struct hl_device *hdev, · 412 unmodified lines
663 hdev->event_queue.check_eqe_index = false;667 hdev->event_queue.check_eqe_index = false;
664 668
665 /* Read FW application security bits again */669 /* Read FW application security bits again */
666 if (hdev->asic_prop.fw_cpu_boot_dev_sts0_valid) {670 if (prop->fw_cpu_boot_dev_sts0_valid) {
667 hdev->asic_prop.fw_app_cpu_boot_dev_sts0 =671 prop->fw_app_cpu_boot_dev_sts0 = RREG32(sts_boot_dev_sts0_reg);
668 RREG32(sts_boot_dev_sts0_reg);672 if (prop->fw_app_cpu_boot_dev_sts0 &
669 if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
670 CPU_BOOT_DEV_STS0_EQ_INDEX_EN)673 CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
671 hdev->event_queue.check_eqe_index = true;674 hdev->event_queue.check_eqe_index = true;
672 }675 }
673 676
674 if (hdev->asic_prop.fw_cpu_boot_dev_sts1_valid)677 if (prop->fw_cpu_boot_dev_sts1_valid)
675 hdev->asic_prop.fw_app_cpu_boot_dev_sts1 =678 prop->fw_app_cpu_boot_dev_sts1 = RREG32(sts_boot_dev_sts1_reg);
676 RREG32(sts_boot_dev_sts1_reg);
677 679
678out:680out:
679 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,681 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
@@ -1008,6 +1010,11 @@ void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev) · 328 unmodified lines
1008 } else {1010 } else {
1009 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);1011 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
1010 msleep(static_loader->cpu_reset_wait_msec);1012 msleep(static_loader->cpu_reset_wait_msec);
1013
1014 /* Must clear this register in order to prevent preboot
1015 * from reading WFE after reboot
1016 */
1017 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_NA);
1011 }1018 }
1012 1019
1013 hdev->device_cpu_is_halted = true;1020 hdev->device_cpu_is_halted = true;
@@ -1055,6 +1062,10 @@ static void detect_cpu_boot_status(struct hl_device *hdev, u32 status) · 41 unmodified lines
1055 dev_err(hdev->dev,1062 dev_err(hdev->dev,
1056 "Device boot progress - Thermal Sensor initialization failed\n");1063 "Device boot progress - Thermal Sensor initialization failed\n");
1057 break;1064 break;
1065 case CPU_BOOT_STATUS_SECURITY_READY:
1066 dev_err(hdev->dev,
1067 "Device boot progress - Stuck in preboot after security initialization\n");
1068 break;
1058 default:1069 default:
1059 dev_err(hdev->dev,1070 dev_err(hdev->dev,
1060 "Device boot progress - Invalid status code %d\n",1071 "Device boot progress - Invalid status code %d\n",
@@ -1238,11 +1249,6 @@ static void hl_fw_preboot_update_state(struct hl_device *hdev) · 177 unmodified lines
1238 * b. Check whether hard reset is done by boot cpu1249 * b. Check whether hard reset is done by boot cpu
1239 * 3. FW application - a. Fetch fw application security status1250 * 3. FW application - a. Fetch fw application security status
1240 * b. Check whether hard reset is done by fw app1251 * b. Check whether hard reset is done by fw app
1241 *
1242 * Preboot:
1243 * Check security status bit (CPU_BOOT_DEV_STS0_ENABLED). If set, then-
1244 * check security enabled bit (CPU_BOOT_DEV_STS0_SECURITY_EN)
1245 * If set, then mark GIC controller to be disabled.
1246 */1252 */
1247 prop->hard_reset_done_by_fw =1253 prop->hard_reset_done_by_fw =
1248 !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);1254 !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
@@ -1953,8 +1959,8 @@ static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev) · 704 unmodified lines
1953 if (!hdev->asic_prop.gic_interrupts_enable &&1959 if (!hdev->asic_prop.gic_interrupts_enable &&
1954 !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &1960 !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
1955 CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {1961 CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
1956 dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_irq_ctrl;1962 dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_pi_upd_irq;
1957 dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_irq_ctrl;1963 dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_pi_upd_irq;
1958 1964
1959 dev_warn(hdev->dev,1965 dev_warn(hdev->dev,
1960 "Using a single interrupt interface towards cpucp");1966 "Using a single interrupt interface towards cpucp");
@@ -2122,8 +2128,7 @@ static void hl_fw_linux_update_state(struct hl_device *hdev, · 161 unmodified lines
2122 2128
2123 /* Read FW application security bits */2129 /* Read FW application security bits */
2124 if (prop->fw_cpu_boot_dev_sts0_valid) {2130 if (prop->fw_cpu_boot_dev_sts0_valid) {
2125 prop->fw_app_cpu_boot_dev_sts0 =2131 prop->fw_app_cpu_boot_dev_sts0 = RREG32(cpu_boot_dev_sts0_reg);
2126 RREG32(cpu_boot_dev_sts0_reg);
2127 2132
2128 if (prop->fw_app_cpu_boot_dev_sts0 &2133 if (prop->fw_app_cpu_boot_dev_sts0 &
2129 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)2134 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
@@ -2143,8 +2148,7 @@ static void hl_fw_linux_update_state(struct hl_device *hdev, · 13 unmodified lines
2143 }2148 }
2144 2149
2145 if (prop->fw_cpu_boot_dev_sts1_valid) {2150 if (prop->fw_cpu_boot_dev_sts1_valid) {
2146 prop->fw_app_cpu_boot_dev_sts1 =2151 prop->fw_app_cpu_boot_dev_sts1 = RREG32(cpu_boot_dev_sts1_reg);
2147 RREG32(cpu_boot_dev_sts1_reg);
2148 2152
2149 dev_dbg(hdev->dev,2153 dev_dbg(hdev->dev,
2150 "Firmware application CPU status1 %#x\n",2154 "Firmware application CPU status1 %#x\n",
@@ -2235,6 +2239,10 @@ static int hl_fw_dynamic_init_cpu(struct hl_device *hdev, · 84 unmodified lines
2235 dev_info(hdev->dev,2239 dev_info(hdev->dev,
2236 "Loading firmware to device, may take some time...\n");2240 "Loading firmware to device, may take some time...\n");
2237 2241
2242 /*
2243 * In this stage, "cpu_dyn_regs" contains only LKD's hard coded values!
2244 * It will be updated from FW after hl_fw_dynamic_request_descriptor().
2245 */
2238 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;2246 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2239 2247
2240 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,2248 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,
drivers/misc/habanalabs/common/habanalabs.h+375−46
@@ -20,6 +20,7 @@ · 19 unmodified lines
20#include <linux/scatterlist.h>20#include <linux/scatterlist.h>
21#include <linux/hashtable.h>21#include <linux/hashtable.h>
22#include <linux/debugfs.h>22#include <linux/debugfs.h>
23#include <linux/rwsem.h>
23#include <linux/bitfield.h>24#include <linux/bitfield.h>
24#include <linux/genalloc.h>25#include <linux/genalloc.h>
25#include <linux/sched/signal.h>26#include <linux/sched/signal.h>
@@ -65,6 +66,11 @@ · 39 unmodified lines
65 66
66#define HL_COMMON_USER_INTERRUPT_ID 0xFFF67#define HL_COMMON_USER_INTERRUPT_ID 0xFFF
67 68
69#define HL_STATE_DUMP_HIST_LEN 5
70
71#define OBJ_NAMES_HASH_TABLE_BITS 7 /* 1 << 7 buckets */
72#define SYNC_TO_ENGINE_HASH_TABLE_BITS 7 /* 1 << 7 buckets */
73
68/* Memory */74/* Memory */
69#define MEM_HASH_TABLE_BITS 7 /* 1 << 7 buckets */75#define MEM_HASH_TABLE_BITS 7 /* 1 << 7 buckets */
70 76
@@ -122,12 +128,17 @@ enum hl_mmu_page_table_location { · 51 unmodified lines
122 *128 *
123 * - HL_RESET_DEVICE_RELEASE129 * - HL_RESET_DEVICE_RELEASE
124 * Set if reset is due to device release130 * Set if reset is due to device release
131 *
132 * - HL_RESET_FW
133 * F/W will perform the reset. No need to ask it to reset the device. This is relevant
134 * only when running with secured f/w
125 */135 */
126#define HL_RESET_HARD (1 << 0)136#define HL_RESET_HARD (1 << 0)
127#define HL_RESET_FROM_RESET_THREAD (1 << 1)137#define HL_RESET_FROM_RESET_THREAD (1 << 1)
128#define HL_RESET_HEARTBEAT (1 << 2)138#define HL_RESET_HEARTBEAT (1 << 2)
129#define HL_RESET_TDR (1 << 3)139#define HL_RESET_TDR (1 << 3)
130#define HL_RESET_DEVICE_RELEASE (1 << 4)140#define HL_RESET_DEVICE_RELEASE (1 << 4)
141#define HL_RESET_FW (1 << 5)
131 142
132#define HL_MAX_SOBS_PER_MONITOR 8143#define HL_MAX_SOBS_PER_MONITOR 8
133 144
@@ -236,7 +247,9 @@ enum hl_cs_type { · 102 unmodified lines
236 CS_TYPE_DEFAULT,247 CS_TYPE_DEFAULT,
237 CS_TYPE_SIGNAL,248 CS_TYPE_SIGNAL,
238 CS_TYPE_WAIT,249 CS_TYPE_WAIT,
239 CS_TYPE_COLLECTIVE_WAIT250 CS_TYPE_COLLECTIVE_WAIT,
251 CS_RESERVE_SIGNALS,
252 CS_UNRESERVE_SIGNALS
240};253};
241 254
242/*255/*
@@ -281,13 +294,17 @@ enum queue_cb_alloc_flags { · 38 unmodified lines
281 * @hdev: habanalabs device structure.294 * @hdev: habanalabs device structure.
282 * @kref: refcount of this SOB. The SOB will reset once the refcount is zero.295 * @kref: refcount of this SOB. The SOB will reset once the refcount is zero.
283 * @sob_id: id of this SOB.296 * @sob_id: id of this SOB.
297 * @sob_addr: the sob offset from the base address.
284 * @q_idx: the H/W queue that uses this SOB.298 * @q_idx: the H/W queue that uses this SOB.
299 * @need_reset: reset indication set when switching to the other sob.
285 */300 */
286struct hl_hw_sob {301struct hl_hw_sob {
287 struct hl_device *hdev;302 struct hl_device *hdev;
288 struct kref kref;303 struct kref kref;
289 u32 sob_id;304 u32 sob_id;
305 u32 sob_addr;
290 u32 q_idx;306 u32 q_idx;
307 bool need_reset;
291};308};
292 309
293enum hl_collective_mode {310enum hl_collective_mode {
@@ -317,11 +334,11 @@ struct hw_queue_properties { · 23 unmodified lines
317};334};
318 335
319/**336/**
320 * enum vm_type_t - virtual memory mapping request information.337 * enum vm_type - virtual memory mapping request information.
321 * @VM_TYPE_USERPTR: mapping of user memory to device virtual address.338 * @VM_TYPE_USERPTR: mapping of user memory to device virtual address.
322 * @VM_TYPE_PHYS_PACK: mapping of DRAM memory to device virtual address.339 * @VM_TYPE_PHYS_PACK: mapping of DRAM memory to device virtual address.
323 */340 */
324enum vm_type_t {341enum vm_type {
325 VM_TYPE_USERPTR = 0x1,342 VM_TYPE_USERPTR = 0x1,
326 VM_TYPE_PHYS_PACK = 0x2343 VM_TYPE_PHYS_PACK = 0x2
327};344};
@@ -381,6 +398,16 @@ struct hl_mmu_properties { · 53 unmodified lines
381 u8 host_resident;398 u8 host_resident;
382};399};
383 400
401/**
402 * struct hl_hints_range - hint addresses reserved va range.
403 * @start_addr: start address of the va range.
404 * @end_addr: end address of the va range.
405 */
406struct hl_hints_range {
407 u64 start_addr;
408 u64 end_addr;
409};
410
384/**411/**
385 * struct asic_fixed_properties - ASIC specific immutable properties.412 * struct asic_fixed_properties - ASIC specific immutable properties.
386 * @hw_queues_props: H/W queues properties.413 * @hw_queues_props: H/W queues properties.
@@ -392,6 +419,10 @@ struct hl_mmu_properties { · 5 unmodified lines
392 * @pmmu: PCI (host) MMU address translation properties.419 * @pmmu: PCI (host) MMU address translation properties.
393 * @pmmu_huge: PCI (host) MMU address translation properties for memory420 * @pmmu_huge: PCI (host) MMU address translation properties for memory
394 * allocated with huge pages.421 * allocated with huge pages.
422 * @hints_dram_reserved_va_range: dram hint addresses reserved range.
423 * @hints_host_reserved_va_range: host hint addresses reserved range.
424 * @hints_host_hpage_reserved_va_range: host huge page hint addresses reserved
425 * range.
395 * @sram_base_address: SRAM physical start address.426 * @sram_base_address: SRAM physical start address.
396 * @sram_end_address: SRAM physical end address.427 * @sram_end_address: SRAM physical end address.
397 * @sram_user_base_address - SRAM physical start address for user access.428 * @sram_user_base_address - SRAM physical start address for user access.
@@ -412,6 +443,10 @@ struct hl_mmu_properties { · 14 unmodified lines
412 * to the device's MMU.443 * to the device's MMU.
413 * @cb_va_end_addr: virtual end address of command buffers which are mapped to444 * @cb_va_end_addr: virtual end address of command buffers which are mapped to
414 * the device's MMU.445 * the device's MMU.
446 * @dram_hints_align_mask: dram va hint addresses alignment mask which is used
447 * for hints validity check.
448 * device_dma_offset_for_host_access: the offset to add to host DMA addresses
449 * to enable the device to access them.
415 * @mmu_pgt_size: MMU page tables total size.450 * @mmu_pgt_size: MMU page tables total size.
416 * @mmu_pte_size: PTE size in MMU page tables.451 * @mmu_pte_size: PTE size in MMU page tables.
417 * @mmu_hop_table_size: MMU hop table size.452 * @mmu_hop_table_size: MMU hop table size.
@@ -459,6 +494,8 @@ struct hl_mmu_properties { · 41 unmodified lines
459 * reserved for the user494 * reserved for the user
460 * @first_available_cq: first available CQ for the user.495 * @first_available_cq: first available CQ for the user.
461 * @user_interrupt_count: number of user interrupts.496 * @user_interrupt_count: number of user interrupts.
497 * @server_type: Server type that the ASIC is currently installed in.
498 * The value is according to enum hl_server_type in uapi file.
462 * @tpc_enabled_mask: which TPCs are enabled.499 * @tpc_enabled_mask: which TPCs are enabled.
463 * @completion_queues_count: number of completion queues.500 * @completion_queues_count: number of completion queues.
464 * @fw_security_enabled: true if security measures are enabled in firmware,501 * @fw_security_enabled: true if security measures are enabled in firmware,
@@ -470,6 +507,7 @@ struct hl_mmu_properties { · 5 unmodified lines
470 * @dram_supports_virtual_memory: is there an MMU towards the DRAM507 * @dram_supports_virtual_memory: is there an MMU towards the DRAM
471 * @hard_reset_done_by_fw: true if firmware is handling hard reset flow508 * @hard_reset_done_by_fw: true if firmware is handling hard reset flow
472 * @num_functional_hbms: number of functional HBMs in each DCORE.509 * @num_functional_hbms: number of functional HBMs in each DCORE.
510 * @hints_range_reservation: device support hint addresses range reservation.
473 * @iatu_done_by_fw: true if iATU configuration is being done by FW.511 * @iatu_done_by_fw: true if iATU configuration is being done by FW.
474 * @dynamic_fw_load: is dynamic FW load is supported.512 * @dynamic_fw_load: is dynamic FW load is supported.
475 * @gic_interrupts_enable: true if FW is not blocking GIC controller,513 * @gic_interrupts_enable: true if FW is not blocking GIC controller,
@@ -483,6 +521,9 @@ struct asic_fixed_properties { · 7 unmodified lines
483 struct hl_mmu_properties dmmu;521 struct hl_mmu_properties dmmu;
484 struct hl_mmu_properties pmmu;522 struct hl_mmu_properties pmmu;
485 struct hl_mmu_properties pmmu_huge;523 struct hl_mmu_properties pmmu_huge;
524 struct hl_hints_range hints_dram_reserved_va_range;
525 struct hl_hints_range hints_host_reserved_va_range;
526 struct hl_hints_range hints_host_hpage_reserved_va_range;
486 u64 sram_base_address;527 u64 sram_base_address;
487 u64 sram_end_address;528 u64 sram_end_address;
488 u64 sram_user_base_address;529 u64 sram_user_base_address;
@@ -500,6 +541,8 @@ struct asic_fixed_properties { · 11 unmodified lines
500 u64 mmu_dram_default_page_addr;541 u64 mmu_dram_default_page_addr;
501 u64 cb_va_start_addr;542 u64 cb_va_start_addr;
502 u64 cb_va_end_addr;543 u64 cb_va_end_addr;
544 u64 dram_hints_align_mask;
545 u64 device_dma_offset_for_host_access;
503 u32 mmu_pgt_size;546 u32 mmu_pgt_size;
504 u32 mmu_pte_size;547 u32 mmu_pte_size;
505 u32 mmu_hop_table_size;548 u32 mmu_hop_table_size;
@@ -534,6 +577,7 @@ struct asic_fixed_properties { · 28 unmodified lines
534 u16 first_available_user_msix_interrupt;577 u16 first_available_user_msix_interrupt;
535 u16 first_available_cq[HL_MAX_DCORES];578 u16 first_available_cq[HL_MAX_DCORES];
536 u16 user_interrupt_count;579 u16 user_interrupt_count;
580 u16 server_type;
537 u8 tpc_enabled_mask;581 u8 tpc_enabled_mask;
538 u8 completion_queues_count;582 u8 completion_queues_count;
539 u8 fw_security_enabled;583 u8 fw_security_enabled;
@@ -542,6 +586,7 @@ struct asic_fixed_properties { · 2 unmodified lines
542 u8 dram_supports_virtual_memory;586 u8 dram_supports_virtual_memory;
543 u8 hard_reset_done_by_fw;587 u8 hard_reset_done_by_fw;
544 u8 num_functional_hbms;588 u8 num_functional_hbms;
589 u8 hints_range_reservation;
545 u8 iatu_done_by_fw;590 u8 iatu_done_by_fw;
546 u8 dynamic_fw_load;591 u8 dynamic_fw_load;
547 u8 gic_interrupts_enable;592 u8 gic_interrupts_enable;
@@ -552,40 +597,45 @@ struct asic_fixed_properties { · 4 unmodified lines
552 * @completion: fence is implemented using completion597 * @completion: fence is implemented using completion
553 * @refcount: refcount for this fence598 * @refcount: refcount for this fence
554 * @cs_sequence: sequence of the corresponding command submission599 * @cs_sequence: sequence of the corresponding command submission
600 * @stream_master_qid_map: streams masters QID bitmap to represent all streams
601 * masters QIDs that multi cs is waiting on
555 * @error: mark this fence with error602 * @error: mark this fence with error
556 * @timestamp: timestamp upon completion603 * @timestamp: timestamp upon completion
557 *
558 */604 */
559struct hl_fence {605struct hl_fence {
560 struct completion completion;606 struct completion completion;
561 struct kref refcount;607 struct kref refcount;
562 u64 cs_sequence;608 u64 cs_sequence;
609 u32 stream_master_qid_map;
563 int error;610 int error;
564 ktime_t timestamp;611 ktime_t timestamp;
565};612};
566 613
567/**614/**
568 * struct hl_cs_compl - command submission completion object.615 * struct hl_cs_compl - command submission completion object.
569 * @sob_reset_work: workqueue object to run SOB reset flow.
570 * @base_fence: hl fence object.616 * @base_fence: hl fence object.
571 * @lock: spinlock to protect fence.617 * @lock: spinlock to protect fence.
572 * @hdev: habanalabs device structure.618 * @hdev: habanalabs device structure.
573 * @hw_sob: the H/W SOB used in this signal/wait CS.619 * @hw_sob: the H/W SOB used in this signal/wait CS.
620 * @encaps_sig_hdl: encaps signals hanlder.
574 * @cs_seq: command submission sequence number.621 * @cs_seq: command submission sequence number.
575 * @type: type of the CS - signal/wait.622 * @type: type of the CS - signal/wait.
576 * @sob_val: the SOB value that is used in this signal/wait CS.623 * @sob_val: the SOB value that is used in this signal/wait CS.
577 * @sob_group: the SOB group that is used in this collective wait CS.624 * @sob_group: the SOB group that is used in this collective wait CS.
625 * @encaps_signals: indication whether it's a completion object of cs with
626 * encaps signals or not.
578 */627 */
579struct hl_cs_compl {628struct hl_cs_compl {
580 struct work_struct sob_reset_work;
581 struct hl_fence base_fence;629 struct hl_fence base_fence;
582 spinlock_t lock;630 spinlock_t lock;
583 struct hl_device *hdev;631 struct hl_device *hdev;
584 struct hl_hw_sob *hw_sob;632 struct hl_hw_sob *hw_sob;
633 struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
585 u64 cs_seq;634 u64 cs_seq;
586 enum hl_cs_type type;635 enum hl_cs_type type;
587 u16 sob_val;636 u16 sob_val;
588 u16 sob_group;637 u16 sob_group;
638 bool encaps_signals;
589};639};
590 640
591/*641/*
@@ -697,6 +747,17 @@ struct hl_sync_stream_properties { · 105 unmodified lines
697 u8 curr_sob_offset;747 u8 curr_sob_offset;
698};748};
699 749
750/**
751 * struct hl_encaps_signals_mgr - describes sync stream encapsulated signals
752 * handlers manager
753 * @lock: protects handles.
754 * @handles: an idr to hold all encapsulated signals handles.
755 */
756struct hl_encaps_signals_mgr {
757 spinlock_t lock;
758 struct idr handles;
759};
760
700/**761/**
701 * struct hl_hw_queue - describes a H/W transport queue.762 * struct hl_hw_queue - describes a H/W transport queue.
702 * @shadow_queue: pointer to a shadow queue that holds pointers to jobs.763 * @shadow_queue: pointer to a shadow queue that holds pointers to jobs.
@@ -875,7 +936,7 @@ struct pci_mem_region { · 172 unmodified lines
875 u64 region_base;936 u64 region_base;
876 u64 region_size;937 u64 region_size;
877 u64 bar_size;938 u64 bar_size;
878 u32 offset_in_bar;939 u64 offset_in_bar;
879 u8 bar_id;940 u8 bar_id;
880 u8 used;941 u8 used;
881};942};
@@ -996,7 +1057,7 @@ struct fw_load_mgr { · 114 unmodified lines
996 * hw_fini and before CS rollback.1057 * hw_fini and before CS rollback.
997 * @suspend: handles IP specific H/W or SW changes for suspend.1058 * @suspend: handles IP specific H/W or SW changes for suspend.
998 * @resume: handles IP specific H/W or SW changes for resume.1059 * @resume: handles IP specific H/W or SW changes for resume.
999 * @cb_mmap: maps a CB.1060 * @mmap: maps a memory.
1000 * @ring_doorbell: increment PI on a given QMAN.1061 * @ring_doorbell: increment PI on a given QMAN.
1001 * @pqe_write: Write the PQ entry to the PQ. This is ASIC-specific1062 * @pqe_write: Write the PQ entry to the PQ. This is ASIC-specific
1002 * function because the PQs are located in different memory areas1063 * function because the PQs are located in different memory areas
@@ -1101,6 +1162,10 @@ struct fw_load_mgr { · 98 unmodified lines
1101 * generic f/w compatible PLL Indexes1162 * generic f/w compatible PLL Indexes
1102 * @init_firmware_loader: initialize data for FW loader.1163 * @init_firmware_loader: initialize data for FW loader.
1103 * @init_cpu_scrambler_dram: Enable CPU specific DRAM scrambling1164 * @init_cpu_scrambler_dram: Enable CPU specific DRAM scrambling
1165 * @state_dump_init: initialize constants required for state dump
1166 * @get_sob_addr: get SOB base address offset.
1167 * @set_pci_memory_regions: setting properties of PCI memory regions
1168 * @get_stream_master_qid_arr: get pointer to stream masters QID array
1104 */1169 */
1105struct hl_asic_funcs {1170struct hl_asic_funcs {
1106 int (*early_init)(struct hl_device *hdev);1171 int (*early_init)(struct hl_device *hdev);
@@ -1110,11 +1175,11 @@ struct hl_asic_funcs { · 3 unmodified lines
1110 int (*sw_init)(struct hl_device *hdev);1175 int (*sw_init)(struct hl_device *hdev);
1111 int (*sw_fini)(struct hl_device *hdev);1176 int (*sw_fini)(struct hl_device *hdev);
1112 int (*hw_init)(struct hl_device *hdev);1177 int (*hw_init)(struct hl_device *hdev);
1113 void (*hw_fini)(struct hl_device *hdev, bool hard_reset);1178 void (*hw_fini)(struct hl_device *hdev, bool hard_reset, bool fw_reset);
1114 void (*halt_engines)(struct hl_device *hdev, bool hard_reset);1179 void (*halt_engines)(struct hl_device *hdev, bool hard_reset, bool fw_reset);
1115 int (*suspend)(struct hl_device *hdev);1180 int (*suspend)(struct hl_device *hdev);
1116 int (*resume)(struct hl_device *hdev);1181 int (*resume)(struct hl_device *hdev);
1117 int (*cb_mmap)(struct hl_device *hdev, struct vm_area_struct *vma,1182 int (*mmap)(struct hl_device *hdev, struct vm_area_struct *vma,
1118 void *cpu_addr, dma_addr_t dma_addr, size_t size);1183 void *cpu_addr, dma_addr_t dma_addr, size_t size);
1119 void (*ring_doorbell)(struct hl_device *hdev, u32 hw_queue_id, u32 pi);1184 void (*ring_doorbell)(struct hl_device *hdev, u32 hw_queue_id, u32 pi);
1120 void (*pqe_write)(struct hl_device *hdev, __le64 *pqe,1185 void (*pqe_write)(struct hl_device *hdev, __le64 *pqe,
@@ -1210,10 +1275,11 @@ struct hl_asic_funcs { · 89 unmodified lines
1210 void (*reset_sob_group)(struct hl_device *hdev, u16 sob_group);1275 void (*reset_sob_group)(struct hl_device *hdev, u16 sob_group);
1211 void (*set_dma_mask_from_fw)(struct hl_device *hdev);1276 void (*set_dma_mask_from_fw)(struct hl_device *hdev);
1212 u64 (*get_device_time)(struct hl_device *hdev);1277 u64 (*get_device_time)(struct hl_device *hdev);
1213 void (*collective_wait_init_cs)(struct hl_cs *cs);1278 int (*collective_wait_init_cs)(struct hl_cs *cs);
1214 int (*collective_wait_create_jobs)(struct hl_device *hdev,1279 int (*collective_wait_create_jobs)(struct hl_device *hdev,
1215 struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id,1280 struct hl_ctx *ctx, struct hl_cs *cs,
1216 u32 collective_engine_id);1281 u32 wait_queue_id, u32 collective_engine_id,
1282 u32 encaps_signal_offset);
1217 u64 (*scramble_addr)(struct hl_device *hdev, u64 addr);1283 u64 (*scramble_addr)(struct hl_device *hdev, u64 addr);
1218 u64 (*descramble_addr)(struct hl_device *hdev, u64 addr);1284 u64 (*descramble_addr)(struct hl_device *hdev, u64 addr);
1219 void (*ack_protection_bits_errors)(struct hl_device *hdev);1285 void (*ack_protection_bits_errors)(struct hl_device *hdev);
@@ -1226,6 +1292,10 @@ struct hl_asic_funcs { · 6 unmodified lines
1226 int (*map_pll_idx_to_fw_idx)(u32 pll_idx);1292 int (*map_pll_idx_to_fw_idx)(u32 pll_idx);
1227 void (*init_firmware_loader)(struct hl_device *hdev);1293 void (*init_firmware_loader)(struct hl_device *hdev);
1228 void (*init_cpu_scrambler_dram)(struct hl_device *hdev);1294 void (*init_cpu_scrambler_dram)(struct hl_device *hdev);
1295 void (*state_dump_init)(struct hl_device *hdev);
1296 u32 (*get_sob_addr)(struct hl_device *hdev, u32 sob_id);
1297 void (*set_pci_memory_regions)(struct hl_device *hdev);
1298 u32* (*get_stream_master_qid_arr)(void);
1229};1299};
1230 1300
1231 1301
@@ -1282,20 +1352,6 @@ struct hl_cs_counters_atomic { · 50 unmodified lines
1282 atomic64_t validation_drop_cnt;1352 atomic64_t validation_drop_cnt;
1283};1353};
1284 1354
1285/**
1286 * struct hl_pending_cb - pending command buffer structure
1287 * @cb_node: cb node in pending cb list
1288 * @cb: command buffer to send in next submission
1289 * @cb_size: command buffer size
1290 * @hw_queue_id: destination queue id
1291 */
1292struct hl_pending_cb {
1293 struct list_head cb_node;
1294 struct hl_cb *cb;
1295 u32 cb_size;
1296 u32 hw_queue_id;
1297};
1298
1299/**1355/**
1300 * struct hl_ctx - user/kernel context.1356 * struct hl_ctx - user/kernel context.
1301 * @mem_hash: holds mapping from virtual address to virtual memory area1357 * @mem_hash: holds mapping from virtual address to virtual memory area
@@ -1312,28 +1368,21 @@ struct hl_pending_cb { · 10 unmodified lines
1312 * MMU hash or walking the PGT requires talking this lock.1368 * MMU hash or walking the PGT requires talking this lock.
1313 * @hw_block_list_lock: protects the HW block memory list.1369 * @hw_block_list_lock: protects the HW block memory list.
1314 * @debugfs_list: node in debugfs list of contexts.1370 * @debugfs_list: node in debugfs list of contexts.
1315 * pending_cb_list: list of pending command buffers waiting to be sent upon
1316 * next user command submission context.
1317 * @hw_block_mem_list: list of HW block virtual mapped addresses.1371 * @hw_block_mem_list: list of HW block virtual mapped addresses.
1318 * @cs_counters: context command submission counters.1372 * @cs_counters: context command submission counters.
1319 * @cb_va_pool: device VA pool for command buffers which are mapped to the1373 * @cb_va_pool: device VA pool for command buffers which are mapped to the
1320 * device's MMU.1374 * device's MMU.
1375 * @sig_mgr: encaps signals handle manager.
1321 * @cs_sequence: sequence number for CS. Value is assigned to a CS and passed1376 * @cs_sequence: sequence number for CS. Value is assigned to a CS and passed
1322 * to user so user could inquire about CS. It is used as1377 * to user so user could inquire about CS. It is used as
1323 * index to cs_pending array.1378 * index to cs_pending array.
1324 * @dram_default_hops: array that holds all hops addresses needed for default1379 * @dram_default_hops: array that holds all hops addresses needed for default
1325 * DRAM mapping.1380 * DRAM mapping.
1326 * @pending_cb_lock: spinlock to protect pending cb list
1327 * @cs_lock: spinlock to protect cs_sequence.1381 * @cs_lock: spinlock to protect cs_sequence.
1328 * @dram_phys_mem: amount of used physical DRAM memory by this context.1382 * @dram_phys_mem: amount of used physical DRAM memory by this context.
1329 * @thread_ctx_switch_token: token to prevent multiple threads of the same1383 * @thread_ctx_switch_token: token to prevent multiple threads of the same
1330 * context from running the context switch phase.1384 * context from running the context switch phase.
1331 * Only a single thread should run it.1385 * Only a single thread should run it.
1332 * @thread_pending_cb_token: token to prevent multiple threads from processing
1333 * the pending CB list. Only a single thread should
1334 * process the list since it is protected by a
1335 * spinlock and we don't want to halt the entire
1336 * command submission sequence.
1337 * @thread_ctx_switch_wait_token: token to prevent the threads that didn't run1386 * @thread_ctx_switch_wait_token: token to prevent the threads that didn't run
1338 * the context switch phase from moving to their1387 * the context switch phase from moving to their
1339 * execution phase before the context switch phase1388 * execution phase before the context switch phase
@@ -1353,17 +1402,15 @@ struct hl_ctx { · 13 unmodified lines
1353 struct mutex mmu_lock;1402 struct mutex mmu_lock;
1354 struct mutex hw_block_list_lock;1403 struct mutex hw_block_list_lock;
1355 struct list_head debugfs_list;1404 struct list_head debugfs_list;
1356 struct list_head pending_cb_list;
1357 struct list_head hw_block_mem_list;1405 struct list_head hw_block_mem_list;
1358 struct hl_cs_counters_atomic cs_counters;1406 struct hl_cs_counters_atomic cs_counters;
1359 struct gen_pool *cb_va_pool;1407 struct gen_pool *cb_va_pool;
1408 struct hl_encaps_signals_mgr sig_mgr;
1360 u64 cs_sequence;1409 u64 cs_sequence;
1361 u64 *dram_default_hops;1410 u64 *dram_default_hops;
1362 spinlock_t pending_cb_lock;
1363 spinlock_t cs_lock;1411 spinlock_t cs_lock;
1364 atomic64_t dram_phys_mem;1412 atomic64_t dram_phys_mem;
1365 atomic_t thread_ctx_switch_token;1413 atomic_t thread_ctx_switch_token;
1366 atomic_t thread_pending_cb_token;
1367 u32 thread_ctx_switch_wait_token;1414 u32 thread_ctx_switch_wait_token;
1368 u32 asid;1415 u32 asid;
1369 u32 handle;1416 u32 handle;
@@ -1394,20 +1441,22 @@ struct hl_ctx_mgr { · 24 unmodified lines
1394 * @sgt: pointer to the scatter-gather table that holds the pages.1441 * @sgt: pointer to the scatter-gather table that holds the pages.
1395 * @dir: for DMA unmapping, the direction must be supplied, so save it.1442 * @dir: for DMA unmapping, the direction must be supplied, so save it.
1396 * @debugfs_list: node in debugfs list of command submissions.1443 * @debugfs_list: node in debugfs list of command submissions.
1444 * @pid: the pid of the user process owning the memory
1397 * @addr: user-space virtual address of the start of the memory area.1445 * @addr: user-space virtual address of the start of the memory area.
1398 * @size: size of the memory area to pin & map.1446 * @size: size of the memory area to pin & map.
1399 * @dma_mapped: true if the SG was mapped to DMA addresses, false otherwise.1447 * @dma_mapped: true if the SG was mapped to DMA addresses, false otherwise.
1400 */1448 */
1401struct hl_userptr {1449struct hl_userptr {
1402 enum vm_type_t vm_type; /* must be first */1450 enum vm_type vm_type; /* must be first */
1403 struct list_head job_node;1451 struct list_head job_node;
1404 struct page **pages;1452 struct page **pages;
1405 unsigned int npages;1453 unsigned int npages;
1406 struct sg_table *sgt;1454 struct sg_table *sgt;
1407 enum dma_data_direction dir;1455 enum dma_data_direction dir;
1408 struct list_head debugfs_list;1456 struct list_head debugfs_list;
1457 pid_t pid;
1409 u64 addr;1458 u64 addr;
1410 u32 size;1459 u64 size;
1411 u8 dma_mapped;1460 u8 dma_mapped;
1412};1461};
1413 1462
@@ -1426,12 +1475,14 @@ struct hl_userptr { · 12 unmodified lines
1426 * @mirror_node : node in device mirror list of command submissions.1475 * @mirror_node : node in device mirror list of command submissions.
1427 * @staged_cs_node: node in the staged cs list.1476 * @staged_cs_node: node in the staged cs list.
1428 * @debugfs_list: node in debugfs list of command submissions.1477 * @debugfs_list: node in debugfs list of command submissions.
1478 * @encaps_sig_hdl: holds the encaps signals handle.
1429 * @sequence: the sequence number of this CS.1479 * @sequence: the sequence number of this CS.
1430 * @staged_sequence: the sequence of the staged submission this CS is part of,1480 * @staged_sequence: the sequence of the staged submission this CS is part of,
1431 * relevant only if staged_cs is set.1481 * relevant only if staged_cs is set.
1432 * @timeout_jiffies: cs timeout in jiffies.1482 * @timeout_jiffies: cs timeout in jiffies.
1433 * @submission_time_jiffies: submission time of the cs1483 * @submission_time_jiffies: submission time of the cs
1434 * @type: CS_TYPE_*.1484 * @type: CS_TYPE_*.
1485 * @encaps_sig_hdl_id: encaps signals handle id, set for the first staged cs.
1435 * @submitted: true if CS was submitted to H/W.1486 * @submitted: true if CS was submitted to H/W.
1436 * @completed: true if CS was completed by device.1487 * @completed: true if CS was completed by device.
1437 * @timedout : true if CS was timedout.1488 * @timedout : true if CS was timedout.
@@ -1445,6 +1496,7 @@ struct hl_userptr { · 7 unmodified lines
1445 * @staged_cs: true if this CS is part of a staged submission.1496 * @staged_cs: true if this CS is part of a staged submission.
1446 * @skip_reset_on_timeout: true if we shall not reset the device in case1497 * @skip_reset_on_timeout: true if we shall not reset the device in case
1447 * timeout occurs (debug scenario).1498 * timeout occurs (debug scenario).
1499 * @encaps_signals: true if this CS has encaps reserved signals.
1448 */1500 */
1449struct hl_cs {1501struct hl_cs {
1450 u16 *jobs_in_queue_cnt;1502 u16 *jobs_in_queue_cnt;
@@ -1459,11 +1511,13 @@ struct hl_cs { · 8 unmodified lines
1459 struct list_head mirror_node;1511 struct list_head mirror_node;
1460 struct list_head staged_cs_node;1512 struct list_head staged_cs_node;
1461 struct list_head debugfs_list;1513 struct list_head debugfs_list;
1514 struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
1462 u64 sequence;1515 u64 sequence;
1463 u64 staged_sequence;1516 u64 staged_sequence;
1464 u64 timeout_jiffies;1517 u64 timeout_jiffies;
1465 u64 submission_time_jiffies;1518 u64 submission_time_jiffies;
1466 enum hl_cs_type type;1519 enum hl_cs_type type;
1520 u32 encaps_sig_hdl_id;
1467 u8 submitted;1521 u8 submitted;
1468 u8 completed;1522 u8 completed;
1469 u8 timedout;1523 u8 timedout;
@@ -1474,6 +1528,7 @@ struct hl_cs { · 4 unmodified lines
1474 u8 staged_first;1528 u8 staged_first;
1475 u8 staged_cs;1529 u8 staged_cs;
1476 u8 skip_reset_on_timeout;1530 u8 skip_reset_on_timeout;
1531 u8 encaps_signals;
1477};1532};
1478 1533
1479/**1534/**
@@ -1493,6 +1548,8 @@ struct hl_cs { · 13 unmodified lines
1493 * @hw_queue_id: the id of the H/W queue this job is submitted to.1548 * @hw_queue_id: the id of the H/W queue this job is submitted to.
1494 * @user_cb_size: the actual size of the CB we got from the user.1549 * @user_cb_size: the actual size of the CB we got from the user.
1495 * @job_cb_size: the actual size of the CB that we put on the queue.1550 * @job_cb_size: the actual size of the CB that we put on the queue.
1551 * @encaps_sig_wait_offset: encapsulated signals offset, which allow user
1552 * to wait on part of the reserved signals.
1496 * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a1553 * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a
1497 * handle to a kernel-allocated CB object, false1554 * handle to a kernel-allocated CB object, false
1498 * otherwise (SRAM/DRAM/host address).1555 * otherwise (SRAM/DRAM/host address).
@@ -1517,6 +1574,7 @@ struct hl_cs_job { · 18 unmodified lines
1517 u32 hw_queue_id;1574 u32 hw_queue_id;
1518 u32 user_cb_size;1575 u32 user_cb_size;
1519 u32 job_cb_size;1576 u32 job_cb_size;
1577 u32 encaps_sig_wait_offset;
1520 u8 is_kernel_allocated_cb;1578 u8 is_kernel_allocated_cb;
1521 u8 contains_dma_pkt;1579 u8 contains_dma_pkt;
1522};1580};
@@ -1613,7 +1671,7 @@ struct hl_vm_hw_block_list_node { · 90 unmodified lines
1613 * @created_from_userptr: is product of host virtual address.1671 * @created_from_userptr: is product of host virtual address.
1614 */1672 */
1615struct hl_vm_phys_pg_pack {1673struct hl_vm_phys_pg_pack {
1616 enum vm_type_t vm_type; /* must be first */1674 enum vm_type vm_type; /* must be first */
1617 u64 *pages;1675 u64 *pages;
1618 u64 npages;1676 u64 npages;
1619 u64 total_size;1677 u64 total_size;
@@ -1759,9 +1817,13 @@ struct hl_debugfs_entry { · 139 unmodified lines
1759 * @ctx_mem_hash_list: list of available contexts with MMU mappings.1817 * @ctx_mem_hash_list: list of available contexts with MMU mappings.
1760 * @ctx_mem_hash_spinlock: protects cb_list.1818 * @ctx_mem_hash_spinlock: protects cb_list.
1761 * @blob_desc: descriptor of blob1819 * @blob_desc: descriptor of blob
1820 * @state_dump: data of the system states in case of a bad cs.
1821 * @state_dump_sem: protects state_dump.
1762 * @addr: next address to read/write from/to in read/write32.1822 * @addr: next address to read/write from/to in read/write32.
1763 * @mmu_addr: next virtual address to translate to physical address in mmu_show.1823 * @mmu_addr: next virtual address to translate to physical address in mmu_show.
1824 * @userptr_lookup: the target user ptr to look up for on demand.
1764 * @mmu_asid: ASID to use while translating in mmu_show.1825 * @mmu_asid: ASID to use while translating in mmu_show.
1826 * @state_dump_head: index of the latest state dump
1765 * @i2c_bus: generic u8 debugfs file for bus value to use in i2c_data_read.1827 * @i2c_bus: generic u8 debugfs file for bus value to use in i2c_data_read.
1766 * @i2c_addr: generic u8 debugfs file for address value to use in i2c_data_read.1828 * @i2c_addr: generic u8 debugfs file for address value to use in i2c_data_read.
1767 * @i2c_reg: generic u8 debugfs file for register value to use in i2c_data_read.1829 * @i2c_reg: generic u8 debugfs file for register value to use in i2c_data_read.
@@ -1783,14 +1845,149 @@ struct hl_dbg_device_entry { · 15 unmodified lines
1783 struct list_head ctx_mem_hash_list;1845 struct list_head ctx_mem_hash_list;
1784 spinlock_t ctx_mem_hash_spinlock;1846 spinlock_t ctx_mem_hash_spinlock;
1785 struct debugfs_blob_wrapper blob_desc;1847 struct debugfs_blob_wrapper blob_desc;
1848 char *state_dump[HL_STATE_DUMP_HIST_LEN];
1849 struct rw_semaphore state_dump_sem;
1786 u64 addr;1850 u64 addr;
1787 u64 mmu_addr;1851 u64 mmu_addr;
1852 u64 userptr_lookup;
1788 u32 mmu_asid;1853 u32 mmu_asid;
1854 u32 state_dump_head;
1789 u8 i2c_bus;1855 u8 i2c_bus;
1790 u8 i2c_addr;1856 u8 i2c_addr;
1791 u8 i2c_reg;1857 u8 i2c_reg;
1792};1858};
1793 1859
1860/**
1861 * struct hl_hw_obj_name_entry - single hw object name, member of
1862 * hl_state_dump_specs
1863 * @node: link to the containing hash table
1864 * @name: hw object name
1865 * @id: object identifier
1866 */
1867struct hl_hw_obj_name_entry {
1868 struct hlist_node node;
1869 const char *name;
1870 u32 id;
1871};
1872
1873enum hl_state_dump_specs_props {
1874 SP_SYNC_OBJ_BASE_ADDR,
1875 SP_NEXT_SYNC_OBJ_ADDR,
1876 SP_SYNC_OBJ_AMOUNT,
1877 SP_MON_OBJ_WR_ADDR_LOW,
1878 SP_MON_OBJ_WR_ADDR_HIGH,
1879 SP_MON_OBJ_WR_DATA,
1880 SP_MON_OBJ_ARM_DATA,
1881 SP_MON_OBJ_STATUS,
1882 SP_MONITORS_AMOUNT,
1883 SP_TPC0_CMDQ,
1884 SP_TPC0_CFG_SO,
1885 SP_NEXT_TPC,
1886 SP_MME_CMDQ,
1887 SP_MME_CFG_SO,
1888 SP_NEXT_MME,
1889 SP_DMA_CMDQ,
1890 SP_DMA_CFG_SO,
1891 SP_DMA_QUEUES_OFFSET,
1892 SP_NUM_OF_MME_ENGINES,
1893 SP_SUB_MME_ENG_NUM,
1894 SP_NUM_OF_DMA_ENGINES,
1895 SP_NUM_OF_TPC_ENGINES,
1896 SP_ENGINE_NUM_OF_QUEUES,
1897 SP_ENGINE_NUM_OF_STREAMS,
1898 SP_ENGINE_NUM_OF_FENCES,
1899 SP_FENCE0_CNT_OFFSET,
1900 SP_FENCE0_RDATA_OFFSET,
1901 SP_CP_STS_OFFSET,
1902 SP_NUM_CORES,
1903
1904 SP_MAX
1905};
1906
1907enum hl_sync_engine_type {
1908 ENGINE_TPC,
1909 ENGINE_DMA,
1910 ENGINE_MME,
1911};
1912
1913/**
1914 * struct hl_mon_state_dump - represents a state dump of a single monitor
1915 * @id: monitor id
1916 * @wr_addr_low: address monitor will write to, low bits
1917 * @wr_addr_high: address monitor will write to, high bits
1918 * @wr_data: data monitor will write
1919 * @arm_data: register value containing monitor configuration
1920 * @status: monitor status
1921 */
1922struct hl_mon_state_dump {
1923 u32 id;
1924 u32 wr_addr_low;
1925 u32 wr_addr_high;
1926 u32 wr_data;
1927 u32 arm_data;
1928 u32 status;
1929};
1930
1931/**
1932 * struct hl_sync_to_engine_map_entry - sync object id to engine mapping entry
1933 * @engine_type: type of the engine
1934 * @engine_id: id of the engine
1935 * @sync_id: id of the sync object
1936 */
1937struct hl_sync_to_engine_map_entry {
1938 struct hlist_node node;
1939 enum hl_sync_engine_type engine_type;
1940 u32 engine_id;
1941 u32 sync_id;
1942};
1943
1944/**
1945 * struct hl_sync_to_engine_map - maps sync object id to associated engine id
1946 * @tb: hash table containing the mapping, each element is of type
1947 * struct hl_sync_to_engine_map_entry
1948 */
1949struct hl_sync_to_engine_map {
1950 DECLARE_HASHTABLE(tb, SYNC_TO_ENGINE_HASH_TABLE_BITS);
1951};
1952
1953/**
1954 * struct hl_state_dump_specs_funcs - virtual functions used by the state dump
1955 * @gen_sync_to_engine_map: generate a hash map from sync obj id to its engine
1956 * @print_single_monitor: format monitor data as string
1957 * @monitor_valid: return true if given monitor dump is valid
1958 * @print_fences_single_engine: format fences data as string
1959 */
1960struct hl_state_dump_specs_funcs {
1961 int (*gen_sync_to_engine_map)(struct hl_device *hdev,
1962 struct hl_sync_to_engine_map *map);
1963 int (*print_single_monitor)(char **buf, size_t *size, size_t *offset,
1964 struct hl_device *hdev,
1965 struct hl_mon_state_dump *mon);
1966 int (*monitor_valid)(struct hl_mon_state_dump *mon);
1967 int (*print_fences_single_engine)(struct hl_device *hdev,
1968 u64 base_offset,
1969 u64 status_base_offset,
1970 enum hl_sync_engine_type engine_type,
1971 u32 engine_id, char **buf,
1972 size_t *size, size_t *offset);
1973};
1974
1975/**
1976 * struct hl_state_dump_specs - defines ASIC known hw objects names
1977 * @so_id_to_str_tb: sync objects names index table
1978 * @monitor_id_to_str_tb: monitors names index table
1979 * @funcs: virtual functions used for state dump
1980 * @sync_namager_names: readable names for sync manager if available (ex: N_E)
1981 * @props: pointer to a per asic const props array required for state dump
1982 */
1983struct hl_state_dump_specs {
1984 DECLARE_HASHTABLE(so_id_to_str_tb, OBJ_NAMES_HASH_TABLE_BITS);
1985 DECLARE_HASHTABLE(monitor_id_to_str_tb, OBJ_NAMES_HASH_TABLE_BITS);
1986 struct hl_state_dump_specs_funcs funcs;
1987 const char * const *sync_namager_names;
1988 s64 *props;
1989};
1990
1794 1991
1795/*1992/*
1796 * DEVICES1993 * DEVICES
@@ -1798,7 +1995,7 @@ struct hl_dbg_device_entry { · 1 unmodified line
1798 1995
1799#define HL_STR_MAX 321996#define HL_STR_MAX 32
1800 1997
1801#define HL_DEV_STS_MAX (HL_DEVICE_STATUS_NEEDS_RESET + 1)1998#define HL_DEV_STS_MAX (HL_DEVICE_STATUS_LAST + 1)
1802 1999
1803/* Theoretical limit only. A single host can only contain up to 4 or 8 PCIe2000/* Theoretical limit only. A single host can only contain up to 4 or 8 PCIe
1804 * x16 cards. In extreme cases, there are hosts that can accommodate 16 cards.2001 * x16 cards. In extreme cases, there are hosts that can accommodate 16 cards.
@@ -1946,11 +2143,13 @@ struct hwmon_chip_info; · 141 unmodified lines
1946 * @wq: work queue for device reset procedure.2143 * @wq: work queue for device reset procedure.
1947 * @reset_work: reset work to be done.2144 * @reset_work: reset work to be done.
1948 * @hdev: habanalabs device structure.2145 * @hdev: habanalabs device structure.
2146 * @fw_reset: whether f/w will do the reset without us sending them a message to do it.
1949 */2147 */
1950struct hl_device_reset_work {2148struct hl_device_reset_work {
1951 struct workqueue_struct *wq;2149 struct workqueue_struct *wq;
1952 struct delayed_work reset_work;2150 struct delayed_work reset_work;
1953 struct hl_device *hdev;2151 struct hl_device *hdev;
2152 bool fw_reset;
1954};2153};
1955 2154
1956/**2155/**
@@ -2064,6 +2263,58 @@ struct hl_mmu_funcs { · 107 unmodified lines
2064 u64 virt_addr, struct hl_mmu_hop_info *hops);2263 u64 virt_addr, struct hl_mmu_hop_info *hops);
2065};2264};
2066 2265
2266/**
2267 * number of user contexts allowed to call wait_for_multi_cs ioctl in
2268 * parallel
2269 */
2270#define MULTI_CS_MAX_USER_CTX 2
2271
2272/**
2273 * struct multi_cs_completion - multi CS wait completion.
2274 * @completion: completion of any of the CS in the list
2275 * @lock: spinlock for the completion structure
2276 * @timestamp: timestamp for the multi-CS completion
2277 * @stream_master_qid_map: bitmap of all stream masters on which the multi-CS
2278 * is waiting
2279 * @used: 1 if in use, otherwise 0
2280 */
2281struct multi_cs_completion {
2282 struct completion completion;
2283 spinlock_t lock;
2284 s64 timestamp;
2285 u32 stream_master_qid_map;
2286 u8 used;
2287};
2288
2289/**
2290 * struct multi_cs_data - internal data for multi CS call
2291 * @ctx: pointer to the context structure
2292 * @fence_arr: array of fences of all CSs
2293 * @seq_arr: array of CS sequence numbers
2294 * @timeout_us: timeout in usec for waiting for CS to complete
2295 * @timestamp: timestamp of first completed CS
2296 * @wait_status: wait for CS status
2297 * @completion_bitmap: bitmap of completed CSs (1- completed, otherwise 0)
2298 * @stream_master_qid_map: bitmap of all stream master QIDs on which the
2299 * multi-CS is waiting
2300 * @arr_len: fence_arr and seq_arr array length
2301 * @gone_cs: indication of gone CS (1- there was gone CS, otherwise 0)
2302 * @update_ts: update timestamp. 1- update the timestamp, otherwise 0.
2303 */
2304struct multi_cs_data {
2305 struct hl_ctx *ctx;
2306 struct hl_fence **fence_arr;
2307 u64 *seq_arr;
2308 s64 timeout_us;
2309 s64 timestamp;
2310 long wait_status;
2311 u32 completion_bitmap;
2312 u32 stream_master_qid_map;
2313 u8 arr_len;
2314 u8 gone_cs;
2315 u8 update_ts;
2316};
2317
2067/**2318/**
2068 * struct hl_device - habanalabs device structure.2319 * struct hl_device - habanalabs device structure.
2069 * @pdev: pointer to PCI device, can be NULL in case of simulator device.2320 * @pdev: pointer to PCI device, can be NULL in case of simulator device.
@@ -2129,6 +2380,8 @@ struct hl_mmu_funcs { · 59 unmodified lines
2129 * @mmu_func: device-related MMU functions.2380 * @mmu_func: device-related MMU functions.
2130 * @fw_loader: FW loader manager.2381 * @fw_loader: FW loader manager.
2131 * @pci_mem_region: array of memory regions in the PCI2382 * @pci_mem_region: array of memory regions in the PCI
2383 * @state_dump_specs: constants and dictionaries needed to dump system state.
2384 * @multi_cs_completion: array of multi-CS completion.
2132 * @dram_used_mem: current DRAM memory consumption.2385 * @dram_used_mem: current DRAM memory consumption.
2133 * @timeout_jiffies: device CS timeout value.2386 * @timeout_jiffies: device CS timeout value.
2134 * @max_power: the max power of the device, as configured by the sysadmin. This2387 * @max_power: the max power of the device, as configured by the sysadmin. This
@@ -2205,6 +2458,7 @@ struct hl_mmu_funcs { · 70 unmodified lines
2205 * halted. We can't halt it again because the COMMS2458 * halted. We can't halt it again because the COMMS
2206 * protocol will throw an error. Relevant only for2459 * protocol will throw an error. Relevant only for
2207 * cases where Linux was not loaded to device CPU2460 * cases where Linux was not loaded to device CPU
2461 * @supports_wait_for_multi_cs: true if wait for multi CS is supported
2208 */2462 */
2209struct hl_device {2463struct hl_device {
2210 struct pci_dev *pdev;2464 struct pci_dev *pdev;
@@ -2273,6 +2527,11 @@ struct hl_device { · 62 unmodified lines
2273 2527
2274 struct pci_mem_region pci_mem_region[PCI_REGION_NUMBER];2528 struct pci_mem_region pci_mem_region[PCI_REGION_NUMBER];
2275 2529
2530 struct hl_state_dump_specs state_dump_specs;
2531
2532 struct multi_cs_completion multi_cs_completion[
2533 MULTI_CS_MAX_USER_CTX];
2534 u32 *stream_master_qid_arr;
2276 atomic64_t dram_used_mem;2535 atomic64_t dram_used_mem;
2277 u64 timeout_jiffies;2536 u64 timeout_jiffies;
2278 u64 max_power;2537 u64 max_power;
@@ -2322,6 +2581,8 @@ struct hl_device { · 43 unmodified lines
2322 u8 curr_reset_cause;2581 u8 curr_reset_cause;
2323 u8 skip_reset_on_timeout;2582 u8 skip_reset_on_timeout;
2324 u8 device_cpu_is_halted;2583 u8 device_cpu_is_halted;
2584 u8 supports_wait_for_multi_cs;
2585 u8 stream_master_qid_arr_size;
2325 2586
2326 /* Parameters for bring-up */2587 /* Parameters for bring-up */
2327 u64 nic_ports_mask;2588 u64 nic_ports_mask;
@@ -2343,6 +2604,29 @@ struct hl_device { · 15 unmodified lines
2343};2604};
2344 2605
2345 2606
2607/**
2608 * struct hl_cs_encaps_sig_handle - encapsulated signals handle structure
2609 * @refcount: refcount used to protect removing this id when several
2610 * wait cs are used to wait of the reserved encaps signals.
2611 * @hdev: pointer to habanalabs device structure.
2612 * @hw_sob: pointer to H/W SOB used in the reservation.
2613 * @cs_seq: staged cs sequence which contains encapsulated signals
2614 * @id: idr handler id to be used to fetch the handler info
2615 * @q_idx: stream queue index
2616 * @pre_sob_val: current SOB value before reservation
2617 * @count: signals number
2618 */
2619struct hl_cs_encaps_sig_handle {
2620 struct kref refcount;
2621 struct hl_device *hdev;
2622 struct hl_hw_sob *hw_sob;
2623 u64 cs_seq;
2624 u32 id;
2625 u32 q_idx;
2626 u32 pre_sob_val;
2627 u32 count;
2628};
2629
2346/*2630/*
2347 * IOCTLs2631 * IOCTLs
2348 */2632 */
@@ -2372,6 +2656,23 @@ struct hl_ioctl_desc { · 23 unmodified lines
2372 * Kernel module functions that can be accessed by entire module2656 * Kernel module functions that can be accessed by entire module
2373 */2657 */
2374 2658
2659/**
2660 * hl_get_sg_info() - get number of pages and the DMA address from SG list.
2661 * @sg: the SG list.
2662 * @dma_addr: pointer to DMA address to return.
2663 *
2664 * Calculate the number of consecutive pages described by the SG list. Take the
2665 * offset of the address in the first page, add to it the length and round it up
2666 * to the number of needed pages.
2667 */
2668static inline u32 hl_get_sg_info(struct scatterlist *sg, dma_addr_t *dma_addr)
2669{
2670 *dma_addr = sg_dma_address(sg);
2671
2672 return ((((*dma_addr) & (PAGE_SIZE - 1)) + sg_dma_len(sg)) +
2673 (PAGE_SIZE - 1)) >> PAGE_SHIFT;
2674}
2675
2375/**2676/**
2376 * hl_mem_area_inside_range() - Checks whether address+size are inside a range.2677 * hl_mem_area_inside_range() - Checks whether address+size are inside a range.
2377 * @address: The start address of the area we want to validate.2678 * @address: The start address of the area we want to validate.
@@ -2436,7 +2737,9 @@ void destroy_hdev(struct hl_device *hdev); · 58 unmodified lines
2436int hl_hw_queues_create(struct hl_device *hdev);2737int hl_hw_queues_create(struct hl_device *hdev);
2437void hl_hw_queues_destroy(struct hl_device *hdev);2738void hl_hw_queues_destroy(struct hl_device *hdev);
2438int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,2739int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
2439 u32 cb_size, u64 cb_ptr);2740 u32 cb_size, u64 cb_ptr);
2741void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,
2742 u32 ctl, u32 len, u64 ptr);
2440int hl_hw_queue_schedule_cs(struct hl_cs *cs);2743int hl_hw_queue_schedule_cs(struct hl_cs *cs);
2441u32 hl_hw_queue_add_ptr(u32 ptr, u16 val);2744u32 hl_hw_queue_add_ptr(u32 ptr, u16 val);
2442void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id);2745void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id);
@@ -2470,6 +2773,8 @@ void hl_ctx_do_release(struct kref *ref); · 27 unmodified lines
2470void hl_ctx_get(struct hl_device *hdev, struct hl_ctx *ctx);2773void hl_ctx_get(struct hl_device *hdev, struct hl_ctx *ctx);
2471int hl_ctx_put(struct hl_ctx *ctx);2774int hl_ctx_put(struct hl_ctx *ctx);
2472struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq);2775struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq);
2776int hl_ctx_get_fences(struct hl_ctx *ctx, u64 *seq_arr,
2777 struct hl_fence **fence, u32 arr_len);
2473void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr);2778void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr);
2474void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr);2779void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr);
2475 2780
@@ -2511,18 +2816,19 @@ int hl_cb_va_pool_init(struct hl_ctx *ctx); · 35 unmodified lines
2511void hl_cb_va_pool_fini(struct hl_ctx *ctx);2816void hl_cb_va_pool_fini(struct hl_ctx *ctx);
2512 2817
2513void hl_cs_rollback_all(struct hl_device *hdev);2818void hl_cs_rollback_all(struct hl_device *hdev);
2514void hl_pending_cb_list_flush(struct hl_ctx *ctx);
2515struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev,2819struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev,
2516 enum hl_queue_type queue_type, bool is_kernel_allocated_cb);2820 enum hl_queue_type queue_type, bool is_kernel_allocated_cb);
2517void hl_sob_reset_error(struct kref *ref);2821void hl_sob_reset_error(struct kref *ref);
2518int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask);2822int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask);
2519void hl_fence_put(struct hl_fence *fence);2823void hl_fence_put(struct hl_fence *fence);
2824void hl_fences_put(struct hl_fence **fence, int len);
2520void hl_fence_get(struct hl_fence *fence);2825void hl_fence_get(struct hl_fence *fence);
2521void cs_get(struct hl_cs *cs);2826void cs_get(struct hl_cs *cs);
2522bool cs_needs_completion(struct hl_cs *cs);2827bool cs_needs_completion(struct hl_cs *cs);
2523bool cs_needs_timeout(struct hl_cs *cs);2828bool cs_needs_timeout(struct hl_cs *cs);
2524bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs);2829bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs);
2525struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq);2830struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq);
2831void hl_multi_cs_completion_init(struct hl_device *hdev);
2526 2832
2527void goya_set_asic_funcs(struct hl_device *hdev);2833void goya_set_asic_funcs(struct hl_device *hdev);
2528void gaudi_set_asic_funcs(struct hl_device *hdev);2834void gaudi_set_asic_funcs(struct hl_device *hdev);
@@ -2650,9 +2956,25 @@ int hl_set_voltage(struct hl_device *hdev, · 121 unmodified lines
2650 int sensor_index, u32 attr, long value);2956 int sensor_index, u32 attr, long value);
2651int hl_set_current(struct hl_device *hdev,2957int hl_set_current(struct hl_device *hdev,
2652 int sensor_index, u32 attr, long value);2958 int sensor_index, u32 attr, long value);
2959void hw_sob_get(struct hl_hw_sob *hw_sob);
2960void hw_sob_put(struct hl_hw_sob *hw_sob);
2961void hl_encaps_handle_do_release(struct kref *ref);
2962void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev,
2963 struct hl_cs *cs, struct hl_cs_job *job,
2964 struct hl_cs_compl *cs_cmpl);
2653void hl_release_pending_user_interrupts(struct hl_device *hdev);2965void hl_release_pending_user_interrupts(struct hl_device *hdev);
2654int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,2966int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,
2655 struct hl_hw_sob **hw_sob, u32 count);2967 struct hl_hw_sob **hw_sob, u32 count, bool encaps_sig);
2968
2969int hl_state_dump(struct hl_device *hdev);
2970const char *hl_state_dump_get_sync_name(struct hl_device *hdev, u32 sync_id);
2971const char *hl_state_dump_get_monitor_name(struct hl_device *hdev,
2972 struct hl_mon_state_dump *mon);
2973void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map);
2974__printf(4, 5) int hl_snprintf_resize(char **buf, size_t *size, size_t *offset,
2975 const char *format, ...);
2976char *hl_format_as_binary(char *buf, size_t buf_len, u32 n);
2977const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type);
2656 2978
2657#ifdef CONFIG_DEBUG_FS2979#ifdef CONFIG_DEBUG_FS
2658 2980
@@ -2673,6 +2995,8 @@ void hl_debugfs_remove_userptr(struct hl_device *hdev, · 14 unmodified lines
2673 struct hl_userptr *userptr);2995 struct hl_userptr *userptr);
2674void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);2996void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);
2675void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);2997void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);
2998void hl_debugfs_set_state_dump(struct hl_device *hdev, char *data,
2999 unsigned long length);
2676 3000
2677#else3001#else
2678 3002
@@ -2746,6 +3070,11 @@ static inline void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, · 67 unmodified lines
2746{3070{
2747}3071}
2748 3072
3073static inline void hl_debugfs_set_state_dump(struct hl_device *hdev,
3074 char *data, unsigned long length)
3075{
3076}
3077
2749#endif3078#endif
2750 3079
2751/* IOCTLs */3080/* IOCTLs */
drivers/misc/habanalabs/common/habanalabs_drv.c+8−5
@@ -141,7 +141,7 @@ int hl_device_open(struct inode *inode, struct file *filp) · 140 unmodified lines
141 hl_cb_mgr_init(&hpriv->cb_mgr);141 hl_cb_mgr_init(&hpriv->cb_mgr);
142 hl_ctx_mgr_init(&hpriv->ctx_mgr);142 hl_ctx_mgr_init(&hpriv->ctx_mgr);
143 143
144 hpriv->taskpid = find_get_pid(current->pid);144 hpriv->taskpid = get_task_pid(current, PIDTYPE_PID);
145 145
146 mutex_lock(&hdev->fpriv_list_lock);146 mutex_lock(&hdev->fpriv_list_lock);
147 147
@@ -194,7 +194,6 @@ int hl_device_open(struct inode *inode, struct file *filp) · 46 unmodified lines
194 194
195out_err:195out_err:
196 mutex_unlock(&hdev->fpriv_list_lock);196 mutex_unlock(&hdev->fpriv_list_lock);
197
198 hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);197 hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
199 hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);198 hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
200 filp->private_data = NULL;199 filp->private_data = NULL;
@@ -318,12 +317,16 @@ int create_hdev(struct hl_device **dev, struct pci_dev *pdev, · 117 unmodified lines
318 hdev->asic_prop.fw_security_enabled = false;317 hdev->asic_prop.fw_security_enabled = false;
319 318
320 /* Assign status description string */319 /* Assign status description string */
321 strncpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION],320 strncpy(hdev->status[HL_DEVICE_STATUS_OPERATIONAL],
322 "disabled", HL_STR_MAX);321 "operational", HL_STR_MAX);
323 strncpy(hdev->status[HL_DEVICE_STATUS_IN_RESET],322 strncpy(hdev->status[HL_DEVICE_STATUS_IN_RESET],
324 "in reset", HL_STR_MAX);323 "in reset", HL_STR_MAX);
324 strncpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION],
325 "disabled", HL_STR_MAX);
325 strncpy(hdev->status[HL_DEVICE_STATUS_NEEDS_RESET],326 strncpy(hdev->status[HL_DEVICE_STATUS_NEEDS_RESET],
326 "needs reset", HL_STR_MAX);327 "needs reset", HL_STR_MAX);
328 strncpy(hdev->status[HL_DEVICE_STATUS_IN_DEVICE_CREATION],
329 "in device creation", HL_STR_MAX);
327 330
328 hdev->major = hl_major;331 hdev->major = hl_major;
329 hdev->reset_on_lockup = reset_on_lockup;332 hdev->reset_on_lockup = reset_on_lockup;
@@ -532,7 +535,7 @@ hl_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t state) · 202 unmodified lines
532 result = PCI_ERS_RESULT_NONE;535 result = PCI_ERS_RESULT_NONE;
533 }536 }
534 537
535 hdev->asic_funcs->halt_engines(hdev, true);538 hdev->asic_funcs->halt_engines(hdev, true, false);
536 539
537 return result;540 return result;
538}541}
drivers/misc/habanalabs/common/habanalabs_ioctl.c+2−0
@@ -94,6 +94,8 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args) · 93 unmodified lines
94 94
95 hw_ip.first_available_interrupt_id =95 hw_ip.first_available_interrupt_id =
96 prop->first_available_user_msix_interrupt;96 prop->first_available_user_msix_interrupt;
97 hw_ip.server_type = prop->server_type;
98
97 return copy_to_user(out, &hw_ip,99 return copy_to_user(out, &hw_ip,
98 min((size_t) size, sizeof(hw_ip))) ? -EFAULT : 0;100 min((size_t) size, sizeof(hw_ip))) ? -EFAULT : 0;
99}101}
drivers/misc/habanalabs/common/hw_queue.c+155−43
@@ -65,7 +65,7 @@ void hl_hw_queue_update_ci(struct hl_cs *cs) · 64 unmodified lines
65}65}
66 66
67/*67/*
68 * ext_and_hw_queue_submit_bd() - Submit a buffer descriptor to an external or a68 * hl_hw_queue_submit_bd() - Submit a buffer descriptor to an external or a
69 * H/W queue.69 * H/W queue.
70 * @hdev: pointer to habanalabs device structure70 * @hdev: pointer to habanalabs device structure
71 * @q: pointer to habanalabs queue structure71 * @q: pointer to habanalabs queue structure
@@ -80,8 +80,8 @@ void hl_hw_queue_update_ci(struct hl_cs *cs) · 8 unmodified lines
80 * This function must be called when the scheduler mutex is taken80 * This function must be called when the scheduler mutex is taken
81 *81 *
82 */82 */
83static void ext_and_hw_queue_submit_bd(struct hl_device *hdev,83void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,
84 struct hl_hw_queue *q, u32 ctl, u32 len, u64 ptr)84 u32 ctl, u32 len, u64 ptr)
85{85{
86 struct hl_bd *bd;86 struct hl_bd *bd;
87 87
@@ -222,8 +222,8 @@ static int hw_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q, · 134 unmodified lines
222 * @cb_size: size of CB222 * @cb_size: size of CB
223 * @cb_ptr: pointer to CB location223 * @cb_ptr: pointer to CB location
224 *224 *
225 * This function sends a single CB, that must NOT generate a completion entry225 * This function sends a single CB, that must NOT generate a completion entry.
226 *226 * Sending CPU messages can be done instead via 'hl_hw_queue_submit_bd()'
227 */227 */
228int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,228int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
229 u32 cb_size, u64 cb_ptr)229 u32 cb_size, u64 cb_ptr)
@@ -231,16 +231,7 @@ int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id, · 1 unmodified line
231 struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];231 struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];
232 int rc = 0;232 int rc = 0;
233 233
234 /*234 hdev->asic_funcs->hw_queues_lock(hdev);
235 * The CPU queue is a synchronous queue with an effective depth of
236 * a single entry (although it is allocated with room for multiple
237 * entries). Therefore, there is a different lock, called
238 * send_cpu_message_lock, that serializes accesses to the CPU queue.
239 * As a result, we don't need to lock the access to the entire H/W
240 * queues module when submitting a JOB to the CPU queue
241 */
242 if (q->queue_type != QUEUE_TYPE_CPU)
243 hdev->asic_funcs->hw_queues_lock(hdev);
244 235
245 if (hdev->disabled) {236 if (hdev->disabled) {
246 rc = -EPERM;237 rc = -EPERM;
@@ -258,11 +249,10 @@ int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id, · 11 unmodified lines
258 goto out;249 goto out;
259 }250 }
260 251
261 ext_and_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);252 hl_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);
262 253
263out:254out:
264 if (q->queue_type != QUEUE_TYPE_CPU)255 hdev->asic_funcs->hw_queues_unlock(hdev);
265 hdev->asic_funcs->hw_queues_unlock(hdev);
266 256
267 return rc;257 return rc;
268}258}
@@ -328,7 +318,7 @@ static void ext_queue_schedule_job(struct hl_cs_job *job) · 59 unmodified lines
328 cq->pi = hl_cq_inc_ptr(cq->pi);318 cq->pi = hl_cq_inc_ptr(cq->pi);
329 319
330submit_bd:320submit_bd:
331 ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr);321 hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
332}322}
333 323
334/*324/*
@@ -407,7 +397,7 @@ static void hw_queue_schedule_job(struct hl_cs_job *job) · 72 unmodified lines
407 else397 else
408 ptr = (u64) (uintptr_t) job->user_cb;398 ptr = (u64) (uintptr_t) job->user_cb;
409 399
410 ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr);400 hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
411}401}
412 402
413static int init_signal_cs(struct hl_device *hdev,403static int init_signal_cs(struct hl_device *hdev,
@@ -426,8 +416,9 @@ static int init_signal_cs(struct hl_device *hdev, · 12 unmodified lines
426 cs_cmpl->sob_val = prop->next_sob_val;416 cs_cmpl->sob_val = prop->next_sob_val;
427 417
428 dev_dbg(hdev->dev,418 dev_dbg(hdev->dev,
429 "generate signal CB, sob_id: %d, sob val: 0x%x, q_idx: %d\n",419 "generate signal CB, sob_id: %d, sob val: %u, q_idx: %d, seq: %llu\n",
430 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, q_idx);420 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, q_idx,
421 cs_cmpl->cs_seq);
431 422
432 /* we set an EB since we must make sure all oeprations are done423 /* we set an EB since we must make sure all oeprations are done
433 * when sending the signal424 * when sending the signal
@@ -435,17 +426,37 @@ static int init_signal_cs(struct hl_device *hdev, · 1 unmodified line
435 hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb,426 hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb,
436 cs_cmpl->hw_sob->sob_id, 0, true);427 cs_cmpl->hw_sob->sob_id, 0, true);
437 428
438 rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1);429 rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1,
430 false);
439 431
440 return rc;432 return rc;
441}433}
442 434
443static void init_wait_cs(struct hl_device *hdev, struct hl_cs *cs,435void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev,
436 struct hl_cs *cs, struct hl_cs_job *job,
437 struct hl_cs_compl *cs_cmpl)
438{
439 struct hl_cs_encaps_sig_handle *handle = cs->encaps_sig_hdl;
440
441 cs_cmpl->hw_sob = handle->hw_sob;
442
443 /* Note that encaps_sig_wait_offset was validated earlier in the flow
444 * for offset value which exceeds the max reserved signal count.
445 * always decrement 1 of the offset since when the user
446 * set offset 1 for example he mean to wait only for the first
447 * signal only, which will be pre_sob_val, and if he set offset 2
448 * then the value required is (pre_sob_val + 1) and so on...
449 */
450 cs_cmpl->sob_val = handle->pre_sob_val +
451 (job->encaps_sig_wait_offset - 1);
452}
453
454static int init_wait_cs(struct hl_device *hdev, struct hl_cs *cs,
444 struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl)455 struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl)
445{456{
446 struct hl_cs_compl *signal_cs_cmpl;
447 struct hl_sync_stream_properties *prop;
448 struct hl_gen_wait_properties wait_prop;457 struct hl_gen_wait_properties wait_prop;
458 struct hl_sync_stream_properties *prop;
459 struct hl_cs_compl *signal_cs_cmpl;
449 u32 q_idx;460 u32 q_idx;
450 461
451 q_idx = job->hw_queue_id;462 q_idx = job->hw_queue_id;
@@ -455,14 +466,51 @@ static void init_wait_cs(struct hl_device *hdev, struct hl_cs *cs, · 3 unmodified lines
455 struct hl_cs_compl,466 struct hl_cs_compl,
456 base_fence);467 base_fence);
457 468
458 /* copy the SOB id and value of the signal CS */469 if (cs->encaps_signals) {
459 cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;470 /* use the encaps signal handle stored earlier in the flow
460 cs_cmpl->sob_val = signal_cs_cmpl->sob_val;471 * and set the SOB information from the encaps
472 * signals handle
473 */
474 hl_hw_queue_encaps_sig_set_sob_info(hdev, cs, job, cs_cmpl);
475
476 dev_dbg(hdev->dev, "Wait for encaps signals handle, qidx(%u), CS sequence(%llu), sob val: 0x%x, offset: %u\n",
477 cs->encaps_sig_hdl->q_idx,
478 cs->encaps_sig_hdl->cs_seq,
479 cs_cmpl->sob_val,
480 job->encaps_sig_wait_offset);
481 } else {
482 /* Copy the SOB id and value of the signal CS */
483 cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;
484 cs_cmpl->sob_val = signal_cs_cmpl->sob_val;
485 }
486
487 /* check again if the signal cs already completed.
488 * if yes then don't send any wait cs since the hw_sob
489 * could be in reset already. if signal is not completed
490 * then get refcount to hw_sob to prevent resetting the sob
491 * while wait cs is not submitted.
492 * note that this check is protected by two locks,
493 * hw queue lock and completion object lock,
494 * and the same completion object lock also protects
495 * the hw_sob reset handler function.
496 * The hw_queue lock prevent out of sync of hw_sob
497 * refcount value, changed by signal/wait flows.
498 */
499 spin_lock(&signal_cs_cmpl->lock);
500
501 if (completion_done(&cs->signal_fence->completion)) {
502 spin_unlock(&signal_cs_cmpl->lock);
503 return -EINVAL;
504 }
505
506 kref_get(&cs_cmpl->hw_sob->kref);
507
508 spin_unlock(&signal_cs_cmpl->lock);
461 509
462 dev_dbg(hdev->dev,510 dev_dbg(hdev->dev,
463 "generate wait CB, sob_id: %d, sob_val: 0x%x, mon_id: %d, q_idx: %d\n",511 "generate wait CB, sob_id: %d, sob_val: 0x%x, mon_id: %d, q_idx: %d, seq: %llu\n",
464 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val,512 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val,
465 prop->base_mon_id, q_idx);513 prop->base_mon_id, q_idx, cs->sequence);
466 514
467 wait_prop.data = (void *) job->patched_cb;515 wait_prop.data = (void *) job->patched_cb;
468 wait_prop.sob_base = cs_cmpl->hw_sob->sob_id;516 wait_prop.sob_base = cs_cmpl->hw_sob->sob_id;
@@ -471,17 +519,14 @@ static void init_wait_cs(struct hl_device *hdev, struct hl_cs *cs, · 2 unmodified lines
471 wait_prop.mon_id = prop->base_mon_id;519 wait_prop.mon_id = prop->base_mon_id;
472 wait_prop.q_idx = q_idx;520 wait_prop.q_idx = q_idx;
473 wait_prop.size = 0;521 wait_prop.size = 0;
522
474 hdev->asic_funcs->gen_wait_cb(hdev, &wait_prop);523 hdev->asic_funcs->gen_wait_cb(hdev, &wait_prop);
475 524
476 kref_get(&cs_cmpl->hw_sob->kref);
477 /*
478 * Must put the signal fence after the SOB refcnt increment so
479 * the SOB refcnt won't turn 0 and reset the SOB before the
480 * wait CS was submitted.
481 */
482 mb();525 mb();
483 hl_fence_put(cs->signal_fence);526 hl_fence_put(cs->signal_fence);
484 cs->signal_fence = NULL;527 cs->signal_fence = NULL;
528
529 return 0;
485}530}
486 531
487/*532/*
@@ -506,7 +551,60 @@ static int init_signal_wait_cs(struct hl_cs *cs) · 18 unmodified lines
506 if (cs->type & CS_TYPE_SIGNAL)551 if (cs->type & CS_TYPE_SIGNAL)
507 rc = init_signal_cs(hdev, job, cs_cmpl);552 rc = init_signal_cs(hdev, job, cs_cmpl);
508 else if (cs->type & CS_TYPE_WAIT)553 else if (cs->type & CS_TYPE_WAIT)
509 init_wait_cs(hdev, cs, job, cs_cmpl);554 rc = init_wait_cs(hdev, cs, job, cs_cmpl);
555
556 return rc;
557}
558
559static int encaps_sig_first_staged_cs_handler
560 (struct hl_device *hdev, struct hl_cs *cs)
561{
562 struct hl_cs_compl *cs_cmpl =
563 container_of(cs->fence,
564 struct hl_cs_compl, base_fence);
565 struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
566 struct hl_encaps_signals_mgr *mgr;
567 int rc = 0;
568
569 mgr = &hdev->compute_ctx->sig_mgr;
570
571 spin_lock(&mgr->lock);
572 encaps_sig_hdl = idr_find(&mgr->handles, cs->encaps_sig_hdl_id);
573 if (encaps_sig_hdl) {
574 /*
575 * Set handler CS sequence,
576 * the CS which contains the encapsulated signals.
577 */
578 encaps_sig_hdl->cs_seq = cs->sequence;
579 /* store the handle and set encaps signal indication,
580 * to be used later in cs_do_release to put the last
581 * reference to encaps signals handlers.
582 */
583 cs_cmpl->encaps_signals = true;
584 cs_cmpl->encaps_sig_hdl = encaps_sig_hdl;
585
586 /* set hw_sob pointer in completion object
587 * since it's used in cs_do_release flow to put
588 * refcount to sob
589 */
590 cs_cmpl->hw_sob = encaps_sig_hdl->hw_sob;
591 cs_cmpl->sob_val = encaps_sig_hdl->pre_sob_val +
592 encaps_sig_hdl->count;
593
594 dev_dbg(hdev->dev, "CS seq (%llu) added to encaps signal handler id (%u), count(%u), qidx(%u), sob(%u), val(%u)\n",
595 cs->sequence, encaps_sig_hdl->id,
596 encaps_sig_hdl->count,
597 encaps_sig_hdl->q_idx,
598 cs_cmpl->hw_sob->sob_id,
599 cs_cmpl->sob_val);
600
601 } else {
602 dev_err(hdev->dev, "encaps handle id(%u) wasn't found!\n",
603 cs->encaps_sig_hdl_id);
604 rc = -EINVAL;
605 }
606
607 spin_unlock(&mgr->lock);
510 608
511 return rc;609 return rc;
512}610}
@@ -581,14 +679,21 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) · 68 unmodified lines
581 679
582 if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT)) {680 if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT)) {
583 rc = init_signal_wait_cs(cs);681 rc = init_signal_wait_cs(cs);
584 if (rc) {682 if (rc)
585 dev_err(hdev->dev, "Failed to submit signal cs\n");
586 goto unroll_cq_resv;683 goto unroll_cq_resv;
587 }684 } else if (cs->type == CS_TYPE_COLLECTIVE_WAIT) {
588 } else if (cs->type == CS_TYPE_COLLECTIVE_WAIT)685 rc = hdev->asic_funcs->collective_wait_init_cs(cs);
589 hdev->asic_funcs->collective_wait_init_cs(cs);686 if (rc)
687 goto unroll_cq_resv;
688 }
590 689
591 690
691 if (cs->encaps_signals && cs->staged_first) {
692 rc = encaps_sig_first_staged_cs_handler(hdev, cs);
693 if (rc)
694 goto unroll_cq_resv;
695 }
696
592 spin_lock(&hdev->cs_mirror_lock);697 spin_lock(&hdev->cs_mirror_lock);
593 698
594 /* Verify staged CS exists and add to the staged list */699 /* Verify staged CS exists and add to the staged list */
@@ -613,6 +718,11 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) · 18 unmodified lines
613 }718 }
614 719
615 list_add_tail(&cs->staged_cs_node, &staged_cs->staged_cs_node);720 list_add_tail(&cs->staged_cs_node, &staged_cs->staged_cs_node);
721
722 /* update stream map of the first CS */
723 if (hdev->supports_wait_for_multi_cs)
724 staged_cs->fence->stream_master_qid_map |=
725 cs->fence->stream_master_qid_map;
616 }726 }
617 727
618 list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list);728 list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list);
@@ -834,6 +944,8 @@ static void sync_stream_queue_init(struct hl_device *hdev, u32 q_idx) · 215 unmodified lines
834 hw_sob = &sync_stream_prop->hw_sob[sob];944 hw_sob = &sync_stream_prop->hw_sob[sob];
835 hw_sob->hdev = hdev;945 hw_sob->hdev = hdev;
836 hw_sob->sob_id = sync_stream_prop->base_sob_id + sob;946 hw_sob->sob_id = sync_stream_prop->base_sob_id + sob;
947 hw_sob->sob_addr =
948 hdev->asic_funcs->get_sob_addr(hdev, hw_sob->sob_id);
837 hw_sob->q_idx = q_idx;949 hw_sob->q_idx = q_idx;
838 kref_init(&hw_sob->kref);950 kref_init(&hw_sob->kref);
839 }951 }
drivers/misc/habanalabs/common/memory.c+123−46
@@ -124,7 +124,7 @@ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args, · 123 unmodified lines
124 124
125 spin_lock(&vm->idr_lock);125 spin_lock(&vm->idr_lock);
126 handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0,126 handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0,
127 GFP_KERNEL);127 GFP_ATOMIC);
128 spin_unlock(&vm->idr_lock);128 spin_unlock(&vm->idr_lock);
129 129
130 if (handle < 0) {130 if (handle < 0) {
@@ -528,6 +528,33 @@ static inline int add_va_block(struct hl_device *hdev, · 397 unmodified lines
528 return rc;528 return rc;
529}529}
530 530
531/**
532 * is_hint_crossing_range() - check if hint address crossing specified reserved
533 * range.
534 */
535static inline bool is_hint_crossing_range(enum hl_va_range_type range_type,
536 u64 start_addr, u32 size, struct asic_fixed_properties *prop) {
537 bool range_cross;
538
539 if (range_type == HL_VA_RANGE_TYPE_DRAM)
540 range_cross =
541 hl_mem_area_crosses_range(start_addr, size,
542 prop->hints_dram_reserved_va_range.start_addr,
543 prop->hints_dram_reserved_va_range.end_addr);
544 else if (range_type == HL_VA_RANGE_TYPE_HOST)
545 range_cross =
546 hl_mem_area_crosses_range(start_addr, size,
547 prop->hints_host_reserved_va_range.start_addr,
548 prop->hints_host_reserved_va_range.end_addr);
549 else
550 range_cross =
551 hl_mem_area_crosses_range(start_addr, size,
552 prop->hints_host_hpage_reserved_va_range.start_addr,
553 prop->hints_host_hpage_reserved_va_range.end_addr);
554
555 return range_cross;
556}
557
531/**558/**
532 * get_va_block() - get a virtual block for the given size and alignment.559 * get_va_block() - get a virtual block for the given size and alignment.
533 *560 *
@@ -536,6 +563,8 @@ static inline int add_va_block(struct hl_device *hdev, · 2 unmodified lines
536 * @size: requested block size.563 * @size: requested block size.
537 * @hint_addr: hint for requested address by the user.564 * @hint_addr: hint for requested address by the user.
538 * @va_block_align: required alignment of the virtual block start address.565 * @va_block_align: required alignment of the virtual block start address.
566 * @range_type: va range type (host, dram)
567 * @flags: additional memory flags, currently only uses HL_MEM_FORCE_HINT
539 *568 *
540 * This function does the following:569 * This function does the following:
541 * - Iterate on the virtual block list to find a suitable virtual block for the570 * - Iterate on the virtual block list to find a suitable virtual block for the
@@ -545,13 +574,19 @@ static inline int add_va_block(struct hl_device *hdev, · 3 unmodified lines
545 */574 */
546static u64 get_va_block(struct hl_device *hdev,575static u64 get_va_block(struct hl_device *hdev,
547 struct hl_va_range *va_range,576 struct hl_va_range *va_range,
548 u64 size, u64 hint_addr, u32 va_block_align)577 u64 size, u64 hint_addr, u32 va_block_align,
578 enum hl_va_range_type range_type,
579 u32 flags)
549{580{
550 struct hl_vm_va_block *va_block, *new_va_block = NULL;581 struct hl_vm_va_block *va_block, *new_va_block = NULL;
582 struct asic_fixed_properties *prop = &hdev->asic_prop;
551 u64 tmp_hint_addr, valid_start, valid_size, prev_start, prev_end,583 u64 tmp_hint_addr, valid_start, valid_size, prev_start, prev_end,
552 align_mask, reserved_valid_start = 0, reserved_valid_size = 0;584 align_mask, reserved_valid_start = 0, reserved_valid_size = 0,
585 dram_hint_mask = prop->dram_hints_align_mask;
553 bool add_prev = false;586 bool add_prev = false;
554 bool is_align_pow_2 = is_power_of_2(va_range->page_size);587 bool is_align_pow_2 = is_power_of_2(va_range->page_size);
588 bool is_hint_dram_addr = hl_is_dram_va(hdev, hint_addr);
589 bool force_hint = flags & HL_MEM_FORCE_HINT;
555 590
556 if (is_align_pow_2)591 if (is_align_pow_2)
557 align_mask = ~((u64)va_block_align - 1);592 align_mask = ~((u64)va_block_align - 1);
@@ -564,12 +599,20 @@ static u64 get_va_block(struct hl_device *hdev, · 6 unmodified lines
564 size = DIV_ROUND_UP_ULL(size, va_range->page_size) *599 size = DIV_ROUND_UP_ULL(size, va_range->page_size) *
565 va_range->page_size;600 va_range->page_size;
566 601
567 tmp_hint_addr = hint_addr;602 tmp_hint_addr = hint_addr & ~dram_hint_mask;
568 603
569 /* Check if we need to ignore hint address */604 /* Check if we need to ignore hint address */
570 if ((is_align_pow_2 && (hint_addr & (va_block_align - 1))) ||605 if ((is_align_pow_2 && (hint_addr & (va_block_align - 1))) ||
571 (!is_align_pow_2 &&606 (!is_align_pow_2 && is_hint_dram_addr &&
572 do_div(tmp_hint_addr, va_range->page_size))) {607 do_div(tmp_hint_addr, va_range->page_size))) {
608
609 if (force_hint) {
610 /* Hint must be respected, so here we just fail */
611 dev_err(hdev->dev,
612 "Hint address 0x%llx is not page aligned - cannot be respected\n",
613 hint_addr);
614 return 0;
615 }
573 616
574 dev_dbg(hdev->dev,617 dev_dbg(hdev->dev,
575 "Hint address 0x%llx will be ignored because it is not aligned\n",618 "Hint address 0x%llx will be ignored because it is not aligned\n",
@@ -596,6 +639,16 @@ static u64 get_va_block(struct hl_device *hdev, · 20 unmodified lines
596 if (valid_size < size)639 if (valid_size < size)
597 continue;640 continue;
598 641
642 /*
643 * In case hint address is 0, and arc_hints_range_reservation
644 * property enabled, then avoid allocating va blocks from the
645 * range reserved for hint addresses
646 */
647 if (prop->hints_range_reservation && !hint_addr)
648 if (is_hint_crossing_range(range_type, valid_start,
649 size, prop))
650 continue;
651
599 /* Pick the minimal length block which has the required size */652 /* Pick the minimal length block which has the required size */
600 if (!new_va_block || (valid_size < reserved_valid_size)) {653 if (!new_va_block || (valid_size < reserved_valid_size)) {
601 new_va_block = va_block;654 new_va_block = va_block;
@@ -618,6 +671,17 @@ static u64 get_va_block(struct hl_device *hdev, · 16 unmodified lines
618 goto out;671 goto out;
619 }672 }
620 673
674 if (force_hint && reserved_valid_start != hint_addr) {
675 /* Hint address must be respected. If we are here - this means
676 * we could not respect it.
677 */
678 dev_err(hdev->dev,
679 "Hint address 0x%llx could not be respected\n",
680 hint_addr);
681 reserved_valid_start = 0;
682 goto out;
683 }
684
621 /*685 /*
622 * Check if there is some leftover range due to reserving the new686 * Check if there is some leftover range due to reserving the new
623 * va block, then return it to the main virtual addresses list.687 * va block, then return it to the main virtual addresses list.
@@ -670,7 +734,8 @@ u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, · 46 unmodified lines
670 enum hl_va_range_type type, u32 size, u32 alignment)734 enum hl_va_range_type type, u32 size, u32 alignment)
671{735{
672 return get_va_block(hdev, ctx->va_range[type], size, 0,736 return get_va_block(hdev, ctx->va_range[type], size, 0,
673 max(alignment, ctx->va_range[type]->page_size));737 max(alignment, ctx->va_range[type]->page_size),
738 type, 0);
674}739}
675 740
676/**741/**
@@ -731,29 +796,16 @@ int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, · 54 unmodified lines
731 return rc;796 return rc;
732}797}
733 798
734/**
735 * get_sg_info() - get number of pages and the DMA address from SG list.
736 * @sg: the SG list.
737 * @dma_addr: pointer to DMA address to return.
738 *
739 * Calculate the number of consecutive pages described by the SG list. Take the
740 * offset of the address in the first page, add to it the length and round it up
741 * to the number of needed pages.
742 */
743static u32 get_sg_info(struct scatterlist *sg, dma_addr_t *dma_addr)
744{
745 *dma_addr = sg_dma_address(sg);
746
747 return ((((*dma_addr) & (PAGE_SIZE - 1)) + sg_dma_len(sg)) +
748 (PAGE_SIZE - 1)) >> PAGE_SHIFT;
749}
750
751/**799/**
752 * init_phys_pg_pack_from_userptr() - initialize physical page pack from host800 * init_phys_pg_pack_from_userptr() - initialize physical page pack from host
753 * memory801 * memory
754 * @ctx: pointer to the context structure.802 * @ctx: pointer to the context structure.
755 * @userptr: userptr to initialize from.803 * @userptr: userptr to initialize from.
756 * @pphys_pg_pack: result pointer.804 * @pphys_pg_pack: result pointer.
805 * @force_regular_page: tell the function to ignore huge page optimization,
806 * even if possible. Needed for cases where the device VA
807 * is allocated before we know the composition of the
808 * physical pages
757 *809 *
758 * This function does the following:810 * This function does the following:
759 * - Pin the physical pages related to the given virtual block.811 * - Pin the physical pages related to the given virtual block.
@@ -762,17 +814,18 @@ static u32 get_sg_info(struct scatterlist *sg, dma_addr_t *dma_addr) · 2 unmodified lines
762 */814 */
763static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx,815static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx,
764 struct hl_userptr *userptr,816 struct hl_userptr *userptr,
765 struct hl_vm_phys_pg_pack **pphys_pg_pack)817 struct hl_vm_phys_pg_pack **pphys_pg_pack,
818 bool force_regular_page)
766{819{
820 u32 npages, page_size = PAGE_SIZE,
821 huge_page_size = ctx->hdev->asic_prop.pmmu_huge.page_size;
822 u32 pgs_in_huge_page = huge_page_size >> __ffs(page_size);
767 struct hl_vm_phys_pg_pack *phys_pg_pack;823 struct hl_vm_phys_pg_pack *phys_pg_pack;
824 bool first = true, is_huge_page_opt;
825 u64 page_mask, total_npages;
768 struct scatterlist *sg;826 struct scatterlist *sg;
769 dma_addr_t dma_addr;827 dma_addr_t dma_addr;
770 u64 page_mask, total_npages;
771 u32 npages, page_size = PAGE_SIZE,
772 huge_page_size = ctx->hdev->asic_prop.pmmu_huge.page_size;
773 bool first = true, is_huge_page_opt = true;
774 int rc, i, j;828 int rc, i, j;
775 u32 pgs_in_huge_page = huge_page_size >> __ffs(page_size);
776 829
777 phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);830 phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);
778 if (!phys_pg_pack)831 if (!phys_pg_pack)
@@ -783,6 +836,8 @@ static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx, · 4 unmodified lines
783 phys_pg_pack->asid = ctx->asid;836 phys_pg_pack->asid = ctx->asid;
784 atomic_set(&phys_pg_pack->mapping_cnt, 1);837 atomic_set(&phys_pg_pack->mapping_cnt, 1);
785 838
839 is_huge_page_opt = (force_regular_page ? false : true);
840
786 /* Only if all dma_addrs are aligned to 2MB and their841 /* Only if all dma_addrs are aligned to 2MB and their
787 * sizes is at least 2MB, we can use huge page mapping.842 * sizes is at least 2MB, we can use huge page mapping.
788 * We limit the 2MB optimization to this condition,843 * We limit the 2MB optimization to this condition,
@@ -791,7 +846,7 @@ static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx, · 2 unmodified lines
791 */846 */
792 total_npages = 0;847 total_npages = 0;
793 for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {848 for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {
794 npages = get_sg_info(sg, &dma_addr);849 npages = hl_get_sg_info(sg, &dma_addr);
795 850
796 total_npages += npages;851 total_npages += npages;
797 852
@@ -820,7 +875,7 @@ static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx, · 22 unmodified lines
820 875
821 j = 0;876 j = 0;
822 for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {877 for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {
823 npages = get_sg_info(sg, &dma_addr);878 npages = hl_get_sg_info(sg, &dma_addr);
824 879
825 /* align down to physical page size and save the offset */880 /* align down to physical page size and save the offset */
826 if (first) {881 if (first) {
@@ -1001,11 +1056,12 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, · 174 unmodified lines
1001 struct hl_userptr *userptr = NULL;1056 struct hl_userptr *userptr = NULL;
1002 struct hl_vm_hash_node *hnode;1057 struct hl_vm_hash_node *hnode;
1003 struct hl_va_range *va_range;1058 struct hl_va_range *va_range;
1004 enum vm_type_t *vm_type;1059 enum vm_type *vm_type;
1005 u64 ret_vaddr, hint_addr;1060 u64 ret_vaddr, hint_addr;
1006 u32 handle = 0, va_block_align;1061 u32 handle = 0, va_block_align;
1007 int rc;1062 int rc;
1008 bool is_userptr = args->flags & HL_MEM_USERPTR;1063 bool is_userptr = args->flags & HL_MEM_USERPTR;
1064 enum hl_va_range_type va_range_type = 0;
1009 1065
1010 /* Assume failure */1066 /* Assume failure */
1011 *device_addr = 0;1067 *device_addr = 0;
@@ -1023,7 +1079,7 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, · 11 unmodified lines
1023 }1079 }
1024 1080
1025 rc = init_phys_pg_pack_from_userptr(ctx, userptr,1081 rc = init_phys_pg_pack_from_userptr(ctx, userptr,
1026 &phys_pg_pack);1082 &phys_pg_pack, false);
1027 if (rc) {1083 if (rc) {
1028 dev_err(hdev->dev,1084 dev_err(hdev->dev,
1029 "unable to init page pack for vaddr 0x%llx\n",1085 "unable to init page pack for vaddr 0x%llx\n",
@@ -1031,14 +1087,14 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, · 1 unmodified line
1031 goto init_page_pack_err;1087 goto init_page_pack_err;
1032 }1088 }
1033 1089
1034 vm_type = (enum vm_type_t *) userptr;1090 vm_type = (enum vm_type *) userptr;
1035 hint_addr = args->map_host.hint_addr;1091 hint_addr = args->map_host.hint_addr;
1036 handle = phys_pg_pack->handle;1092 handle = phys_pg_pack->handle;
1037 1093
1038 /* get required alignment */1094 /* get required alignment */
1039 if (phys_pg_pack->page_size == page_size) {1095 if (phys_pg_pack->page_size == page_size) {
1040 va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];1096 va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];
1041 1097 va_range_type = HL_VA_RANGE_TYPE_HOST;
1042 /*1098 /*
1043 * huge page alignment may be needed in case of regular1099 * huge page alignment may be needed in case of regular
1044 * page mapping, depending on the host VA alignment1100 * page mapping, depending on the host VA alignment
@@ -1053,6 +1109,7 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, · 8 unmodified lines
1053 * mapping1109 * mapping
1054 */1110 */
1055 va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];1111 va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];
1112 va_range_type = HL_VA_RANGE_TYPE_HOST_HUGE;
1056 va_block_align = huge_page_size;1113 va_block_align = huge_page_size;
1057 }1114 }
1058 } else {1115 } else {
@@ -1072,12 +1129,13 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, · 13 unmodified lines
1072 1129
1073 spin_unlock(&vm->idr_lock);1130 spin_unlock(&vm->idr_lock);
1074 1131
1075 vm_type = (enum vm_type_t *) phys_pg_pack;1132 vm_type = (enum vm_type *) phys_pg_pack;
1076 1133
1077 hint_addr = args->map_device.hint_addr;1134 hint_addr = args->map_device.hint_addr;
1078 1135
1079 /* DRAM VA alignment is the same as the MMU page size */1136 /* DRAM VA alignment is the same as the MMU page size */
1080 va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];1137 va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];
1138 va_range_type = HL_VA_RANGE_TYPE_DRAM;
1081 va_block_align = hdev->asic_prop.dmmu.page_size;1139 va_block_align = hdev->asic_prop.dmmu.page_size;
1082 }1140 }
1083 1141
@@ -1100,8 +1158,23 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, · 16 unmodified lines
1100 goto hnode_err;1158 goto hnode_err;
1101 }1159 }
1102 1160
1161 if (hint_addr && phys_pg_pack->offset) {
1162 if (args->flags & HL_MEM_FORCE_HINT) {
1163 /* Fail if hint must be respected but it can't be */
1164 dev_err(hdev->dev,
1165 "Hint address 0x%llx cannot be respected because source memory is not aligned 0x%x\n",
1166 hint_addr, phys_pg_pack->offset);
1167 rc = -EINVAL;
1168 goto va_block_err;
1169 }
1170 dev_dbg(hdev->dev,
1171 "Hint address 0x%llx will be ignored because source memory is not aligned 0x%x\n",
1172 hint_addr, phys_pg_pack->offset);
1173 }
1174
1103 ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size,1175 ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size,
1104 hint_addr, va_block_align);1176 hint_addr, va_block_align,
1177 va_range_type, args->flags);
1105 if (!ret_vaddr) {1178 if (!ret_vaddr) {
1106 dev_err(hdev->dev, "no available va block for handle %u\n",1179 dev_err(hdev->dev, "no available va block for handle %u\n",
1107 handle);1180 handle);
@@ -1181,17 +1254,19 @@ init_page_pack_err: · 73 unmodified lines
1181static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,1254static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
1182 bool ctx_free)1255 bool ctx_free)
1183{1256{
1184 struct hl_device *hdev = ctx->hdev;
1185 struct asic_fixed_properties *prop = &hdev->asic_prop;
1186 struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;1257 struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;
1258 u64 vaddr = args->unmap.device_virt_addr;
1187 struct hl_vm_hash_node *hnode = NULL;1259 struct hl_vm_hash_node *hnode = NULL;
1260 struct asic_fixed_properties *prop;
1261 struct hl_device *hdev = ctx->hdev;
1188 struct hl_userptr *userptr = NULL;1262 struct hl_userptr *userptr = NULL;
1189 struct hl_va_range *va_range;1263 struct hl_va_range *va_range;
1190 u64 vaddr = args->unmap.device_virt_addr;1264 enum vm_type *vm_type;
1191 enum vm_type_t *vm_type;
1192 bool is_userptr;1265 bool is_userptr;
1193 int rc = 0;1266 int rc = 0;
1194 1267
1268 prop = &hdev->asic_prop;
1269
1195 /* protect from double entrance */1270 /* protect from double entrance */
1196 mutex_lock(&ctx->mem_hash_lock);1271 mutex_lock(&ctx->mem_hash_lock);
1197 hash_for_each_possible(ctx->mem_hash, hnode, node, (unsigned long)vaddr)1272 hash_for_each_possible(ctx->mem_hash, hnode, node, (unsigned long)vaddr)
@@ -1214,8 +1289,9 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, · 16 unmodified lines
1214 if (*vm_type == VM_TYPE_USERPTR) {1289 if (*vm_type == VM_TYPE_USERPTR) {
1215 is_userptr = true;1290 is_userptr = true;
1216 userptr = hnode->ptr;1291 userptr = hnode->ptr;
1217 rc = init_phys_pg_pack_from_userptr(ctx, userptr,1292
1218 &phys_pg_pack);1293 rc = init_phys_pg_pack_from_userptr(ctx, userptr, &phys_pg_pack,
1294 false);
1219 if (rc) {1295 if (rc) {
1220 dev_err(hdev->dev,1296 dev_err(hdev->dev,
1221 "unable to init page pack for vaddr 0x%llx\n",1297 "unable to init page pack for vaddr 0x%llx\n",
@@ -1299,7 +1375,7 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, · 77 unmodified lines
1299 kfree(hnode);1375 kfree(hnode);
1300 1376
1301 if (is_userptr) {1377 if (is_userptr) {
1302 rc = free_phys_pg_pack(hdev, phys_pg_pack);1378 free_phys_pg_pack(hdev, phys_pg_pack);
1303 dma_unmap_host_va(hdev, userptr);1379 dma_unmap_host_va(hdev, userptr);
1304 }1380 }
1305 1381
@@ -1669,6 +1745,7 @@ int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size, · 363 unmodified lines
1669 return -EINVAL;1745 return -EINVAL;
1670 }1746 }
1671 1747
1748 userptr->pid = current->pid;
1672 userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_KERNEL);1749 userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_KERNEL);
1673 if (!userptr->sgt)1750 if (!userptr->sgt)
1674 return -ENOMEM;1751 return -ENOMEM;
@@ -2033,7 +2110,7 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx) · 358 unmodified lines
2033 * another side effect error2110 * another side effect error
2034 */2111 */
2035 if (!hdev->hard_reset_pending && !hash_empty(ctx->mem_hash))2112 if (!hdev->hard_reset_pending && !hash_empty(ctx->mem_hash))
2036 dev_notice(hdev->dev,2113 dev_dbg(hdev->dev,
2037 "user released device without removing its memory mappings\n");2114 "user released device without removing its memory mappings\n");
2038 2115
2039 hash_for_each_safe(ctx->mem_hash, i, tmp_node, hnode, node) {2116 hash_for_each_safe(ctx->mem_hash, i, tmp_node, hnode, node) {
drivers/misc/habanalabs/common/mmu/mmu_v1.c+6−6
@@ -470,13 +470,13 @@ static void hl_mmu_v1_fini(struct hl_device *hdev) · 469 unmodified lines
470 if (!ZERO_OR_NULL_PTR(hdev->mmu_priv.hr.mmu_shadow_hop0)) {470 if (!ZERO_OR_NULL_PTR(hdev->mmu_priv.hr.mmu_shadow_hop0)) {
471 kvfree(hdev->mmu_priv.dr.mmu_shadow_hop0);471 kvfree(hdev->mmu_priv.dr.mmu_shadow_hop0);
472 gen_pool_destroy(hdev->mmu_priv.dr.mmu_pgt_pool);472 gen_pool_destroy(hdev->mmu_priv.dr.mmu_pgt_pool);
473 }
474 473
475 /* Make sure that if we arrive here again without init was called we474 /* Make sure that if we arrive here again without init was
476 * won't cause kernel panic. This can happen for example if we fail475 * called we won't cause kernel panic. This can happen for
477 * during hard reset code at certain points476 * example if we fail during hard reset code at certain points
478 */477 */
479 hdev->mmu_priv.dr.mmu_shadow_hop0 = NULL;478 hdev->mmu_priv.dr.mmu_shadow_hop0 = NULL;
479 }
480}480}
481 481
482/**482/**
drivers/misc/habanalabs/common/pci/pci.c+2−0
@@ -436,6 +436,8 @@ int hl_pci_init(struct hl_device *hdev) · 435 unmodified lines
436 goto unmap_pci_bars;436 goto unmap_pci_bars;
437 }437 }
438 438
439 dma_set_max_seg_size(&pdev->dev, U32_MAX);
440
439 return 0;441 return 0;
440 442
441unmap_pci_bars:443unmap_pci_bars:
drivers/misc/habanalabs/common/state_dump.c+718−0
@@ -0,0 +1,718 @@
1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Copyright 2021 HabanaLabs, Ltd.
5 * All Rights Reserved.
6 */
7
8#include <linux/vmalloc.h>
9#include <uapi/misc/habanalabs.h>
10#include "habanalabs.h"
11
12/**
13 * hl_format_as_binary - helper function, format an integer as binary
14 * using supplied scratch buffer
15 * @buf: the buffer to use
16 * @buf_len: buffer capacity
17 * @n: number to format
18 *
19 * Returns pointer to buffer
20 */
21char *hl_format_as_binary(char *buf, size_t buf_len, u32 n)
22{
23 int i;
24 u32 bit;
25 bool leading0 = true;
26 char *wrptr = buf;
27
28 if (buf_len > 0 && buf_len < 3) {
29 *wrptr = '\0';
30 return buf;
31 }
32
33 wrptr[0] = '0';
34 wrptr[1] = 'b';
35 wrptr += 2;
36 /* Remove 3 characters from length for '0b' and '\0' termination */
37 buf_len -= 3;
38
39 for (i = 0; i < sizeof(n) * BITS_PER_BYTE && buf_len; ++i, n <<= 1) {
40 /* Writing bit calculation in one line would cause a false
41 * positive static code analysis error, so splitting.
42 */
43 bit = n & (1 << (sizeof(n) * BITS_PER_BYTE - 1));
44 bit = !!bit;
45 leading0 &= !bit;
46 if (!leading0) {
47 *wrptr = '0' + bit;
48 ++wrptr;
49 }
50 }
51
52 *wrptr = '\0';
53
54 return buf;
55}
56
57/**
58 * resize_to_fit - helper function, resize buffer to fit given amount of data
59 * @buf: destination buffer double pointer
60 * @size: pointer to the size container
61 * @desired_size: size the buffer must contain
62 *
63 * Returns 0 on success or error code on failure.
64 * On success, the size of buffer is at least desired_size. Buffer is allocated
65 * via vmalloc and must be freed with vfree.
66 */
67static int resize_to_fit(char **buf, size_t *size, size_t desired_size)
68{
69 char *resized_buf;
70 size_t new_size;
71
72 if (*size >= desired_size)
73 return 0;
74
75 /* Not enough space to print all, have to resize */
76 new_size = max_t(size_t, PAGE_SIZE, round_up(desired_size, PAGE_SIZE));
77 resized_buf = vmalloc(new_size);
78 if (!resized_buf)
79 return -ENOMEM;
80 memcpy(resized_buf, *buf, *size);
81 vfree(*buf);
82 *buf = resized_buf;
83 *size = new_size;
84
85 return 1;
86}
87
88/**
89 * hl_snprintf_resize() - print formatted data to buffer, resize as needed
90 * @buf: buffer double pointer, to be written to and resized, must be either
91 * NULL or allocated with vmalloc.
92 * @size: current size of the buffer
93 * @offset: current offset to write to
94 * @format: format of the data
95 *
96 * This function will write formatted data into the buffer. If buffer is not
97 * large enough, it will be resized using vmalloc. Size may be modified if the
98 * buffer was resized, offset will be advanced by the number of bytes written
99 * not including the terminating character
100 *
101 * Returns 0 on success or error code on failure
102 *
103 * Note that the buffer has to be manually released using vfree.
104 */
105int hl_snprintf_resize(char **buf, size_t *size, size_t *offset,
106 const char *format, ...)
107{
108 va_list args;
109 size_t length;
110 int rc;
111
112 if (*buf == NULL && (*size != 0 || *offset != 0))
113 return -EINVAL;
114
115 va_start(args, format);
116 length = vsnprintf(*buf + *offset, *size - *offset, format, args);
117 va_end(args);
118
119 rc = resize_to_fit(buf, size, *offset + length + 1);
120 if (rc < 0)
121 return rc;
122 else if (rc > 0) {
123 /* Resize was needed, write again */
124 va_start(args, format);
125 length = vsnprintf(*buf + *offset, *size - *offset, format,
126 args);
127 va_end(args);
128 }
129
130 *offset += length;
131
132 return 0;
133}
134
135/**
136 * hl_sync_engine_to_string - convert engine type enum to string literal
137 * @engine_type: engine type (TPC/MME/DMA)
138 *
139 * Return the resolved string literal
140 */
141const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type)
142{
143 switch (engine_type) {
144 case ENGINE_DMA:
145 return "DMA";
146 case ENGINE_MME:
147 return "MME";
148 case ENGINE_TPC:
149 return "TPC";
150 }
151 return "Invalid Engine Type";
152}
153
154/**
155 * hl_print_resize_sync_engine - helper function, format engine name and ID
156 * using hl_snprintf_resize
157 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
158 * @size: pointer to the size container
159 * @offset: pointer to the offset container
160 * @engine_type: engine type (TPC/MME/DMA)
161 * @engine_id: engine numerical id
162 *
163 * Returns 0 on success or error code on failure
164 */
165static int hl_print_resize_sync_engine(char **buf, size_t *size, size_t *offset,
166 enum hl_sync_engine_type engine_type,
167 u32 engine_id)
168{
169 return hl_snprintf_resize(buf, size, offset, "%s%u",
170 hl_sync_engine_to_string(engine_type), engine_id);
171}
172
173/**
174 * hl_state_dump_get_sync_name - transform sync object id to name if available
175 * @hdev: pointer to the device
176 * @sync_id: sync object id
177 *
178 * Returns a name literal or NULL if not resolved.
179 * Note: returning NULL shall not be considered as a failure, as not all
180 * sync objects are named.
181 */
182const char *hl_state_dump_get_sync_name(struct hl_device *hdev, u32 sync_id)
183{
184 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
185 struct hl_hw_obj_name_entry *entry;
186
187 hash_for_each_possible(sds->so_id_to_str_tb, entry,
188 node, sync_id)
189 if (sync_id == entry->id)
190 return entry->name;
191
192 return NULL;
193}
194
195/**
196 * hl_state_dump_get_monitor_name - transform monitor object dump to monitor
197 * name if available
198 * @hdev: pointer to the device
199 * @mon: monitor state dump
200 *
201 * Returns a name literal or NULL if not resolved.
202 * Note: returning NULL shall not be considered as a failure, as not all
203 * monitors are named.
204 */
205const char *hl_state_dump_get_monitor_name(struct hl_device *hdev,
206 struct hl_mon_state_dump *mon)
207{
208 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
209 struct hl_hw_obj_name_entry *entry;
210
211 hash_for_each_possible(sds->monitor_id_to_str_tb,
212 entry, node, mon->id)
213 if (mon->id == entry->id)
214 return entry->name;
215
216 return NULL;
217}
218
219/**
220 * hl_state_dump_free_sync_to_engine_map - free sync object to engine map
221 * @map: sync object to engine map
222 *
223 * Note: generic free implementation, the allocation is implemented per ASIC.
224 */
225void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map)
226{
227 struct hl_sync_to_engine_map_entry *entry;
228 struct hlist_node *tmp_node;
229 int i;
230
231 hash_for_each_safe(map->tb, i, tmp_node, entry, node) {
232 hash_del(&entry->node);
233 kfree(entry);
234 }
235}
236
237/**
238 * hl_state_dump_get_sync_to_engine - transform sync_id to
239 * hl_sync_to_engine_map_entry if available for current id
240 * @map: sync object to engine map
241 * @sync_id: sync object id
242 *
243 * Returns the translation entry if found or NULL if not.
244 * Note, returned NULL shall not be considered as a failure as the map
245 * does not cover all possible, it is a best effort sync ids.
246 */
247static struct hl_sync_to_engine_map_entry *
248hl_state_dump_get_sync_to_engine(struct hl_sync_to_engine_map *map, u32 sync_id)
249{
250 struct hl_sync_to_engine_map_entry *entry;
251
252 hash_for_each_possible(map->tb, entry, node, sync_id)
253 if (entry->sync_id == sync_id)
254 return entry;
255 return NULL;
256}
257
258/**
259 * hl_state_dump_read_sync_objects - read sync objects array
260 * @hdev: pointer to the device
261 * @index: sync manager block index starting with E_N
262 *
263 * Returns array of size SP_SYNC_OBJ_AMOUNT on success or NULL on failure
264 */
265static u32 *hl_state_dump_read_sync_objects(struct hl_device *hdev, u32 index)
266{
267 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
268 u32 *sync_objects;
269 s64 base_addr; /* Base addr can be negative */
270 int i;
271
272 base_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +
273 sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;
274
275 sync_objects = vmalloc(sds->props[SP_SYNC_OBJ_AMOUNT] * sizeof(u32));
276 if (!sync_objects)
277 return NULL;
278
279 for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i)
280 sync_objects[i] = RREG32(base_addr + i * sizeof(u32));
281
282 return sync_objects;
283}
284
285/**
286 * hl_state_dump_free_sync_objects - free sync objects array allocated by
287 * hl_state_dump_read_sync_objects
288 * @sync_objects: sync objects array
289 */
290static void hl_state_dump_free_sync_objects(u32 *sync_objects)
291{
292 vfree(sync_objects);
293}
294
295
296/**
297 * hl_state_dump_print_syncs_single_block - print active sync objects on a
298 * single block
299 * @hdev: pointer to the device
300 * @index: sync manager block index starting with E_N
301 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
302 * @size: pointer to the size container
303 * @offset: pointer to the offset container
304 * @map: sync engines names map
305 *
306 * Returns 0 on success or error code on failure
307 */
308static int
309hl_state_dump_print_syncs_single_block(struct hl_device *hdev, u32 index,
310 char **buf, size_t *size, size_t *offset,
311 struct hl_sync_to_engine_map *map)
312{
313 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
314 const char *sync_name;
315 u32 *sync_objects = NULL;
316 int rc = 0, i;
317
318 if (sds->sync_namager_names) {
319 rc = hl_snprintf_resize(
320 buf, size, offset, "%s\n",
321 sds->sync_namager_names[index]);
322 if (rc)
323 goto out;
324 }
325
326 sync_objects = hl_state_dump_read_sync_objects(hdev, index);
327 if (!sync_objects) {
328 rc = -ENOMEM;
329 goto out;
330 }
331
332 for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i) {
333 struct hl_sync_to_engine_map_entry *entry;
334 u64 sync_object_addr;
335
336 if (!sync_objects[i])
337 continue;
338
339 sync_object_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +
340 sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index +
341 i * sizeof(u32);
342
343 rc = hl_snprintf_resize(buf, size, offset, "sync id: %u", i);
344 if (rc)
345 goto free_sync_objects;
346 sync_name = hl_state_dump_get_sync_name(hdev, i);
347 if (sync_name) {
348 rc = hl_snprintf_resize(buf, size, offset, " %s",
349 sync_name);
350 if (rc)
351 goto free_sync_objects;
352 }
353 rc = hl_snprintf_resize(buf, size, offset, ", value: %u",
354 sync_objects[i]);
355 if (rc)
356 goto free_sync_objects;
357
358 /* Append engine string */
359 entry = hl_state_dump_get_sync_to_engine(map,
360 (u32)sync_object_addr);
361 if (entry) {
362 rc = hl_snprintf_resize(buf, size, offset,
363 ", Engine: ");
364 if (rc)
365 goto free_sync_objects;
366 rc = hl_print_resize_sync_engine(buf, size, offset,
367 entry->engine_type,
368 entry->engine_id);
369 if (rc)
370 goto free_sync_objects;
371 }
372
373 rc = hl_snprintf_resize(buf, size, offset, "\n");
374 if (rc)
375 goto free_sync_objects;
376 }
377
378free_sync_objects:
379 hl_state_dump_free_sync_objects(sync_objects);
380out:
381 return rc;
382}
383
384/**
385 * hl_state_dump_print_syncs - print active sync objects
386 * @hdev: pointer to the device
387 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
388 * @size: pointer to the size container
389 * @offset: pointer to the offset container
390 *
391 * Returns 0 on success or error code on failure
392 */
393static int hl_state_dump_print_syncs(struct hl_device *hdev,
394 char **buf, size_t *size,
395 size_t *offset)
396
397{
398 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
399 struct hl_sync_to_engine_map *map;
400 u32 index;
401 int rc = 0;
402
403 map = kzalloc(sizeof(*map), GFP_KERNEL);
404 if (!map)
405 return -ENOMEM;
406
407 rc = sds->funcs.gen_sync_to_engine_map(hdev, map);
408 if (rc)
409 goto free_map_mem;
410
411 rc = hl_snprintf_resize(buf, size, offset, "Non zero sync objects:\n");
412 if (rc)
413 goto out;
414
415 if (sds->sync_namager_names) {
416 for (index = 0; sds->sync_namager_names[index]; ++index) {
417 rc = hl_state_dump_print_syncs_single_block(
418 hdev, index, buf, size, offset, map);
419 if (rc)
420 goto out;
421 }
422 } else {
423 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {
424 rc = hl_state_dump_print_syncs_single_block(
425 hdev, index, buf, size, offset, map);
426 if (rc)
427 goto out;
428 }
429 }
430
431out:
432 hl_state_dump_free_sync_to_engine_map(map);
433free_map_mem:
434 kfree(map);
435
436 return rc;
437}
438
439/**
440 * hl_state_dump_alloc_read_sm_block_monitors - read monitors for a specific
441 * block
442 * @hdev: pointer to the device
443 * @index: sync manager block index starting with E_N
444 *
445 * Returns an array of monitor data of size SP_MONITORS_AMOUNT or NULL
446 * on error
447 */
448static struct hl_mon_state_dump *
449hl_state_dump_alloc_read_sm_block_monitors(struct hl_device *hdev, u32 index)
450{
451 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
452 struct hl_mon_state_dump *monitors;
453 s64 base_addr; /* Base addr can be negative */
454 int i;
455
456 monitors = vmalloc(sds->props[SP_MONITORS_AMOUNT] *
457 sizeof(struct hl_mon_state_dump));
458 if (!monitors)
459 return NULL;
460
461 base_addr = sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;
462
463 for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {
464 monitors[i].id = i;
465 monitors[i].wr_addr_low =
466 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_LOW] +
467 i * sizeof(u32));
468
469 monitors[i].wr_addr_high =
470 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_HIGH] +
471 i * sizeof(u32));
472
473 monitors[i].wr_data =
474 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_DATA] +
475 i * sizeof(u32));
476
477 monitors[i].arm_data =
478 RREG32(base_addr + sds->props[SP_MON_OBJ_ARM_DATA] +
479 i * sizeof(u32));
480
481 monitors[i].status =
482 RREG32(base_addr + sds->props[SP_MON_OBJ_STATUS] +
483 i * sizeof(u32));
484 }
485
486 return monitors;
487}
488
489/**
490 * hl_state_dump_free_monitors - free the monitors structure
491 * @monitors: monitors array created with
492 * hl_state_dump_alloc_read_sm_block_monitors
493 */
494static void hl_state_dump_free_monitors(struct hl_mon_state_dump *monitors)
495{
496 vfree(monitors);
497}
498
499/**
500 * hl_state_dump_print_monitors_single_block - print active monitors on a
501 * single block
502 * @hdev: pointer to the device
503 * @index: sync manager block index starting with E_N
504 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
505 * @size: pointer to the size container
506 * @offset: pointer to the offset container
507 *
508 * Returns 0 on success or error code on failure
509 */
510static int hl_state_dump_print_monitors_single_block(struct hl_device *hdev,
511 u32 index,
512 char **buf, size_t *size,
513 size_t *offset)
514{
515 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
516 struct hl_mon_state_dump *monitors = NULL;
517 int rc = 0, i;
518
519 if (sds->sync_namager_names) {
520 rc = hl_snprintf_resize(
521 buf, size, offset, "%s\n",
522 sds->sync_namager_names[index]);
523 if (rc)
524 goto out;
525 }
526
527 monitors = hl_state_dump_alloc_read_sm_block_monitors(hdev, index);
528 if (!monitors) {
529 rc = -ENOMEM;
530 goto out;
531 }
532
533 for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {
534 if (!(sds->funcs.monitor_valid(&monitors[i])))
535 continue;
536
537 /* Monitor is valid, dump it */
538 rc = sds->funcs.print_single_monitor(buf, size, offset, hdev,
539 &monitors[i]);
540 if (rc)
541 goto free_monitors;
542
543 hl_snprintf_resize(buf, size, offset, "\n");
544 }
545
546free_monitors:
547 hl_state_dump_free_monitors(monitors);
548out:
549 return rc;
550}
551
552/**
553 * hl_state_dump_print_monitors - print active monitors
554 * @hdev: pointer to the device
555 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
556 * @size: pointer to the size container
557 * @offset: pointer to the offset container
558 *
559 * Returns 0 on success or error code on failure
560 */
561static int hl_state_dump_print_monitors(struct hl_device *hdev,
562 char **buf, size_t *size,
563 size_t *offset)
564{
565 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
566 u32 index;
567 int rc = 0;
568
569 rc = hl_snprintf_resize(buf, size, offset,
570 "Valid (armed) monitor objects:\n");
571 if (rc)
572 goto out;
573
574 if (sds->sync_namager_names) {
575 for (index = 0; sds->sync_namager_names[index]; ++index) {
576 rc = hl_state_dump_print_monitors_single_block(
577 hdev, index, buf, size, offset);
578 if (rc)
579 goto out;
580 }
581 } else {
582 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {
583 rc = hl_state_dump_print_monitors_single_block(
584 hdev, index, buf, size, offset);
585 if (rc)
586 goto out;
587 }
588 }
589
590out:
591 return rc;
592}
593
594/**
595 * hl_state_dump_print_engine_fences - print active fences for a specific
596 * engine
597 * @hdev: pointer to the device
598 * @engine_type: engine type to use
599 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
600 * @size: pointer to the size container
601 * @offset: pointer to the offset container
602 */
603static int
604hl_state_dump_print_engine_fences(struct hl_device *hdev,
605 enum hl_sync_engine_type engine_type,
606 char **buf, size_t *size, size_t *offset)
607{
608 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
609 int rc = 0, i, n_fences;
610 u64 base_addr, next_fence;
611
612 switch (engine_type) {
613 case ENGINE_TPC:
614 n_fences = sds->props[SP_NUM_OF_TPC_ENGINES];
615 base_addr = sds->props[SP_TPC0_CMDQ];
616 next_fence = sds->props[SP_NEXT_TPC];
617 break;
618 case ENGINE_MME:
619 n_fences = sds->props[SP_NUM_OF_MME_ENGINES];
620 base_addr = sds->props[SP_MME_CMDQ];
621 next_fence = sds->props[SP_NEXT_MME];
622 break;
623 case ENGINE_DMA:
624 n_fences = sds->props[SP_NUM_OF_DMA_ENGINES];
625 base_addr = sds->props[SP_DMA_CMDQ];
626 next_fence = sds->props[SP_DMA_QUEUES_OFFSET];
627 break;
628 default:
629 return -EINVAL;
630 }
631 for (i = 0; i < n_fences; ++i) {
632 rc = sds->funcs.print_fences_single_engine(
633 hdev,
634 base_addr + next_fence * i +
635 sds->props[SP_FENCE0_CNT_OFFSET],
636 base_addr + next_fence * i +
637 sds->props[SP_CP_STS_OFFSET],
638 engine_type, i, buf, size, offset);
639 if (rc)
640 goto out;
641 }
642out:
643 return rc;
644}
645
646/**
647 * hl_state_dump_print_fences - print active fences
648 * @hdev: pointer to the device
649 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
650 * @size: pointer to the size container
651 * @offset: pointer to the offset container
652 */
653static int hl_state_dump_print_fences(struct hl_device *hdev, char **buf,
654 size_t *size, size_t *offset)
655{
656 int rc = 0;
657
658 rc = hl_snprintf_resize(buf, size, offset, "Valid (armed) fences:\n");
659 if (rc)
660 goto out;
661
662 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_TPC, buf, size, offset);
663 if (rc)
664 goto out;
665
666 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_MME, buf, size, offset);
667 if (rc)
668 goto out;
669
670 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_DMA, buf, size, offset);
671 if (rc)
672 goto out;
673
674out:
675 return rc;
676}
677
678/**
679 * hl_state_dump() - dump system state
680 * @hdev: pointer to device structure
681 */
682int hl_state_dump(struct hl_device *hdev)
683{
684 char *buf = NULL;
685 size_t offset = 0, size = 0;
686 int rc;
687
688 rc = hl_snprintf_resize(&buf, &size, &offset,
689 "Timestamp taken on: %llu\n\n",
690 ktime_to_ns(ktime_get()));
691 if (rc)
692 goto err;
693
694 rc = hl_state_dump_print_syncs(hdev, &buf, &size, &offset);
695 if (rc)
696 goto err;
697
698 hl_snprintf_resize(&buf, &size, &offset, "\n");
699
700 rc = hl_state_dump_print_monitors(hdev, &buf, &size, &offset);
701 if (rc)
702 goto err;
703
704 hl_snprintf_resize(&buf, &size, &offset, "\n");
705
706 rc = hl_state_dump_print_fences(hdev, &buf, &size, &offset);
707 if (rc)
708 goto err;
709
710 hl_snprintf_resize(&buf, &size, &offset, "\n");
711
712 hl_debugfs_set_state_dump(hdev, buf, size);
713
714 return 0;
715err:
716 vfree(buf);
717 return rc;
718}
drivers/misc/habanalabs/common/sysfs.c+7−13
@@ -9,8 +9,7 @@ · 8 unmodified lines
9 9
10#include <linux/pci.h>10#include <linux/pci.h>
11 11
12long hl_get_frequency(struct hl_device *hdev, u32 pll_index,12long hl_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr)
13 bool curr)
14{13{
15 struct cpucp_packet pkt;14 struct cpucp_packet pkt;
16 u32 used_pll_idx;15 u32 used_pll_idx;
@@ -44,8 +43,7 @@ long hl_get_frequency(struct hl_device *hdev, u32 pll_index, · 27 unmodified lines
44 return (long) result;43 return (long) result;
45}44}
46 45
47void hl_set_frequency(struct hl_device *hdev, u32 pll_index,46void hl_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq)
48 u64 freq)
49{47{
50 struct cpucp_packet pkt;48 struct cpucp_packet pkt;
51 u32 used_pll_idx;49 u32 used_pll_idx;
@@ -285,16 +283,12 @@ static ssize_t status_show(struct device *dev, struct device_attribute *attr, · 233 unmodified lines
285 char *buf)283 char *buf)
286{284{
287 struct hl_device *hdev = dev_get_drvdata(dev);285 struct hl_device *hdev = dev_get_drvdata(dev);
288 char *str;286 char str[HL_STR_MAX];
289 287
290 if (atomic_read(&hdev->in_reset))288 strscpy(str, hdev->status[hl_device_status(hdev)], HL_STR_MAX);
291 str = "In reset";289
292 else if (hdev->disabled)290 /* use uppercase for backward compatibility */
293 str = "Malfunction";291 str[0] = 'A' + (str[0] - 'a');
294 else if (hdev->needs_reset)
295 str = "Needs Reset";
296 else
297 str = "Operational";
298 292
299 return sprintf(buf, "%s\n", str);293 return sprintf(buf, "%s\n", str);
300}294}
drivers/misc/habanalabs/gaudi/gaudi.c+571−145
@@ -76,7 +76,7 @@ · 75 unmodified lines
76#define GAUDI_PLDM_MMU_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 100)76#define GAUDI_PLDM_MMU_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 100)
77#define GAUDI_PLDM_QMAN0_TIMEOUT_USEC (HL_DEVICE_TIMEOUT_USEC * 30)77#define GAUDI_PLDM_QMAN0_TIMEOUT_USEC (HL_DEVICE_TIMEOUT_USEC * 30)
78#define GAUDI_PLDM_TPC_KERNEL_WAIT_USEC (HL_DEVICE_TIMEOUT_USEC * 30)78#define GAUDI_PLDM_TPC_KERNEL_WAIT_USEC (HL_DEVICE_TIMEOUT_USEC * 30)
79#define GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC 1000000 /* 1s */79#define GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC 4000000 /* 4s */
80#define GAUDI_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */80#define GAUDI_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */
81#define GAUDI_WAIT_FOR_BL_TIMEOUT_USEC 15000000 /* 15s */81#define GAUDI_WAIT_FOR_BL_TIMEOUT_USEC 15000000 /* 15s */
82 82
@@ -106,6 +106,21 @@ · 23 unmodified lines
106 106
107#define GAUDI_PLL_MAX 10107#define GAUDI_PLL_MAX 10
108 108
109#define BIN_REG_STRING_SIZE sizeof("0b10101010101010101010101010101010")
110
111#define MONITOR_SOB_STRING_SIZE 256
112
113static u32 gaudi_stream_master[GAUDI_STREAM_MASTER_ARR_SIZE] = {
114 GAUDI_QUEUE_ID_DMA_0_0,
115 GAUDI_QUEUE_ID_DMA_0_1,
116 GAUDI_QUEUE_ID_DMA_0_2,
117 GAUDI_QUEUE_ID_DMA_0_3,
118 GAUDI_QUEUE_ID_DMA_1_0,
119 GAUDI_QUEUE_ID_DMA_1_1,
120 GAUDI_QUEUE_ID_DMA_1_2,
121 GAUDI_QUEUE_ID_DMA_1_3
122};
123
109static const char gaudi_irq_name[GAUDI_MSI_ENTRIES][GAUDI_MAX_STRING_LEN] = {124static const char gaudi_irq_name[GAUDI_MSI_ENTRIES][GAUDI_MAX_STRING_LEN] = {
110 "gaudi cq 0_0", "gaudi cq 0_1", "gaudi cq 0_2", "gaudi cq 0_3",125 "gaudi cq 0_0", "gaudi cq 0_1", "gaudi cq 0_2", "gaudi cq 0_3",
111 "gaudi cq 1_0", "gaudi cq 1_1", "gaudi cq 1_2", "gaudi cq 1_3",126 "gaudi cq 1_0", "gaudi cq 1_1", "gaudi cq 1_2", "gaudi cq 1_3",
@@ -348,6 +363,97 @@ static enum hl_queue_type gaudi_queue_type[GAUDI_QUEUE_ID_SIZE] = { · 236 unmodified lines
348 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_3 */363 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_3 */
349};364};
350 365
366static struct hl_hw_obj_name_entry gaudi_so_id_to_str[] = {
367 { .id = 0, .name = "SYNC_OBJ_DMA_DOWN_FEEDBACK" },
368 { .id = 1, .name = "SYNC_OBJ_DMA_UP_FEEDBACK" },
369 { .id = 2, .name = "SYNC_OBJ_DMA_STATIC_DRAM_SRAM_FEEDBACK" },
370 { .id = 3, .name = "SYNC_OBJ_DMA_SRAM_DRAM_FEEDBACK" },
371 { .id = 4, .name = "SYNC_OBJ_FIRST_COMPUTE_FINISH" },
372 { .id = 5, .name = "SYNC_OBJ_HOST_DRAM_DONE" },
373 { .id = 6, .name = "SYNC_OBJ_DBG_CTR_DEPRECATED" },
374 { .id = 7, .name = "SYNC_OBJ_DMA_ACTIVATIONS_DRAM_SRAM_FEEDBACK" },
375 { .id = 8, .name = "SYNC_OBJ_ENGINE_SEM_MME_0" },
376 { .id = 9, .name = "SYNC_OBJ_ENGINE_SEM_MME_1" },
377 { .id = 10, .name = "SYNC_OBJ_ENGINE_SEM_TPC_0" },
378 { .id = 11, .name = "SYNC_OBJ_ENGINE_SEM_TPC_1" },
379 { .id = 12, .name = "SYNC_OBJ_ENGINE_SEM_TPC_2" },
380 { .id = 13, .name = "SYNC_OBJ_ENGINE_SEM_TPC_3" },
381 { .id = 14, .name = "SYNC_OBJ_ENGINE_SEM_TPC_4" },
382 { .id = 15, .name = "SYNC_OBJ_ENGINE_SEM_TPC_5" },
383 { .id = 16, .name = "SYNC_OBJ_ENGINE_SEM_TPC_6" },
384 { .id = 17, .name = "SYNC_OBJ_ENGINE_SEM_TPC_7" },
385 { .id = 18, .name = "SYNC_OBJ_ENGINE_SEM_DMA_1" },
386 { .id = 19, .name = "SYNC_OBJ_ENGINE_SEM_DMA_2" },
387 { .id = 20, .name = "SYNC_OBJ_ENGINE_SEM_DMA_3" },
388 { .id = 21, .name = "SYNC_OBJ_ENGINE_SEM_DMA_4" },
389 { .id = 22, .name = "SYNC_OBJ_ENGINE_SEM_DMA_5" },
390 { .id = 23, .name = "SYNC_OBJ_ENGINE_SEM_DMA_6" },
391 { .id = 24, .name = "SYNC_OBJ_ENGINE_SEM_DMA_7" },
392 { .id = 25, .name = "SYNC_OBJ_DBG_CTR_0" },
393 { .id = 26, .name = "SYNC_OBJ_DBG_CTR_1" },
394};
395
396static struct hl_hw_obj_name_entry gaudi_monitor_id_to_str[] = {
397 { .id = 200, .name = "MON_OBJ_DMA_DOWN_FEEDBACK_RESET" },
398 { .id = 201, .name = "MON_OBJ_DMA_UP_FEADBACK_RESET" },
399 { .id = 203, .name = "MON_OBJ_DRAM_TO_SRAM_QUEUE_FENCE" },
400 { .id = 204, .name = "MON_OBJ_TPC_0_CLK_GATE" },
401 { .id = 205, .name = "MON_OBJ_TPC_1_CLK_GATE" },
402 { .id = 206, .name = "MON_OBJ_TPC_2_CLK_GATE" },
403 { .id = 207, .name = "MON_OBJ_TPC_3_CLK_GATE" },
404 { .id = 208, .name = "MON_OBJ_TPC_4_CLK_GATE" },
405 { .id = 209, .name = "MON_OBJ_TPC_5_CLK_GATE" },
406 { .id = 210, .name = "MON_OBJ_TPC_6_CLK_GATE" },
407 { .id = 211, .name = "MON_OBJ_TPC_7_CLK_GATE" },
408};
409
410static s64 gaudi_state_dump_specs_props[] = {
411 [SP_SYNC_OBJ_BASE_ADDR] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0,
412 [SP_NEXT_SYNC_OBJ_ADDR] = NEXT_SYNC_OBJ_ADDR_INTERVAL,
413 [SP_SYNC_OBJ_AMOUNT] = NUM_OF_SOB_IN_BLOCK,
414 [SP_MON_OBJ_WR_ADDR_LOW] =
415 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0,
416 [SP_MON_OBJ_WR_ADDR_HIGH] =
417 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0,
418 [SP_MON_OBJ_WR_DATA] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_DATA_0,
419 [SP_MON_OBJ_ARM_DATA] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_ARM_0,
420 [SP_MON_OBJ_STATUS] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0,
421 [SP_MONITORS_AMOUNT] = NUM_OF_MONITORS_IN_BLOCK,
422 [SP_TPC0_CMDQ] = mmTPC0_QM_GLBL_CFG0,
423 [SP_TPC0_CFG_SO] = mmTPC0_CFG_QM_SYNC_OBJECT_ADDR,
424 [SP_NEXT_TPC] = mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0,
425 [SP_MME_CMDQ] = mmMME0_QM_GLBL_CFG0,
426 [SP_MME_CFG_SO] = mmMME0_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL,
427 [SP_NEXT_MME] = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0,
428 [SP_DMA_CMDQ] = mmDMA0_QM_GLBL_CFG0,
429 [SP_DMA_CFG_SO] = mmDMA0_CORE_WR_COMP_ADDR_LO,
430 [SP_DMA_QUEUES_OFFSET] = mmDMA1_QM_GLBL_CFG0 - mmDMA0_QM_GLBL_CFG0,
431 [SP_NUM_OF_MME_ENGINES] = NUM_OF_MME_ENGINES,
432 [SP_SUB_MME_ENG_NUM] = NUM_OF_MME_SUB_ENGINES,
433 [SP_NUM_OF_DMA_ENGINES] = NUM_OF_DMA_ENGINES,
434 [SP_NUM_OF_TPC_ENGINES] = NUM_OF_TPC_ENGINES,
435 [SP_ENGINE_NUM_OF_QUEUES] = NUM_OF_QUEUES,
436 [SP_ENGINE_NUM_OF_STREAMS] = NUM_OF_STREAMS,
437 [SP_ENGINE_NUM_OF_FENCES] = NUM_OF_FENCES,
438 [SP_FENCE0_CNT_OFFSET] =
439 mmDMA0_QM_CP_FENCE0_CNT_0 - mmDMA0_QM_GLBL_CFG0,
440 [SP_FENCE0_RDATA_OFFSET] =
441 mmDMA0_QM_CP_FENCE0_RDATA_0 - mmDMA0_QM_GLBL_CFG0,
442 [SP_CP_STS_OFFSET] = mmDMA0_QM_CP_STS_0 - mmDMA0_QM_GLBL_CFG0,
443 [SP_NUM_CORES] = 1,
444};
445
446/* The order here is opposite to the order of the indexing in the h/w.
447 * i.e. SYNC_MGR_W_S is actually 0, SYNC_MGR_E_S is 1, etc.
448 */
449static const char * const gaudi_sync_manager_names[] = {
450 "SYNC_MGR_E_N",
451 "SYNC_MGR_W_N",
452 "SYNC_MGR_E_S",
453 "SYNC_MGR_W_S",
454 NULL
455};
456
351struct ecc_info_extract_params {457struct ecc_info_extract_params {
352 u64 block_address;458 u64 block_address;
353 u32 num_memories;459 u32 num_memories;
@@ -363,8 +469,6 @@ static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr, · 9 unmodified lines
363 u32 size, u64 val);469 u32 size, u64 val);
364static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base,470static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base,
365 u32 num_regs, u32 val);471 u32 num_regs, u32 val);
366static int gaudi_schedule_register_memset(struct hl_device *hdev,
367 u32 hw_queue_id, u64 reg_base, u32 num_regs, u32 val);
368static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel,472static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel,
369 u32 tpc_id);473 u32 tpc_id);
370static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev);474static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev);
@@ -375,7 +479,6 @@ static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, · 4 unmodified lines
375 u32 size, bool eb);479 u32 size, bool eb);
376static u32 gaudi_gen_wait_cb(struct hl_device *hdev,480static u32 gaudi_gen_wait_cb(struct hl_device *hdev,
377 struct hl_gen_wait_properties *prop);481 struct hl_gen_wait_properties *prop);
378
379static inline enum hl_collective_mode482static inline enum hl_collective_mode
380get_collective_mode(struct hl_device *hdev, u32 queue_id)483get_collective_mode(struct hl_device *hdev, u32 queue_id)
381{484{
@@ -403,7 +506,11 @@ static inline void set_default_power_values(struct hl_device *hdev) · 21 unmodified lines
403 506
404 if (hdev->card_type == cpucp_card_type_pmc) {507 if (hdev->card_type == cpucp_card_type_pmc) {
405 prop->max_power_default = MAX_POWER_DEFAULT_PMC;508 prop->max_power_default = MAX_POWER_DEFAULT_PMC;
406 prop->dc_power_default = DC_POWER_DEFAULT_PMC;509
510 if (prop->fw_security_enabled)
511 prop->dc_power_default = DC_POWER_DEFAULT_PMC_SEC;
512 else
513 prop->dc_power_default = DC_POWER_DEFAULT_PMC;
407 } else {514 } else {
408 prop->max_power_default = MAX_POWER_DEFAULT_PCI;515 prop->max_power_default = MAX_POWER_DEFAULT_PCI;
409 prop->dc_power_default = DC_POWER_DEFAULT_PCI;516 prop->dc_power_default = DC_POWER_DEFAULT_PCI;
@@ -450,6 +557,7 @@ static int gaudi_set_fixed_properties(struct hl_device *hdev) · 40 unmodified lines
450 get_collective_mode(hdev, i);557 get_collective_mode(hdev, i);
451 }558 }
452 559
560 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE;
453 prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;561 prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;
454 prop->collective_first_sob = 0;562 prop->collective_first_sob = 0;
455 prop->collective_first_mon = 0;563 prop->collective_first_mon = 0;
@@ -551,6 +659,8 @@ static int gaudi_set_fixed_properties(struct hl_device *hdev) · 95 unmodified lines
551 prop->hard_reset_done_by_fw = false;659 prop->hard_reset_done_by_fw = false;
552 prop->gic_interrupts_enable = true;660 prop->gic_interrupts_enable = true;
553 661
662 prop->server_type = HL_SERVER_TYPE_UNKNOWN;
663
554 return 0;664 return 0;
555}665}
556 666
@@ -723,14 +833,14 @@ pci_init: · 166 unmodified lines
723 GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC);833 GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC);
724 if (rc) {834 if (rc) {
725 if (hdev->reset_on_preboot_fail)835 if (hdev->reset_on_preboot_fail)
726 hdev->asic_funcs->hw_fini(hdev, true);836 hdev->asic_funcs->hw_fini(hdev, true, false);
727 goto pci_fini;837 goto pci_fini;
728 }838 }
729 839
730 if (gaudi_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {840 if (gaudi_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
731 dev_info(hdev->dev,841 dev_info(hdev->dev,
732 "H/W state is dirty, must reset before initializing\n");842 "H/W state is dirty, must reset before initializing\n");
733 hdev->asic_funcs->hw_fini(hdev, true);843 hdev->asic_funcs->hw_fini(hdev, true, false);
734 }844 }
735 845
736 return 0;846 return 0;
@@ -974,17 +1084,11 @@ static void gaudi_sob_group_hw_reset(struct kref *ref) · 237 unmodified lines
974 struct gaudi_hw_sob_group *hw_sob_group =1084 struct gaudi_hw_sob_group *hw_sob_group =
975 container_of(ref, struct gaudi_hw_sob_group, kref);1085 container_of(ref, struct gaudi_hw_sob_group, kref);
976 struct hl_device *hdev = hw_sob_group->hdev;1086 struct hl_device *hdev = hw_sob_group->hdev;
977 u64 base_addr;1087 int i;
978 int rc;
979 1088
980 base_addr = CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 +1089 for (i = 0 ; i < NUMBER_OF_SOBS_IN_GRP ; i++)
981 hw_sob_group->base_sob_id * 4;1090 WREG32((mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 +
982 rc = gaudi_schedule_register_memset(hdev, hw_sob_group->queue_id,1091 (hw_sob_group->base_sob_id * 4) + (i * 4)), 0);
983 base_addr, NUMBER_OF_SOBS_IN_GRP, 0);
984 if (rc)
985 dev_err(hdev->dev,
986 "failed resetting sob group - sob base %u, count %u",
987 hw_sob_group->base_sob_id, NUMBER_OF_SOBS_IN_GRP);
988 1092
989 kref_init(&hw_sob_group->kref);1093 kref_init(&hw_sob_group->kref);
990}1094}
@@ -1121,6 +1225,20 @@ static void gaudi_collective_slave_init_job(struct hl_device *hdev, · 130 unmodified lines
1121 queue_id = job->hw_queue_id;1225 queue_id = job->hw_queue_id;
1122 prop = &hdev->kernel_queues[queue_id].sync_stream_prop;1226 prop = &hdev->kernel_queues[queue_id].sync_stream_prop;
1123 1227
1228 if (job->cs->encaps_signals) {
1229 /* use the encaps signal handle store earlier in the flow
1230 * and set the SOB information from the encaps
1231 * signals handle
1232 */
1233 hl_hw_queue_encaps_sig_set_sob_info(hdev, job->cs, job,
1234 cs_cmpl);
1235
1236 dev_dbg(hdev->dev, "collective wait: Sequence %llu found, sob_id: %u, wait for sob_val: %u\n",
1237 job->cs->sequence,
1238 cs_cmpl->hw_sob->sob_id,
1239 cs_cmpl->sob_val);
1240 }
1241
1124 /* Add to wait CBs using slave monitor */1242 /* Add to wait CBs using slave monitor */
1125 wait_prop.data = (void *) job->user_cb;1243 wait_prop.data = (void *) job->user_cb;
1126 wait_prop.sob_base = cs_cmpl->hw_sob->sob_id;1244 wait_prop.sob_base = cs_cmpl->hw_sob->sob_id;
@@ -1131,7 +1249,7 @@ static void gaudi_collective_slave_init_job(struct hl_device *hdev, · 4 unmodified lines
1131 wait_prop.size = cb_size;1249 wait_prop.size = cb_size;
1132 1250
1133 dev_dbg(hdev->dev,1251 dev_dbg(hdev->dev,
1134 "Generate slave wait CB, sob %d, val:0x%x, mon %d, q %d\n",1252 "Generate slave wait CB, sob %d, val:%x, mon %d, q %d\n",
1135 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val,1253 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val,
1136 prop->collective_slave_mon_id, queue_id);1254 prop->collective_slave_mon_id, queue_id);
1137 1255
@@ -1145,7 +1263,7 @@ static void gaudi_collective_slave_init_job(struct hl_device *hdev, · 7 unmodified lines
1145 prop->collective_sob_id, cb_size, false);1263 prop->collective_sob_id, cb_size, false);
1146}1264}
1147 1265
1148static void gaudi_collective_wait_init_cs(struct hl_cs *cs)1266static int gaudi_collective_wait_init_cs(struct hl_cs *cs)
1149{1267{
1150 struct hl_cs_compl *signal_cs_cmpl =1268 struct hl_cs_compl *signal_cs_cmpl =
1151 container_of(cs->signal_fence, struct hl_cs_compl, base_fence);1269 container_of(cs->signal_fence, struct hl_cs_compl, base_fence);
@@ -1163,9 +1281,37 @@ static void gaudi_collective_wait_init_cs(struct hl_cs *cs) · 11 unmodified lines
1163 gaudi = hdev->asic_specific;1281 gaudi = hdev->asic_specific;
1164 cprop = &gaudi->collective_props;1282 cprop = &gaudi->collective_props;
1165 1283
1166 /* copy the SOB id and value of the signal CS */1284 /* In encaps signals case the SOB info will be retrieved from
1167 cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;1285 * the handle in gaudi_collective_slave_init_job.
1168 cs_cmpl->sob_val = signal_cs_cmpl->sob_val;1286 */
1287 if (!cs->encaps_signals) {
1288 /* copy the SOB id and value of the signal CS */
1289 cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;
1290 cs_cmpl->sob_val = signal_cs_cmpl->sob_val;
1291 }
1292
1293 /* check again if the signal cs already completed.
1294 * if yes then don't send any wait cs since the hw_sob
1295 * could be in reset already. if signal is not completed
1296 * then get refcount to hw_sob to prevent resetting the sob
1297 * while wait cs is not submitted.
1298 * note that this check is protected by two locks,
1299 * hw queue lock and completion object lock,
1300 * and the same completion object lock also protects
1301 * the hw_sob reset handler function.
1302 * The hw_queue lock prevent out of sync of hw_sob
1303 * refcount value, changed by signal/wait flows.
1304 */
1305 spin_lock(&signal_cs_cmpl->lock);
1306
1307 if (completion_done(&cs->signal_fence->completion)) {
1308 spin_unlock(&signal_cs_cmpl->lock);
1309 return -EINVAL;
1310 }
1311 /* Increment kref since all slave queues are now waiting on it */
1312 kref_get(&cs_cmpl->hw_sob->kref);
1313
1314 spin_unlock(&signal_cs_cmpl->lock);
1169 1315
1170 /* Calculate the stream from collective master queue (1st job) */1316 /* Calculate the stream from collective master queue (1st job) */
1171 job = list_first_entry(&cs->job_list, struct hl_cs_job, cs_node);1317 job = list_first_entry(&cs->job_list, struct hl_cs_job, cs_node);
@@ -1210,21 +1356,17 @@ static void gaudi_collective_wait_init_cs(struct hl_cs *cs) · 38 unmodified lines
1210 cprop->curr_sob_group_idx[stream], stream);1356 cprop->curr_sob_group_idx[stream], stream);
1211 }1357 }
1212 1358
1213 /* Increment kref since all slave queues are now waiting on it */
1214 kref_get(&cs_cmpl->hw_sob->kref);
1215 /*
1216 * Must put the signal fence after the SOB refcnt increment so
1217 * the SOB refcnt won't turn 0 and reset the SOB before the
1218 * wait CS was submitted.
1219 */
1220 mb();1359 mb();
1221 hl_fence_put(cs->signal_fence);1360 hl_fence_put(cs->signal_fence);
1222 cs->signal_fence = NULL;1361 cs->signal_fence = NULL;
1362
1363 return 0;
1223}1364}
1224 1365
1225static int gaudi_collective_wait_create_job(struct hl_device *hdev,1366static int gaudi_collective_wait_create_job(struct hl_device *hdev,
1226 struct hl_ctx *ctx, struct hl_cs *cs,1367 struct hl_ctx *ctx, struct hl_cs *cs,
1227 enum hl_collective_mode mode, u32 queue_id, u32 wait_queue_id)1368 enum hl_collective_mode mode, u32 queue_id, u32 wait_queue_id,
1369 u32 encaps_signal_offset)
1228{1370{
1229 struct hw_queue_properties *hw_queue_prop;1371 struct hw_queue_properties *hw_queue_prop;
1230 struct hl_cs_counters_atomic *cntr;1372 struct hl_cs_counters_atomic *cntr;
@@ -1284,6 +1426,13 @@ static int gaudi_collective_wait_create_job(struct hl_device *hdev, · 53 unmodified lines
1284 job->user_cb_size = cb_size;1426 job->user_cb_size = cb_size;
1285 job->hw_queue_id = queue_id;1427 job->hw_queue_id = queue_id;
1286 1428
1429 /* since its guaranteed to have only one chunk in the collective wait
1430 * cs, we can use this chunk to set the encapsulated signal offset
1431 * in the jobs.
1432 */
1433 if (cs->encaps_signals)
1434 job->encaps_sig_wait_offset = encaps_signal_offset;
1435
1287 /*1436 /*
1288 * No need in parsing, user CB is the patched CB.1437 * No need in parsing, user CB is the patched CB.
1289 * We call hl_cb_destroy() out of two reasons - we don't need1438 * We call hl_cb_destroy() out of two reasons - we don't need
@@ -1312,8 +1461,9 @@ static int gaudi_collective_wait_create_job(struct hl_device *hdev, · 22 unmodified lines
1312}1461}
1313 1462
1314static int gaudi_collective_wait_create_jobs(struct hl_device *hdev,1463static int gaudi_collective_wait_create_jobs(struct hl_device *hdev,
1315 struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id,1464 struct hl_ctx *ctx, struct hl_cs *cs,
1316 u32 collective_engine_id)1465 u32 wait_queue_id, u32 collective_engine_id,
1466 u32 encaps_signal_offset)
1317{1467{
1318 struct gaudi_device *gaudi = hdev->asic_specific;1468 struct gaudi_device *gaudi = hdev->asic_specific;
1319 struct hw_queue_properties *hw_queue_prop;1469 struct hw_queue_properties *hw_queue_prop;
@@ -1363,7 +1513,8 @@ static int gaudi_collective_wait_create_jobs(struct hl_device *hdev, · 43 unmodified lines
1363 if (i == 0) {1513 if (i == 0) {
1364 queue_id = wait_queue_id;1514 queue_id = wait_queue_id;
1365 rc = gaudi_collective_wait_create_job(hdev, ctx, cs,1515 rc = gaudi_collective_wait_create_job(hdev, ctx, cs,
1366 HL_COLLECTIVE_MASTER, queue_id, wait_queue_id);1516 HL_COLLECTIVE_MASTER, queue_id,
1517 wait_queue_id, encaps_signal_offset);
1367 } else {1518 } else {
1368 if (nic_idx < NIC_NUMBER_OF_ENGINES) {1519 if (nic_idx < NIC_NUMBER_OF_ENGINES) {
1369 if (gaudi->hw_cap_initialized &1520 if (gaudi->hw_cap_initialized &
@@ -1383,7 +1534,8 @@ static int gaudi_collective_wait_create_jobs(struct hl_device *hdev, · 13 unmodified lines
1383 }1534 }
1384 1535
1385 rc = gaudi_collective_wait_create_job(hdev, ctx, cs,1536 rc = gaudi_collective_wait_create_job(hdev, ctx, cs,
1386 HL_COLLECTIVE_SLAVE, queue_id, wait_queue_id);1537 HL_COLLECTIVE_SLAVE, queue_id,
1538 wait_queue_id, encaps_signal_offset);
1387 }1539 }
1388 1540
1389 if (rc)1541 if (rc)
@@ -1431,6 +1583,11 @@ static int gaudi_late_init(struct hl_device *hdev) · 41 unmodified lines
1431 return rc;1583 return rc;
1432 }1584 }
1433 1585
1586 /* Scrub both SRAM and DRAM */
1587 rc = hdev->asic_funcs->scrub_device_mem(hdev, 0, 0);
1588 if (rc)
1589 goto disable_pci_access;
1590
1434 rc = gaudi_fetch_psoc_frequency(hdev);1591 rc = gaudi_fetch_psoc_frequency(hdev);
1435 if (rc) {1592 if (rc) {
1436 dev_err(hdev->dev, "Failed to fetch psoc frequency\n");1593 dev_err(hdev->dev, "Failed to fetch psoc frequency\n");
@@ -1455,6 +1612,11 @@ static int gaudi_late_init(struct hl_device *hdev) · 18 unmodified lines
1455 goto disable_pci_access;1612 goto disable_pci_access;
1456 }1613 }
1457 1614
1615 /* We only support a single ASID for the user, so for the sake of optimization, just
1616 * initialize the ASID one time during device initialization with the fixed value of 1
1617 */
1618 gaudi_mmu_prepare(hdev, 1);
1619
1458 return 0;1620 return 0;
1459 1621
1460disable_pci_access:1622disable_pci_access:
@@ -1720,8 +1882,12 @@ static int gaudi_sw_init(struct hl_device *hdev) · 259 unmodified lines
1720 hdev->supports_sync_stream = true;1882 hdev->supports_sync_stream = true;
1721 hdev->supports_coresight = true;1883 hdev->supports_coresight = true;
1722 hdev->supports_staged_submission = true;1884 hdev->supports_staged_submission = true;
1885 hdev->supports_wait_for_multi_cs = true;
1723 1886
1724 gaudi_set_pci_memory_regions(hdev);1887 hdev->asic_funcs->set_pci_memory_regions(hdev);
1888 hdev->stream_master_qid_arr =
1889 hdev->asic_funcs->get_stream_master_qid_arr();
1890 hdev->stream_master_qid_arr_size = GAUDI_STREAM_MASTER_ARR_SIZE;
1725 1891
1726 return 0;1892 return 0;
1727 1893
@@ -2523,7 +2689,7 @@ static void gaudi_init_golden_registers(struct hl_device *hdev) · 795 unmodified lines
2523 tpc_id < TPC_NUMBER_OF_ENGINES;2689 tpc_id < TPC_NUMBER_OF_ENGINES;
2524 tpc_id++, tpc_offset += TPC_CFG_OFFSET) {2690 tpc_id++, tpc_offset += TPC_CFG_OFFSET) {
2525 /* Mask all arithmetic interrupts from TPC */2691 /* Mask all arithmetic interrupts from TPC */
2526 WREG32(mmTPC0_CFG_TPC_INTR_MASK + tpc_offset, 0x8FFF);2692 WREG32(mmTPC0_CFG_TPC_INTR_MASK + tpc_offset, 0x8FFE);
2527 /* Set 16 cache lines */2693 /* Set 16 cache lines */
2528 WREG32_FIELD(TPC0_CFG_MSS_CONFIG, tpc_offset,2694 WREG32_FIELD(TPC0_CFG_MSS_CONFIG, tpc_offset,
2529 ICACHE_FETCH_LINE_NUM, 2);2695 ICACHE_FETCH_LINE_NUM, 2);
@@ -3670,7 +3836,7 @@ static void gaudi_disable_timestamp(struct hl_device *hdev) · 1140 unmodified lines
3670 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);3836 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
3671}3837}
3672 3838
3673static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset)3839static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset)
3674{3840{
3675 u32 wait_timeout_ms;3841 u32 wait_timeout_ms;
3676 3842
@@ -3682,6 +3848,9 @@ static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset) · 5 unmodified lines
3682 else3848 else
3683 wait_timeout_ms = GAUDI_RESET_WAIT_MSEC;3849 wait_timeout_ms = GAUDI_RESET_WAIT_MSEC;
3684 3850
3851 if (fw_reset)
3852 goto skip_engines;
3853
3685 gaudi_stop_nic_qmans(hdev);3854 gaudi_stop_nic_qmans(hdev);
3686 gaudi_stop_mme_qmans(hdev);3855 gaudi_stop_mme_qmans(hdev);
3687 gaudi_stop_tpc_qmans(hdev);3856 gaudi_stop_tpc_qmans(hdev);
@@ -3707,6 +3876,7 @@ static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset) · 19 unmodified lines
3707 3876
3708 gaudi_disable_timestamp(hdev);3877 gaudi_disable_timestamp(hdev);
3709 3878
3879skip_engines:
3710 gaudi_disable_msi(hdev);3880 gaudi_disable_msi(hdev);
3711}3881}
3712 3882
@@ -3739,6 +3909,9 @@ static int gaudi_mmu_init(struct hl_device *hdev) · 26 unmodified lines
3739 WREG32(mmSTLB_CACHE_INV_BASE_39_8, MMU_CACHE_MNG_ADDR >> 8);3909 WREG32(mmSTLB_CACHE_INV_BASE_39_8, MMU_CACHE_MNG_ADDR >> 8);
3740 WREG32(mmSTLB_CACHE_INV_BASE_49_40, MMU_CACHE_MNG_ADDR >> 40);3910 WREG32(mmSTLB_CACHE_INV_BASE_49_40, MMU_CACHE_MNG_ADDR >> 40);
3741 3911
3912 /* mem cache invalidation */
3913 WREG32(mmSTLB_MEM_CACHE_INVALIDATION, 1);
3914
3742 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, 0);3915 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, 0);
3743 3916
3744 WREG32(mmMMU_UP_MMU_ENABLE, 1);3917 WREG32(mmMMU_UP_MMU_ENABLE, 1);
@@ -4071,7 +4244,7 @@ disable_queues: · 326 unmodified lines
4071 return rc;4244 return rc;
4072}4245}
4073 4246
4074static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset)4247static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
4075{4248{
4076 struct cpu_dyn_regs *dyn_regs =4249 struct cpu_dyn_regs *dyn_regs =
4077 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;4250 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
@@ -4092,6 +4265,14 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset) · 14 unmodified lines
4092 cpu_timeout_ms = GAUDI_CPU_RESET_WAIT_MSEC;4265 cpu_timeout_ms = GAUDI_CPU_RESET_WAIT_MSEC;
4093 }4266 }
4094 4267
4268 if (fw_reset) {
4269 dev_info(hdev->dev,
4270 "Firmware performs HARD reset, going to wait %dms\n",
4271 reset_timeout_ms);
4272
4273 goto skip_reset;
4274 }
4275
4095 driver_performs_reset = !!(!hdev->asic_prop.fw_security_enabled &&4276 driver_performs_reset = !!(!hdev->asic_prop.fw_security_enabled &&
4096 !hdev->asic_prop.hard_reset_done_by_fw);4277 !hdev->asic_prop.hard_reset_done_by_fw);
4097 4278
@@ -4168,6 +4349,7 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset) · 70 unmodified lines
4168 reset_timeout_ms);4349 reset_timeout_ms);
4169 }4350 }
4170 4351
4352skip_reset:
4171 /*4353 /*
4172 * After hard reset, we can't poll the BTM_FSM register because the PSOC4354 * After hard reset, we can't poll the BTM_FSM register because the PSOC
4173 * itself is in reset. Need to wait until the reset is deasserted4355 * itself is in reset. Need to wait until the reset is deasserted
@@ -4212,7 +4394,7 @@ static int gaudi_resume(struct hl_device *hdev) · 38 unmodified lines
4212 return gaudi_init_iatu(hdev);4394 return gaudi_init_iatu(hdev);
4213}4395}
4214 4396
4215static int gaudi_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma,4397static int gaudi_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
4216 void *cpu_addr, dma_addr_t dma_addr, size_t size)4398 void *cpu_addr, dma_addr_t dma_addr, size_t size)
4217{4399{
4218 int rc;4400 int rc;
@@ -4621,8 +4803,8 @@ static int gaudi_hbm_scrubbing(struct hl_device *hdev) · 402 unmodified lines
4621 "Doing HBM scrubbing for 0x%09llx - 0x%09llx\n",4803 "Doing HBM scrubbing for 0x%09llx - 0x%09llx\n",
4622 cur_addr, cur_addr + chunk_size);4804 cur_addr, cur_addr + chunk_size);
4623 4805
4624 WREG32(mmDMA0_CORE_SRC_BASE_LO + dma_offset, 0);4806 WREG32(mmDMA0_CORE_SRC_BASE_LO + dma_offset, 0xdeadbeaf);
4625 WREG32(mmDMA0_CORE_SRC_BASE_HI + dma_offset, 0);4807 WREG32(mmDMA0_CORE_SRC_BASE_HI + dma_offset, 0xdeadbeaf);
4626 WREG32(mmDMA0_CORE_DST_BASE_LO + dma_offset,4808 WREG32(mmDMA0_CORE_DST_BASE_LO + dma_offset,
4627 lower_32_bits(cur_addr));4809 lower_32_bits(cur_addr));
4628 WREG32(mmDMA0_CORE_DST_BASE_HI + dma_offset,4810 WREG32(mmDMA0_CORE_DST_BASE_HI + dma_offset,
@@ -5796,78 +5978,6 @@ release_cb: · 1167 unmodified lines
5796 return rc;5978 return rc;
5797}5979}
5798 5980
5799static int gaudi_schedule_register_memset(struct hl_device *hdev,
5800 u32 hw_queue_id, u64 reg_base, u32 num_regs, u32 val)
5801{
5802 struct hl_ctx *ctx;
5803 struct hl_pending_cb *pending_cb;
5804 struct packet_msg_long *pkt;
5805 u32 cb_size, ctl;
5806 struct hl_cb *cb;
5807 int i, rc;
5808
5809 mutex_lock(&hdev->fpriv_list_lock);
5810 ctx = hdev->compute_ctx;
5811
5812 /* If no compute context available or context is going down
5813 * memset registers directly
5814 */
5815 if (!ctx || kref_read(&ctx->refcount) == 0) {
5816 rc = gaudi_memset_registers(hdev, reg_base, num_regs, val);
5817 mutex_unlock(&hdev->fpriv_list_lock);
5818 return rc;
5819 }
5820
5821 mutex_unlock(&hdev->fpriv_list_lock);
5822
5823 cb_size = (sizeof(*pkt) * num_regs) +
5824 sizeof(struct packet_msg_prot) * 2;
5825
5826 if (cb_size > SZ_2M) {
5827 dev_err(hdev->dev, "CB size must be smaller than %uMB", SZ_2M);
5828 return -ENOMEM;
5829 }
5830
5831 pending_cb = kzalloc(sizeof(*pending_cb), GFP_KERNEL);
5832 if (!pending_cb)
5833 return -ENOMEM;
5834
5835 cb = hl_cb_kernel_create(hdev, cb_size, false);
5836 if (!cb) {
5837 kfree(pending_cb);
5838 return -EFAULT;
5839 }
5840
5841 pkt = cb->kernel_address;
5842
5843 ctl = FIELD_PREP(GAUDI_PKT_LONG_CTL_OP_MASK, 0); /* write the value */
5844 ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_LONG);
5845 ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1);
5846 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1);
5847 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);
5848
5849 for (i = 0; i < num_regs ; i++, pkt++) {
5850 pkt->ctl = cpu_to_le32(ctl);
5851 pkt->value = cpu_to_le32(val);
5852 pkt->addr = cpu_to_le64(reg_base + (i * 4));
5853 }
5854
5855 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
5856
5857 pending_cb->cb = cb;
5858 pending_cb->cb_size = cb_size;
5859 /* The queue ID MUST be an external queue ID. Otherwise, we will
5860 * have undefined behavior
5861 */
5862 pending_cb->hw_queue_id = hw_queue_id;
5863
5864 spin_lock(&ctx->pending_cb_lock);
5865 list_add_tail(&pending_cb->cb_node, &ctx->pending_cb_list);
5866 spin_unlock(&ctx->pending_cb_lock);
5867
5868 return 0;
5869}
5870
5871static int gaudi_restore_sm_registers(struct hl_device *hdev)5981static int gaudi_restore_sm_registers(struct hl_device *hdev)
5872{5982{
5873 u64 base_addr;5983 u64 base_addr;
@@ -6013,7 +6123,7 @@ static int gaudi_restore_user_registers(struct hl_device *hdev) · 139 unmodified lines
6013 6123
6014static int gaudi_context_switch(struct hl_device *hdev, u32 asid)6124static int gaudi_context_switch(struct hl_device *hdev, u32 asid)
6015{6125{
6016 return gaudi_restore_user_registers(hdev);6126 return 0;
6017}6127}
6018 6128
6019static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev)6129static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev)
@@ -6723,6 +6833,9 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) · 703 unmodified lines
6723 asid);6833 asid);
6724 }6834 }
6725 6835
6836 gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_ARUSER, asid);
6837 gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_AWUSER, asid);
6838
6726 hdev->asic_funcs->set_clock_gating(hdev);6839 hdev->asic_funcs->set_clock_gating(hdev);
6727 6840
6728 mutex_unlock(&gaudi->clk_gate_mutex);6841 mutex_unlock(&gaudi->clk_gate_mutex);
@@ -6772,7 +6885,8 @@ static int gaudi_send_job_on_qman0(struct hl_device *hdev, · 43 unmodified lines
6772 6885
6773 dma_offset = gaudi_dma_assignment[GAUDI_PCI_DMA_1] * DMA_CORE_OFFSET;6886 dma_offset = gaudi_dma_assignment[GAUDI_PCI_DMA_1] * DMA_CORE_OFFSET;
6774 6887
6775 WREG32_OR(mmDMA0_CORE_PROT + dma_offset, BIT(DMA0_CORE_PROT_VAL_SHIFT));6888 WREG32(mmDMA0_CORE_PROT + dma_offset,
6889 BIT(DMA0_CORE_PROT_ERR_VAL_SHIFT) | BIT(DMA0_CORE_PROT_VAL_SHIFT));
6776 6890
6777 rc = hl_hw_queue_send_cb_no_cmpl(hdev, GAUDI_QUEUE_ID_DMA_0_0,6891 rc = hl_hw_queue_send_cb_no_cmpl(hdev, GAUDI_QUEUE_ID_DMA_0_0,
6778 job->job_cb_size, cb->bus_address);6892 job->job_cb_size, cb->bus_address);
@@ -6793,8 +6907,7 @@ static int gaudi_send_job_on_qman0(struct hl_device *hdev, · 14 unmodified lines
6793 }6907 }
6794 6908
6795free_fence_ptr:6909free_fence_ptr:
6796 WREG32_AND(mmDMA0_CORE_PROT + dma_offset,6910 WREG32(mmDMA0_CORE_PROT + dma_offset, BIT(DMA0_CORE_PROT_ERR_VAL_SHIFT));
6797 ~BIT(DMA0_CORE_PROT_VAL_SHIFT));
6798 6911
6799 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,6912 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
6800 fence_dma_addr);6913 fence_dma_addr);
@@ -7168,7 +7281,7 @@ static void gaudi_print_sw_config_stream_data(struct hl_device *hdev, u32 stream · 367 unmodified lines
7168 7281
7169 cq_ptr = (((u64) RREG32(cq_ptr_hi)) << 32) | RREG32(cq_ptr_lo);7282 cq_ptr = (((u64) RREG32(cq_ptr_hi)) << 32) | RREG32(cq_ptr_lo);
7170 size = RREG32(cq_tsize);7283 size = RREG32(cq_tsize);
7171 dev_info(hdev->dev, "stop on err: stream: %u, addr: %#llx, size: %x\n",7284 dev_info(hdev->dev, "stop on err: stream: %u, addr: %#llx, size: %u\n",
7172 stream, cq_ptr, size);7285 stream, cq_ptr, size);
7173}7286}
7174 7287
@@ -7224,7 +7337,7 @@ static void gaudi_print_last_pqes_on_err(struct hl_device *hdev, u32 qid_base, · 49 unmodified lines
7224 7337
7225 addr = le64_to_cpu(bd->ptr);7338 addr = le64_to_cpu(bd->ptr);
7226 7339
7227 dev_info(hdev->dev, "stop on err PQE(stream %u): ci: %u, addr: %#llx, size: %x\n",7340 dev_info(hdev->dev, "stop on err PQE(stream %u): ci: %u, addr: %#llx, size: %u\n",
7228 stream, ci, addr, len);7341 stream, ci, addr, len);
7229 7342
7230 /* get previous ci, wrap if needed */7343 /* get previous ci, wrap if needed */
@@ -7326,24 +7439,30 @@ static void gaudi_print_sm_sei_info(struct hl_device *hdev, u16 event_type, · 95 unmodified lines
7326{7439{
7327 u32 index = event_type - GAUDI_EVENT_DMA_IF_SEI_0;7440 u32 index = event_type - GAUDI_EVENT_DMA_IF_SEI_0;
7328 7441
7442 /* Flip the bits as the enum is ordered in the opposite way */
7443 index = (index ^ 0x3) & 0x3;
7444
7329 switch (sei_data->sei_cause) {7445 switch (sei_data->sei_cause) {
7330 case SM_SEI_SO_OVERFLOW:7446 case SM_SEI_SO_OVERFLOW:
7331 dev_err(hdev->dev,7447 dev_err_ratelimited(hdev->dev,
7332 "SM %u SEI Error: SO %u overflow/underflow",7448 "%s SEI Error: SOB Group %u overflow/underflow",
7333 index, le32_to_cpu(sei_data->sei_log));7449 gaudi_sync_manager_names[index],
7450 le32_to_cpu(sei_data->sei_log));
7334 break;7451 break;
7335 case SM_SEI_LBW_4B_UNALIGNED:7452 case SM_SEI_LBW_4B_UNALIGNED:
7336 dev_err(hdev->dev,7453 dev_err_ratelimited(hdev->dev,
7337 "SM %u SEI Error: Unaligned 4B LBW access, monitor agent address low - %#x",7454 "%s SEI Error: Unaligned 4B LBW access, monitor agent address low - %#x",
7338 index, le32_to_cpu(sei_data->sei_log));7455 gaudi_sync_manager_names[index],
7456 le32_to_cpu(sei_data->sei_log));
7339 break;7457 break;
7340 case SM_SEI_AXI_RESPONSE_ERR:7458 case SM_SEI_AXI_RESPONSE_ERR:
7341 dev_err(hdev->dev,7459 dev_err_ratelimited(hdev->dev,
7342 "SM %u SEI Error: AXI ID %u response error",7460 "%s SEI Error: AXI ID %u response error",
7343 index, le32_to_cpu(sei_data->sei_log));7461 gaudi_sync_manager_names[index],
7462 le32_to_cpu(sei_data->sei_log));
7344 break;7463 break;
7345 default:7464 default:
7346 dev_err(hdev->dev, "Unknown SM SEI cause %u",7465 dev_err_ratelimited(hdev->dev, "Unknown SM SEI cause %u",
7347 le32_to_cpu(sei_data->sei_log));7466 le32_to_cpu(sei_data->sei_log));
7348 break;7467 break;
7349 }7468 }
@@ -7358,6 +7477,11 @@ static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type, · 8 unmodified lines
7358 bool extract_info_from_fw;7477 bool extract_info_from_fw;
7359 int rc;7478 int rc;
7360 7479
7480 if (hdev->asic_prop.fw_security_enabled) {
7481 extract_info_from_fw = true;
7482 goto extract_ecc_info;
7483 }
7484
7361 switch (event_type) {7485 switch (event_type) {
7362 case GAUDI_EVENT_PCIE_CORE_SERR ... GAUDI_EVENT_PCIE_PHY_DERR:7486 case GAUDI_EVENT_PCIE_CORE_SERR ... GAUDI_EVENT_PCIE_PHY_DERR:
7363 case GAUDI_EVENT_DMA0_SERR_ECC ... GAUDI_EVENT_MMU_DERR:7487 case GAUDI_EVENT_DMA0_SERR_ECC ... GAUDI_EVENT_MMU_DERR:
@@ -7430,6 +7554,7 @@ static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type, · 66 unmodified lines
7430 return;7554 return;
7431 }7555 }
7432 7556
7557extract_ecc_info:
7433 if (extract_info_from_fw) {7558 if (extract_info_from_fw) {
7434 ecc_address = le64_to_cpu(ecc_data->ecc_address);7559 ecc_address = le64_to_cpu(ecc_data->ecc_address);
7435 ecc_syndrom = le64_to_cpu(ecc_data->ecc_syndrom);7560 ecc_syndrom = le64_to_cpu(ecc_data->ecc_syndrom);
@@ -7806,8 +7931,15 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 370 unmodified lines
7806 u32 ctl = le32_to_cpu(eq_entry->hdr.ctl);7931 u32 ctl = le32_to_cpu(eq_entry->hdr.ctl);
7807 u16 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK)7932 u16 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK)
7808 >> EQ_CTL_EVENT_TYPE_SHIFT);7933 >> EQ_CTL_EVENT_TYPE_SHIFT);
7809 u8 cause;
7810 bool reset_required;7934 bool reset_required;
7935 u8 cause;
7936 int rc;
7937
7938 if (event_type >= GAUDI_EVENT_SIZE) {
7939 dev_err(hdev->dev, "Event type %u exceeds maximum of %u",
7940 event_type, GAUDI_EVENT_SIZE - 1);
7941 return;
7942 }
7811 7943
7812 gaudi->events_stat[event_type]++;7944 gaudi->events_stat[event_type]++;
7813 gaudi->events_stat_aggregate[event_type]++;7945 gaudi->events_stat_aggregate[event_type]++;
@@ -7880,10 +8012,10 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 66 unmodified lines
7880 tpc_dec_event_to_tpc_id(event_type),8012 tpc_dec_event_to_tpc_id(event_type),
7881 "AXI_SLV_DEC_Error");8013 "AXI_SLV_DEC_Error");
7882 if (reset_required) {8014 if (reset_required) {
7883 dev_err(hdev->dev, "hard reset required due to %s\n",8015 dev_err(hdev->dev, "reset required due to %s\n",
7884 gaudi_irq_map_table[event_type].name);8016 gaudi_irq_map_table[event_type].name);
7885 8017
7886 goto reset_device;8018 hl_device_reset(hdev, 0);
7887 } else {8019 } else {
7888 hl_fw_unmask_irq(hdev, event_type);8020 hl_fw_unmask_irq(hdev, event_type);
7889 }8021 }
@@ -7902,10 +8034,10 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 12 unmodified lines
7902 tpc_krn_event_to_tpc_id(event_type),8034 tpc_krn_event_to_tpc_id(event_type),
7903 "KRN_ERR");8035 "KRN_ERR");
7904 if (reset_required) {8036 if (reset_required) {
7905 dev_err(hdev->dev, "hard reset required due to %s\n",8037 dev_err(hdev->dev, "reset required due to %s\n",
7906 gaudi_irq_map_table[event_type].name);8038 gaudi_irq_map_table[event_type].name);
7907 8039
7908 goto reset_device;8040 hl_device_reset(hdev, 0);
7909 } else {8041 } else {
7910 hl_fw_unmask_irq(hdev, event_type);8042 hl_fw_unmask_irq(hdev, event_type);
7911 }8043 }
@@ -7993,6 +8125,10 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 81 unmodified lines
7993 gaudi_print_irq_info(hdev, event_type, false);8125 gaudi_print_irq_info(hdev, event_type, false);
7994 gaudi_print_sm_sei_info(hdev, event_type,8126 gaudi_print_sm_sei_info(hdev, event_type,
7995 &eq_entry->sm_sei_data);8127 &eq_entry->sm_sei_data);
8128 rc = hl_state_dump(hdev);
8129 if (rc)
8130 dev_err(hdev->dev,
8131 "Error during system state dump %d\n", rc);
7996 hl_fw_unmask_irq(hdev, event_type);8132 hl_fw_unmask_irq(hdev, event_type);
7997 break;8133 break;
7998 8134
@@ -8031,7 +8167,9 @@ static void gaudi_handle_eqe(struct hl_device *hdev, · 32 unmodified lines
8031 return;8167 return;
8032 8168
8033reset_device:8169reset_device:
8034 if (hdev->hard_reset_on_fw_events)8170 if (hdev->asic_prop.fw_security_enabled)
8171 hl_device_reset(hdev, HL_RESET_HARD | HL_RESET_FW);
8172 else if (hdev->hard_reset_on_fw_events)
8035 hl_device_reset(hdev, HL_RESET_HARD);8173 hl_device_reset(hdev, HL_RESET_HARD);
8036 else8174 else
8037 hl_fw_unmask_irq(hdev, event_type);8175 hl_fw_unmask_irq(hdev, event_type);
@@ -8563,11 +8701,20 @@ static void gaudi_internal_cb_pool_fini(struct hl_device *hdev, · 525 unmodified lines
8563 8701
8564static int gaudi_ctx_init(struct hl_ctx *ctx)8702static int gaudi_ctx_init(struct hl_ctx *ctx)
8565{8703{
8704 int rc;
8705
8566 if (ctx->asid == HL_KERNEL_ASID_ID)8706 if (ctx->asid == HL_KERNEL_ASID_ID)
8567 return 0;8707 return 0;
8568 8708
8569 gaudi_mmu_prepare(ctx->hdev, ctx->asid);8709 rc = gaudi_internal_cb_pool_init(ctx->hdev, ctx);
8570 return gaudi_internal_cb_pool_init(ctx->hdev, ctx);8710 if (rc)
8711 return rc;
8712
8713 rc = gaudi_restore_user_registers(ctx->hdev);
8714 if (rc)
8715 gaudi_internal_cb_pool_fini(ctx->hdev, ctx);
8716
8717 return rc;
8571}8718}
8572 8719
8573static void gaudi_ctx_fini(struct hl_ctx *ctx)8720static void gaudi_ctx_fini(struct hl_ctx *ctx)
@@ -8596,6 +8743,11 @@ static u32 gaudi_get_wait_cb_size(struct hl_device *hdev) · 22 unmodified lines
8596 sizeof(struct packet_msg_prot) * 2;8743 sizeof(struct packet_msg_prot) * 2;
8597}8744}
8598 8745
8746static u32 gaudi_get_sob_addr(struct hl_device *hdev, u32 sob_id)
8747{
8748 return mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + (sob_id * 4);
8749}
8750
8599static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id,8751static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id,
8600 u32 size, bool eb)8752 u32 size, bool eb)
8601{8753{
@@ -8902,16 +9054,12 @@ static u32 gaudi_gen_wait_cb(struct hl_device *hdev, · 300 unmodified lines
8902static void gaudi_reset_sob(struct hl_device *hdev, void *data)9054static void gaudi_reset_sob(struct hl_device *hdev, void *data)
8903{9055{
8904 struct hl_hw_sob *hw_sob = (struct hl_hw_sob *) data;9056 struct hl_hw_sob *hw_sob = (struct hl_hw_sob *) data;
8905 int rc;
8906 9057
8907 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx,9058 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx,
8908 hw_sob->sob_id);9059 hw_sob->sob_id);
8909 9060
8910 rc = gaudi_schedule_register_memset(hdev, hw_sob->q_idx,9061 WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 +
8911 CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 +9062 hw_sob->sob_id * 4, 0);
8912 hw_sob->sob_id * 4, 1, 0);
8913 if (rc)
8914 dev_err(hdev->dev, "failed resetting sob %u", hw_sob->sob_id);
8915 9063
8916 kref_init(&hw_sob->kref);9064 kref_init(&hw_sob->kref);
8917}9065}
@@ -8977,6 +9125,280 @@ static int gaudi_map_pll_idx_to_fw_idx(u32 pll_idx) · 59 unmodified lines
8977 }9125 }
8978}9126}
8979 9127
9128static int gaudi_add_sync_to_engine_map_entry(
9129 struct hl_sync_to_engine_map *map, u32 reg_value,
9130 enum hl_sync_engine_type engine_type, u32 engine_id)
9131{
9132 struct hl_sync_to_engine_map_entry *entry;
9133
9134 /* Reg value represents a partial address of sync object,
9135 * it is used as unique identifier. For this we need to
9136 * clear the cutoff cfg base bits from the value.
9137 */
9138 if (reg_value == 0 || reg_value == 0xffffffff)
9139 return 0;
9140 reg_value -= (u32)CFG_BASE;
9141
9142 /* create a new hash entry */
9143 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
9144 if (!entry)
9145 return -ENOMEM;
9146 entry->engine_type = engine_type;
9147 entry->engine_id = engine_id;
9148 entry->sync_id = reg_value;
9149 hash_add(map->tb, &entry->node, reg_value);
9150
9151 return 0;
9152}
9153
9154static int gaudi_gen_sync_to_engine_map(struct hl_device *hdev,
9155 struct hl_sync_to_engine_map *map)
9156{
9157 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
9158 struct gaudi_device *gaudi = hdev->asic_specific;
9159 int i, j, rc;
9160 u32 reg_value;
9161
9162 /* Iterate over TPC engines */
9163 for (i = 0; i < sds->props[SP_NUM_OF_TPC_ENGINES]; ++i) {
9164 /* TPC registered must be accessed with clock gating disabled */
9165 mutex_lock(&gaudi->clk_gate_mutex);
9166 hdev->asic_funcs->disable_clock_gating(hdev);
9167
9168 reg_value = RREG32(sds->props[SP_TPC0_CFG_SO] +
9169 sds->props[SP_NEXT_TPC] * i);
9170
9171 /* We can reenable clock_gating */
9172 hdev->asic_funcs->set_clock_gating(hdev);
9173 mutex_unlock(&gaudi->clk_gate_mutex);
9174
9175 rc = gaudi_add_sync_to_engine_map_entry(map, reg_value,
9176 ENGINE_TPC, i);
9177 if (rc)
9178 goto free_sync_to_engine_map;
9179 }
9180
9181 /* Iterate over MME engines */
9182 for (i = 0; i < sds->props[SP_NUM_OF_MME_ENGINES]; ++i) {
9183 for (j = 0; j < sds->props[SP_SUB_MME_ENG_NUM]; ++j) {
9184 /* MME registered must be accessed with clock gating
9185 * disabled
9186 */
9187 mutex_lock(&gaudi->clk_gate_mutex);
9188 hdev->asic_funcs->disable_clock_gating(hdev);
9189
9190 reg_value = RREG32(sds->props[SP_MME_CFG_SO] +
9191 sds->props[SP_NEXT_MME] * i +
9192 j * sizeof(u32));
9193
9194 /* We can reenable clock_gating */
9195 hdev->asic_funcs->set_clock_gating(hdev);
9196 mutex_unlock(&gaudi->clk_gate_mutex);
9197
9198 rc = gaudi_add_sync_to_engine_map_entry(
9199 map, reg_value, ENGINE_MME,
9200 i * sds->props[SP_SUB_MME_ENG_NUM] + j);
9201 if (rc)
9202 goto free_sync_to_engine_map;
9203 }
9204 }
9205
9206 /* Iterate over DMA engines */
9207 for (i = 0; i < sds->props[SP_NUM_OF_DMA_ENGINES]; ++i) {
9208 reg_value = RREG32(sds->props[SP_DMA_CFG_SO] +
9209 sds->props[SP_DMA_QUEUES_OFFSET] * i);
9210 rc = gaudi_add_sync_to_engine_map_entry(map, reg_value,
9211 ENGINE_DMA, i);
9212 if (rc)
9213 goto free_sync_to_engine_map;
9214 }
9215
9216 return 0;
9217
9218free_sync_to_engine_map:
9219 hl_state_dump_free_sync_to_engine_map(map);
9220
9221 return rc;
9222}
9223
9224static int gaudi_monitor_valid(struct hl_mon_state_dump *mon)
9225{
9226 return FIELD_GET(
9227 SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_VALID_MASK,
9228 mon->status);
9229}
9230
9231static void gaudi_fill_sobs_from_mon(char *sobs, struct hl_mon_state_dump *mon)
9232{
9233 const size_t max_write = 10;
9234 u32 gid, mask, sob;
9235 int i, offset;
9236
9237 /* Sync object ID is calculated as follows:
9238 * (8 * group_id + cleared bits in mask)
9239 */
9240 gid = FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SID_MASK,
9241 mon->arm_data);
9242 mask = FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_MASK_MASK,
9243 mon->arm_data);
9244
9245 for (i = 0, offset = 0; mask && offset < MONITOR_SOB_STRING_SIZE -
9246 max_write; mask >>= 1, i++) {
9247 if (!(mask & 1)) {
9248 sob = gid * MONITOR_MAX_SOBS + i;
9249
9250 if (offset > 0)
9251 offset += snprintf(sobs + offset, max_write,
9252 ", ");
9253
9254 offset += snprintf(sobs + offset, max_write, "%u", sob);
9255 }
9256 }
9257}
9258
9259static int gaudi_print_single_monitor(char **buf, size_t *size, size_t *offset,
9260 struct hl_device *hdev,
9261 struct hl_mon_state_dump *mon)
9262{
9263 const char *name;
9264 char scratch_buf1[BIN_REG_STRING_SIZE],
9265 scratch_buf2[BIN_REG_STRING_SIZE];
9266 char monitored_sobs[MONITOR_SOB_STRING_SIZE] = {0};
9267
9268 name = hl_state_dump_get_monitor_name(hdev, mon);
9269 if (!name)
9270 name = "";
9271
9272 gaudi_fill_sobs_from_mon(monitored_sobs, mon);
9273
9274 return hl_snprintf_resize(
9275 buf, size, offset,
9276 "Mon id: %u%s, wait for group id: %u mask %s to reach val: %u and write %u to address 0x%llx. Pending: %s. Means sync objects [%s] are being monitored.",
9277 mon->id, name,
9278 FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SID_MASK,
9279 mon->arm_data),
9280 hl_format_as_binary(
9281 scratch_buf1, sizeof(scratch_buf1),
9282 FIELD_GET(
9283 SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_MASK_MASK,
9284 mon->arm_data)),
9285 FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SOD_MASK,
9286 mon->arm_data),
9287 mon->wr_data,
9288 (((u64)mon->wr_addr_high) << 32) | mon->wr_addr_low,
9289 hl_format_as_binary(
9290 scratch_buf2, sizeof(scratch_buf2),
9291 FIELD_GET(
9292 SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_PENDING_MASK,
9293 mon->status)),
9294 monitored_sobs);
9295}
9296
9297
9298static int gaudi_print_fences_single_engine(
9299 struct hl_device *hdev, u64 base_offset, u64 status_base_offset,
9300 enum hl_sync_engine_type engine_type, u32 engine_id, char **buf,
9301 size_t *size, size_t *offset)
9302{
9303 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
9304 int rc = -ENOMEM, i;
9305 u32 *statuses, *fences;
9306
9307 statuses = kcalloc(sds->props[SP_ENGINE_NUM_OF_QUEUES],
9308 sizeof(*statuses), GFP_KERNEL);
9309 if (!statuses)
9310 goto out;
9311
9312 fences = kcalloc(sds->props[SP_ENGINE_NUM_OF_FENCES] *
9313 sds->props[SP_ENGINE_NUM_OF_QUEUES],
9314 sizeof(*fences), GFP_KERNEL);
9315 if (!fences)
9316 goto free_status;
9317
9318 for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_FENCES]; ++i)
9319 statuses[i] = RREG32(status_base_offset + i * sizeof(u32));
9320
9321 for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_FENCES] *
9322 sds->props[SP_ENGINE_NUM_OF_QUEUES]; ++i)
9323 fences[i] = RREG32(base_offset + i * sizeof(u32));
9324
9325 /* The actual print */
9326 for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_QUEUES]; ++i) {
9327 u32 fence_id;
9328 u64 fence_cnt, fence_rdata;
9329 const char *engine_name;
9330
9331 if (!FIELD_GET(TPC0_QM_CP_STS_0_FENCE_IN_PROGRESS_MASK,
9332 statuses[i]))
9333 continue;
9334
9335 fence_id =
9336 FIELD_GET(TPC0_QM_CP_STS_0_FENCE_ID_MASK, statuses[i]);
9337 fence_cnt = base_offset + CFG_BASE +
9338 sizeof(u32) *
9339 (i + fence_id * sds->props[SP_ENGINE_NUM_OF_QUEUES]);
9340 fence_rdata = fence_cnt - sds->props[SP_FENCE0_CNT_OFFSET] +
9341 sds->props[SP_FENCE0_RDATA_OFFSET];
9342 engine_name = hl_sync_engine_to_string(engine_type);
9343
9344 rc = hl_snprintf_resize(
9345 buf, size, offset,
9346 "%s%u, stream %u: fence id %u cnt = 0x%llx (%s%u_QM.CP_FENCE%u_CNT_%u) rdata = 0x%llx (%s%u_QM.CP_FENCE%u_RDATA_%u) value = %u, cp_status = %u\n",
9347 engine_name, engine_id,
9348 i, fence_id,
9349 fence_cnt, engine_name, engine_id, fence_id, i,
9350 fence_rdata, engine_name, engine_id, fence_id, i,
9351 fences[fence_id],
9352 statuses[i]);
9353 if (rc)
9354 goto free_fences;
9355 }
9356
9357 rc = 0;
9358
9359free_fences:
9360 kfree(fences);
9361free_status:
9362 kfree(statuses);
9363out:
9364 return rc;
9365}
9366
9367
9368static struct hl_state_dump_specs_funcs gaudi_state_dump_funcs = {
9369 .monitor_valid = gaudi_monitor_valid,
9370 .print_single_monitor = gaudi_print_single_monitor,
9371 .gen_sync_to_engine_map = gaudi_gen_sync_to_engine_map,
9372 .print_fences_single_engine = gaudi_print_fences_single_engine,
9373};
9374
9375static void gaudi_state_dump_init(struct hl_device *hdev)
9376{
9377 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
9378 int i;
9379
9380 for (i = 0; i < ARRAY_SIZE(gaudi_so_id_to_str); ++i)
9381 hash_add(sds->so_id_to_str_tb,
9382 &gaudi_so_id_to_str[i].node,
9383 gaudi_so_id_to_str[i].id);
9384
9385 for (i = 0; i < ARRAY_SIZE(gaudi_monitor_id_to_str); ++i)
9386 hash_add(sds->monitor_id_to_str_tb,
9387 &gaudi_monitor_id_to_str[i].node,
9388 gaudi_monitor_id_to_str[i].id);
9389
9390 sds->props = gaudi_state_dump_specs_props;
9391
9392 sds->sync_namager_names = gaudi_sync_manager_names;
9393
9394 sds->funcs = gaudi_state_dump_funcs;
9395}
9396
9397static u32 *gaudi_get_stream_master_qid_arr(void)
9398{
9399 return gaudi_stream_master;
9400}
9401
8980static const struct hl_asic_funcs gaudi_funcs = {9402static const struct hl_asic_funcs gaudi_funcs = {
8981 .early_init = gaudi_early_init,9403 .early_init = gaudi_early_init,
8982 .early_fini = gaudi_early_fini,9404 .early_fini = gaudi_early_fini,
@@ -8989,7 +9411,7 @@ static const struct hl_asic_funcs gaudi_funcs = { · 6 unmodified lines
8989 .halt_engines = gaudi_halt_engines,9411 .halt_engines = gaudi_halt_engines,
8990 .suspend = gaudi_suspend,9412 .suspend = gaudi_suspend,
8991 .resume = gaudi_resume,9413 .resume = gaudi_resume,
8992 .cb_mmap = gaudi_cb_mmap,9414 .mmap = gaudi_mmap,
8993 .ring_doorbell = gaudi_ring_doorbell,9415 .ring_doorbell = gaudi_ring_doorbell,
8994 .pqe_write = gaudi_pqe_write,9416 .pqe_write = gaudi_pqe_write,
8995 .asic_dma_alloc_coherent = gaudi_dma_alloc_coherent,9417 .asic_dma_alloc_coherent = gaudi_dma_alloc_coherent,
@@ -9062,7 +9484,11 @@ static const struct hl_asic_funcs gaudi_funcs = { · 66 unmodified lines
9062 .enable_events_from_fw = gaudi_enable_events_from_fw,9484 .enable_events_from_fw = gaudi_enable_events_from_fw,
9063 .map_pll_idx_to_fw_idx = gaudi_map_pll_idx_to_fw_idx,9485 .map_pll_idx_to_fw_idx = gaudi_map_pll_idx_to_fw_idx,
9064 .init_firmware_loader = gaudi_init_firmware_loader,9486 .init_firmware_loader = gaudi_init_firmware_loader,
9065 .init_cpu_scrambler_dram = gaudi_init_scrambler_hbm9487 .init_cpu_scrambler_dram = gaudi_init_scrambler_hbm,
9488 .state_dump_init = gaudi_state_dump_init,
9489 .get_sob_addr = gaudi_get_sob_addr,
9490 .set_pci_memory_regions = gaudi_set_pci_memory_regions,
9491 .get_stream_master_qid_arr = gaudi_get_stream_master_qid_arr
9066};9492};
9067 9493
9068/**9494/**
drivers/misc/habanalabs/gaudi/gaudiP.h+18−1
@@ -36,6 +36,8 @@ · 35 unmodified lines
36#define NUMBER_OF_INTERRUPTS (NUMBER_OF_CMPLT_QUEUES + \36#define NUMBER_OF_INTERRUPTS (NUMBER_OF_CMPLT_QUEUES + \
37 NUMBER_OF_CPU_HW_QUEUES)37 NUMBER_OF_CPU_HW_QUEUES)
38 38
39#define GAUDI_STREAM_MASTER_ARR_SIZE 8
40
39#if (NUMBER_OF_INTERRUPTS > GAUDI_MSI_ENTRIES)41#if (NUMBER_OF_INTERRUPTS > GAUDI_MSI_ENTRIES)
40#error "Number of MSI interrupts must be smaller or equal to GAUDI_MSI_ENTRIES"42#error "Number of MSI interrupts must be smaller or equal to GAUDI_MSI_ENTRIES"
41#endif43#endif
@@ -50,6 +52,8 @@ · 8 unmodified lines
50#define DC_POWER_DEFAULT_PCI 60000 /* 60W */52#define DC_POWER_DEFAULT_PCI 60000 /* 60W */
51#define DC_POWER_DEFAULT_PMC 60000 /* 60W */53#define DC_POWER_DEFAULT_PMC 60000 /* 60W */
52 54
55#define DC_POWER_DEFAULT_PMC_SEC 97000 /* 97W */
56
53#define GAUDI_CPU_TIMEOUT_USEC 30000000 /* 30s */57#define GAUDI_CPU_TIMEOUT_USEC 30000000 /* 30s */
54 58
55#define TPC_ENABLED_MASK 0xFF59#define TPC_ENABLED_MASK 0xFF
@@ -62,7 +66,7 @@ · 6 unmodified lines
62 66
63#define DMA_MAX_TRANSFER_SIZE U32_MAX67#define DMA_MAX_TRANSFER_SIZE U32_MAX
64 68
65#define GAUDI_DEFAULT_CARD_NAME "HL2000"69#define GAUDI_DEFAULT_CARD_NAME "HL205"
66 70
67#define GAUDI_MAX_PENDING_CS SZ_16K71#define GAUDI_MAX_PENDING_CS SZ_16K
68 72
@@ -117,6 +121,7 @@ · 48 unmodified lines
117 (((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_511 - \121 (((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_511 - \
118 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0) + 4) >> 2)122 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0) + 4) >> 2)
119 123
124#define MONITOR_MAX_SOBS 8
120 125
121/* DRAM Memory Map */126/* DRAM Memory Map */
122 127
@@ -200,6 +205,18 @@ · 77 unmodified lines
200#define HW_CAP_TPC_MASK GENMASK(31, 24)205#define HW_CAP_TPC_MASK GENMASK(31, 24)
201#define HW_CAP_TPC_SHIFT 24206#define HW_CAP_TPC_SHIFT 24
202 207
208#define NEXT_SYNC_OBJ_ADDR_INTERVAL \
209 (mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0 - \
210 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0)
211#define NUM_OF_MME_ENGINES 2
212#define NUM_OF_MME_SUB_ENGINES 2
213#define NUM_OF_TPC_ENGINES 8
214#define NUM_OF_DMA_ENGINES 8
215#define NUM_OF_QUEUES 5
216#define NUM_OF_STREAMS 4
217#define NUM_OF_FENCES 4
218
219
203#define GAUDI_CPU_PCI_MSB_ADDR(addr) (((addr) & GENMASK_ULL(49, 39)) >> 39)220#define GAUDI_CPU_PCI_MSB_ADDR(addr) (((addr) & GENMASK_ULL(49, 39)) >> 39)
204#define GAUDI_PCI_TO_CPU_ADDR(addr) \221#define GAUDI_PCI_TO_CPU_ADDR(addr) \
205 do { \222 do { \
drivers/misc/habanalabs/gaudi/gaudi_coresight.c+0−5
@@ -622,11 +622,6 @@ static int gaudi_config_etr(struct hl_device *hdev, · 621 unmodified lines
622 return -EINVAL;622 return -EINVAL;
623 }623 }
624 624
625 gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_ARUSER,
626 hdev->compute_ctx->asid);
627 gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_AWUSER,
628 hdev->compute_ctx->asid);
629
630 msb = upper_32_bits(input->buffer_address) >> 8;625 msb = upper_32_bits(input->buffer_address) >> 8;
631 msb &= PSOC_GLOBAL_CONF_TRACE_ADDR_MSB_MASK;626 msb &= PSOC_GLOBAL_CONF_TRACE_ADDR_MSB_MASK;
632 WREG32(mmPSOC_GLOBAL_CONF_TRACE_ADDR, msb);627 WREG32(mmPSOC_GLOBAL_CONF_TRACE_ADDR, msb);
drivers/misc/habanalabs/gaudi/gaudi_security.c+8−0
@@ -9559,6 +9559,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 9558 unmodified lines
9559 mask |= 1U << ((mmTPC0_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);9559 mask |= 1U << ((mmTPC0_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
9560 mask |= 1U << ((mmTPC0_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);9560 mask |= 1U << ((mmTPC0_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
9561 mask |= 1U << ((mmTPC0_CFG_TPC_STALL & 0x7F) >> 2);9561 mask |= 1U << ((mmTPC0_CFG_TPC_STALL & 0x7F) >> 2);
9562 mask |= 1U << ((mmTPC0_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
9562 mask |= 1U << ((mmTPC0_CFG_RD_RATE_LIMIT & 0x7F) >> 2);9563 mask |= 1U << ((mmTPC0_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
9563 mask |= 1U << ((mmTPC0_CFG_WR_RATE_LIMIT & 0x7F) >> 2);9564 mask |= 1U << ((mmTPC0_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
9564 mask |= 1U << ((mmTPC0_CFG_MSS_CONFIG & 0x7F) >> 2);9565 mask |= 1U << ((mmTPC0_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -10013,6 +10014,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 448 unmodified lines
10013 mask |= 1U << ((mmTPC1_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);10014 mask |= 1U << ((mmTPC1_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
10014 mask |= 1U << ((mmTPC1_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);10015 mask |= 1U << ((mmTPC1_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
10015 mask |= 1U << ((mmTPC1_CFG_TPC_STALL & 0x7F) >> 2);10016 mask |= 1U << ((mmTPC1_CFG_TPC_STALL & 0x7F) >> 2);
10017 mask |= 1U << ((mmTPC1_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
10016 mask |= 1U << ((mmTPC1_CFG_RD_RATE_LIMIT & 0x7F) >> 2);10018 mask |= 1U << ((mmTPC1_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
10017 mask |= 1U << ((mmTPC1_CFG_WR_RATE_LIMIT & 0x7F) >> 2);10019 mask |= 1U << ((mmTPC1_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
10018 mask |= 1U << ((mmTPC1_CFG_MSS_CONFIG & 0x7F) >> 2);10020 mask |= 1U << ((mmTPC1_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -10466,6 +10468,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 447 unmodified lines
10466 mask |= 1U << ((mmTPC2_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);10468 mask |= 1U << ((mmTPC2_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
10467 mask |= 1U << ((mmTPC2_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);10469 mask |= 1U << ((mmTPC2_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
10468 mask |= 1U << ((mmTPC2_CFG_TPC_STALL & 0x7F) >> 2);10470 mask |= 1U << ((mmTPC2_CFG_TPC_STALL & 0x7F) >> 2);
10471 mask |= 1U << ((mmTPC2_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
10469 mask |= 1U << ((mmTPC2_CFG_RD_RATE_LIMIT & 0x7F) >> 2);10472 mask |= 1U << ((mmTPC2_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
10470 mask |= 1U << ((mmTPC2_CFG_WR_RATE_LIMIT & 0x7F) >> 2);10473 mask |= 1U << ((mmTPC2_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
10471 mask |= 1U << ((mmTPC2_CFG_MSS_CONFIG & 0x7F) >> 2);10474 mask |= 1U << ((mmTPC2_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -10919,6 +10922,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 447 unmodified lines
10919 mask |= 1U << ((mmTPC3_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);10922 mask |= 1U << ((mmTPC3_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
10920 mask |= 1U << ((mmTPC3_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);10923 mask |= 1U << ((mmTPC3_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
10921 mask |= 1U << ((mmTPC3_CFG_TPC_STALL & 0x7F) >> 2);10924 mask |= 1U << ((mmTPC3_CFG_TPC_STALL & 0x7F) >> 2);
10925 mask |= 1U << ((mmTPC3_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
10922 mask |= 1U << ((mmTPC3_CFG_RD_RATE_LIMIT & 0x7F) >> 2);10926 mask |= 1U << ((mmTPC3_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
10923 mask |= 1U << ((mmTPC3_CFG_WR_RATE_LIMIT & 0x7F) >> 2);10927 mask |= 1U << ((mmTPC3_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
10924 mask |= 1U << ((mmTPC3_CFG_MSS_CONFIG & 0x7F) >> 2);10928 mask |= 1U << ((mmTPC3_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -11372,6 +11376,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 447 unmodified lines
11372 mask |= 1U << ((mmTPC4_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);11376 mask |= 1U << ((mmTPC4_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
11373 mask |= 1U << ((mmTPC4_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);11377 mask |= 1U << ((mmTPC4_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
11374 mask |= 1U << ((mmTPC4_CFG_TPC_STALL & 0x7F) >> 2);11378 mask |= 1U << ((mmTPC4_CFG_TPC_STALL & 0x7F) >> 2);
11379 mask |= 1U << ((mmTPC4_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
11375 mask |= 1U << ((mmTPC4_CFG_RD_RATE_LIMIT & 0x7F) >> 2);11380 mask |= 1U << ((mmTPC4_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
11376 mask |= 1U << ((mmTPC4_CFG_WR_RATE_LIMIT & 0x7F) >> 2);11381 mask |= 1U << ((mmTPC4_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
11377 mask |= 1U << ((mmTPC4_CFG_MSS_CONFIG & 0x7F) >> 2);11382 mask |= 1U << ((mmTPC4_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -11825,6 +11830,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 447 unmodified lines
11825 mask |= 1U << ((mmTPC5_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);11830 mask |= 1U << ((mmTPC5_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
11826 mask |= 1U << ((mmTPC5_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);11831 mask |= 1U << ((mmTPC5_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
11827 mask |= 1U << ((mmTPC5_CFG_TPC_STALL & 0x7F) >> 2);11832 mask |= 1U << ((mmTPC5_CFG_TPC_STALL & 0x7F) >> 2);
11833 mask |= 1U << ((mmTPC5_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
11828 mask |= 1U << ((mmTPC5_CFG_RD_RATE_LIMIT & 0x7F) >> 2);11834 mask |= 1U << ((mmTPC5_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
11829 mask |= 1U << ((mmTPC5_CFG_WR_RATE_LIMIT & 0x7F) >> 2);11835 mask |= 1U << ((mmTPC5_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
11830 mask |= 1U << ((mmTPC5_CFG_MSS_CONFIG & 0x7F) >> 2);11836 mask |= 1U << ((mmTPC5_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -12280,6 +12286,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 449 unmodified lines
12280 mask |= 1U << ((mmTPC6_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);12286 mask |= 1U << ((mmTPC6_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
12281 mask |= 1U << ((mmTPC6_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);12287 mask |= 1U << ((mmTPC6_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
12282 mask |= 1U << ((mmTPC6_CFG_TPC_STALL & 0x7F) >> 2);12288 mask |= 1U << ((mmTPC6_CFG_TPC_STALL & 0x7F) >> 2);
12289 mask |= 1U << ((mmTPC6_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
12283 mask |= 1U << ((mmTPC6_CFG_RD_RATE_LIMIT & 0x7F) >> 2);12290 mask |= 1U << ((mmTPC6_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
12284 mask |= 1U << ((mmTPC6_CFG_WR_RATE_LIMIT & 0x7F) >> 2);12291 mask |= 1U << ((mmTPC6_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
12285 mask |= 1U << ((mmTPC6_CFG_MSS_CONFIG & 0x7F) >> 2);12292 mask |= 1U << ((mmTPC6_CFG_MSS_CONFIG & 0x7F) >> 2);
@@ -12735,6 +12742,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) · 449 unmodified lines
12735 mask |= 1U << ((mmTPC7_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);12742 mask |= 1U << ((mmTPC7_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
12736 mask |= 1U << ((mmTPC7_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);12743 mask |= 1U << ((mmTPC7_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
12737 mask |= 1U << ((mmTPC7_CFG_TPC_STALL & 0x7F) >> 2);12744 mask |= 1U << ((mmTPC7_CFG_TPC_STALL & 0x7F) >> 2);
12745 mask |= 1U << ((mmTPC7_CFG_ICACHE_BASE_ADDERESS_HIGH & 0x7F) >> 2);
12738 mask |= 1U << ((mmTPC7_CFG_RD_RATE_LIMIT & 0x7F) >> 2);12746 mask |= 1U << ((mmTPC7_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
12739 mask |= 1U << ((mmTPC7_CFG_WR_RATE_LIMIT & 0x7F) >> 2);12747 mask |= 1U << ((mmTPC7_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
12740 mask |= 1U << ((mmTPC7_CFG_MSS_CONFIG & 0x7F) >> 2);12748 mask |= 1U << ((mmTPC7_CFG_MSS_CONFIG & 0x7F) >> 2);
drivers/misc/habanalabs/goya/goya.c+84−18
@@ -350,6 +350,8 @@ static u32 goya_all_events[] = { · 349 unmodified lines
350 GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E350 GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E
351};351};
352 352
353static s64 goya_state_dump_specs_props[SP_MAX] = {0};
354
353static int goya_mmu_clear_pgt_range(struct hl_device *hdev);355static int goya_mmu_clear_pgt_range(struct hl_device *hdev);
354static int goya_mmu_set_dram_default_page(struct hl_device *hdev);356static int goya_mmu_set_dram_default_page(struct hl_device *hdev);
355static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev);357static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev);
@@ -387,6 +389,7 @@ int goya_set_fixed_properties(struct hl_device *hdev) · 31 unmodified lines
387 prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_USER;389 prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_USER;
388 }390 }
389 391
392 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE;
390 prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;393 prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;
391 394
392 prop->dram_base_address = DRAM_PHYS_BASE;395 prop->dram_base_address = DRAM_PHYS_BASE;
@@ -466,6 +469,8 @@ int goya_set_fixed_properties(struct hl_device *hdev) · 73 unmodified lines
466 prop->hard_reset_done_by_fw = false;469 prop->hard_reset_done_by_fw = false;
467 prop->gic_interrupts_enable = true;470 prop->gic_interrupts_enable = true;
468 471
472 prop->server_type = HL_SERVER_TYPE_UNKNOWN;
473
469 return 0;474 return 0;
470}475}
471 476
@@ -649,14 +654,14 @@ pci_init: · 177 unmodified lines
649 GOYA_BOOT_FIT_REQ_TIMEOUT_USEC);654 GOYA_BOOT_FIT_REQ_TIMEOUT_USEC);
650 if (rc) {655 if (rc) {
651 if (hdev->reset_on_preboot_fail)656 if (hdev->reset_on_preboot_fail)
652 hdev->asic_funcs->hw_fini(hdev, true);657 hdev->asic_funcs->hw_fini(hdev, true, false);
653 goto pci_fini;658 goto pci_fini;
654 }659 }
655 660
656 if (goya_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {661 if (goya_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
657 dev_info(hdev->dev,662 dev_info(hdev->dev,
658 "H/W state is dirty, must reset before initializing\n");663 "H/W state is dirty, must reset before initializing\n");
659 hdev->asic_funcs->hw_fini(hdev, true);664 hdev->asic_funcs->hw_fini(hdev, true, false);
660 }665 }
661 666
662 if (!hdev->pldm) {667 if (!hdev->pldm) {
@@ -955,8 +960,9 @@ static int goya_sw_init(struct hl_device *hdev) · 292 unmodified lines
955 hdev->supports_coresight = true;960 hdev->supports_coresight = true;
956 hdev->supports_soft_reset = true;961 hdev->supports_soft_reset = true;
957 hdev->allow_external_soft_reset = true;962 hdev->allow_external_soft_reset = true;
963 hdev->supports_wait_for_multi_cs = false;
958 964
959 goya_set_pci_memory_regions(hdev);965 hdev->asic_funcs->set_pci_memory_regions(hdev);
960 966
961 return 0;967 return 0;
962 968
@@ -2374,7 +2380,7 @@ static void goya_disable_timestamp(struct hl_device *hdev) · 1411 unmodified lines
2374 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);2380 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
2375}2381}
2376 2382
2377static void goya_halt_engines(struct hl_device *hdev, bool hard_reset)2383static void goya_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset)
2378{2384{
2379 u32 wait_timeout_ms;2385 u32 wait_timeout_ms;
2380 2386
@@ -2493,6 +2499,7 @@ static void goya_init_firmware_loader(struct hl_device *hdev) · 112 unmodified lines
2493 struct fw_load_mgr *fw_loader = &hdev->fw_loader;2499 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
2494 2500
2495 /* fill common fields */2501 /* fill common fields */
2502 fw_loader->linux_loaded = false;
2496 fw_loader->boot_fit_img.image_name = GOYA_BOOT_FIT_FILE;2503 fw_loader->boot_fit_img.image_name = GOYA_BOOT_FIT_FILE;
2497 fw_loader->linux_img.image_name = GOYA_LINUX_FW_FILE;2504 fw_loader->linux_img.image_name = GOYA_LINUX_FW_FILE;
2498 fw_loader->cpu_timeout = GOYA_CPU_TIMEOUT_USEC;2505 fw_loader->cpu_timeout = GOYA_CPU_TIMEOUT_USEC;
@@ -2696,14 +2703,7 @@ disable_queues: · 197 unmodified lines
2696 return rc;2703 return rc;
2697}2704}
2698 2705
2699/*2706static void goya_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
2700 * goya_hw_fini - Goya hardware tear-down code
2701 *
2702 * @hdev: pointer to hl_device structure
2703 * @hard_reset: should we do hard reset to all engines or just reset the
2704 * compute/dma engines
2705 */
2706static void goya_hw_fini(struct hl_device *hdev, bool hard_reset)
2707{2707{
2708 struct goya_device *goya = hdev->asic_specific;2708 struct goya_device *goya = hdev->asic_specific;
2709 u32 reset_timeout_ms, cpu_timeout_ms, status;2709 u32 reset_timeout_ms, cpu_timeout_ms, status;
@@ -2796,7 +2796,7 @@ int goya_resume(struct hl_device *hdev) · 86 unmodified lines
2796 return goya_init_iatu(hdev);2796 return goya_init_iatu(hdev);
2797}2797}
2798 2798
2799static int goya_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma,2799static int goya_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
2800 void *cpu_addr, dma_addr_t dma_addr, size_t size)2800 void *cpu_addr, dma_addr_t dma_addr, size_t size)
2801{2801{
2802 int rc;2802 int rc;
@@ -4797,6 +4797,12 @@ void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) · 1994 unmodified lines
4797 >> EQ_CTL_EVENT_TYPE_SHIFT);4797 >> EQ_CTL_EVENT_TYPE_SHIFT);
4798 struct goya_device *goya = hdev->asic_specific;4798 struct goya_device *goya = hdev->asic_specific;
4799 4799
4800 if (event_type >= GOYA_ASYNC_EVENT_ID_SIZE) {
4801 dev_err(hdev->dev, "Event type %u exceeds maximum of %u",
4802 event_type, GOYA_ASYNC_EVENT_ID_SIZE - 1);
4803 return;
4804 }
4805
4800 goya->events_stat[event_type]++;4806 goya->events_stat[event_type]++;
4801 goya->events_stat_aggregate[event_type]++;4807 goya->events_stat_aggregate[event_type]++;
4802 4808
@@ -5475,14 +5481,14 @@ u64 goya_get_device_time(struct hl_device *hdev) · 672 unmodified lines
5475 return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL);5481 return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL);
5476}5482}
5477 5483
5478static void goya_collective_wait_init_cs(struct hl_cs *cs)5484static int goya_collective_wait_init_cs(struct hl_cs *cs)
5479{5485{
5480 5486 return 0;
5481}5487}
5482 5488
5483static int goya_collective_wait_create_jobs(struct hl_device *hdev,5489static int goya_collective_wait_create_jobs(struct hl_device *hdev,
5484 struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id,5490 struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id,
5485 u32 collective_engine_id)5491 u32 collective_engine_id, u32 encaps_signal_offset)
5486{5492{
5487 return -EINVAL;5493 return -EINVAL;
5488}5494}
@@ -5524,6 +5530,62 @@ static int goya_map_pll_idx_to_fw_idx(u32 pll_idx) · 35 unmodified lines
5524 }5530 }
5525}5531}
5526 5532
5533static int goya_gen_sync_to_engine_map(struct hl_device *hdev,
5534 struct hl_sync_to_engine_map *map)
5535{
5536 /* Not implemented */
5537 return 0;
5538}
5539
5540static int goya_monitor_valid(struct hl_mon_state_dump *mon)
5541{
5542 /* Not implemented */
5543 return 0;
5544}
5545
5546static int goya_print_single_monitor(char **buf, size_t *size, size_t *offset,
5547 struct hl_device *hdev,
5548 struct hl_mon_state_dump *mon)
5549{
5550 /* Not implemented */
5551 return 0;
5552}
5553
5554
5555static int goya_print_fences_single_engine(
5556 struct hl_device *hdev, u64 base_offset, u64 status_base_offset,
5557 enum hl_sync_engine_type engine_type, u32 engine_id, char **buf,
5558 size_t *size, size_t *offset)
5559{
5560 /* Not implemented */
5561 return 0;
5562}
5563
5564
5565static struct hl_state_dump_specs_funcs goya_state_dump_funcs = {
5566 .monitor_valid = goya_monitor_valid,
5567 .print_single_monitor = goya_print_single_monitor,
5568 .gen_sync_to_engine_map = goya_gen_sync_to_engine_map,
5569 .print_fences_single_engine = goya_print_fences_single_engine,
5570};
5571
5572static void goya_state_dump_init(struct hl_device *hdev)
5573{
5574 /* Not implemented */
5575 hdev->state_dump_specs.props = goya_state_dump_specs_props;
5576 hdev->state_dump_specs.funcs = goya_state_dump_funcs;
5577}
5578
5579static u32 goya_get_sob_addr(struct hl_device *hdev, u32 sob_id)
5580{
5581 return 0;
5582}
5583
5584static u32 *goya_get_stream_master_qid_arr(void)
5585{
5586 return NULL;
5587}
5588
5527static const struct hl_asic_funcs goya_funcs = {5589static const struct hl_asic_funcs goya_funcs = {
5528 .early_init = goya_early_init,5590 .early_init = goya_early_init,
5529 .early_fini = goya_early_fini,5591 .early_fini = goya_early_fini,
@@ -5536,7 +5598,7 @@ static const struct hl_asic_funcs goya_funcs = { · 6 unmodified lines
5536 .halt_engines = goya_halt_engines,5598 .halt_engines = goya_halt_engines,
5537 .suspend = goya_suspend,5599 .suspend = goya_suspend,
5538 .resume = goya_resume,5600 .resume = goya_resume,
5539 .cb_mmap = goya_cb_mmap,5601 .mmap = goya_mmap,
5540 .ring_doorbell = goya_ring_doorbell,5602 .ring_doorbell = goya_ring_doorbell,
5541 .pqe_write = goya_pqe_write,5603 .pqe_write = goya_pqe_write,
5542 .asic_dma_alloc_coherent = goya_dma_alloc_coherent,5604 .asic_dma_alloc_coherent = goya_dma_alloc_coherent,
@@ -5609,7 +5671,11 @@ static const struct hl_asic_funcs goya_funcs = { · 66 unmodified lines
5609 .enable_events_from_fw = goya_enable_events_from_fw,5671 .enable_events_from_fw = goya_enable_events_from_fw,
5610 .map_pll_idx_to_fw_idx = goya_map_pll_idx_to_fw_idx,5672 .map_pll_idx_to_fw_idx = goya_map_pll_idx_to_fw_idx,
5611 .init_firmware_loader = goya_init_firmware_loader,5673 .init_firmware_loader = goya_init_firmware_loader,
5612 .init_cpu_scrambler_dram = goya_cpu_init_scrambler_dram5674 .init_cpu_scrambler_dram = goya_cpu_init_scrambler_dram,
5675 .state_dump_init = goya_state_dump_init,
5676 .get_sob_addr = &goya_get_sob_addr,
5677 .set_pci_memory_regions = goya_set_pci_memory_regions,
5678 .get_stream_master_qid_arr = goya_get_stream_master_qid_arr,
5613};5679};
5614 5680
5615/*5681/*
drivers/misc/habanalabs/include/common/cpucp_if.h+109−6
@@ -98,6 +98,18 @@ struct hl_eq_fw_alive { · 97 unmodified lines
98 __u8 pad[7];98 __u8 pad[7];
99};99};
100 100
101enum hl_pcie_addr_dec_cause {
102 PCIE_ADDR_DEC_HBW_ERR_RESP,
103 PCIE_ADDR_DEC_LBW_ERR_RESP,
104 PCIE_ADDR_DEC_TLP_BLOCKED_BY_RR
105};
106
107struct hl_eq_pcie_addr_dec_data {
108 /* enum hl_pcie_addr_dec_cause */
109 __u8 addr_dec_cause;
110 __u8 pad[7];
111};
112
101struct hl_eq_entry {113struct hl_eq_entry {
102 struct hl_eq_header hdr;114 struct hl_eq_header hdr;
103 union {115 union {
@@ -106,6 +118,7 @@ struct hl_eq_entry { · 2 unmodified lines
106 struct hl_eq_sm_sei_data sm_sei_data;118 struct hl_eq_sm_sei_data sm_sei_data;
107 struct cpucp_pkt_sync_err pkt_sync_err;119 struct cpucp_pkt_sync_err pkt_sync_err;
108 struct hl_eq_fw_alive fw_alive;120 struct hl_eq_fw_alive fw_alive;
121 struct hl_eq_pcie_addr_dec_data pcie_addr_dec_data;
109 __le64 data[7];122 __le64 data[7];
110 };123 };
111};124};
@@ -116,7 +129,7 @@ struct hl_eq_entry { · 4 unmodified lines
116#define EQ_CTL_READY_MASK 0x80000000129#define EQ_CTL_READY_MASK 0x80000000
117 130
118#define EQ_CTL_EVENT_TYPE_SHIFT 16131#define EQ_CTL_EVENT_TYPE_SHIFT 16
119#define EQ_CTL_EVENT_TYPE_MASK 0x03FF0000132#define EQ_CTL_EVENT_TYPE_MASK 0x0FFF0000
120 133
121#define EQ_CTL_INDEX_SHIFT 0134#define EQ_CTL_INDEX_SHIFT 0
122#define EQ_CTL_INDEX_MASK 0x0000FFFF135#define EQ_CTL_INDEX_MASK 0x0000FFFF
@@ -300,7 +313,7 @@ enum pq_init_status { · 177 unmodified lines
300 * The packet's arguments specify the desired sensor and the field to313 * The packet's arguments specify the desired sensor and the field to
301 * set.314 * set.
302 *315 *
303 * CPUCP_PACKET_PCIE_THROUGHPUT_GET316 * CPUCP_PACKET_PCIE_THROUGHPUT_GET -
304 * Get throughput of PCIe.317 * Get throughput of PCIe.
305 * The packet's arguments specify the transaction direction (TX/RX).318 * The packet's arguments specify the transaction direction (TX/RX).
306 * The window measurement is 10[msec], and the return value is in KB/sec.319 * The window measurement is 10[msec], and the return value is in KB/sec.
@@ -309,19 +322,19 @@ enum pq_init_status { · 2 unmodified lines
309 * Replay count measures number of "replay" events, which is basicly322 * Replay count measures number of "replay" events, which is basicly
310 * number of retries done by PCIe.323 * number of retries done by PCIe.
311 *324 *
312 * CPUCP_PACKET_TOTAL_ENERGY_GET325 * CPUCP_PACKET_TOTAL_ENERGY_GET -
313 * Total Energy is measurement of energy from the time FW Linux326 * Total Energy is measurement of energy from the time FW Linux
314 * is loaded. It is calculated by multiplying the average power327 * is loaded. It is calculated by multiplying the average power
315 * by time (passed from armcp start). The units are in MilliJouls.328 * by time (passed from armcp start). The units are in MilliJouls.
316 *329 *
317 * CPUCP_PACKET_PLL_INFO_GET330 * CPUCP_PACKET_PLL_INFO_GET -
318 * Fetch frequencies of PLL from the required PLL IP.331 * Fetch frequencies of PLL from the required PLL IP.
319 * The packet's arguments specify the device PLL type332 * The packet's arguments specify the device PLL type
320 * Pll type is the PLL from device pll_index enum.333 * Pll type is the PLL from device pll_index enum.
321 * The result is composed of 4 outputs, each is 16-bit334 * The result is composed of 4 outputs, each is 16-bit
322 * frequency in MHz.335 * frequency in MHz.
323 *336 *
324 * CPUCP_PACKET_POWER_GET337 * CPUCP_PACKET_POWER_GET -
325 * Fetch the present power consumption of the device (Current * Voltage).338 * Fetch the present power consumption of the device (Current * Voltage).
326 *339 *
327 * CPUCP_PACKET_NIC_PFC_SET -340 * CPUCP_PACKET_NIC_PFC_SET -
@@ -345,6 +358,24 @@ enum pq_init_status { · 17 unmodified lines
345 * CPUCP_PACKET_MSI_INFO_SET -358 * CPUCP_PACKET_MSI_INFO_SET -
346 * set the index number for each supported msi type going from359 * set the index number for each supported msi type going from
347 * host to device360 * host to device
361 *
362 * CPUCP_PACKET_NIC_XPCS91_REGS_GET -
363 * Fetch the un/correctable counters values from the NIC MAC.
364 *
365 * CPUCP_PACKET_NIC_STAT_REGS_GET -
366 * Fetch various NIC MAC counters from the NIC STAT.
367 *
368 * CPUCP_PACKET_NIC_STAT_REGS_CLR -
369 * Clear the various NIC MAC counters in the NIC STAT.
370 *
371 * CPUCP_PACKET_NIC_STAT_REGS_ALL_GET -
372 * Fetch all NIC MAC counters from the NIC STAT.
373 *
374 * CPUCP_PACKET_IS_IDLE_CHECK -
375 * Check if the device is IDLE in regard to the DMA/compute engines
376 * and QMANs. The f/w will return a bitmask where each bit represents
377 * a different engine or QMAN according to enum cpucp_idle_mask.
378 * The bit will be 1 if the engine is NOT idle.
348 */379 */
349 380
350enum cpucp_packet_id {381enum cpucp_packet_id {
@@ -385,6 +416,11 @@ enum cpucp_packet_id { · 34 unmodified lines
385 CPUCP_PACKET_NIC_LPBK_SET, /* internal */416 CPUCP_PACKET_NIC_LPBK_SET, /* internal */
386 CPUCP_PACKET_NIC_MAC_CFG, /* internal */417 CPUCP_PACKET_NIC_MAC_CFG, /* internal */
387 CPUCP_PACKET_MSI_INFO_SET, /* internal */418 CPUCP_PACKET_MSI_INFO_SET, /* internal */
419 CPUCP_PACKET_NIC_XPCS91_REGS_GET, /* internal */
420 CPUCP_PACKET_NIC_STAT_REGS_GET, /* internal */
421 CPUCP_PACKET_NIC_STAT_REGS_CLR, /* internal */
422 CPUCP_PACKET_NIC_STAT_REGS_ALL_GET, /* internal */
423 CPUCP_PACKET_IS_IDLE_CHECK, /* internal */
388};424};
389 425
390#define CPUCP_PACKET_FENCE_VAL 0xFE8CE7A5426#define CPUCP_PACKET_FENCE_VAL 0xFE8CE7A5
@@ -414,6 +450,11 @@ enum cpucp_packet_id { · 23 unmodified lines
414#define CPUCP_PKT_VAL_LPBK_IN2_SHIFT 1450#define CPUCP_PKT_VAL_LPBK_IN2_SHIFT 1
415#define CPUCP_PKT_VAL_LPBK_IN2_MASK 0x000000000000001Eull451#define CPUCP_PKT_VAL_LPBK_IN2_MASK 0x000000000000001Eull
416 452
453#define CPUCP_PKT_VAL_MAC_CNT_IN1_SHIFT 0
454#define CPUCP_PKT_VAL_MAC_CNT_IN1_MASK 0x0000000000000001ull
455#define CPUCP_PKT_VAL_MAC_CNT_IN2_SHIFT 1
456#define CPUCP_PKT_VAL_MAC_CNT_IN2_MASK 0x00000000FFFFFFFEull
457
417/* heartbeat status bits */458/* heartbeat status bits */
418#define CPUCP_PKT_HB_STATUS_EQ_FAULT_SHIFT 0459#define CPUCP_PKT_HB_STATUS_EQ_FAULT_SHIFT 0
419#define CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK 0x00000001460#define CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK 0x00000001
@@ -467,7 +508,8 @@ struct cpucp_packet { · 47 unmodified lines
467 __le32 status_mask;508 __le32 status_mask;
468 };509 };
469 510
470 __le32 reserved;511 /* For NIC requests */
512 __le32 port_index;
471};513};
472 514
473struct cpucp_unmask_irq_arr_packet {515struct cpucp_unmask_irq_arr_packet {
@@ -476,6 +518,12 @@ struct cpucp_unmask_irq_arr_packet { · 2 unmodified lines
476 __le32 irqs[0];518 __le32 irqs[0];
477};519};
478 520
521struct cpucp_nic_status_packet {
522 struct cpucp_packet cpucp_pkt;
523 __le32 length;
524 __le32 data[0];
525};
526
479struct cpucp_array_data_packet {527struct cpucp_array_data_packet {
480 struct cpucp_packet cpucp_pkt;528 struct cpucp_packet cpucp_pkt;
481 __le32 length;529 __le32 length;
@@ -595,6 +643,18 @@ enum pll_index { · 113 unmodified lines
595 PLL_MAX643 PLL_MAX
596};644};
597 645
646enum rl_index {
647 TPC_RL = 0,
648 MME_RL,
649};
650
651enum pvt_index {
652 PVT_SW,
653 PVT_SE,
654 PVT_NW,
655 PVT_NE
656};
657
598/* Event Queue Packets */658/* Event Queue Packets */
599 659
600struct eq_generic_event {660struct eq_generic_event {
@@ -700,6 +760,15 @@ struct cpucp_mac_addr { · 99 unmodified lines
700 __u8 mac_addr[ETH_ALEN];760 __u8 mac_addr[ETH_ALEN];
701};761};
702 762
763enum cpucp_serdes_type {
764 TYPE_1_SERDES_TYPE,
765 TYPE_2_SERDES_TYPE,
766 HLS1_SERDES_TYPE,
767 HLS1H_SERDES_TYPE,
768 UNKNOWN_SERDES_TYPE,
769 MAX_NUM_SERDES_TYPE = UNKNOWN_SERDES_TYPE
770};
771
703struct cpucp_nic_info {772struct cpucp_nic_info {
704 struct cpucp_mac_addr mac_addrs[CPUCP_MAX_NICS];773 struct cpucp_mac_addr mac_addrs[CPUCP_MAX_NICS];
705 __le64 link_mask[CPUCP_NIC_MASK_ARR_LEN];774 __le64 link_mask[CPUCP_NIC_MASK_ARR_LEN];
@@ -708,6 +777,40 @@ struct cpucp_nic_info { · 2 unmodified lines
708 __le64 link_ext_mask[CPUCP_NIC_MASK_ARR_LEN];777 __le64 link_ext_mask[CPUCP_NIC_MASK_ARR_LEN];
709 __u8 qsfp_eeprom[CPUCP_NIC_QSFP_EEPROM_MAX_LEN];778 __u8 qsfp_eeprom[CPUCP_NIC_QSFP_EEPROM_MAX_LEN];
710 __le64 auto_neg_mask[CPUCP_NIC_MASK_ARR_LEN];779 __le64 auto_neg_mask[CPUCP_NIC_MASK_ARR_LEN];
780 __le16 serdes_type; /* enum cpucp_serdes_type */
781 __u8 reserved[6];
782};
783
784/*
785 * struct cpucp_nic_status - describes the status of a NIC port.
786 * @port: NIC port index.
787 * @bad_format_cnt: e.g. CRC.
788 * @responder_out_of_sequence_psn_cnt: e.g NAK.
789 * @high_ber_reinit_cnt: link reinit due to high BER.
790 * @correctable_err_cnt: e.g. bit-flip.
791 * @uncorrectable_err_cnt: e.g. MAC errors.
792 * @retraining_cnt: re-training counter.
793 * @up: is port up.
794 * @pcs_link: has PCS link.
795 * @phy_ready: is PHY ready.
796 * @auto_neg: is Autoneg enabled.
797 * @timeout_retransmission_cnt: timeout retransmission events
798 * @high_ber_cnt: high ber events
799 */
800struct cpucp_nic_status {
801 __le32 port;
802 __le32 bad_format_cnt;
803 __le32 responder_out_of_sequence_psn_cnt;
804 __le32 high_ber_reinit;
805 __le32 correctable_err_cnt;
806 __le32 uncorrectable_err_cnt;
807 __le32 retraining_cnt;
808 __u8 up;
809 __u8 pcs_link;
810 __u8 phy_ready;
811 __u8 auto_neg;
812 __le32 timeout_retransmission_cnt;
813 __le32 high_ber_cnt;
711};814};
712 815
713#endif /* CPUCP_IF_H */816#endif /* CPUCP_IF_H */
drivers/misc/habanalabs/include/common/hl_boot_if.h+57−5
@@ -78,6 +78,26 @@ · 77 unmodified lines
78 * CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL Device is unusable and customer support78 * CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL Device is unusable and customer support
79 * should be contacted.79 * should be contacted.
80 *80 *
81 * CPU_BOOT_ERR0_ARC0_HALT_ACK_NOT_RCVD HALT ACK from ARC0 is not received
82 * within specified retries after issuing
83 * HALT request. ARC0 appears to be in bad
84 * reset.
85 *
86 * CPU_BOOT_ERR0_ARC1_HALT_ACK_NOT_RCVD HALT ACK from ARC1 is not received
87 * within specified retries after issuing
88 * HALT request. ARC1 appears to be in bad
89 * reset.
90 *
91 * CPU_BOOT_ERR0_ARC0_RUN_ACK_NOT_RCVD RUN ACK from ARC0 is not received
92 * within specified timeout after issuing
93 * RUN request. ARC0 appears to be in bad
94 * reset.
95 *
96 * CPU_BOOT_ERR0_ARC1_RUN_ACK_NOT_RCVD RUN ACK from ARC1 is not received
97 * within specified timeout after issuing
98 * RUN request. ARC1 appears to be in bad
99 * reset.
100 *
81 * CPU_BOOT_ERR0_ENABLED Error registers enabled.101 * CPU_BOOT_ERR0_ENABLED Error registers enabled.
82 * This is a main indication that the102 * This is a main indication that the
83 * running FW populates the error103 * running FW populates the error
@@ -98,6 +118,10 @@ · 14 unmodified lines
98#define CPU_BOOT_ERR0_SEC_IMG_VER_FAIL (1 << 11)118#define CPU_BOOT_ERR0_SEC_IMG_VER_FAIL (1 << 11)
99#define CPU_BOOT_ERR0_PLL_FAIL (1 << 12)119#define CPU_BOOT_ERR0_PLL_FAIL (1 << 12)
100#define CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL (1 << 13)120#define CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL (1 << 13)
121#define CPU_BOOT_ERR0_ARC0_HALT_ACK_NOT_RCVD (1 << 14)
122#define CPU_BOOT_ERR0_ARC1_HALT_ACK_NOT_RCVD (1 << 15)
123#define CPU_BOOT_ERR0_ARC0_RUN_ACK_NOT_RCVD (1 << 16)
124#define CPU_BOOT_ERR0_ARC1_RUN_ACK_NOT_RCVD (1 << 17)
101#define CPU_BOOT_ERR0_ENABLED (1 << 31)125#define CPU_BOOT_ERR0_ENABLED (1 << 31)
102#define CPU_BOOT_ERR1_ENABLED (1 << 31)126#define CPU_BOOT_ERR1_ENABLED (1 << 31)
103 127
@@ -186,6 +210,10 @@ · 82 unmodified lines
186 * configured and is ready for use.210 * configured and is ready for use.
187 * Initialized in: ppboot211 * Initialized in: ppboot
188 *212 *
213 * CPU_BOOT_DEV_STS0_FW_NIC_MAC_EN NIC MAC channels init is done by FW and
214 * any access to them is done via the FW.
215 * Initialized in: linux
216 *
189 * CPU_BOOT_DEV_STS0_DYN_PLL_EN Dynamic PLL configuration is enabled.217 * CPU_BOOT_DEV_STS0_DYN_PLL_EN Dynamic PLL configuration is enabled.
190 * FW sends to host a bitmap of supported218 * FW sends to host a bitmap of supported
191 * PLLs.219 * PLLs.
@@ -209,6 +237,21 @@ · 17 unmodified lines
209 * prevent IRQs overriding each other.237 * prevent IRQs overriding each other.
210 * Initialized in: linux238 * Initialized in: linux
211 *239 *
240 * CPU_BOOT_DEV_STS0_FW_NIC_STAT_XPCS91_EN
241 * NIC STAT and XPCS91 access is restricted
242 * and is done via FW only.
243 * Initialized in: linux
244 *
245 * CPU_BOOT_DEV_STS0_FW_NIC_STAT_EXT_EN
246 * NIC STAT get all is supported.
247 * Initialized in: linux
248 *
249 * CPU_BOOT_DEV_STS0_IS_IDLE_CHECK_EN
250 * F/W checks if the device is idle by reading defined set
251 * of registers. It returns a bitmask of all the engines,
252 * where a bit is set if the engine is not idle.
253 * Initialized in: linux
254 *
212 * CPU_BOOT_DEV_STS0_ENABLED Device status register enabled.255 * CPU_BOOT_DEV_STS0_ENABLED Device status register enabled.
213 * This is a main indication that the256 * This is a main indication that the
214 * running FW populates the device status257 * running FW populates the device status
@@ -236,10 +279,14 @@ · 21 unmodified lines
236#define CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN (1 << 15)279#define CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN (1 << 15)
237#define CPU_BOOT_DEV_STS0_FW_LD_COM_EN (1 << 16)280#define CPU_BOOT_DEV_STS0_FW_LD_COM_EN (1 << 16)
238#define CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN (1 << 17)281#define CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN (1 << 17)
282#define CPU_BOOT_DEV_STS0_FW_NIC_MAC_EN (1 << 18)
239#define CPU_BOOT_DEV_STS0_DYN_PLL_EN (1 << 19)283#define CPU_BOOT_DEV_STS0_DYN_PLL_EN (1 << 19)
240#define CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN (1 << 20)284#define CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN (1 << 20)
241#define CPU_BOOT_DEV_STS0_EQ_INDEX_EN (1 << 21)285#define CPU_BOOT_DEV_STS0_EQ_INDEX_EN (1 << 21)
242#define CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN (1 << 22)286#define CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN (1 << 22)
287#define CPU_BOOT_DEV_STS0_FW_NIC_STAT_XPCS91_EN (1 << 23)
288#define CPU_BOOT_DEV_STS0_FW_NIC_STAT_EXT_EN (1 << 24)
289#define CPU_BOOT_DEV_STS0_IS_IDLE_CHECK_EN (1 << 25)
243#define CPU_BOOT_DEV_STS0_ENABLED (1 << 31)290#define CPU_BOOT_DEV_STS0_ENABLED (1 << 31)
244#define CPU_BOOT_DEV_STS1_ENABLED (1 << 31)291#define CPU_BOOT_DEV_STS1_ENABLED (1 << 31)
245 292
@@ -313,10 +360,7 @@ struct cpu_dyn_regs { · 67 unmodified lines
313 __le32 hw_state;360 __le32 hw_state;
314 __le32 kmd_msg_to_cpu;361 __le32 kmd_msg_to_cpu;
315 __le32 cpu_cmd_status_to_host;362 __le32 cpu_cmd_status_to_host;
316 union {363 __le32 gic_host_pi_upd_irq;
317 __le32 gic_host_irq_ctrl;
318 __le32 gic_host_pi_upd_irq;
319 };
320 __le32 gic_tpc_qm_irq_ctrl;364 __le32 gic_tpc_qm_irq_ctrl;
321 __le32 gic_mme_qm_irq_ctrl;365 __le32 gic_mme_qm_irq_ctrl;
322 __le32 gic_dma_qm_irq_ctrl;366 __le32 gic_dma_qm_irq_ctrl;
@@ -324,7 +368,9 @@ struct cpu_dyn_regs { · 1 unmodified line
324 __le32 gic_dma_core_irq_ctrl;368 __le32 gic_dma_core_irq_ctrl;
325 __le32 gic_host_halt_irq;369 __le32 gic_host_halt_irq;
326 __le32 gic_host_ints_irq;370 __le32 gic_host_ints_irq;
327 __le32 reserved1[24]; /* reserve for future use */371 __le32 gic_host_soft_rst_irq;
372 __le32 gic_rot_qm_irq_ctrl;
373 __le32 reserved1[22]; /* reserve for future use */
328};374};
329 375
330/* TODO: remove the desc magic after the code is updated to use message */376/* TODO: remove the desc magic after the code is updated to use message */
@@ -462,6 +508,11 @@ struct lkd_fw_comms_msg { · 131 unmodified lines
462 * Do not wait for BMC response.508 * Do not wait for BMC response.
463 *509 *
464 * COMMS_LOW_PLL_OPP Initialize PLLs for low OPP.510 * COMMS_LOW_PLL_OPP Initialize PLLs for low OPP.
511 *
512 * COMMS_PREP_DESC_ELBI Same as COMMS_PREP_DESC only that the memory
513 * space is allocated in a ELBI access only
514 * address range.
515 *
465 */516 */
466enum comms_cmd {517enum comms_cmd {
467 COMMS_NOOP = 0,518 COMMS_NOOP = 0,
@@ -474,6 +525,7 @@ enum comms_cmd { · 6 unmodified lines
474 COMMS_GOTO_WFE = 7,525 COMMS_GOTO_WFE = 7,
475 COMMS_SKIP_BMC = 8,526 COMMS_SKIP_BMC = 8,
476 COMMS_LOW_PLL_OPP = 9,527 COMMS_LOW_PLL_OPP = 9,
528 COMMS_PREP_DESC_ELBI = 10,
477 COMMS_INVLD_LAST529 COMMS_INVLD_LAST
478};530};
479 531
drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_regs.h+3−0
@@ -126,6 +126,9 @@ · 125 unmodified lines
126#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_1 0x4F2004126#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_1 0x4F2004
127#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_2047 0x4F3FFC127#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_2047 0x4F3FFC
128#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 0x4F4000128#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 0x4F4000
129#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 0x4F4800
130#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_DATA_0 0x4F5000
131#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_ARM_0 0x4F5800
129#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0 0x4F6000132#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0 0x4F6000
130#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_511 0x4F67FC133#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_511 0x4F67FC
131 134
drivers/misc/habanalabs/include/gaudi/gaudi_masks.h+17−0
@@ -449,4 +449,21 @@ enum axi_id { · 448 unmodified lines
449#define PCIE_AUX_FLR_CTRL_HW_CTRL_MASK 0x1449#define PCIE_AUX_FLR_CTRL_HW_CTRL_MASK 0x1
450#define PCIE_AUX_FLR_CTRL_INT_MASK_MASK 0x2450#define PCIE_AUX_FLR_CTRL_INT_MASK_MASK 0x2
451 451
452#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_VALID_SHIFT 0
453#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_VALID_MASK 0x1
454#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_PENDING_SHIFT 1
455#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_PENDING_MASK 0x1FE
456#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SID_SHIFT 0
457#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SID_MASK 0xFF
458#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_MASK_SHIFT 8
459#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_MASK_MASK 0xFF00
460#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SOP_SHIFT 16
461#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SOP_MASK 0x10000
462#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SOD_SHIFT 17
463#define SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SOD_MASK 0xFFFE0000
464#define TPC0_QM_CP_STS_0_FENCE_ID_SHIFT 20
465#define TPC0_QM_CP_STS_0_FENCE_ID_MASK 0x300000
466#define TPC0_QM_CP_STS_0_FENCE_IN_PROGRESS_SHIFT 22
467#define TPC0_QM_CP_STS_0_FENCE_IN_PROGRESS_MASK 0x400000
468
452#endif /* GAUDI_MASKS_H_ */469#endif /* GAUDI_MASKS_H_ */
drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h+0−2
@@ -12,8 +12,6 @@ · 11 unmodified lines
12 * PSOC scratch-pad registers12 * PSOC scratch-pad registers
13 */13 */
14#define mmHW_STATE mmPSOC_GLOBAL_CONF_SCRATCHPAD_014#define mmHW_STATE mmPSOC_GLOBAL_CONF_SCRATCHPAD_0
15/* TODO: remove mmGIC_HOST_IRQ_CTRL_POLL_REG */
16#define mmGIC_HOST_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_1
17#define mmGIC_HOST_PI_UPD_IRQ_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_115#define mmGIC_HOST_PI_UPD_IRQ_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_1
18#define mmGIC_TPC_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_216#define mmGIC_TPC_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_2
19#define mmGIC_MME_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_317#define mmGIC_MME_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_3
include/uapi/misc/habanalabs.h+166−20
@@ -276,7 +276,17 @@ enum hl_device_status { · 275 unmodified lines
276 HL_DEVICE_STATUS_OPERATIONAL,276 HL_DEVICE_STATUS_OPERATIONAL,
277 HL_DEVICE_STATUS_IN_RESET,277 HL_DEVICE_STATUS_IN_RESET,
278 HL_DEVICE_STATUS_MALFUNCTION,278 HL_DEVICE_STATUS_MALFUNCTION,
279 HL_DEVICE_STATUS_NEEDS_RESET279 HL_DEVICE_STATUS_NEEDS_RESET,
280 HL_DEVICE_STATUS_IN_DEVICE_CREATION,
281 HL_DEVICE_STATUS_LAST = HL_DEVICE_STATUS_IN_DEVICE_CREATION
282};
283
284enum hl_server_type {
285 HL_SERVER_TYPE_UNKNOWN = 0,
286 HL_SERVER_GAUDI_HLS1 = 1,
287 HL_SERVER_GAUDI_HLS1H = 2,
288 HL_SERVER_GAUDI_TYPE1 = 3,
289 HL_SERVER_GAUDI_TYPE2 = 4
280};290};
281 291
282/* Opcode for management ioctl292/* Opcode for management ioctl
@@ -337,17 +347,49 @@ enum hl_device_status { · 54 unmodified lines
337#define HL_INFO_VERSION_MAX_LEN 128347#define HL_INFO_VERSION_MAX_LEN 128
338#define HL_INFO_CARD_NAME_MAX_LEN 16348#define HL_INFO_CARD_NAME_MAX_LEN 16
339 349
350/**
351 * struct hl_info_hw_ip_info - hardware information on various IPs in the ASIC
352 * @sram_base_address: The first SRAM physical base address that is free to be
353 * used by the user.
354 * @dram_base_address: The first DRAM virtual or physical base address that is
355 * free to be used by the user.
356 * @dram_size: The DRAM size that is available to the user.
357 * @sram_size: The SRAM size that is available to the user.
358 * @num_of_events: The number of events that can be received from the f/w. This
359 * is needed so the user can what is the size of the h/w events
360 * array he needs to pass to the kernel when he wants to fetch
361 * the event counters.
362 * @device_id: PCI device ID of the ASIC.
363 * @module_id: Module ID of the ASIC for mezzanine cards in servers
364 * (From OCP spec).
365 * @first_available_interrupt_id: The first available interrupt ID for the user
366 * to be used when it works with user interrupts.
367 * @server_type: Server type that the Gaudi ASIC is currently installed in.
368 * The value is according to enum hl_server_type
369 * @cpld_version: CPLD version on the board.
370 * @psoc_pci_pll_nr: PCI PLL NR value. Needed by the profiler in some ASICs.
371 * @psoc_pci_pll_nf: PCI PLL NF value. Needed by the profiler in some ASICs.
372 * @psoc_pci_pll_od: PCI PLL OD value. Needed by the profiler in some ASICs.
373 * @psoc_pci_pll_div_factor: PCI PLL DIV factor value. Needed by the profiler
374 * in some ASICs.
375 * @tpc_enabled_mask: Bit-mask that represents which TPCs are enabled. Relevant
376 * for Goya/Gaudi only.
377 * @dram_enabled: Whether the DRAM is enabled.
378 * @cpucp_version: The CPUCP f/w version.
379 * @card_name: The card name as passed by the f/w.
380 * @dram_page_size: The DRAM physical page size.
381 */
340struct hl_info_hw_ip_info {382struct hl_info_hw_ip_info {
341 __u64 sram_base_address;383 __u64 sram_base_address;
342 __u64 dram_base_address;384 __u64 dram_base_address;
343 __u64 dram_size;385 __u64 dram_size;
344 __u32 sram_size;386 __u32 sram_size;
345 __u32 num_of_events;387 __u32 num_of_events;
346 __u32 device_id; /* PCI Device ID */388 __u32 device_id;
347 __u32 module_id; /* For mezzanine cards in servers (From OCP spec.) */389 __u32 module_id;
348 __u32 reserved;390 __u32 reserved;
349 __u16 first_available_interrupt_id;391 __u16 first_available_interrupt_id;
350 __u16 reserved2;392 __u16 server_type;
351 __u32 cpld_version;393 __u32 cpld_version;
352 __u32 psoc_pci_pll_nr;394 __u32 psoc_pci_pll_nr;
353 __u32 psoc_pci_pll_nf;395 __u32 psoc_pci_pll_nf;
@@ -358,7 +400,7 @@ struct hl_info_hw_ip_info { · 4 unmodified lines
358 __u8 pad[2];400 __u8 pad[2];
359 __u8 cpucp_version[HL_INFO_VERSION_MAX_LEN];401 __u8 cpucp_version[HL_INFO_VERSION_MAX_LEN];
360 __u8 card_name[HL_INFO_CARD_NAME_MAX_LEN];402 __u8 card_name[HL_INFO_CARD_NAME_MAX_LEN];
361 __u64 reserved3;403 __u64 reserved2;
362 __u64 dram_page_size;404 __u64 dram_page_size;
363};405};
364 406
@@ -628,12 +670,21 @@ struct hl_cs_chunk { · 263 unmodified lines
628 __u64 cb_handle;670 __u64 cb_handle;
629 671
630 /* Relevant only when HL_CS_FLAGS_WAIT or672 /* Relevant only when HL_CS_FLAGS_WAIT or
631 * HL_CS_FLAGS_COLLECTIVE_WAIT is set.673 * HL_CS_FLAGS_COLLECTIVE_WAIT is set
632 * This holds address of array of u64 values that contain674 * This holds address of array of u64 values that contain
633 * signal CS sequence numbers. The wait described by this job675 * signal CS sequence numbers. The wait described by
634 * will listen on all those signals (wait event per signal)676 * this job will listen on all those signals
677 * (wait event per signal)
635 */678 */
636 __u64 signal_seq_arr;679 __u64 signal_seq_arr;
680
681 /*
682 * Relevant only when HL_CS_FLAGS_WAIT or
683 * HL_CS_FLAGS_COLLECTIVE_WAIT is set
684 * along with HL_CS_FLAGS_ENCAP_SIGNALS.
685 * This is the CS sequence which has the encapsulated signals.
686 */
687 __u64 encaps_signal_seq;
637 };688 };
638 689
639 /* Index of queue to put the CB on */690 /* Index of queue to put the CB on */
@@ -651,6 +702,17 @@ struct hl_cs_chunk { · 11 unmodified lines
651 * Number of entries in signal_seq_arr702 * Number of entries in signal_seq_arr
652 */703 */
653 __u32 num_signal_seq_arr;704 __u32 num_signal_seq_arr;
705
706 /* Relevant only when HL_CS_FLAGS_WAIT or
707 * HL_CS_FLAGS_COLLECTIVE_WAIT is set along
708 * with HL_CS_FLAGS_ENCAP_SIGNALS
709 * This set the signals range that the user want to wait for
710 * out of the whole reserved signals range.
711 * e.g if the signals range is 20, and user don't want
712 * to wait for signal 8, so he set this offset to 7, then
713 * he call the API again with 9 and so on till 20.
714 */
715 __u32 encaps_signal_offset;
654 };716 };
655 717
656 /* HL_CS_CHUNK_FLAGS_* */718 /* HL_CS_CHUNK_FLAGS_* */
@@ -678,6 +740,28 @@ struct hl_cs_chunk { · 21 unmodified lines
678#define HL_CS_FLAGS_CUSTOM_TIMEOUT 0x200740#define HL_CS_FLAGS_CUSTOM_TIMEOUT 0x200
679#define HL_CS_FLAGS_SKIP_RESET_ON_TIMEOUT 0x400741#define HL_CS_FLAGS_SKIP_RESET_ON_TIMEOUT 0x400
680 742
743/*
744 * The encapsulated signals CS is merged into the existing CS ioctls.
745 * In order to use this feature need to follow the below procedure:
746 * 1. Reserve signals, set the CS type to HL_CS_FLAGS_RESERVE_SIGNALS_ONLY
747 * the output of this API will be the SOB offset from CFG_BASE.
748 * this address will be used to patch CB cmds to do the signaling for this
749 * SOB by incrementing it's value.
750 * for reverting the reservation use HL_CS_FLAGS_UNRESERVE_SIGNALS_ONLY
751 * CS type, note that this might fail if out-of-sync happened to the SOB
752 * value, in case other signaling request to the same SOB occurred between
753 * reserve-unreserve calls.
754 * 2. Use the staged CS to do the encapsulated signaling jobs.
755 * use HL_CS_FLAGS_STAGED_SUBMISSION and HL_CS_FLAGS_STAGED_SUBMISSION_FIRST
756 * along with HL_CS_FLAGS_ENCAP_SIGNALS flag, and set encaps_signal_offset
757 * field. This offset allows app to wait on part of the reserved signals.
758 * 3. Use WAIT/COLLECTIVE WAIT CS along with HL_CS_FLAGS_ENCAP_SIGNALS flag
759 * to wait for the encapsulated signals.
760 */
761#define HL_CS_FLAGS_ENCAP_SIGNALS 0x800
762#define HL_CS_FLAGS_RESERVE_SIGNALS_ONLY 0x1000
763#define HL_CS_FLAGS_UNRESERVE_SIGNALS_ONLY 0x2000
764
681#define HL_CS_STATUS_SUCCESS 0765#define HL_CS_STATUS_SUCCESS 0
682 766
683#define HL_MAX_JOBS_PER_CS 512767#define HL_MAX_JOBS_PER_CS 512
@@ -690,10 +774,35 @@ struct hl_cs_in { · 6 unmodified lines
690 /* holds address of array of hl_cs_chunk for execution phase */774 /* holds address of array of hl_cs_chunk for execution phase */
691 __u64 chunks_execute;775 __u64 chunks_execute;
692 776
693 /* Sequence number of a staged submission CS777 union {
694 * valid only if HL_CS_FLAGS_STAGED_SUBMISSION is set778 /*
695 */779 * Sequence number of a staged submission CS
696 __u64 seq;780 * valid only if HL_CS_FLAGS_STAGED_SUBMISSION is set and
781 * HL_CS_FLAGS_STAGED_SUBMISSION_FIRST is unset.
782 */
783 __u64 seq;
784
785 /*
786 * Encapsulated signals handle id
787 * Valid for two flows:
788 * 1. CS with encapsulated signals:
789 * when HL_CS_FLAGS_STAGED_SUBMISSION and
790 * HL_CS_FLAGS_STAGED_SUBMISSION_FIRST
791 * and HL_CS_FLAGS_ENCAP_SIGNALS are set.
792 * 2. unreserve signals:
793 * valid when HL_CS_FLAGS_UNRESERVE_SIGNALS_ONLY is set.
794 */
795 __u32 encaps_sig_handle_id;
796
797 /* Valid only when HL_CS_FLAGS_RESERVE_SIGNALS_ONLY is set */
798 struct {
799 /* Encapsulated signals number */
800 __u32 encaps_signals_count;
801
802 /* Encapsulated signals queue index (stream) */
803 __u32 encaps_signals_q_idx;
804 };
805 };
697 806
698 /* Number of chunks in restore phase array. Maximum number is807 /* Number of chunks in restore phase array. Maximum number is
699 * HL_MAX_JOBS_PER_CS808 * HL_MAX_JOBS_PER_CS
@@ -718,14 +827,31 @@ struct hl_cs_in { · 18 unmodified lines
718};827};
719 828
720struct hl_cs_out {829struct hl_cs_out {
830 union {
831 /*
832 * seq holds the sequence number of the CS to pass to wait
833 * ioctl. All values are valid except for 0 and ULLONG_MAX
834 */
835 __u64 seq;
836
837 /* Valid only when HL_CS_FLAGS_RESERVE_SIGNALS_ONLY is set */
838 struct {
839 /* This is the resereved signal handle id */
840 __u32 handle_id;
841
842 /* This is the signals count */
843 __u32 count;
844 };
845 };
846
847 /* HL_CS_STATUS */
848 __u32 status;
849
721 /*850 /*
722 * seq holds the sequence number of the CS to pass to wait ioctl. All851 * SOB base address offset
723 * values are valid except for 0 and ULLONG_MAX852 * Valid only when HL_CS_FLAGS_RESERVE_SIGNALS_ONLY is set
724 */853 */
725 __u64 seq;854 __u32 sob_base_addr_offset;
726 /* HL_CS_STATUS_* */
727 __u32 status;
728 __u32 pad;
729};855};
730 856
731union hl_cs_args {857union hl_cs_args {
@@ -735,11 +861,18 @@ union hl_cs_args { · 3 unmodified lines
735 861
736#define HL_WAIT_CS_FLAGS_INTERRUPT 0x2862#define HL_WAIT_CS_FLAGS_INTERRUPT 0x2
737#define HL_WAIT_CS_FLAGS_INTERRUPT_MASK 0xFFF00000863#define HL_WAIT_CS_FLAGS_INTERRUPT_MASK 0xFFF00000
864#define HL_WAIT_CS_FLAGS_MULTI_CS 0x4
865
866#define HL_WAIT_MULTI_CS_LIST_MAX_LEN 32
738 867
739struct hl_wait_cs_in {868struct hl_wait_cs_in {
740 union {869 union {
741 struct {870 struct {
742 /* Command submission sequence number */871 /*
872 * In case of wait_cs holds the CS sequence number.
873 * In case of wait for multi CS hold a user pointer to
874 * an array of CS sequence numbers
875 */
743 __u64 seq;876 __u64 seq;
744 /* Absolute timeout to wait for command submission877 /* Absolute timeout to wait for command submission
745 * in microseconds878 * in microseconds
@@ -767,12 +900,17 @@ struct hl_wait_cs_in { · 21 unmodified lines
767 900
768 /* Context ID - Currently not in use */901 /* Context ID - Currently not in use */
769 __u32 ctx_id;902 __u32 ctx_id;
903
770 /* HL_WAIT_CS_FLAGS_*904 /* HL_WAIT_CS_FLAGS_*
771 * If HL_WAIT_CS_FLAGS_INTERRUPT is set, this field should include905 * If HL_WAIT_CS_FLAGS_INTERRUPT is set, this field should include
772 * interrupt id according to HL_WAIT_CS_FLAGS_INTERRUPT_MASK, in order906 * interrupt id according to HL_WAIT_CS_FLAGS_INTERRUPT_MASK, in order
773 * not to specify an interrupt id ,set mask to all 1s.907 * not to specify an interrupt id ,set mask to all 1s.
774 */908 */
775 __u32 flags;909 __u32 flags;
910
911 /* Multi CS API info- valid entries in multi-CS array */
912 __u8 seq_arr_len;
913 __u8 pad[7];
776};914};
777 915
778#define HL_WAIT_CS_STATUS_COMPLETED 0916#define HL_WAIT_CS_STATUS_COMPLETED 0
@@ -789,8 +927,15 @@ struct hl_wait_cs_out { · 10 unmodified lines
789 __u32 status;927 __u32 status;
790 /* HL_WAIT_CS_STATUS_FLAG* */928 /* HL_WAIT_CS_STATUS_FLAG* */
791 __u32 flags;929 __u32 flags;
792 /* valid only if HL_WAIT_CS_STATUS_FLAG_TIMESTAMP_VLD is set */930 /*
931 * valid only if HL_WAIT_CS_STATUS_FLAG_TIMESTAMP_VLD is set
932 * for wait_cs: timestamp of CS completion
933 * for wait_multi_cs: timestamp of FIRST CS completion
934 */
793 __s64 timestamp_nsec;935 __s64 timestamp_nsec;
936 /* multi CS completion bitmap */
937 __u32 cs_completion_map;
938 __u32 pad;
794};939};
795 940
796union hl_wait_cs_args {941union hl_wait_cs_args {
@@ -813,6 +958,7 @@ union hl_wait_cs_args { · 16 unmodified lines
813#define HL_MEM_CONTIGUOUS 0x1958#define HL_MEM_CONTIGUOUS 0x1
814#define HL_MEM_SHARED 0x2959#define HL_MEM_SHARED 0x2
815#define HL_MEM_USERPTR 0x4960#define HL_MEM_USERPTR 0x4
961#define HL_MEM_FORCE_HINT 0x8
816 962
817struct hl_mem_in {963struct hl_mem_in {
818 union {964 union {