accel/amdxdna: Add command doorbell and wait support

Expose the command doorbell register to userspace on a per-hardware context basis, enabling applications to notify the firmware of pending commands via doorbell writes. Introduce DRM_IOCTL_AMDXDNA_WAIT_CMD to allow userspace to wait for completion of individual commands.

David Zhang committed 6 months ago

Commit 91f4da81 parent 193612711 files · +198 −7Browse files

Unified · Split

11 files

drivers/accel/amdxdna/aie4_ctx.c +75 −0
drivers/accel/amdxdna/aie4_host_queue.h +2 −0
drivers/accel/amdxdna/aie4_pci.c +34 −0
drivers/accel/amdxdna/aie4_pci.h +3 −0
drivers/accel/amdxdna/amdxdna_ctx.c +34 −0
drivers/accel/amdxdna/amdxdna_ctx.h +1 −3
drivers/accel/amdxdna/amdxdna_gem.c +3 −2
drivers/accel/amdxdna/amdxdna_pci_drv.c +17 −1
drivers/accel/amdxdna/amdxdna_pci_drv.h +3 −0
drivers/accel/amdxdna/npu3_regs.c +5 −0
include/uapi/drm/amdxdna_accel.h +21 −1

drivers/accel/amdxdna/aie4_ctx.c+75−0

@@ -256,3 +256,78 @@ void aie4_hwctx_fini(struct amdxdna_hwctx *hwctx) · 255 unmodified lines
256	256	aie4_hwctx_umq_fini(hwctx);
257	257	kfree(hwctx->priv);
258	258	}
	259	+
	260	+static inline bool valid_queue_index(u64 read, u64 write, u32 capacity)
	261	+{
	262	+ return (write >= read) && ((write - read) <= capacity);
	263	+}
	264	+
	265	+static u64 get_read_index(struct amdxdna_hwctx *hwctx)
	266	+{
	267	+ u64 wi = READ_ONCE(*hwctx->priv->umq_write_index);
	268	+ u64 ri = READ_ONCE(*hwctx->priv->umq_read_index);
	269	+ struct amdxdna_dev *xdna = hwctx->client->xdna;
	270	+
	271	+ /*
	272	+ * CERT cannot update read index as uint64 atomically. Driver may read
	273	+ * half-updated read index when it has bits in high 32bit. In case read
	274	+ * index is not valid, wait for some time and retry once. It should
	275	+ * allow CERT to complete the read index update.
	276	+ */
	277	+ if (!valid_queue_index(ri, wi, CTX_MAX_CMDS)) {
	278	+ XDNA_WARN(xdna, "Invalid index, ri %llu, wi %llu", ri, wi);
	279	+ usleep_range(100, 200);
	280	+ ri = READ_ONCE(*hwctx->priv->umq_read_index);
	281	+ if (!valid_queue_index(ri, wi, CTX_MAX_CMDS)) {
	282	+ XDNA_ERR(xdna, "Invalid index after retry, ri %llu, wi %llu", ri, wi);
	283	+ ri = 0;
	284	+ }
	285	+ }
	286	+
	287	+ return ri;
	288	+}
	289	+
	290	+static inline bool check_cmd_done(struct amdxdna_hwctx *hwctx, u64 seq)
	291	+{
	292	+ u64 read_idx = get_read_index(hwctx);
	293	+
	294	+ return read_idx > seq;
	295	+}
	296	+
	297	+int aie4_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq, u32 timeout)
	298	+{
	299	+ unsigned long wait_jifs = MAX_SCHEDULE_TIMEOUT;
	300	+ struct amdxdna_hwctx_priv *priv = hwctx->priv;
	301	+ struct cert_comp *cert_comp = priv->cert_comp;
	302	+ long ret;
	303	+
	304	+ if (timeout)
	305	+ wait_jifs = msecs_to_jiffies(timeout);
	306	+
	307	+ ret = wait_event_interruptible_timeout(cert_comp->waitq,
	308	+ (check_cmd_done(hwctx, seq)),
	309	+ wait_jifs);
	310	+
	311	+ if (!ret)
	312	+ ret = -ETIME;
	313	+
	314	+ return ret <= 0 ? ret : 0;
	315	+}
	316	+
	317	+int aie4_hwctx_valid_doorbell(struct amdxdna_client *client, u32 vm_pgoff)
	318	+{
	319	+ struct amdxdna_hwctx *hwctx;
	320	+ unsigned long hwctx_id;
	321	+ int idx;
	322	+
	323	+ idx = srcu_read_lock(&client->hwctx_srcu);
	324	+ amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
	325	+ if (vm_pgoff == (hwctx->doorbell_offset >> PAGE_SHIFT)) {
	326	+ srcu_read_unlock(&client->hwctx_srcu, idx);
	327	+ return 1;
	328	+ }
	329	+ }
	330	+ srcu_read_unlock(&client->hwctx_srcu, idx);
	331	+
	332	+ return 0;
	333	+}

drivers/accel/amdxdna/aie4_host_queue.h+2−0

@@ -8,6 +8,8 @@ · 7 unmodified lines
8	8
9	9	#include <linux/types.h>
10	10
	11	+#define CTX_MAX_CMDS 32
	12	+
11	13	struct host_queue_header {
12	14	__u64 read_index;
13	15	struct {

drivers/accel/amdxdna/aie4_pci.c+34−0

@@ -503,6 +503,38 @@ static int aie4m_pcidev_init(struct amdxdna_dev *xdna) · 502 unmodified lines
503	503	return 0;
504	504	}
505	505
	506	+static int aie4_doorbell_mmap(struct amdxdna_client client, struct vm_area_struct vma)
	507	+{
	508	+ struct amdxdna_dev *xdna = client->xdna;
	509	+ struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
	510	+ const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv;
	511	+ phys_addr_t res_start;
	512	+ unsigned long pfn;
	513	+ int ret;
	514	+
	515	+ if (!aie4_hwctx_valid_doorbell(client, vma->vm_pgoff)) {
	516	+ XDNA_ERR(xdna, "Invalid doorbell page offset 0x%lx", vma->vm_pgoff);
	517	+ return -EINVAL;
	518	+ }
	519	+
	520	+ if (vma_pages(vma) != 1) {
	521	+ XDNA_ERR(xdna, "can only map one page, got %ld", vma_pages(vma));
	522	+ return -EINVAL;
	523	+ }
	524	+
	525	+ res_start = pci_resource_start(pdev, xdna->dev_info->doorbell_bar) + npriv->doorbell_off;
	526	+ pfn = PHYS_PFN(res_start) + vma->vm_pgoff;
	527	+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
	528	+ vm_flags_set(vma, VM_IO \| VM_DONTEXPAND \| VM_DONTDUMP);
	529	+ ret = io_remap_pfn_range(vma, vma->vm_start,
	530	+ pfn,
	531	+ PAGE_SIZE,
	532	+ vma->vm_page_prot);
	533	+
	534	+ XDNA_DBG(xdna, "doorbell ret %d", ret);
	535	+ return ret;
	536	+}
	537	+
506	538	static int aie4_pf_init(struct amdxdna_dev *xdna)
507	539	{
508	540	int ret;
@@ -547,4 +579,6 @@ const struct amdxdna_dev_ops aie4_vf_ops = { · 38 unmodified lines
547	579	.fini = aie4_vf_fini,
548	580	.hwctx_init = aie4_hwctx_init,
549	581	.hwctx_fini = aie4_hwctx_fini,
	582	+ .mmap = aie4_doorbell_mmap,
	583	+ .cmd_wait = aie4_cmd_wait,
550	584	};

drivers/accel/amdxdna/aie4_pci.h+3−0

@@ -36,6 +36,7 @@ struct amdxdna_dev_priv { · 35 unmodified lines
36	36	u32 mbox_bar;
37	37	u32 mbox_rbuf_bar;
38	38	u64 mbox_info_off;
	39	+ u32 doorbell_off;
39	40
40	41	struct aie_bar_off_pair psp_regs_off[PSP_MAX_REGS];
41	42	struct aie_bar_off_pair smu_regs_off[SMU_MAX_REGS];
@@ -60,6 +61,8 @@ int aie4_suspend_fw(struct amdxdna_dev_hdl *ndev); · 18 unmodified lines
60	61	/* aie4_ctx.c */
61	62	int aie4_hwctx_init(struct amdxdna_hwctx *hwctx);
62	63	void aie4_hwctx_fini(struct amdxdna_hwctx *hwctx);
	64	+int aie4_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq, u32 timeout);
	65	+int aie4_hwctx_valid_doorbell(struct amdxdna_client *client, u32 vm_pgoff);
63	66
64	67	/* aie4_sriov.c */
65	68	#if IS_ENABLED(CONFIG_PCI_IOV)

drivers/accel/amdxdna/amdxdna_ctx.c+34−0

@@ -627,3 +627,37 @@ int amdxdna_drm_submit_cmd_ioctl(struct drm_device dev, void data, struct drm_ · 626 unmodified lines
627	627	XDNA_ERR(client->xdna, "Invalid command type %d", args->type);
628	628	return -EINVAL;
629	629	}
	630	+
	631	+int amdxdna_drm_wait_cmd_ioctl(struct drm_device dev, void data, struct drm_file *filp)
	632	+{
	633	+ struct amdxdna_client *client = filp->driver_priv;
	634	+ struct amdxdna_dev *xdna = to_xdna_dev(dev);
	635	+ struct amdxdna_drm_wait_cmd *args = data;
	636	+ struct amdxdna_hwctx *hwctx;
	637	+ int ret, idx;
	638	+
	639	+ XDNA_DBG(xdna, "PID %d ctx %d timeout set %d ms for cmd %llu",
	640	+ client->pid, args->hwctx, args->timeout, args->seq);
	641	+
	642	+ if (!xdna->dev_info->ops->cmd_wait)
	643	+ return -EOPNOTSUPP;
	644	+
	645	+ idx = srcu_read_lock(&client->hwctx_srcu);
	646	+ hwctx = xa_load(&client->hwctx_xa, args->hwctx);
	647	+ if (!hwctx) {
	648	+ XDNA_DBG(xdna, "PID %d failed to get ctx %d", client->pid, args->hwctx);
	649	+ ret = -EINVAL;
	650	+ goto unlock_ctx_srcu;
	651	+ }
	652	+
	653	+ ret = xdna->dev_info->ops->cmd_wait(hwctx, args->seq, args->timeout);
	654	+
	655	+ XDNA_DBG(xdna, "PID %d ctx %d cmd %lld wait finished, ret %d",
	656	+ client->pid, args->hwctx, args->seq, ret);
	657	+
	658	+ trace_amdxdna_debug_point(current->comm, args->seq, "job returned to user");
	659	+
	660	+unlock_ctx_srcu:
	661	+ srcu_read_unlock(&client->hwctx_srcu, idx);
	662	+ return ret;
	663	+}

drivers/accel/amdxdna/amdxdna_ctx.h+1−3

@@ -211,12 +211,10 @@ int amdxdna_cmd_submit(struct amdxdna_client *client, · 210 unmodified lines
211	211	u32 *arg_bo_hdls, u32 arg_bo_cnt,
212	212	u32 hwctx_hdl, u64 *seq);
213	213
214		−int amdxdna_cmd_wait(struct amdxdna_client *client, u32 hwctx_hdl,
215		− u64 seq, u32 timeout);
216		−
217	214	int amdxdna_drm_create_hwctx_ioctl(struct drm_device dev, void data, struct drm_file *filp);
218	215	int amdxdna_drm_config_hwctx_ioctl(struct drm_device dev, void data, struct drm_file *filp);
219	216	int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device dev, void data, struct drm_file *filp);
220	217	int amdxdna_drm_submit_cmd_ioctl(struct drm_device dev, void data, struct drm_file *filp);
	218	+int amdxdna_drm_wait_cmd_ioctl(struct drm_device dev, void data, struct drm_file *filp);
221	219
222	220	#endif /* _AMDXDNA_CTX_H_ */

drivers/accel/amdxdna/amdxdna_gem.c+3−2

@@ -212,7 +212,8 @@ static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni, · 211 unmodified lines
212	212	mmu_interval_set_seq(&mapp->notifier, cur_seq);
213	213	up_write(&xdna->notifier_lock);
214	214
215		− xdna->dev_info->ops->hmm_invalidate(abo, cur_seq);
	215	+ if (xdna->dev_info->ops->hmm_invalidate)
	216	+ xdna->dev_info->ops->hmm_invalidate(abo, cur_seq);
216	217
217	218	if (range->event == MMU_NOTIFY_UNMAP) {
218	219	down_write(&xdna->notifier_lock);
@@ -295,7 +296,7 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo, · 76 unmodified lines
295	296	u32 nr_pages;
296	297	int ret;
297	298
298		− if (!xdna->dev_info->ops->hmm_invalidate)
	299	+ if (!amdxdna_pasid_on(abo->client))
299	300	return 0;
300	301
301	302	mapp = kzalloc_obj(*mapp);

drivers/accel/amdxdna/amdxdna_pci_drv.c+17−1

@@ -224,6 +224,21 @@ static int amdxdna_drm_set_state_ioctl(struct drm_device dev, void data, struc · 223 unmodified lines
224	224	return ret;
225	225	}
226	226
	227	+static int amdxdna_drm_gem_mmap(struct file filp, struct vm_area_struct vma)
	228	+{
	229	+ struct drm_file *drm_filp = filp->private_data;
	230	+ struct amdxdna_client *client = drm_filp->driver_priv;
	231	+ struct amdxdna_dev *xdna = client->xdna;
	232	+
	233	+ if (likely(vma->vm_pgoff >= DRM_FILE_PAGE_OFFSET_START))
	234	+ return drm_gem_mmap(filp, vma);
	235	+
	236	+ if (!xdna->dev_info->ops->mmap)
	237	+ return -EOPNOTSUPP;
	238	+
	239	+ return xdna->dev_info->ops->mmap(client, vma);
	240	+}
	241	+
227	242	static const struct drm_ioctl_desc amdxdna_drm_ioctls[] = {
228	243	/* Context */
229	244	DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_HWCTX, amdxdna_drm_create_hwctx_ioctl, 0),
@@ -235,6 +250,7 @@ static const struct drm_ioctl_desc amdxdna_drm_ioctls[] = { · 5 unmodified lines
235	250	DRM_IOCTL_DEF_DRV(AMDXDNA_SYNC_BO, amdxdna_drm_sync_bo_ioctl, 0),
236	251	/* Execution */
237	252	DRM_IOCTL_DEF_DRV(AMDXDNA_EXEC_CMD, amdxdna_drm_submit_cmd_ioctl, 0),
	253	+ DRM_IOCTL_DEF_DRV(AMDXDNA_WAIT_CMD, amdxdna_drm_wait_cmd_ioctl, 0),
238	254	/* AIE hardware */
239	255	DRM_IOCTL_DEF_DRV(AMDXDNA_GET_INFO, amdxdna_drm_get_info_ioctl, 0),
240	256	DRM_IOCTL_DEF_DRV(AMDXDNA_GET_ARRAY, amdxdna_drm_get_array_ioctl, 0),
@@ -281,7 +297,7 @@ static const struct file_operations amdxdna_fops = { · 40 unmodified lines
281	297	.poll = drm_poll,
282	298	.read = drm_read,
283	299	.llseek = noop_llseek,
284		− .mmap = drm_gem_mmap,
	300	+ .mmap = amdxdna_drm_gem_mmap,
285	301	.show_fdinfo = drm_show_fdinfo,
286	302	.fop_flags = FOP_UNSIGNED_OFFSET,
287	303	};

drivers/accel/amdxdna/amdxdna_pci_drv.h+3−0

@@ -56,12 +56,14 @@ struct amdxdna_dev_ops { · 55 unmodified lines
56	56	int (resume)(struct amdxdna_dev xdna);
57	57	int (suspend)(struct amdxdna_dev xdna);
58	58	int (sriov_configure)(struct amdxdna_dev xdna, int num_vfs);
	59	+ int (mmap)(struct amdxdna_client client, struct vm_area_struct *vma);
59	60	int (hwctx_init)(struct amdxdna_hwctx hwctx);
60	61	void (hwctx_fini)(struct amdxdna_hwctx hwctx);
61	62	int (hwctx_config)(struct amdxdna_hwctx hwctx, u32 type, u64 value, void *buf, u32 size);
62	63	int (hwctx_sync_debug_bo)(struct amdxdna_hwctx hwctx, u32 debug_bo_hdl);
63	64	void (hmm_invalidate)(struct amdxdna_gem_obj abo, unsigned long cur_seq);
64	65	int (cmd_submit)(struct amdxdna_hwctx hwctx, struct amdxdna_sched_job job, u64 seq);
	66	+ int (cmd_wait)(struct amdxdna_hwctx hwctx, u64 seq, u32 timeout);
65	67	int (get_aie_info)(struct amdxdna_client client, struct amdxdna_drm_get_info *args);
66	68	int (set_aie_state)(struct amdxdna_client client, struct amdxdna_drm_set_state *args);
67	69	int (get_array)(struct amdxdna_client client, struct amdxdna_drm_get_array *args);
@@ -85,6 +87,7 @@ struct amdxdna_dev_info { · 17 unmodified lines
85	87	int sram_bar;
86	88	int psp_bar;
87	89	int smu_bar;
	90	+ int doorbell_bar;
88	91	int device_type;
89	92	int first_col;
90	93	u32 dev_mem_buf_shift;

drivers/accel/amdxdna/npu3_regs.c+5−0

@@ -14,6 +14,9 @@ · 13 unmodified lines
14	14	#define NPU3_MBOX_BUFFER_BAR 2
15	15	#define NPU3_MBOX_INFO_OFF 0x0
16	16
	17	+#define NPU3_DOORBELL_BAR 2
	18	+#define NPU3_DOORBELL_OFF 0x0
	19	+
17	20	/* PCIe BAR Index for NPU3 */
18	21	#define NPU3_REG_BAR_INDEX 0
19	22	#define NPU3_PSP_BAR_INDEX 4
@@ -45,6 +48,7 @@ static const struct amdxdna_dev_priv npu3_dev_priv = { · 25 unmodified lines
45	48	.mbox_bar = NPU3_MBOX_BAR,
46	49	.mbox_rbuf_bar = NPU3_MBOX_BUFFER_BAR,
47	50	.mbox_info_off = NPU3_MBOX_INFO_OFF,
	51	+ .doorbell_off = NPU3_DOORBELL_OFF,
48	52	.psp_regs_off = {
49	53	DEFINE_BAR_OFFSET(PSP_CMD_REG, NPU3_PSP, MPASP_C2PMSG_123_ALT_1),
50	54	DEFINE_BAR_OFFSET(PSP_ARG0_REG, NPU3_PSP, MPASP_C2PMSG_156_ALT_1),
@@ -87,6 +91,7 @@ const struct amdxdna_dev_info dev_npu3_pf_info = { · 36 unmodified lines
87	91	const struct amdxdna_dev_info dev_npu3_vf_info = {
88	92	.mbox_bar = NPU3_MBOX_BAR,
89	93	.sram_bar = NPU3_MBOX_BUFFER_BAR,
	94	+ .doorbell_bar = NPU3_DOORBELL_BAR,
90	95	.default_vbnv = "RyzenAI-npu3-vf",
91	96	.device_type = AMDXDNA_DEV_TYPE_UMQ,
92	97	.dev_priv = &npu3_dev_vf_priv,

include/uapi/drm/amdxdna_accel.h+21−1

@@ -45,7 +45,8 @@ enum amdxdna_drm_ioctl_id { · 44 unmodified lines
45	45	DRM_AMDXDNA_EXEC_CMD,
46	46	DRM_AMDXDNA_GET_INFO,
47	47	DRM_AMDXDNA_SET_STATE,
48		− DRM_AMDXDNA_GET_ARRAY = 10,
	48	+ DRM_AMDXDNA_WAIT_CMD,
	49	+ DRM_AMDXDNA_GET_ARRAY,
49	50	};
50	51
51	52	/**
@@ -274,6 +275,21 @@ struct amdxdna_drm_exec_cmd { · 222 unmodified lines
274	275	__u64 seq;
275	276	};
276	277
	278	+/**
	279	+ * struct amdxdna_drm_wait_cmd - Wait execution command.
	280	+ *
	281	+ * @hwctx: Context handle.
	282	+ * @timeout: timeout in ms, 0 implies infinite wait.
	283	+ * @seq: sequence number of the command returned by execute command.
	284	+ *
	285	+ * Wait a command specified by seq to be completed.
	286	+ */
	287	+struct amdxdna_drm_wait_cmd {
	288	+ __u32 hwctx;
	289	+ __u32 timeout;
	290	+ __u64 seq;
	291	+};
	292	+
277	293	/**
278	294	* struct amdxdna_drm_query_aie_status - Query the status of the AIE hardware
279	295	* @buffer: The user space buffer that will return the AIE status.
@@ -739,6 +755,10 @@ struct amdxdna_drm_set_power_mode { · 459 unmodified lines
739	755	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDXDNA_GET_ARRAY, \
740	756	struct amdxdna_drm_get_array)
741	757
	758	+#define DRM_IOCTL_AMDXDNA_WAIT_CMD \
	759	+ DRM_IOW(DRM_COMMAND_BASE + DRM_AMDXDNA_WAIT_CMD, \
	760	+ struct amdxdna_drm_wait_cmd)
	761	+
742	762	#if defined(__cplusplus)
743	763	} /* extern c end */
744	764	#endif

accel/amdxdna: Add command doorbell and wait support

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code