History

Merge branch 'amdkfd-v6' of git://people.freedesktop.org/~gabbayo/linux into drm-next

Merge AMDKFD it seems clean enough. * 'amdkfd-v6' of git://people.freedesktop.org/~gabbayo/linux: (29 commits) amdkfd: Implement the Get Version IOCTL amdkfd: Implement the Get Process Aperture IOCTL amdkfd: Implement the Get Clock Counters IOCTL amdkfd: Implement the Set Memory Policy IOCTL amdkfd: Implement the create/destroy/update queue IOCTLs amdkfd: Add interrupt handling module amdkfd: Add device queue manager module amdkfd: Add process queue manager module amdkfd: Add packet manager module amdkfd: Add module parameter of scheduling policy amdkfd: Add kernel queue module amdkfd: Add mqd_manager module amdkfd: Add queue module amdkfd: Add binding/unbinding calls to amd_iommu driver amdkfd: Add basic modules to amdkfd amdkfd: Add topology module to amdkfd amdkfd: Add amdkfd skeleton driver amdkfd: Add IOCTL set definitions of amdkfd Update MAINTAINERS and CREDITS files with amdkfd info drm/radeon: Add radeon <--> amdkfd interface ...

Dave Airlie committed 11 years ago

Commit cc5ac1c2 parents 8975626 ecd5c9853 files · +9853 −170Browse files

Unified · Split

53 files
  1. CREDITS +7 −0
  2. MAINTAINERS +10 −0
  3. drivers/gpu/drm/Kconfig +2 −0
  4. drivers/gpu/drm/Makefile +1 −0
  5. drivers/gpu/drm/amd/amdkfd/Kconfig +9 −0
  6. drivers/gpu/drm/amd/amdkfd/Makefile +14 −0
  7. drivers/gpu/drm/amd/amdkfd/cik_regs.h +221 −0
  8. drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +576 −0
  9. drivers/gpu/drm/amd/amdkfd/kfd_crat.h +294 −0
  10. drivers/gpu/drm/amd/amdkfd/kfd_device.c +307 −0
  11. drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +1059 −0
  12. drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +146 −0
  13. drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c +255 −0
  14. drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c +355 −0
  15. drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c +176 −0
  16. drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c +347 −0
  17. drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h +69 −0
  18. drivers/gpu/drm/amd/amdkfd/kfd_module.c +159 −0
  19. drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c +346 −0
  20. drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h +91 −0
  21. drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c +565 −0
  22. drivers/gpu/drm/amd/amdkfd/kfd_pasid.c +97 −0
  23. drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h +405 −0
  24. drivers/gpu/drm/amd/amdkfd/kfd_pm4_opcodes.h +107 −0
  25. drivers/gpu/drm/amd/amdkfd/kfd_priv.h +598 −0
  26. drivers/gpu/drm/amd/amdkfd/kfd_process.c +415 −0
  27. drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +342 −0
  28. drivers/gpu/drm/amd/amdkfd/kfd_queue.c +85 −0
  29. drivers/gpu/drm/amd/amdkfd/kfd_topology.c +1235 −0
  30. drivers/gpu/drm/amd/amdkfd/kfd_topology.h +168 −0
  31. drivers/gpu/drm/amd/include/kgd_kfd_interface.h +185 −0
  32. drivers/gpu/drm/radeon/Makefile +1 −0
  33. drivers/gpu/drm/radeon/cik.c +61 −94
  34. drivers/gpu/drm/radeon/cik_reg.h +136 −0
  35. drivers/gpu/drm/radeon/cikd.h +52 −1
  36. drivers/gpu/drm/radeon/radeon.h +10 −0
  37. drivers/gpu/drm/radeon/radeon_device.c +32 −0
  38. drivers/gpu/drm/radeon/radeon_drv.c +5 −0
  39. drivers/gpu/drm/radeon/radeon_kfd.c +563 −0
  40. drivers/gpu/drm/radeon/radeon_kfd.h +47 −0
  41. drivers/gpu/drm/radeon/radeon_kms.c +7 −0
  42. drivers/iommu/amd_iommu_v2.c +11 −58
  43. include/linux/mmu_notifier.h +83 −5
  44. include/uapi/linux/kfd_ioctl.h +154 −0
  45. kernel/events/uprobes.c +1 −1
  46. mm/fremap.c +1 −1
  47. mm/huge_memory.c +5 −4
  48. mm/hugetlb.c +6 −1
  49. mm/ksm.c +2 −2
  50. mm/memory.c +2 −1
  51. mm/migrate.c +2 −1
  52. mm/mmu_notifier.c +25 −0
  53. mm/rmap.c +1 −1
CREDITS+7−0
@@ -1197,6 +1197,13 @@ S: R. Tocantins, 89 - Cristo Rei · 1196 unmodified lines
11971197 S: 80050-430 - Curitiba - Paraná
11981198 S: Brazil
11991199
1200+N: Oded Gabbay
1201+E: oded.gabbay@gmail.com
1202+D: AMD KFD maintainer
1203+S: 12 Shraga Raphaeli
1204+S: Petah-Tikva, 4906418
1205+S: Israel
1206+
12001207 N: Kumar Gala
12011208 E: galak@kernel.crashing.org
12021209 D: Embedded PowerPC 6xx/7xx/74xx/82xx/83xx/85xx support
MAINTAINERS+10−0
@@ -618,6 +618,16 @@ S: Maintained · 617 unmodified lines
618618 F: drivers/iommu/amd_iommu*.[ch]
619619 F: include/linux/amd-iommu.h
620620
621+AMD KFD
622+M: Oded Gabbay <oded.gabbay@amd.com>
623+L: dri-devel@lists.freedesktop.org
624+T: git git://people.freedesktop.org/~gabbayo/linux.git
625+S: Supported
626+F: drivers/gpu/drm/amd/amdkfd/
627+F: drivers/gpu/drm/radeon/radeon_kfd.c
628+F: drivers/gpu/drm/radeon/radeon_kfd.h
629+F: include/uapi/linux/kfd_ioctl.h
630+
621631 AMD MICROCODE UPDATE SUPPORT
622632 M: Andreas Herrmann <herrmann.der.user@googlemail.com>
623633 L: amd64-microcode@amd64.org
drivers/gpu/drm/Kconfig+2−0
@@ -200,3 +200,5 @@ source "drivers/gpu/drm/tegra/Kconfig" · 199 unmodified lines
200200 source "drivers/gpu/drm/panel/Kconfig"
201201
202202 source "drivers/gpu/drm/sti/Kconfig"
203+
204+source "drivers/gpu/drm/amd/amdkfd/Kconfig"
drivers/gpu/drm/Makefile+1−0
@@ -65,3 +65,4 @@ obj-$(CONFIG_DRM_STI) += sti/ · 64 unmodified lines
6565 obj-y += i2c/
6666 obj-y += panel/
6767 obj-y += bridge/
68+obj-$(CONFIG_HSA_AMD) += amd/amdkfd/
drivers/gpu/drm/amd/amdkfd/Kconfig+9−0
@@ -0,0 +1,9 @@
1+#
2+# Heterogenous system architecture configuration
3+#
4+
5+config HSA_AMD
6+ tristate "HSA kernel driver for AMD GPU devices"
7+ depends on (DRM_RADEON || DRM_AMDGPU) && AMD_IOMMU_V2 && X86_64
8+ help
9+ Enable this if you want to use HSA features on AMD GPU devices.
drivers/gpu/drm/amd/amdkfd/Makefile+14−0
@@ -0,0 +1,14 @@
1+#
2+# Makefile for Heterogenous System Architecture support for AMD GPU devices
3+#
4+
5+ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/amd/include/
6+
7+amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \
8+ kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \
9+ kfd_process.o kfd_queue.o kfd_mqd_manager.o \
10+ kfd_kernel_queue.o kfd_packet_manager.o \
11+ kfd_process_queue_manager.o kfd_device_queue_manager.o \
12+ kfd_interrupt.o
13+
14+obj-$(CONFIG_HSA_AMD) += amdkfd.o
drivers/gpu/drm/amd/amdkfd/cik_regs.h+221−0
@@ -0,0 +1,221 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+#ifndef CIK_REGS_H
24+#define CIK_REGS_H
25+
26+#define IH_VMID_0_LUT 0x3D40u
27+
28+#define BIF_DOORBELL_CNTL 0x530Cu
29+
30+#define SRBM_GFX_CNTL 0xE44
31+#define PIPEID(x) ((x) << 0)
32+#define MEID(x) ((x) << 2)
33+#define VMID(x) ((x) << 4)
34+#define QUEUEID(x) ((x) << 8)
35+
36+#define SQ_CONFIG 0x8C00
37+
38+#define SH_MEM_BASES 0x8C28
39+/* if PTR32, these are the bases for scratch and lds */
40+#define PRIVATE_BASE(x) ((x) << 0) /* scratch */
41+#define SHARED_BASE(x) ((x) << 16) /* LDS */
42+#define SH_MEM_APE1_BASE 0x8C2C
43+/* if PTR32, this is the base location of GPUVM */
44+#define SH_MEM_APE1_LIMIT 0x8C30
45+/* if PTR32, this is the upper limit of GPUVM */
46+#define SH_MEM_CONFIG 0x8C34
47+#define PTR32 (1 << 0)
48+#define PRIVATE_ATC (1 << 1)
49+#define ALIGNMENT_MODE(x) ((x) << 2)
50+#define SH_MEM_ALIGNMENT_MODE_DWORD 0
51+#define SH_MEM_ALIGNMENT_MODE_DWORD_STRICT 1
52+#define SH_MEM_ALIGNMENT_MODE_STRICT 2
53+#define SH_MEM_ALIGNMENT_MODE_UNALIGNED 3
54+#define DEFAULT_MTYPE(x) ((x) << 4)
55+#define APE1_MTYPE(x) ((x) << 7)
56+
57+/* valid for both DEFAULT_MTYPE and APE1_MTYPE */
58+#define MTYPE_CACHED 0
59+#define MTYPE_NONCACHED 3
60+
61+
62+#define SH_STATIC_MEM_CONFIG 0x9604u
63+
64+#define TC_CFG_L1_LOAD_POLICY0 0xAC68
65+#define TC_CFG_L1_LOAD_POLICY1 0xAC6C
66+#define TC_CFG_L1_STORE_POLICY 0xAC70
67+#define TC_CFG_L2_LOAD_POLICY0 0xAC74
68+#define TC_CFG_L2_LOAD_POLICY1 0xAC78
69+#define TC_CFG_L2_STORE_POLICY0 0xAC7C
70+#define TC_CFG_L2_STORE_POLICY1 0xAC80
71+#define TC_CFG_L2_ATOMIC_POLICY 0xAC84
72+#define TC_CFG_L1_VOLATILE 0xAC88
73+#define TC_CFG_L2_VOLATILE 0xAC8C
74+
75+#define CP_PQ_WPTR_POLL_CNTL 0xC20C
76+#define WPTR_POLL_EN (1 << 31)
77+
78+#define CPC_INT_CNTL 0xC2D0
79+#define CP_ME1_PIPE0_INT_CNTL 0xC214
80+#define CP_ME1_PIPE1_INT_CNTL 0xC218
81+#define CP_ME1_PIPE2_INT_CNTL 0xC21C
82+#define CP_ME1_PIPE3_INT_CNTL 0xC220
83+#define CP_ME2_PIPE0_INT_CNTL 0xC224
84+#define CP_ME2_PIPE1_INT_CNTL 0xC228
85+#define CP_ME2_PIPE2_INT_CNTL 0xC22C
86+#define CP_ME2_PIPE3_INT_CNTL 0xC230
87+#define DEQUEUE_REQUEST_INT_ENABLE (1 << 13)
88+#define WRM_POLL_TIMEOUT_INT_ENABLE (1 << 17)
89+#define PRIV_REG_INT_ENABLE (1 << 23)
90+#define TIME_STAMP_INT_ENABLE (1 << 26)
91+#define GENERIC2_INT_ENABLE (1 << 29)
92+#define GENERIC1_INT_ENABLE (1 << 30)
93+#define GENERIC0_INT_ENABLE (1 << 31)
94+#define CP_ME1_PIPE0_INT_STATUS 0xC214
95+#define CP_ME1_PIPE1_INT_STATUS 0xC218
96+#define CP_ME1_PIPE2_INT_STATUS 0xC21C
97+#define CP_ME1_PIPE3_INT_STATUS 0xC220
98+#define CP_ME2_PIPE0_INT_STATUS 0xC224
99+#define CP_ME2_PIPE1_INT_STATUS 0xC228
100+#define CP_ME2_PIPE2_INT_STATUS 0xC22C
101+#define CP_ME2_PIPE3_INT_STATUS 0xC230
102+#define DEQUEUE_REQUEST_INT_STATUS (1 << 13)
103+#define WRM_POLL_TIMEOUT_INT_STATUS (1 << 17)
104+#define PRIV_REG_INT_STATUS (1 << 23)
105+#define TIME_STAMP_INT_STATUS (1 << 26)
106+#define GENERIC2_INT_STATUS (1 << 29)
107+#define GENERIC1_INT_STATUS (1 << 30)
108+#define GENERIC0_INT_STATUS (1 << 31)
109+
110+#define CP_HPD_EOP_BASE_ADDR 0xC904
111+#define CP_HPD_EOP_BASE_ADDR_HI 0xC908
112+#define CP_HPD_EOP_VMID 0xC90C
113+#define CP_HPD_EOP_CONTROL 0xC910
114+#define EOP_SIZE(x) ((x) << 0)
115+#define EOP_SIZE_MASK (0x3f << 0)
116+#define CP_MQD_BASE_ADDR 0xC914
117+#define CP_MQD_BASE_ADDR_HI 0xC918
118+#define CP_HQD_ACTIVE 0xC91C
119+#define CP_HQD_VMID 0xC920
120+
121+#define CP_HQD_PERSISTENT_STATE 0xC924u
122+#define DEFAULT_CP_HQD_PERSISTENT_STATE (0x33U << 8)
123+#define PRELOAD_REQ (1 << 0)
124+
125+#define CP_HQD_PIPE_PRIORITY 0xC928u
126+#define CP_HQD_QUEUE_PRIORITY 0xC92Cu
127+#define CP_HQD_QUANTUM 0xC930u
128+#define QUANTUM_EN 1U
129+#define QUANTUM_SCALE_1MS (1U << 4)
130+#define QUANTUM_DURATION(x) ((x) << 8)
131+
132+#define CP_HQD_PQ_BASE 0xC934
133+#define CP_HQD_PQ_BASE_HI 0xC938
134+#define CP_HQD_PQ_RPTR 0xC93C
135+#define CP_HQD_PQ_RPTR_REPORT_ADDR 0xC940
136+#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI 0xC944
137+#define CP_HQD_PQ_WPTR_POLL_ADDR 0xC948
138+#define CP_HQD_PQ_WPTR_POLL_ADDR_HI 0xC94C
139+#define CP_HQD_PQ_DOORBELL_CONTROL 0xC950
140+#define DOORBELL_OFFSET(x) ((x) << 2)
141+#define DOORBELL_OFFSET_MASK (0x1fffff << 2)
142+#define DOORBELL_SOURCE (1 << 28)
143+#define DOORBELL_SCHD_HIT (1 << 29)
144+#define DOORBELL_EN (1 << 30)
145+#define DOORBELL_HIT (1 << 31)
146+#define CP_HQD_PQ_WPTR 0xC954
147+#define CP_HQD_PQ_CONTROL 0xC958
148+#define QUEUE_SIZE(x) ((x) << 0)
149+#define QUEUE_SIZE_MASK (0x3f << 0)
150+#define RPTR_BLOCK_SIZE(x) ((x) << 8)
151+#define RPTR_BLOCK_SIZE_MASK (0x3f << 8)
152+#define MIN_AVAIL_SIZE(x) ((x) << 20)
153+#define PQ_ATC_EN (1 << 23)
154+#define PQ_VOLATILE (1 << 26)
155+#define NO_UPDATE_RPTR (1 << 27)
156+#define UNORD_DISPATCH (1 << 28)
157+#define ROQ_PQ_IB_FLIP (1 << 29)
158+#define PRIV_STATE (1 << 30)
159+#define KMD_QUEUE (1 << 31)
160+
161+#define DEFAULT_RPTR_BLOCK_SIZE RPTR_BLOCK_SIZE(5)
162+#define DEFAULT_MIN_AVAIL_SIZE MIN_AVAIL_SIZE(3)
163+
164+#define CP_HQD_IB_BASE_ADDR 0xC95Cu
165+#define CP_HQD_IB_BASE_ADDR_HI 0xC960u
166+#define CP_HQD_IB_RPTR 0xC964u
167+#define CP_HQD_IB_CONTROL 0xC968u
168+#define IB_ATC_EN (1U << 23)
169+#define DEFAULT_MIN_IB_AVAIL_SIZE (3U << 20)
170+
171+#define CP_HQD_DEQUEUE_REQUEST 0xC974
172+#define DEQUEUE_REQUEST_DRAIN 1
173+#define DEQUEUE_REQUEST_RESET 2
174+#define DEQUEUE_INT (1U << 8)
175+
176+#define CP_HQD_SEMA_CMD 0xC97Cu
177+#define CP_HQD_MSG_TYPE 0xC980u
178+#define CP_HQD_ATOMIC0_PREOP_LO 0xC984u
179+#define CP_HQD_ATOMIC0_PREOP_HI 0xC988u
180+#define CP_HQD_ATOMIC1_PREOP_LO 0xC98Cu
181+#define CP_HQD_ATOMIC1_PREOP_HI 0xC990u
182+#define CP_HQD_HQ_SCHEDULER0 0xC994u
183+#define CP_HQD_HQ_SCHEDULER1 0xC998u
184+
185+
186+#define CP_MQD_CONTROL 0xC99C
187+#define MQD_VMID(x) ((x) << 0)
188+#define MQD_VMID_MASK (0xf << 0)
189+#define MQD_CONTROL_PRIV_STATE_EN (1U << 8)
190+
191+#define GRBM_GFX_INDEX 0x30800
192+#define INSTANCE_INDEX(x) ((x) << 0)
193+#define SH_INDEX(x) ((x) << 8)
194+#define SE_INDEX(x) ((x) << 16)
195+#define SH_BROADCAST_WRITES (1 << 29)
196+#define INSTANCE_BROADCAST_WRITES (1 << 30)
197+#define SE_BROADCAST_WRITES (1 << 31)
198+
199+#define SQC_CACHES 0x30d20
200+#define SQC_POLICY 0x8C38u
201+#define SQC_VOLATILE 0x8C3Cu
202+
203+#define CP_PERFMON_CNTL 0x36020
204+
205+#define ATC_VMID0_PASID_MAPPING 0x339Cu
206+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS 0x3398u
207+#define ATC_VMID_PASID_MAPPING_VALID (1U << 31)
208+
209+#define ATC_VM_APERTURE0_CNTL 0x3310u
210+#define ATS_ACCESS_MODE_NEVER 0
211+#define ATS_ACCESS_MODE_ALWAYS 1
212+
213+#define ATC_VM_APERTURE0_CNTL2 0x3318u
214+#define ATC_VM_APERTURE0_HIGH_ADDR 0x3308u
215+#define ATC_VM_APERTURE0_LOW_ADDR 0x3300u
216+#define ATC_VM_APERTURE1_CNTL 0x3314u
217+#define ATC_VM_APERTURE1_CNTL2 0x331Cu
218+#define ATC_VM_APERTURE1_HIGH_ADDR 0x330Cu
219+#define ATC_VM_APERTURE1_LOW_ADDR 0x3304u
220+
221+#endif
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c+576−0
@@ -0,0 +1,576 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+#include <linux/device.h>
24+#include <linux/export.h>
25+#include <linux/err.h>
26+#include <linux/fs.h>
27+#include <linux/sched.h>
28+#include <linux/slab.h>
29+#include <linux/uaccess.h>
30+#include <linux/compat.h>
31+#include <uapi/linux/kfd_ioctl.h>
32+#include <linux/time.h>
33+#include <linux/mm.h>
34+#include <linux/uaccess.h>
35+#include <uapi/asm-generic/mman-common.h>
36+#include <asm/processor.h>
37+#include "kfd_priv.h"
38+#include "kfd_device_queue_manager.h"
39+
40+static long kfd_ioctl(struct file *, unsigned int, unsigned long);
41+static int kfd_open(struct inode *, struct file *);
42+static int kfd_mmap(struct file *, struct vm_area_struct *);
43+
44+static const char kfd_dev_name[] = "kfd";
45+
46+static const struct file_operations kfd_fops = {
47+ .owner = THIS_MODULE,
48+ .unlocked_ioctl = kfd_ioctl,
49+ .compat_ioctl = kfd_ioctl,
50+ .open = kfd_open,
51+ .mmap = kfd_mmap,
52+};
53+
54+static int kfd_char_dev_major = -1;
55+static struct class *kfd_class;
56+struct device *kfd_device;
57+
58+int kfd_chardev_init(void)
59+{
60+ int err = 0;
61+
62+ kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops);
63+ err = kfd_char_dev_major;
64+ if (err < 0)
65+ goto err_register_chrdev;
66+
67+ kfd_class = class_create(THIS_MODULE, kfd_dev_name);
68+ err = PTR_ERR(kfd_class);
69+ if (IS_ERR(kfd_class))
70+ goto err_class_create;
71+
72+ kfd_device = device_create(kfd_class, NULL,
73+ MKDEV(kfd_char_dev_major, 0),
74+ NULL, kfd_dev_name);
75+ err = PTR_ERR(kfd_device);
76+ if (IS_ERR(kfd_device))
77+ goto err_device_create;
78+
79+ return 0;
80+
81+err_device_create:
82+ class_destroy(kfd_class);
83+err_class_create:
84+ unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
85+err_register_chrdev:
86+ return err;
87+}
88+
89+void kfd_chardev_exit(void)
90+{
91+ device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0));
92+ class_destroy(kfd_class);
93+ unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
94+}
95+
96+struct device *kfd_chardev(void)
97+{
98+ return kfd_device;
99+}
100+
101+
102+static int kfd_open(struct inode *inode, struct file *filep)
103+{
104+ struct kfd_process *process;
105+
106+ if (iminor(inode) != 0)
107+ return -ENODEV;
108+
109+ process = kfd_create_process(current);
110+ if (IS_ERR(process))
111+ return PTR_ERR(process);
112+
113+ process->is_32bit_user_mode = is_compat_task();
114+
115+ dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
116+ process->pasid, process->is_32bit_user_mode);
117+
118+ kfd_init_apertures(process);
119+
120+ return 0;
121+}
122+
123+static long kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
124+ void __user *arg)
125+{
126+ struct kfd_ioctl_get_version_args args;
127+ int err = 0;
128+
129+ args.major_version = KFD_IOCTL_MAJOR_VERSION;
130+ args.minor_version = KFD_IOCTL_MINOR_VERSION;
131+
132+ if (copy_to_user(arg, &args, sizeof(args)))
133+ err = -EFAULT;
134+
135+ return err;
136+}
137+
138+static int set_queue_properties_from_user(struct queue_properties *q_properties,
139+ struct kfd_ioctl_create_queue_args *args)
140+{
141+ if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
142+ pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
143+ return -EINVAL;
144+ }
145+
146+ if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
147+ pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
148+ return -EINVAL;
149+ }
150+
151+ if ((args->ring_base_address) &&
152+ (!access_ok(VERIFY_WRITE, args->ring_base_address, sizeof(uint64_t)))) {
153+ pr_err("kfd: can't access ring base address\n");
154+ return -EFAULT;
155+ }
156+
157+ if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
158+ pr_err("kfd: ring size must be a power of 2 or 0\n");
159+ return -EINVAL;
160+ }
161+
162+ if (!access_ok(VERIFY_WRITE, args->read_pointer_address, sizeof(uint32_t))) {
163+ pr_err("kfd: can't access read pointer\n");
164+ return -EFAULT;
165+ }
166+
167+ if (!access_ok(VERIFY_WRITE, args->write_pointer_address, sizeof(uint32_t))) {
168+ pr_err("kfd: can't access write pointer\n");
169+ return -EFAULT;
170+ }
171+
172+ q_properties->is_interop = false;
173+ q_properties->queue_percent = args->queue_percentage;
174+ q_properties->priority = args->queue_priority;
175+ q_properties->queue_address = args->ring_base_address;
176+ q_properties->queue_size = args->ring_size;
177+ q_properties->read_ptr = (uint32_t *) args->read_pointer_address;
178+ q_properties->write_ptr = (uint32_t *) args->write_pointer_address;
179+ if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
180+ args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
181+ q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
182+ else
183+ return -ENOTSUPP;
184+
185+ if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
186+ q_properties->format = KFD_QUEUE_FORMAT_AQL;
187+ else
188+ q_properties->format = KFD_QUEUE_FORMAT_PM4;
189+
190+ pr_debug("Queue Percentage (%d, %d)\n",
191+ q_properties->queue_percent, args->queue_percentage);
192+
193+ pr_debug("Queue Priority (%d, %d)\n",
194+ q_properties->priority, args->queue_priority);
195+
196+ pr_debug("Queue Address (0x%llX, 0x%llX)\n",
197+ q_properties->queue_address, args->ring_base_address);
198+
199+ pr_debug("Queue Size (0x%llX, %u)\n",
200+ q_properties->queue_size, args->ring_size);
201+
202+ pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n",
203+ (uint64_t) q_properties->read_ptr,
204+ (uint64_t) q_properties->write_ptr);
205+
206+ pr_debug("Queue Format (%d)\n", q_properties->format);
207+
208+ return 0;
209+}
210+
211+static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
212+ void __user *arg)
213+{
214+ struct kfd_ioctl_create_queue_args args;
215+ struct kfd_dev *dev;
216+ int err = 0;
217+ unsigned int queue_id;
218+ struct kfd_process_device *pdd;
219+ struct queue_properties q_properties;
220+
221+ memset(&q_properties, 0, sizeof(struct queue_properties));
222+
223+ if (copy_from_user(&args, arg, sizeof(args)))
224+ return -EFAULT;
225+
226+ pr_debug("kfd: creating queue ioctl\n");
227+
228+ err = set_queue_properties_from_user(&q_properties, &args);
229+ if (err)
230+ return err;
231+
232+ dev = kfd_device_by_id(args.gpu_id);
233+ if (dev == NULL)
234+ return -EINVAL;
235+
236+ mutex_lock(&p->mutex);
237+
238+ pdd = kfd_bind_process_to_device(dev, p);
239+ if (IS_ERR(pdd) < 0) {
240+ err = PTR_ERR(pdd);
241+ goto err_bind_process;
242+ }
243+
244+ pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n",
245+ p->pasid,
246+ dev->id);
247+
248+ err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, 0,
249+ KFD_QUEUE_TYPE_COMPUTE, &queue_id);
250+ if (err != 0)
251+ goto err_create_queue;
252+
253+ args.queue_id = queue_id;
254+
255+ /* Return gpu_id as doorbell offset for mmap usage */
256+ args.doorbell_offset = args.gpu_id << PAGE_SHIFT;
257+
258+ if (copy_to_user(arg, &args, sizeof(args))) {
259+ err = -EFAULT;
260+ goto err_copy_args_out;
261+ }
262+
263+ mutex_unlock(&p->mutex);
264+
265+ pr_debug("kfd: queue id %d was created successfully\n", args.queue_id);
266+
267+ pr_debug("ring buffer address == 0x%016llX\n",
268+ args.ring_base_address);
269+
270+ pr_debug("read ptr address == 0x%016llX\n",
271+ args.read_pointer_address);
272+
273+ pr_debug("write ptr address == 0x%016llX\n",
274+ args.write_pointer_address);
275+
276+ return 0;
277+
278+err_copy_args_out:
279+ pqm_destroy_queue(&p->pqm, queue_id);
280+err_create_queue:
281+err_bind_process:
282+ mutex_unlock(&p->mutex);
283+ return err;
284+}
285+
286+static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
287+ void __user *arg)
288+{
289+ int retval;
290+ struct kfd_ioctl_destroy_queue_args args;
291+
292+ if (copy_from_user(&args, arg, sizeof(args)))
293+ return -EFAULT;
294+
295+ pr_debug("kfd: destroying queue id %d for PASID %d\n",
296+ args.queue_id,
297+ p->pasid);
298+
299+ mutex_lock(&p->mutex);
300+
301+ retval = pqm_destroy_queue(&p->pqm, args.queue_id);
302+
303+ mutex_unlock(&p->mutex);
304+ return retval;
305+}
306+
307+static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
308+ void __user *arg)
309+{
310+ int retval;
311+ struct kfd_ioctl_update_queue_args args;
312+ struct queue_properties properties;
313+
314+ if (copy_from_user(&args, arg, sizeof(args)))
315+ return -EFAULT;
316+
317+ if (args.queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
318+ pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
319+ return -EINVAL;
320+ }
321+
322+ if (args.queue_priority > KFD_MAX_QUEUE_PRIORITY) {
323+ pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
324+ return -EINVAL;
325+ }
326+
327+ if ((args.ring_base_address) &&
328+ (!access_ok(VERIFY_WRITE, args.ring_base_address, sizeof(uint64_t)))) {
329+ pr_err("kfd: can't access ring base address\n");
330+ return -EFAULT;
331+ }
332+
333+ if (!is_power_of_2(args.ring_size) && (args.ring_size != 0)) {
334+ pr_err("kfd: ring size must be a power of 2 or 0\n");
335+ return -EINVAL;
336+ }
337+
338+ properties.queue_address = args.ring_base_address;
339+ properties.queue_size = args.ring_size;
340+ properties.queue_percent = args.queue_percentage;
341+ properties.priority = args.queue_priority;
342+
343+ pr_debug("kfd: updating queue id %d for PASID %d\n",
344+ args.queue_id, p->pasid);
345+
346+ mutex_lock(&p->mutex);
347+
348+ retval = pqm_update_queue(&p->pqm, args.queue_id, &properties);
349+
350+ mutex_unlock(&p->mutex);
351+
352+ return retval;
353+}
354+
355+static long kfd_ioctl_set_memory_policy(struct file *filep,
356+ struct kfd_process *p, void __user *arg)
357+{
358+ struct kfd_ioctl_set_memory_policy_args args;
359+ struct kfd_dev *dev;
360+ int err = 0;
361+ struct kfd_process_device *pdd;
362+ enum cache_policy default_policy, alternate_policy;
363+
364+ if (copy_from_user(&args, arg, sizeof(args)))
365+ return -EFAULT;
366+
367+ if (args.default_policy != KFD_IOC_CACHE_POLICY_COHERENT
368+ && args.default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
369+ return -EINVAL;
370+ }
371+
372+ if (args.alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
373+ && args.alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
374+ return -EINVAL;
375+ }
376+
377+ dev = kfd_device_by_id(args.gpu_id);
378+ if (dev == NULL)
379+ return -EINVAL;
380+
381+ mutex_lock(&p->mutex);
382+
383+ pdd = kfd_bind_process_to_device(dev, p);
384+ if (IS_ERR(pdd) < 0) {
385+ err = PTR_ERR(pdd);
386+ goto out;
387+ }
388+
389+ default_policy = (args.default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
390+ ? cache_policy_coherent : cache_policy_noncoherent;
391+
392+ alternate_policy =
393+ (args.alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
394+ ? cache_policy_coherent : cache_policy_noncoherent;
395+
396+ if (!dev->dqm->set_cache_memory_policy(dev->dqm,
397+ &pdd->qpd,
398+ default_policy,
399+ alternate_policy,
400+ (void __user *)args.alternate_aperture_base,
401+ args.alternate_aperture_size))
402+ err = -EINVAL;
403+
404+out:
405+ mutex_unlock(&p->mutex);
406+
407+ return err;
408+}
409+
410+static long kfd_ioctl_get_clock_counters(struct file *filep,
411+ struct kfd_process *p, void __user *arg)
412+{
413+ struct kfd_ioctl_get_clock_counters_args args;
414+ struct kfd_dev *dev;
415+ struct timespec time;
416+
417+ if (copy_from_user(&args, arg, sizeof(args)))
418+ return -EFAULT;
419+
420+ dev = kfd_device_by_id(args.gpu_id);
421+ if (dev == NULL)
422+ return -EINVAL;
423+
424+ /* Reading GPU clock counter from KGD */
425+ args.gpu_clock_counter = kfd2kgd->get_gpu_clock_counter(dev->kgd);
426+
427+ /* No access to rdtsc. Using raw monotonic time */
428+ getrawmonotonic(&time);
429+ args.cpu_clock_counter = (uint64_t)timespec_to_ns(&time);
430+
431+ get_monotonic_boottime(&time);
432+ args.system_clock_counter = (uint64_t)timespec_to_ns(&time);
433+
434+ /* Since the counter is in nano-seconds we use 1GHz frequency */
435+ args.system_clock_freq = 1000000000;
436+
437+ if (copy_to_user(arg, &args, sizeof(args)))
438+ return -EFAULT;
439+
440+ return 0;
441+}
442+
443+
444+static int kfd_ioctl_get_process_apertures(struct file *filp,
445+ struct kfd_process *p, void __user *arg)
446+{
447+ struct kfd_ioctl_get_process_apertures_args args;
448+ struct kfd_process_device_apertures *pAperture;
449+ struct kfd_process_device *pdd;
450+
451+ dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid);
452+
453+ if (copy_from_user(&args, arg, sizeof(args)))
454+ return -EFAULT;
455+
456+ args.num_of_nodes = 0;
457+
458+ mutex_lock(&p->mutex);
459+
460+ /*if the process-device list isn't empty*/
461+ if (kfd_has_process_device_data(p)) {
462+ /* Run over all pdd of the process */
463+ pdd = kfd_get_first_process_device_data(p);
464+ do {
465+ pAperture = &args.process_apertures[args.num_of_nodes];
466+ pAperture->gpu_id = pdd->dev->id;
467+ pAperture->lds_base = pdd->lds_base;
468+ pAperture->lds_limit = pdd->lds_limit;
469+ pAperture->gpuvm_base = pdd->gpuvm_base;
470+ pAperture->gpuvm_limit = pdd->gpuvm_limit;
471+ pAperture->scratch_base = pdd->scratch_base;
472+ pAperture->scratch_limit = pdd->scratch_limit;
473+
474+ dev_dbg(kfd_device,
475+ "node id %u\n", args.num_of_nodes);
476+ dev_dbg(kfd_device,
477+ "gpu id %u\n", pdd->dev->id);
478+ dev_dbg(kfd_device,
479+ "lds_base %llX\n", pdd->lds_base);
480+ dev_dbg(kfd_device,
481+ "lds_limit %llX\n", pdd->lds_limit);
482+ dev_dbg(kfd_device,
483+ "gpuvm_base %llX\n", pdd->gpuvm_base);
484+ dev_dbg(kfd_device,
485+ "gpuvm_limit %llX\n", pdd->gpuvm_limit);
486+ dev_dbg(kfd_device,
487+ "scratch_base %llX\n", pdd->scratch_base);
488+ dev_dbg(kfd_device,
489+ "scratch_limit %llX\n", pdd->scratch_limit);
490+
491+ args.num_of_nodes++;
492+ } while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL &&
493+ (args.num_of_nodes < NUM_OF_SUPPORTED_GPUS));
494+ }
495+
496+ mutex_unlock(&p->mutex);
497+
498+ if (copy_to_user(arg, &args, sizeof(args)))
499+ return -EFAULT;
500+
501+ return 0;
502+}
503+
504+static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
505+{
506+ struct kfd_process *process;
507+ long err = -EINVAL;
508+
509+ dev_dbg(kfd_device,
510+ "ioctl cmd 0x%x (#%d), arg 0x%lx\n",
511+ cmd, _IOC_NR(cmd), arg);
512+
513+ process = kfd_get_process(current);
514+ if (IS_ERR(process))
515+ return PTR_ERR(process);
516+
517+ switch (cmd) {
518+ case KFD_IOC_GET_VERSION:
519+ err = kfd_ioctl_get_version(filep, process, (void __user *)arg);
520+ break;
521+ case KFD_IOC_CREATE_QUEUE:
522+ err = kfd_ioctl_create_queue(filep, process,
523+ (void __user *)arg);
524+ break;
525+
526+ case KFD_IOC_DESTROY_QUEUE:
527+ err = kfd_ioctl_destroy_queue(filep, process,
528+ (void __user *)arg);
529+ break;
530+
531+ case KFD_IOC_SET_MEMORY_POLICY:
532+ err = kfd_ioctl_set_memory_policy(filep, process,
533+ (void __user *)arg);
534+ break;
535+
536+ case KFD_IOC_GET_CLOCK_COUNTERS:
537+ err = kfd_ioctl_get_clock_counters(filep, process,
538+ (void __user *)arg);
539+ break;
540+
541+ case KFD_IOC_GET_PROCESS_APERTURES:
542+ err = kfd_ioctl_get_process_apertures(filep, process,
543+ (void __user *)arg);
544+ break;
545+
546+ case KFD_IOC_UPDATE_QUEUE:
547+ err = kfd_ioctl_update_queue(filep, process,
548+ (void __user *)arg);
549+ break;
550+
551+ default:
552+ dev_err(kfd_device,
553+ "unknown ioctl cmd 0x%x, arg 0x%lx)\n",
554+ cmd, arg);
555+ err = -EINVAL;
556+ break;
557+ }
558+
559+ if (err < 0)
560+ dev_err(kfd_device,
561+ "ioctl error %ld for ioctl cmd 0x%x (#%d)\n",
562+ err, cmd, _IOC_NR(cmd));
563+
564+ return err;
565+}
566+
567+static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
568+{
569+ struct kfd_process *process;
570+
571+ process = kfd_get_process(current);
572+ if (IS_ERR(process))
573+ return PTR_ERR(process);
574+
575+ return kfd_doorbell_mmap(process, vma);
576+}
drivers/gpu/drm/amd/amdkfd/kfd_crat.h+294−0
@@ -0,0 +1,294 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+#ifndef KFD_CRAT_H_INCLUDED
24+#define KFD_CRAT_H_INCLUDED
25+
26+#include <linux/types.h>
27+
28+#pragma pack(1)
29+
30+/*
31+ * 4CC signature values for the CRAT and CDIT ACPI tables
32+ */
33+
34+#define CRAT_SIGNATURE "CRAT"
35+#define CDIT_SIGNATURE "CDIT"
36+
37+/*
38+ * Component Resource Association Table (CRAT)
39+ */
40+
41+#define CRAT_OEMID_LENGTH 6
42+#define CRAT_OEMTABLEID_LENGTH 8
43+#define CRAT_RESERVED_LENGTH 6
44+
45+#define CRAT_OEMID_64BIT_MASK ((1ULL << (CRAT_OEMID_LENGTH * 8)) - 1)
46+
47+struct crat_header {
48+ uint32_t signature;
49+ uint32_t length;
50+ uint8_t revision;
51+ uint8_t checksum;
52+ uint8_t oem_id[CRAT_OEMID_LENGTH];
53+ uint8_t oem_table_id[CRAT_OEMTABLEID_LENGTH];
54+ uint32_t oem_revision;
55+ uint32_t creator_id;
56+ uint32_t creator_revision;
57+ uint32_t total_entries;
58+ uint16_t num_domains;
59+ uint8_t reserved[CRAT_RESERVED_LENGTH];
60+};
61+
62+/*
63+ * The header structure is immediately followed by total_entries of the
64+ * data definitions
65+ */
66+
67+/*
68+ * The currently defined subtype entries in the CRAT
69+ */
70+#define CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY 0
71+#define CRAT_SUBTYPE_MEMORY_AFFINITY 1
72+#define CRAT_SUBTYPE_CACHE_AFFINITY 2
73+#define CRAT_SUBTYPE_TLB_AFFINITY 3
74+#define CRAT_SUBTYPE_CCOMPUTE_AFFINITY 4
75+#define CRAT_SUBTYPE_IOLINK_AFFINITY 5
76+#define CRAT_SUBTYPE_MAX 6
77+
78+#define CRAT_SIBLINGMAP_SIZE 32
79+
80+/*
81+ * ComputeUnit Affinity structure and definitions
82+ */
83+#define CRAT_CU_FLAGS_ENABLED 0x00000001
84+#define CRAT_CU_FLAGS_HOT_PLUGGABLE 0x00000002
85+#define CRAT_CU_FLAGS_CPU_PRESENT 0x00000004
86+#define CRAT_CU_FLAGS_GPU_PRESENT 0x00000008
87+#define CRAT_CU_FLAGS_IOMMU_PRESENT 0x00000010
88+#define CRAT_CU_FLAGS_RESERVED 0xffffffe0
89+
90+#define CRAT_COMPUTEUNIT_RESERVED_LENGTH 4
91+
92+struct crat_subtype_computeunit {
93+ uint8_t type;
94+ uint8_t length;
95+ uint16_t reserved;
96+ uint32_t flags;
97+ uint32_t proximity_domain;
98+ uint32_t processor_id_low;
99+ uint16_t num_cpu_cores;
100+ uint16_t num_simd_cores;
101+ uint16_t max_waves_simd;
102+ uint16_t io_count;
103+ uint16_t hsa_capability;
104+ uint16_t lds_size_in_kb;
105+ uint8_t wave_front_size;
106+ uint8_t num_banks;
107+ uint16_t micro_engine_id;
108+ uint8_t num_arrays;
109+ uint8_t num_cu_per_array;
110+ uint8_t num_simd_per_cu;
111+ uint8_t max_slots_scatch_cu;
112+ uint8_t reserved2[CRAT_COMPUTEUNIT_RESERVED_LENGTH];
113+};
114+
115+/*
116+ * HSA Memory Affinity structure and definitions
117+ */
118+#define CRAT_MEM_FLAGS_ENABLED 0x00000001
119+#define CRAT_MEM_FLAGS_HOT_PLUGGABLE 0x00000002
120+#define CRAT_MEM_FLAGS_NON_VOLATILE 0x00000004
121+#define CRAT_MEM_FLAGS_RESERVED 0xfffffff8
122+
123+#define CRAT_MEMORY_RESERVED_LENGTH 8
124+
125+struct crat_subtype_memory {
126+ uint8_t type;
127+ uint8_t length;
128+ uint16_t reserved;
129+ uint32_t flags;
130+ uint32_t promixity_domain;
131+ uint32_t base_addr_low;
132+ uint32_t base_addr_high;
133+ uint32_t length_low;
134+ uint32_t length_high;
135+ uint32_t width;
136+ uint8_t reserved2[CRAT_MEMORY_RESERVED_LENGTH];
137+};
138+
139+/*
140+ * HSA Cache Affinity structure and definitions
141+ */
142+#define CRAT_CACHE_FLAGS_ENABLED 0x00000001
143+#define CRAT_CACHE_FLAGS_DATA_CACHE 0x00000002
144+#define CRAT_CACHE_FLAGS_INST_CACHE 0x00000004
145+#define CRAT_CACHE_FLAGS_CPU_CACHE 0x00000008
146+#define CRAT_CACHE_FLAGS_SIMD_CACHE 0x00000010
147+#define CRAT_CACHE_FLAGS_RESERVED 0xffffffe0
148+
149+#define CRAT_CACHE_RESERVED_LENGTH 8
150+
151+struct crat_subtype_cache {
152+ uint8_t type;
153+ uint8_t length;
154+ uint16_t reserved;
155+ uint32_t flags;
156+ uint32_t processor_id_low;
157+ uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE];
158+ uint32_t cache_size;
159+ uint8_t cache_level;
160+ uint8_t lines_per_tag;
161+ uint16_t cache_line_size;
162+ uint8_t associativity;
163+ uint8_t cache_properties;
164+ uint16_t cache_latency;
165+ uint8_t reserved2[CRAT_CACHE_RESERVED_LENGTH];
166+};
167+
168+/*
169+ * HSA TLB Affinity structure and definitions
170+ */
171+#define CRAT_TLB_FLAGS_ENABLED 0x00000001
172+#define CRAT_TLB_FLAGS_DATA_TLB 0x00000002
173+#define CRAT_TLB_FLAGS_INST_TLB 0x00000004
174+#define CRAT_TLB_FLAGS_CPU_TLB 0x00000008
175+#define CRAT_TLB_FLAGS_SIMD_TLB 0x00000010
176+#define CRAT_TLB_FLAGS_RESERVED 0xffffffe0
177+
178+#define CRAT_TLB_RESERVED_LENGTH 4
179+
180+struct crat_subtype_tlb {
181+ uint8_t type;
182+ uint8_t length;
183+ uint16_t reserved;
184+ uint32_t flags;
185+ uint32_t processor_id_low;
186+ uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE];
187+ uint32_t tlb_level;
188+ uint8_t data_tlb_associativity_2mb;
189+ uint8_t data_tlb_size_2mb;
190+ uint8_t instruction_tlb_associativity_2mb;
191+ uint8_t instruction_tlb_size_2mb;
192+ uint8_t data_tlb_associativity_4k;
193+ uint8_t data_tlb_size_4k;
194+ uint8_t instruction_tlb_associativity_4k;
195+ uint8_t instruction_tlb_size_4k;
196+ uint8_t data_tlb_associativity_1gb;
197+ uint8_t data_tlb_size_1gb;
198+ uint8_t instruction_tlb_associativity_1gb;
199+ uint8_t instruction_tlb_size_1gb;
200+ uint8_t reserved2[CRAT_TLB_RESERVED_LENGTH];
201+};
202+
203+/*
204+ * HSA CCompute/APU Affinity structure and definitions
205+ */
206+#define CRAT_CCOMPUTE_FLAGS_ENABLED 0x00000001
207+#define CRAT_CCOMPUTE_FLAGS_RESERVED 0xfffffffe
208+
209+#define CRAT_CCOMPUTE_RESERVED_LENGTH 16
210+
211+struct crat_subtype_ccompute {
212+ uint8_t type;
213+ uint8_t length;
214+ uint16_t reserved;
215+ uint32_t flags;
216+ uint32_t processor_id_low;
217+ uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE];
218+ uint32_t apu_size;
219+ uint8_t reserved2[CRAT_CCOMPUTE_RESERVED_LENGTH];
220+};
221+
222+/*
223+ * HSA IO Link Affinity structure and definitions
224+ */
225+#define CRAT_IOLINK_FLAGS_ENABLED 0x00000001
226+#define CRAT_IOLINK_FLAGS_COHERENCY 0x00000002
227+#define CRAT_IOLINK_FLAGS_RESERVED 0xfffffffc
228+
229+/*
230+ * IO interface types
231+ */
232+#define CRAT_IOLINK_TYPE_UNDEFINED 0
233+#define CRAT_IOLINK_TYPE_HYPERTRANSPORT 1
234+#define CRAT_IOLINK_TYPE_PCIEXPRESS 2
235+#define CRAT_IOLINK_TYPE_OTHER 3
236+#define CRAT_IOLINK_TYPE_MAX 255
237+
238+#define CRAT_IOLINK_RESERVED_LENGTH 24
239+
240+struct crat_subtype_iolink {
241+ uint8_t type;
242+ uint8_t length;
243+ uint16_t reserved;
244+ uint32_t flags;
245+ uint32_t proximity_domain_from;
246+ uint32_t proximity_domain_to;
247+ uint8_t io_interface_type;
248+ uint8_t version_major;
249+ uint16_t version_minor;
250+ uint32_t minimum_latency;
251+ uint32_t maximum_latency;
252+ uint32_t minimum_bandwidth_mbs;
253+ uint32_t maximum_bandwidth_mbs;
254+ uint32_t recommended_transfer_size;
255+ uint8_t reserved2[CRAT_IOLINK_RESERVED_LENGTH];
256+};
257+
258+/*
259+ * HSA generic sub-type header
260+ */
261+
262+#define CRAT_SUBTYPE_FLAGS_ENABLED 0x00000001
263+
264+struct crat_subtype_generic {
265+ uint8_t type;
266+ uint8_t length;
267+ uint16_t reserved;
268+ uint32_t flags;
269+};
270+
271+/*
272+ * Component Locality Distance Information Table (CDIT)
273+ */
274+#define CDIT_OEMID_LENGTH 6
275+#define CDIT_OEMTABLEID_LENGTH 8
276+
277+struct cdit_header {
278+ uint32_t signature;
279+ uint32_t length;
280+ uint8_t revision;
281+ uint8_t checksum;
282+ uint8_t oem_id[CDIT_OEMID_LENGTH];
283+ uint8_t oem_table_id[CDIT_OEMTABLEID_LENGTH];
284+ uint32_t oem_revision;
285+ uint32_t creator_id;
286+ uint32_t creator_revision;
287+ uint32_t total_entries;
288+ uint16_t num_domains;
289+ uint8_t entry[1];
290+};
291+
292+#pragma pack()
293+
294+#endif /* KFD_CRAT_H_INCLUDED */
drivers/gpu/drm/amd/amdkfd/kfd_device.c+307−0
@@ -0,0 +1,307 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+#include <linux/amd-iommu.h>
24+#include <linux/bsearch.h>
25+#include <linux/pci.h>
26+#include <linux/slab.h>
27+#include "kfd_priv.h"
28+#include "kfd_device_queue_manager.h"
29+
30+#define MQD_SIZE_ALIGNED 768
31+
32+static const struct kfd_device_info kaveri_device_info = {
33+ .max_pasid_bits = 16,
34+ .ih_ring_entry_size = 4 * sizeof(uint32_t),
35+ .mqd_size_aligned = MQD_SIZE_ALIGNED
36+};
37+
38+struct kfd_deviceid {
39+ unsigned short did;
40+ const struct kfd_device_info *device_info;
41+};
42+
43+/* Please keep this sorted by increasing device id. */
44+static const struct kfd_deviceid supported_devices[] = {
45+ { 0x1304, &kaveri_device_info }, /* Kaveri */
46+ { 0x1305, &kaveri_device_info }, /* Kaveri */
47+ { 0x1306, &kaveri_device_info }, /* Kaveri */
48+ { 0x1307, &kaveri_device_info }, /* Kaveri */
49+ { 0x1309, &kaveri_device_info }, /* Kaveri */
50+ { 0x130A, &kaveri_device_info }, /* Kaveri */
51+ { 0x130B, &kaveri_device_info }, /* Kaveri */
52+ { 0x130C, &kaveri_device_info }, /* Kaveri */
53+ { 0x130D, &kaveri_device_info }, /* Kaveri */
54+ { 0x130E, &kaveri_device_info }, /* Kaveri */
55+ { 0x130F, &kaveri_device_info }, /* Kaveri */
56+ { 0x1310, &kaveri_device_info }, /* Kaveri */
57+ { 0x1311, &kaveri_device_info }, /* Kaveri */
58+ { 0x1312, &kaveri_device_info }, /* Kaveri */
59+ { 0x1313, &kaveri_device_info }, /* Kaveri */
60+ { 0x1315, &kaveri_device_info }, /* Kaveri */
61+ { 0x1316, &kaveri_device_info }, /* Kaveri */
62+ { 0x1317, &kaveri_device_info }, /* Kaveri */
63+ { 0x1318, &kaveri_device_info }, /* Kaveri */
64+ { 0x131B, &kaveri_device_info }, /* Kaveri */
65+ { 0x131C, &kaveri_device_info }, /* Kaveri */
66+ { 0x131D, &kaveri_device_info }, /* Kaveri */
67+};
68+
69+static const struct kfd_device_info *lookup_device_info(unsigned short did)
70+{
71+ size_t i;
72+
73+ for (i = 0; i < ARRAY_SIZE(supported_devices); i++) {
74+ if (supported_devices[i].did == did) {
75+ BUG_ON(supported_devices[i].device_info == NULL);
76+ return supported_devices[i].device_info;
77+ }
78+ }
79+
80+ return NULL;
81+}
82+
83+struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev)
84+{
85+ struct kfd_dev *kfd;
86+
87+ const struct kfd_device_info *device_info =
88+ lookup_device_info(pdev->device);
89+
90+ if (!device_info)
91+ return NULL;
92+
93+ kfd = kzalloc(sizeof(*kfd), GFP_KERNEL);
94+ if (!kfd)
95+ return NULL;
96+
97+ kfd->kgd = kgd;
98+ kfd->device_info = device_info;
99+ kfd->pdev = pdev;
100+ kfd->init_complete = false;
101+
102+ return kfd;
103+}
104+
105+static bool device_iommu_pasid_init(struct kfd_dev *kfd)
106+{
107+ const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
108+ AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
109+ AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
110+
111+ struct amd_iommu_device_info iommu_info;
112+ unsigned int pasid_limit;
113+ int err;
114+
115+ err = amd_iommu_device_info(kfd->pdev, &iommu_info);
116+ if (err < 0) {
117+ dev_err(kfd_device,
118+ "error getting iommu info. is the iommu enabled?\n");
119+ return false;
120+ }
121+
122+ if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
123+ dev_err(kfd_device, "error required iommu flags ats(%i), pri(%i), pasid(%i)\n",
124+ (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
125+ (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
126+ (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) != 0);
127+ return false;
128+ }
129+
130+ pasid_limit = min_t(unsigned int,
131+ (unsigned int)1 << kfd->device_info->max_pasid_bits,
132+ iommu_info.max_pasids);
133+ /*
134+ * last pasid is used for kernel queues doorbells
135+ * in the future the last pasid might be used for a kernel thread.
136+ */
137+ pasid_limit = min_t(unsigned int,
138+ pasid_limit,
139+ kfd->doorbell_process_limit - 1);
140+
141+ err = amd_iommu_init_device(kfd->pdev, pasid_limit);
142+ if (err < 0) {
143+ dev_err(kfd_device, "error initializing iommu device\n");
144+ return false;
145+ }
146+
147+ if (!kfd_set_pasid_limit(pasid_limit)) {
148+ dev_err(kfd_device, "error setting pasid limit\n");
149+ amd_iommu_free_device(kfd->pdev);
150+ return false;
151+ }
152+
153+ return true;
154+}
155+
156+static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int pasid)
157+{
158+ struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
159+
160+ if (dev)
161+ kfd_unbind_process_from_device(dev, pasid);
162+}
163+
164+bool kgd2kfd_device_init(struct kfd_dev *kfd,
165+ const struct kgd2kfd_shared_resources *gpu_resources)
166+{
167+ unsigned int size;
168+
169+ kfd->shared_resources = *gpu_resources;
170+
171+ /* calculate max size of mqds needed for queues */
172+ size = max_num_of_processes *
173+ max_num_of_queues_per_process *
174+ kfd->device_info->mqd_size_aligned;
175+
176+ /* add another 512KB for all other allocations on gart */
177+ size += 512 * 1024;
178+
179+ if (kfd2kgd->init_sa_manager(kfd->kgd, size)) {
180+ dev_err(kfd_device,
181+ "Error initializing sa manager for device (%x:%x)\n",
182+ kfd->pdev->vendor, kfd->pdev->device);
183+ goto out;
184+ }
185+
186+ kfd_doorbell_init(kfd);
187+
188+ if (kfd_topology_add_device(kfd) != 0) {
189+ dev_err(kfd_device,
190+ "Error adding device (%x:%x) to topology\n",
191+ kfd->pdev->vendor, kfd->pdev->device);
192+ goto kfd_topology_add_device_error;
193+ }
194+
195+ if (kfd_interrupt_init(kfd)) {
196+ dev_err(kfd_device,
197+ "Error initializing interrupts for device (%x:%x)\n",
198+ kfd->pdev->vendor, kfd->pdev->device);
199+ goto kfd_interrupt_error;
200+ }
201+
202+ if (!device_iommu_pasid_init(kfd)) {
203+ dev_err(kfd_device,
204+ "Error initializing iommuv2 for device (%x:%x)\n",
205+ kfd->pdev->vendor, kfd->pdev->device);
206+ goto device_iommu_pasid_error;
207+ }
208+ amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
209+ iommu_pasid_shutdown_callback);
210+
211+ kfd->dqm = device_queue_manager_init(kfd);
212+ if (!kfd->dqm) {
213+ dev_err(kfd_device,
214+ "Error initializing queue manager for device (%x:%x)\n",
215+ kfd->pdev->vendor, kfd->pdev->device);
216+ goto device_queue_manager_error;
217+ }
218+
219+ if (kfd->dqm->start(kfd->dqm) != 0) {
220+ dev_err(kfd_device,
221+ "Error starting queuen manager for device (%x:%x)\n",
222+ kfd->pdev->vendor, kfd->pdev->device);
223+ goto dqm_start_error;
224+ }
225+
226+ kfd->init_complete = true;
227+ dev_info(kfd_device, "added device (%x:%x)\n", kfd->pdev->vendor,
228+ kfd->pdev->device);
229+
230+ pr_debug("kfd: Starting kfd with the following scheduling policy %d\n",
231+ sched_policy);
232+
233+ goto out;
234+
235+dqm_start_error:
236+ device_queue_manager_uninit(kfd->dqm);
237+device_queue_manager_error:
238+ amd_iommu_free_device(kfd->pdev);
239+device_iommu_pasid_error:
240+ kfd_interrupt_exit(kfd);
241+kfd_interrupt_error:
242+ kfd_topology_remove_device(kfd);
243+kfd_topology_add_device_error:
244+ kfd2kgd->fini_sa_manager(kfd->kgd);
245+ dev_err(kfd_device,
246+ "device (%x:%x) NOT added due to errors\n",
247+ kfd->pdev->vendor, kfd->pdev->device);
248+out:
249+ return kfd->init_complete;
250+}
251+
252+void kgd2kfd_device_exit(struct kfd_dev *kfd)
253+{
254+ if (kfd->init_complete) {
255+ device_queue_manager_uninit(kfd->dqm);
256+ amd_iommu_free_device(kfd->pdev);
257+ kfd_interrupt_exit(kfd);
258+ kfd_topology_remove_device(kfd);
259+ }
260+
261+ kfree(kfd);
262+}
263+
264+void kgd2kfd_suspend(struct kfd_dev *kfd)
265+{
266+ BUG_ON(kfd == NULL);
267+
268+ if (kfd->init_complete) {
269+ kfd->dqm->stop(kfd->dqm);
270+ amd_iommu_free_device(kfd->pdev);
271+ }
272+}
273+
274+int kgd2kfd_resume(struct kfd_dev *kfd)
275+{
276+ unsigned int pasid_limit;
277+ int err;
278+
279+ BUG_ON(kfd == NULL);
280+
281+ pasid_limit = kfd_get_pasid_limit();
282+
283+ if (kfd->init_complete) {
284+ err = amd_iommu_init_device(kfd->pdev, pasid_limit);
285+ if (err < 0)
286+ return -ENXIO;
287+ amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
288+ iommu_pasid_shutdown_callback);
289+ kfd->dqm->start(kfd->dqm);
290+ }
291+
292+ return 0;
293+}
294+
295+/* This is called directly from KGD at ISR. */
296+void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
297+{
298+ if (kfd->init_complete) {
299+ spin_lock(&kfd->interrupt_lock);
300+
301+ if (kfd->interrupts_active
302+ && enqueue_ih_ring_entry(kfd, ih_ring_entry))
303+ schedule_work(&kfd->interrupt_work);
304+
305+ spin_unlock(&kfd->interrupt_lock);
306+ }
307+}
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c+1059−0
@@ -0,0 +1,1059 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ *
22+ */
23+
24+#include <linux/slab.h>
25+#include <linux/list.h>
26+#include <linux/types.h>
27+#include <linux/printk.h>
28+#include <linux/bitops.h>
29+#include "kfd_priv.h"
30+#include "kfd_device_queue_manager.h"
31+#include "kfd_mqd_manager.h"
32+#include "cik_regs.h"
33+#include "kfd_kernel_queue.h"
34+#include "../../radeon/cik_reg.h"
35+
36+/* Size of the per-pipe EOP queue */
37+#define CIK_HPD_EOP_BYTES_LOG2 11
38+#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
39+
40+static bool is_mem_initialized;
41+
42+static int init_memory(struct device_queue_manager *dqm);
43+static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
44+ unsigned int pasid, unsigned int vmid);
45+
46+static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
47+ struct queue *q,
48+ struct qcm_process_device *qpd);
49+static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock);
50+static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock);
51+
52+
53+static inline unsigned int get_pipes_num(struct device_queue_manager *dqm)
54+{
55+ BUG_ON(!dqm || !dqm->dev);
56+ return dqm->dev->shared_resources.compute_pipe_count;
57+}
58+
59+static inline unsigned int get_first_pipe(struct device_queue_manager *dqm)
60+{
61+ BUG_ON(!dqm);
62+ return dqm->dev->shared_resources.first_compute_pipe;
63+}
64+
65+static inline unsigned int get_pipes_num_cpsch(void)
66+{
67+ return PIPE_PER_ME_CP_SCHEDULING;
68+}
69+
70+static unsigned int get_sh_mem_bases_nybble_64(struct kfd_process *process,
71+ struct kfd_dev *dev)
72+{
73+ struct kfd_process_device *pdd;
74+ uint32_t nybble;
75+
76+ pdd = kfd_get_process_device_data(dev, process, 1);
77+ nybble = (pdd->lds_base >> 60) & 0x0E;
78+
79+ return nybble;
80+
81+}
82+
83+static unsigned int get_sh_mem_bases_32(struct kfd_process *process,
84+ struct kfd_dev *dev)
85+{
86+ struct kfd_process_device *pdd;
87+ unsigned int shared_base;
88+
89+ pdd = kfd_get_process_device_data(dev, process, 1);
90+ shared_base = (pdd->lds_base >> 16) & 0xFF;
91+
92+ return shared_base;
93+}
94+
95+static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble);
96+static void init_process_memory(struct device_queue_manager *dqm,
97+ struct qcm_process_device *qpd)
98+{
99+ unsigned int temp;
100+
101+ BUG_ON(!dqm || !qpd);
102+
103+ /* check if sh_mem_config register already configured */
104+ if (qpd->sh_mem_config == 0) {
105+ qpd->sh_mem_config =
106+ ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) |
107+ DEFAULT_MTYPE(MTYPE_NONCACHED) |
108+ APE1_MTYPE(MTYPE_NONCACHED);
109+ qpd->sh_mem_ape1_limit = 0;
110+ qpd->sh_mem_ape1_base = 0;
111+ }
112+
113+ if (qpd->pqm->process->is_32bit_user_mode) {
114+ temp = get_sh_mem_bases_32(qpd->pqm->process, dqm->dev);
115+ qpd->sh_mem_bases = SHARED_BASE(temp);
116+ qpd->sh_mem_config |= PTR32;
117+ } else {
118+ temp = get_sh_mem_bases_nybble_64(qpd->pqm->process, dqm->dev);
119+ qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
120+ }
121+
122+ pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
123+ qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
124+}
125+
126+static void program_sh_mem_settings(struct device_queue_manager *dqm,
127+ struct qcm_process_device *qpd)
128+{
129+ return kfd2kgd->program_sh_mem_settings(dqm->dev->kgd, qpd->vmid,
130+ qpd->sh_mem_config,
131+ qpd->sh_mem_ape1_base,
132+ qpd->sh_mem_ape1_limit,
133+ qpd->sh_mem_bases);
134+}
135+
136+static int allocate_vmid(struct device_queue_manager *dqm,
137+ struct qcm_process_device *qpd,
138+ struct queue *q)
139+{
140+ int bit, allocated_vmid;
141+
142+ if (dqm->vmid_bitmap == 0)
143+ return -ENOMEM;
144+
145+ bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM);
146+ clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
147+
148+ /* Kaveri kfd vmid's starts from vmid 8 */
149+ allocated_vmid = bit + KFD_VMID_START_OFFSET;
150+ pr_debug("kfd: vmid allocation %d\n", allocated_vmid);
151+ qpd->vmid = allocated_vmid;
152+ q->properties.vmid = allocated_vmid;
153+
154+ set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
155+ program_sh_mem_settings(dqm, qpd);
156+
157+ return 0;
158+}
159+
160+static void deallocate_vmid(struct device_queue_manager *dqm,
161+ struct qcm_process_device *qpd,
162+ struct queue *q)
163+{
164+ int bit = qpd->vmid - KFD_VMID_START_OFFSET;
165+
166+ set_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
167+ qpd->vmid = 0;
168+ q->properties.vmid = 0;
169+}
170+
171+static int create_queue_nocpsch(struct device_queue_manager *dqm,
172+ struct queue *q,
173+ struct qcm_process_device *qpd,
174+ int *allocated_vmid)
175+{
176+ int retval;
177+
178+ BUG_ON(!dqm || !q || !qpd || !allocated_vmid);
179+
180+ pr_debug("kfd: In func %s\n", __func__);
181+ print_queue(q);
182+
183+ mutex_lock(&dqm->lock);
184+
185+ if (list_empty(&qpd->queues_list)) {
186+ retval = allocate_vmid(dqm, qpd, q);
187+ if (retval != 0) {
188+ mutex_unlock(&dqm->lock);
189+ return retval;
190+ }
191+ }
192+ *allocated_vmid = qpd->vmid;
193+ q->properties.vmid = qpd->vmid;
194+
195+ retval = create_compute_queue_nocpsch(dqm, q, qpd);
196+
197+ if (retval != 0) {
198+ if (list_empty(&qpd->queues_list)) {
199+ deallocate_vmid(dqm, qpd, q);
200+ *allocated_vmid = 0;
201+ }
202+ mutex_unlock(&dqm->lock);
203+ return retval;
204+ }
205+
206+ list_add(&q->list, &qpd->queues_list);
207+ dqm->queue_count++;
208+
209+ mutex_unlock(&dqm->lock);
210+ return 0;
211+}
212+
213+static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
214+{
215+ bool set;
216+ int pipe, bit;
217+
218+ set = false;
219+
220+ for (pipe = dqm->next_pipe_to_allocate; pipe < get_pipes_num(dqm);
221+ pipe = (pipe + 1) % get_pipes_num(dqm)) {
222+ if (dqm->allocated_queues[pipe] != 0) {
223+ bit = find_first_bit(
224+ (unsigned long *)&dqm->allocated_queues[pipe],
225+ QUEUES_PER_PIPE);
226+
227+ clear_bit(bit,
228+ (unsigned long *)&dqm->allocated_queues[pipe]);
229+ q->pipe = pipe;
230+ q->queue = bit;
231+ set = true;
232+ break;
233+ }
234+ }
235+
236+ if (set == false)
237+ return -EBUSY;
238+
239+ pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n",
240+ __func__, q->pipe, q->queue);
241+ /* horizontal hqd allocation */
242+ dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm);
243+
244+ return 0;
245+}
246+
247+static inline void deallocate_hqd(struct device_queue_manager *dqm,
248+ struct queue *q)
249+{
250+ set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]);
251+}
252+
253+static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
254+ struct queue *q,
255+ struct qcm_process_device *qpd)
256+{
257+ int retval;
258+ struct mqd_manager *mqd;
259+
260+ BUG_ON(!dqm || !q || !qpd);
261+
262+ mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
263+ if (mqd == NULL)
264+ return -ENOMEM;
265+
266+ retval = allocate_hqd(dqm, q);
267+ if (retval != 0)
268+ return retval;
269+
270+ retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
271+ &q->gart_mqd_addr, &q->properties);
272+ if (retval != 0) {
273+ deallocate_hqd(dqm, q);
274+ return retval;
275+ }
276+
277+ return 0;
278+}
279+
280+static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
281+ struct qcm_process_device *qpd,
282+ struct queue *q)
283+{
284+ int retval;
285+ struct mqd_manager *mqd;
286+
287+ BUG_ON(!dqm || !q || !q->mqd || !qpd);
288+
289+ retval = 0;
290+
291+ pr_debug("kfd: In Func %s\n", __func__);
292+
293+ mutex_lock(&dqm->lock);
294+ mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
295+ if (mqd == NULL) {
296+ retval = -ENOMEM;
297+ goto out;
298+ }
299+
300+ retval = mqd->destroy_mqd(mqd, q->mqd,
301+ KFD_PREEMPT_TYPE_WAVEFRONT,
302+ QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
303+ q->pipe, q->queue);
304+
305+ if (retval != 0)
306+ goto out;
307+
308+ deallocate_hqd(dqm, q);
309+
310+ mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
311+
312+ list_del(&q->list);
313+ if (list_empty(&qpd->queues_list))
314+ deallocate_vmid(dqm, qpd, q);
315+ dqm->queue_count--;
316+out:
317+ mutex_unlock(&dqm->lock);
318+ return retval;
319+}
320+
321+static int update_queue(struct device_queue_manager *dqm, struct queue *q)
322+{
323+ int retval;
324+ struct mqd_manager *mqd;
325+
326+ BUG_ON(!dqm || !q || !q->mqd);
327+
328+ mutex_lock(&dqm->lock);
329+ mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
330+ if (mqd == NULL) {
331+ mutex_unlock(&dqm->lock);
332+ return -ENOMEM;
333+ }
334+
335+ retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
336+ if (q->properties.is_active == true)
337+ dqm->queue_count++;
338+ else
339+ dqm->queue_count--;
340+
341+ if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
342+ retval = execute_queues_cpsch(dqm, false);
343+
344+ mutex_unlock(&dqm->lock);
345+ return retval;
346+}
347+
348+static struct mqd_manager *get_mqd_manager_nocpsch(
349+ struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
350+{
351+ struct mqd_manager *mqd;
352+
353+ BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX);
354+
355+ pr_debug("kfd: In func %s mqd type %d\n", __func__, type);
356+
357+ mqd = dqm->mqds[type];
358+ if (!mqd) {
359+ mqd = mqd_manager_init(type, dqm->dev);
360+ if (mqd == NULL)
361+ pr_err("kfd: mqd manager is NULL");
362+ dqm->mqds[type] = mqd;
363+ }
364+
365+ return mqd;
366+}
367+
368+static int register_process_nocpsch(struct device_queue_manager *dqm,
369+ struct qcm_process_device *qpd)
370+{
371+ struct device_process_node *n;
372+
373+ BUG_ON(!dqm || !qpd);
374+
375+ pr_debug("kfd: In func %s\n", __func__);
376+
377+ n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL);
378+ if (!n)
379+ return -ENOMEM;
380+
381+ n->qpd = qpd;
382+
383+ mutex_lock(&dqm->lock);
384+ list_add(&n->list, &dqm->queues);
385+
386+ init_process_memory(dqm, qpd);
387+ dqm->processes_count++;
388+
389+ mutex_unlock(&dqm->lock);
390+
391+ return 0;
392+}
393+
394+static int unregister_process_nocpsch(struct device_queue_manager *dqm,
395+ struct qcm_process_device *qpd)
396+{
397+ int retval;
398+ struct device_process_node *cur, *next;
399+
400+ BUG_ON(!dqm || !qpd);
401+
402+ BUG_ON(!list_empty(&qpd->queues_list));
403+
404+ pr_debug("kfd: In func %s\n", __func__);
405+
406+ retval = 0;
407+ mutex_lock(&dqm->lock);
408+
409+ list_for_each_entry_safe(cur, next, &dqm->queues, list) {
410+ if (qpd == cur->qpd) {
411+ list_del(&cur->list);
412+ dqm->processes_count--;
413+ goto out;
414+ }
415+ }
416+ /* qpd not found in dqm list */
417+ retval = 1;
418+out:
419+ mutex_unlock(&dqm->lock);
420+ return retval;
421+}
422+
423+static int
424+set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
425+ unsigned int vmid)
426+{
427+ uint32_t pasid_mapping;
428+
429+ pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
430+ ATC_VMID_PASID_MAPPING_VALID;
431+ return kfd2kgd->set_pasid_vmid_mapping(dqm->dev->kgd, pasid_mapping,
432+ vmid);
433+}
434+
435+static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
436+{
437+ /* In 64-bit mode, we can only control the top 3 bits of the LDS,
438+ * scratch and GPUVM apertures.
439+ * The hardware fills in the remaining 59 bits according to the
440+ * following pattern:
441+ * LDS: X0000000'00000000 - X0000001'00000000 (4GB)
442+ * Scratch: X0000001'00000000 - X0000002'00000000 (4GB)
443+ * GPUVM: Y0010000'00000000 - Y0020000'00000000 (1TB)
444+ *
445+ * (where X/Y is the configurable nybble with the low-bit 0)
446+ *
447+ * LDS and scratch will have the same top nybble programmed in the
448+ * top 3 bits of SH_MEM_BASES.PRIVATE_BASE.
449+ * GPUVM can have a different top nybble programmed in the
450+ * top 3 bits of SH_MEM_BASES.SHARED_BASE.
451+ * We don't bother to support different top nybbles
452+ * for LDS/Scratch and GPUVM.
453+ */
454+
455+ BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
456+ top_address_nybble == 0);
457+
458+ return PRIVATE_BASE(top_address_nybble << 12) |
459+ SHARED_BASE(top_address_nybble << 12);
460+}
461+
462+static int init_memory(struct device_queue_manager *dqm)
463+{
464+ int i, retval;
465+
466+ for (i = 8; i < 16; i++)
467+ set_pasid_vmid_mapping(dqm, 0, i);
468+
469+ retval = kfd2kgd->init_memory(dqm->dev->kgd);
470+ if (retval == 0)
471+ is_mem_initialized = true;
472+ return retval;
473+}
474+
475+
476+static int init_pipelines(struct device_queue_manager *dqm,
477+ unsigned int pipes_num, unsigned int first_pipe)
478+{
479+ void *hpdptr;
480+ struct mqd_manager *mqd;
481+ unsigned int i, err, inx;
482+ uint64_t pipe_hpd_addr;
483+
484+ BUG_ON(!dqm || !dqm->dev);
485+
486+ pr_debug("kfd: In func %s\n", __func__);
487+
488+ /*
489+ * Allocate memory for the HPDs. This is hardware-owned per-pipe data.
490+ * The driver never accesses this memory after zeroing it.
491+ * It doesn't even have to be saved/restored on suspend/resume
492+ * because it contains no data when there are no active queues.
493+ */
494+
495+ err = kfd2kgd->allocate_mem(dqm->dev->kgd,
496+ CIK_HPD_EOP_BYTES * pipes_num,
497+ PAGE_SIZE,
498+ KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
499+ (struct kgd_mem **) &dqm->pipeline_mem);
500+
501+ if (err) {
502+ pr_err("kfd: error allocate vidmem num pipes: %d\n",
503+ pipes_num);
504+ return -ENOMEM;
505+ }
506+
507+ hpdptr = dqm->pipeline_mem->cpu_ptr;
508+ dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr;
509+
510+ memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num);
511+
512+ mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
513+ if (mqd == NULL) {
514+ kfd2kgd->free_mem(dqm->dev->kgd,
515+ (struct kgd_mem *) dqm->pipeline_mem);
516+ return -ENOMEM;
517+ }
518+
519+ for (i = 0; i < pipes_num; i++) {
520+ inx = i + first_pipe;
521+ pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES;
522+ pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr);
523+ /* = log2(bytes/4)-1 */
524+ kfd2kgd->init_pipeline(dqm->dev->kgd, i,
525+ CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr);
526+ }
527+
528+ return 0;
529+}
530+
531+
532+static int init_scheduler(struct device_queue_manager *dqm)
533+{
534+ int retval;
535+
536+ BUG_ON(!dqm);
537+
538+ pr_debug("kfd: In %s\n", __func__);
539+
540+ retval = init_pipelines(dqm, get_pipes_num(dqm), KFD_DQM_FIRST_PIPE);
541+ if (retval != 0)
542+ return retval;
543+
544+ retval = init_memory(dqm);
545+
546+ return retval;
547+}
548+
549+static int initialize_nocpsch(struct device_queue_manager *dqm)
550+{
551+ int i;
552+
553+ BUG_ON(!dqm);
554+
555+ pr_debug("kfd: In func %s num of pipes: %d\n",
556+ __func__, get_pipes_num(dqm));
557+
558+ mutex_init(&dqm->lock);
559+ INIT_LIST_HEAD(&dqm->queues);
560+ dqm->queue_count = dqm->next_pipe_to_allocate = 0;
561+ dqm->allocated_queues = kcalloc(get_pipes_num(dqm),
562+ sizeof(unsigned int), GFP_KERNEL);
563+ if (!dqm->allocated_queues) {
564+ mutex_destroy(&dqm->lock);
565+ return -ENOMEM;
566+ }
567+
568+ for (i = 0; i < get_pipes_num(dqm); i++)
569+ dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1;
570+
571+ dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1;
572+
573+ init_scheduler(dqm);
574+ return 0;
575+}
576+
577+static void uninitialize_nocpsch(struct device_queue_manager *dqm)
578+{
579+ BUG_ON(!dqm);
580+
581+ BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
582+
583+ kfree(dqm->allocated_queues);
584+ mutex_destroy(&dqm->lock);
585+ kfd2kgd->free_mem(dqm->dev->kgd,
586+ (struct kgd_mem *) dqm->pipeline_mem);
587+}
588+
589+static int start_nocpsch(struct device_queue_manager *dqm)
590+{
591+ return 0;
592+}
593+
594+static int stop_nocpsch(struct device_queue_manager *dqm)
595+{
596+ return 0;
597+}
598+
599+/*
600+ * Device Queue Manager implementation for cp scheduler
601+ */
602+
603+static int set_sched_resources(struct device_queue_manager *dqm)
604+{
605+ struct scheduling_resources res;
606+ unsigned int queue_num, queue_mask;
607+
608+ BUG_ON(!dqm);
609+
610+ pr_debug("kfd: In func %s\n", __func__);
611+
612+ queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE;
613+ queue_mask = (1 << queue_num) - 1;
614+ res.vmid_mask = (1 << VMID_PER_DEVICE) - 1;
615+ res.vmid_mask <<= KFD_VMID_START_OFFSET;
616+ res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE);
617+ res.gws_mask = res.oac_mask = res.gds_heap_base =
618+ res.gds_heap_size = 0;
619+
620+ pr_debug("kfd: scheduling resources:\n"
621+ " vmid mask: 0x%8X\n"
622+ " queue mask: 0x%8llX\n",
623+ res.vmid_mask, res.queue_mask);
624+
625+ return pm_send_set_resources(&dqm->packets, &res);
626+}
627+
628+static int initialize_cpsch(struct device_queue_manager *dqm)
629+{
630+ int retval;
631+
632+ BUG_ON(!dqm);
633+
634+ pr_debug("kfd: In func %s num of pipes: %d\n",
635+ __func__, get_pipes_num_cpsch());
636+
637+ mutex_init(&dqm->lock);
638+ INIT_LIST_HEAD(&dqm->queues);
639+ dqm->queue_count = dqm->processes_count = 0;
640+ dqm->active_runlist = false;
641+ retval = init_pipelines(dqm, get_pipes_num(dqm), 0);
642+ if (retval != 0)
643+ goto fail_init_pipelines;
644+
645+ return 0;
646+
647+fail_init_pipelines:
648+ mutex_destroy(&dqm->lock);
649+ return retval;
650+}
651+
652+static int start_cpsch(struct device_queue_manager *dqm)
653+{
654+ struct device_process_node *node;
655+ int retval;
656+
657+ BUG_ON(!dqm);
658+
659+ retval = 0;
660+
661+ retval = pm_init(&dqm->packets, dqm);
662+ if (retval != 0)
663+ goto fail_packet_manager_init;
664+
665+ retval = set_sched_resources(dqm);
666+ if (retval != 0)
667+ goto fail_set_sched_resources;
668+
669+ pr_debug("kfd: allocating fence memory\n");
670+
671+ /* allocate fence memory on the gart */
672+ retval = kfd2kgd->allocate_mem(dqm->dev->kgd,
673+ sizeof(*dqm->fence_addr),
674+ 32,
675+ KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
676+ (struct kgd_mem **) &dqm->fence_mem);
677+
678+ if (retval != 0)
679+ goto fail_allocate_vidmem;
680+
681+ dqm->fence_addr = dqm->fence_mem->cpu_ptr;
682+ dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
683+
684+ list_for_each_entry(node, &dqm->queues, list)
685+ if (node->qpd->pqm->process && dqm->dev)
686+ kfd_bind_process_to_device(dqm->dev,
687+ node->qpd->pqm->process);
688+
689+ execute_queues_cpsch(dqm, true);
690+
691+ return 0;
692+fail_allocate_vidmem:
693+fail_set_sched_resources:
694+ pm_uninit(&dqm->packets);
695+fail_packet_manager_init:
696+ return retval;
697+}
698+
699+static int stop_cpsch(struct device_queue_manager *dqm)
700+{
701+ struct device_process_node *node;
702+ struct kfd_process_device *pdd;
703+
704+ BUG_ON(!dqm);
705+
706+ destroy_queues_cpsch(dqm, true);
707+
708+ list_for_each_entry(node, &dqm->queues, list) {
709+ pdd = kfd_get_process_device_data(dqm->dev,
710+ node->qpd->pqm->process, 1);
711+ pdd->bound = false;
712+ }
713+ kfd2kgd->free_mem(dqm->dev->kgd,
714+ (struct kgd_mem *) dqm->fence_mem);
715+ pm_uninit(&dqm->packets);
716+
717+ return 0;
718+}
719+
720+static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
721+ struct kernel_queue *kq,
722+ struct qcm_process_device *qpd)
723+{
724+ BUG_ON(!dqm || !kq || !qpd);
725+
726+ pr_debug("kfd: In func %s\n", __func__);
727+
728+ mutex_lock(&dqm->lock);
729+ list_add(&kq->list, &qpd->priv_queue_list);
730+ dqm->queue_count++;
731+ qpd->is_debug = true;
732+ execute_queues_cpsch(dqm, false);
733+ mutex_unlock(&dqm->lock);
734+
735+ return 0;
736+}
737+
738+static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
739+ struct kernel_queue *kq,
740+ struct qcm_process_device *qpd)
741+{
742+ BUG_ON(!dqm || !kq);
743+
744+ pr_debug("kfd: In %s\n", __func__);
745+
746+ mutex_lock(&dqm->lock);
747+ destroy_queues_cpsch(dqm, false);
748+ list_del(&kq->list);
749+ dqm->queue_count--;
750+ qpd->is_debug = false;
751+ execute_queues_cpsch(dqm, false);
752+ mutex_unlock(&dqm->lock);
753+}
754+
755+static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
756+ struct qcm_process_device *qpd, int *allocate_vmid)
757+{
758+ int retval;
759+ struct mqd_manager *mqd;
760+
761+ BUG_ON(!dqm || !q || !qpd);
762+
763+ retval = 0;
764+
765+ if (allocate_vmid)
766+ *allocate_vmid = 0;
767+
768+ mutex_lock(&dqm->lock);
769+
770+ mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_CP);
771+ if (mqd == NULL) {
772+ mutex_unlock(&dqm->lock);
773+ return -ENOMEM;
774+ }
775+
776+ retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
777+ &q->gart_mqd_addr, &q->properties);
778+ if (retval != 0)
779+ goto out;
780+
781+ list_add(&q->list, &qpd->queues_list);
782+ if (q->properties.is_active) {
783+ dqm->queue_count++;
784+ retval = execute_queues_cpsch(dqm, false);
785+ }
786+
787+out:
788+ mutex_unlock(&dqm->lock);
789+ return retval;
790+}
791+
792+int fence_wait_timeout(unsigned int *fence_addr, unsigned int fence_value,
793+ unsigned long timeout)
794+{
795+ BUG_ON(!fence_addr);
796+ timeout += jiffies;
797+
798+ while (*fence_addr != fence_value) {
799+ if (time_after(jiffies, timeout)) {
800+ pr_err("kfd: qcm fence wait loop timeout expired\n");
801+ return -ETIME;
802+ }
803+ cpu_relax();
804+ }
805+
806+ return 0;
807+}
808+
809+static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock)
810+{
811+ int retval;
812+
813+ BUG_ON(!dqm);
814+
815+ retval = 0;
816+
817+ if (lock)
818+ mutex_lock(&dqm->lock);
819+ if (dqm->active_runlist == false)
820+ goto out;
821+ retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
822+ KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0);
823+ if (retval != 0)
824+ goto out;
825+
826+ *dqm->fence_addr = KFD_FENCE_INIT;
827+ pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
828+ KFD_FENCE_COMPLETED);
829+ /* should be timed out */
830+ fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
831+ QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
832+ pm_release_ib(&dqm->packets);
833+ dqm->active_runlist = false;
834+
835+out:
836+ if (lock)
837+ mutex_unlock(&dqm->lock);
838+ return retval;
839+}
840+
841+static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
842+{
843+ int retval;
844+
845+ BUG_ON(!dqm);
846+
847+ if (lock)
848+ mutex_lock(&dqm->lock);
849+
850+ retval = destroy_queues_cpsch(dqm, false);
851+ if (retval != 0) {
852+ pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption");
853+ goto out;
854+ }
855+
856+ if (dqm->queue_count <= 0 || dqm->processes_count <= 0) {
857+ retval = 0;
858+ goto out;
859+ }
860+
861+ if (dqm->active_runlist) {
862+ retval = 0;
863+ goto out;
864+ }
865+
866+ retval = pm_send_runlist(&dqm->packets, &dqm->queues);
867+ if (retval != 0) {
868+ pr_err("kfd: failed to execute runlist");
869+ goto out;
870+ }
871+ dqm->active_runlist = true;
872+
873+out:
874+ if (lock)
875+ mutex_unlock(&dqm->lock);
876+ return retval;
877+}
878+
879+static int destroy_queue_cpsch(struct device_queue_manager *dqm,
880+ struct qcm_process_device *qpd,
881+ struct queue *q)
882+{
883+ int retval;
884+ struct mqd_manager *mqd;
885+
886+ BUG_ON(!dqm || !qpd || !q);
887+
888+ retval = 0;
889+
890+ /* remove queue from list to prevent rescheduling after preemption */
891+ mutex_lock(&dqm->lock);
892+
893+ mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_CP);
894+ if (!mqd) {
895+ retval = -ENOMEM;
896+ goto failed;
897+ }
898+
899+ list_del(&q->list);
900+ dqm->queue_count--;
901+
902+ execute_queues_cpsch(dqm, false);
903+
904+ mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
905+
906+ mutex_unlock(&dqm->lock);
907+
908+ return 0;
909+
910+failed:
911+ mutex_unlock(&dqm->lock);
912+ return retval;
913+}
914+
915+/*
916+ * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
917+ * stay in user mode.
918+ */
919+#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
920+/* APE1 limit is inclusive and 64K aligned. */
921+#define APE1_LIMIT_ALIGNMENT 0xFFFF
922+
923+static bool set_cache_memory_policy(struct device_queue_manager *dqm,
924+ struct qcm_process_device *qpd,
925+ enum cache_policy default_policy,
926+ enum cache_policy alternate_policy,
927+ void __user *alternate_aperture_base,
928+ uint64_t alternate_aperture_size)
929+{
930+ uint32_t default_mtype;
931+ uint32_t ape1_mtype;
932+
933+ pr_debug("kfd: In func %s\n", __func__);
934+
935+ mutex_lock(&dqm->lock);
936+
937+ if (alternate_aperture_size == 0) {
938+ /* base > limit disables APE1 */
939+ qpd->sh_mem_ape1_base = 1;
940+ qpd->sh_mem_ape1_limit = 0;
941+ } else {
942+ /*
943+ * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
944+ * SH_MEM_APE1_BASE[31:0], 0x0000 }
945+ * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
946+ * SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
947+ * Verify that the base and size parameters can be
948+ * represented in this format and convert them.
949+ * Additionally restrict APE1 to user-mode addresses.
950+ */
951+
952+ uint64_t base = (uintptr_t)alternate_aperture_base;
953+ uint64_t limit = base + alternate_aperture_size - 1;
954+
955+ if (limit <= base)
956+ goto out;
957+
958+ if ((base & APE1_FIXED_BITS_MASK) != 0)
959+ goto out;
960+
961+ if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT)
962+ goto out;
963+
964+ qpd->sh_mem_ape1_base = base >> 16;
965+ qpd->sh_mem_ape1_limit = limit >> 16;
966+ }
967+
968+ default_mtype = (default_policy == cache_policy_coherent) ?
969+ MTYPE_NONCACHED :
970+ MTYPE_CACHED;
971+
972+ ape1_mtype = (alternate_policy == cache_policy_coherent) ?
973+ MTYPE_NONCACHED :
974+ MTYPE_CACHED;
975+
976+ qpd->sh_mem_config = (qpd->sh_mem_config & PTR32)
977+ | ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
978+ | DEFAULT_MTYPE(default_mtype)
979+ | APE1_MTYPE(ape1_mtype);
980+
981+ if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
982+ program_sh_mem_settings(dqm, qpd);
983+
984+ pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
985+ qpd->sh_mem_config, qpd->sh_mem_ape1_base,
986+ qpd->sh_mem_ape1_limit);
987+
988+ mutex_unlock(&dqm->lock);
989+ return true;
990+
991+out:
992+ mutex_unlock(&dqm->lock);
993+ return false;
994+}
995+
996+struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
997+{
998+ struct device_queue_manager *dqm;
999+
1000+ BUG_ON(!dev);
1001+
1002+ dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL);
1003+ if (!dqm)
1004+ return NULL;
1005+
1006+ dqm->dev = dev;
1007+ switch (sched_policy) {
1008+ case KFD_SCHED_POLICY_HWS:
1009+ case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1010+ /* initialize dqm for cp scheduling */
1011+ dqm->create_queue = create_queue_cpsch;
1012+ dqm->initialize = initialize_cpsch;
1013+ dqm->start = start_cpsch;
1014+ dqm->stop = stop_cpsch;
1015+ dqm->destroy_queue = destroy_queue_cpsch;
1016+ dqm->update_queue = update_queue;
1017+ dqm->get_mqd_manager = get_mqd_manager_nocpsch;
1018+ dqm->register_process = register_process_nocpsch;
1019+ dqm->unregister_process = unregister_process_nocpsch;
1020+ dqm->uninitialize = uninitialize_nocpsch;
1021+ dqm->create_kernel_queue = create_kernel_queue_cpsch;
1022+ dqm->destroy_kernel_queue = destroy_kernel_queue_cpsch;
1023+ dqm->set_cache_memory_policy = set_cache_memory_policy;
1024+ break;
1025+ case KFD_SCHED_POLICY_NO_HWS:
1026+ /* initialize dqm for no cp scheduling */
1027+ dqm->start = start_nocpsch;
1028+ dqm->stop = stop_nocpsch;
1029+ dqm->create_queue = create_queue_nocpsch;
1030+ dqm->destroy_queue = destroy_queue_nocpsch;
1031+ dqm->update_queue = update_queue;
1032+ dqm->get_mqd_manager = get_mqd_manager_nocpsch;
1033+ dqm->register_process = register_process_nocpsch;
1034+ dqm->unregister_process = unregister_process_nocpsch;
1035+ dqm->initialize = initialize_nocpsch;
1036+ dqm->uninitialize = uninitialize_nocpsch;
1037+ dqm->set_cache_memory_policy = set_cache_memory_policy;
1038+ break;
1039+ default:
1040+ BUG();
1041+ break;
1042+ }
1043+
1044+ if (dqm->initialize(dqm) != 0) {
1045+ kfree(dqm);
1046+ return NULL;
1047+ }
1048+
1049+ return dqm;
1050+}
1051+
1052+void device_queue_manager_uninit(struct device_queue_manager *dqm)
1053+{
1054+ BUG_ON(!dqm);
1055+
1056+ dqm->uninitialize(dqm);
1057+ kfree(dqm);
1058+}
1059+
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h+146−0
@@ -0,0 +1,146 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ *
22+ */
23+
24+#ifndef KFD_DEVICE_QUEUE_MANAGER_H_
25+#define KFD_DEVICE_QUEUE_MANAGER_H_
26+
27+#include <linux/rwsem.h>
28+#include <linux/list.h>
29+#include "kfd_priv.h"
30+#include "kfd_mqd_manager.h"
31+
32+#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (500)
33+#define QUEUES_PER_PIPE (8)
34+#define PIPE_PER_ME_CP_SCHEDULING (3)
35+#define CIK_VMID_NUM (8)
36+#define KFD_VMID_START_OFFSET (8)
37+#define VMID_PER_DEVICE CIK_VMID_NUM
38+#define KFD_DQM_FIRST_PIPE (0)
39+
40+struct device_process_node {
41+ struct qcm_process_device *qpd;
42+ struct list_head list;
43+};
44+
45+/**
46+ * struct device_queue_manager
47+ *
48+ * @create_queue: Queue creation routine.
49+ *
50+ * @destroy_queue: Queue destruction routine.
51+ *
52+ * @update_queue: Queue update routine.
53+ *
54+ * @get_mqd_manager: Returns the mqd manager according to the mqd type.
55+ *
56+ * @exeute_queues: Dispatches the queues list to the H/W.
57+ *
58+ * @register_process: This routine associates a specific process with device.
59+ *
60+ * @unregister_process: destroys the associations between process to device.
61+ *
62+ * @initialize: Initializes the pipelines and memory module for that device.
63+ *
64+ * @start: Initializes the resources/modules the the device needs for queues
65+ * execution. This function is called on device initialization and after the
66+ * system woke up after suspension.
67+ *
68+ * @stop: This routine stops execution of all the active queue running on the
69+ * H/W and basically this function called on system suspend.
70+ *
71+ * @uninitialize: Destroys all the device queue manager resources allocated in
72+ * initialize routine.
73+ *
74+ * @create_kernel_queue: Creates kernel queue. Used for debug queue.
75+ *
76+ * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
77+ *
78+ * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
79+ * memory apertures.
80+ *
81+ * This struct is a base class for the kfd queues scheduler in the
82+ * device level. The device base class should expose the basic operations
83+ * for queue creation and queue destruction. This base class hides the
84+ * scheduling mode of the driver and the specific implementation of the
85+ * concrete device. This class is the only class in the queues scheduler
86+ * that configures the H/W.
87+ */
88+
89+struct device_queue_manager {
90+ int (*create_queue)(struct device_queue_manager *dqm,
91+ struct queue *q,
92+ struct qcm_process_device *qpd,
93+ int *allocate_vmid);
94+ int (*destroy_queue)(struct device_queue_manager *dqm,
95+ struct qcm_process_device *qpd,
96+ struct queue *q);
97+ int (*update_queue)(struct device_queue_manager *dqm,
98+ struct queue *q);
99+
100+ struct mqd_manager * (*get_mqd_manager)
101+ (struct device_queue_manager *dqm,
102+ enum KFD_MQD_TYPE type);
103+
104+ int (*register_process)(struct device_queue_manager *dqm,
105+ struct qcm_process_device *qpd);
106+ int (*unregister_process)(struct device_queue_manager *dqm,
107+ struct qcm_process_device *qpd);
108+ int (*initialize)(struct device_queue_manager *dqm);
109+ int (*start)(struct device_queue_manager *dqm);
110+ int (*stop)(struct device_queue_manager *dqm);
111+ void (*uninitialize)(struct device_queue_manager *dqm);
112+ int (*create_kernel_queue)(struct device_queue_manager *dqm,
113+ struct kernel_queue *kq,
114+ struct qcm_process_device *qpd);
115+ void (*destroy_kernel_queue)(struct device_queue_manager *dqm,
116+ struct kernel_queue *kq,
117+ struct qcm_process_device *qpd);
118+ bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
119+ struct qcm_process_device *qpd,
120+ enum cache_policy default_policy,
121+ enum cache_policy alternate_policy,
122+ void __user *alternate_aperture_base,
123+ uint64_t alternate_aperture_size);
124+
125+
126+ struct mqd_manager *mqds[KFD_MQD_TYPE_MAX];
127+ struct packet_manager packets;
128+ struct kfd_dev *dev;
129+ struct mutex lock;
130+ struct list_head queues;
131+ unsigned int processes_count;
132+ unsigned int queue_count;
133+ unsigned int next_pipe_to_allocate;
134+ unsigned int *allocated_queues;
135+ unsigned int vmid_bitmap;
136+ uint64_t pipelines_addr;
137+ struct kfd_mem_obj *pipeline_mem;
138+ uint64_t fence_gpu_addr;
139+ unsigned int *fence_addr;
140+ struct kfd_mem_obj *fence_mem;
141+ bool active_runlist;
142+};
143+
144+
145+
146+#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */
drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c+255−0
@@ -0,0 +1,255 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+#include "kfd_priv.h"
23+#include <linux/mm.h>
24+#include <linux/mman.h>
25+#include <linux/slab.h>
26+
27+/*
28+ * This extension supports a kernel level doorbells management for
29+ * the kernel queues.
30+ * Basically the last doorbells page is devoted to kernel queues
31+ * and that's assures that any user process won't get access to the
32+ * kernel doorbells page
33+ */
34+static DEFINE_MUTEX(doorbell_mutex);
35+static unsigned long doorbell_available_index[
36+ DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)] = { 0 };
37+
38+#define KERNEL_DOORBELL_PASID 1
39+#define KFD_SIZE_OF_DOORBELL_IN_BYTES 4
40+
41+/*
42+ * Each device exposes a doorbell aperture, a PCI MMIO aperture that
43+ * receives 32-bit writes that are passed to queues as wptr values.
44+ * The doorbells are intended to be written by applications as part
45+ * of queueing work on user-mode queues.
46+ * We assign doorbells to applications in PAGE_SIZE-sized and aligned chunks.
47+ * We map the doorbell address space into user-mode when a process creates
48+ * its first queue on each device.
49+ * Although the mapping is done by KFD, it is equivalent to an mmap of
50+ * the /dev/kfd with the particular device encoded in the mmap offset.
51+ * There will be other uses for mmap of /dev/kfd, so only a range of
52+ * offsets (KFD_MMAP_DOORBELL_START-END) is used for doorbells.
53+ */
54+
55+/* # of doorbell bytes allocated for each process. */
56+static inline size_t doorbell_process_allocation(void)
57+{
58+ return roundup(KFD_SIZE_OF_DOORBELL_IN_BYTES *
59+ KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
60+ PAGE_SIZE);
61+}
62+
63+/* Doorbell calculations for device init. */
64+void kfd_doorbell_init(struct kfd_dev *kfd)
65+{
66+ size_t doorbell_start_offset;
67+ size_t doorbell_aperture_size;
68+ size_t doorbell_process_limit;
69+
70+ /*
71+ * We start with calculations in bytes because the input data might
72+ * only be byte-aligned.
73+ * Only after we have done the rounding can we assume any alignment.
74+ */
75+
76+ doorbell_start_offset =
77+ roundup(kfd->shared_resources.doorbell_start_offset,
78+ doorbell_process_allocation());
79+
80+ doorbell_aperture_size =
81+ rounddown(kfd->shared_resources.doorbell_aperture_size,
82+ doorbell_process_allocation());
83+
84+ if (doorbell_aperture_size > doorbell_start_offset)
85+ doorbell_process_limit =
86+ (doorbell_aperture_size - doorbell_start_offset) /
87+ doorbell_process_allocation();
88+ else
89+ doorbell_process_limit = 0;
90+
91+ kfd->doorbell_base = kfd->shared_resources.doorbell_physical_address +
92+ doorbell_start_offset;
93+
94+ kfd->doorbell_id_offset = doorbell_start_offset / sizeof(u32);
95+ kfd->doorbell_process_limit = doorbell_process_limit - 1;
96+
97+ kfd->doorbell_kernel_ptr = ioremap(kfd->doorbell_base,
98+ doorbell_process_allocation());
99+
100+ BUG_ON(!kfd->doorbell_kernel_ptr);
101+
102+ pr_debug("kfd: doorbell initialization:\n");
103+ pr_debug("kfd: doorbell base == 0x%08lX\n",
104+ (uintptr_t)kfd->doorbell_base);
105+
106+ pr_debug("kfd: doorbell_id_offset == 0x%08lX\n",
107+ kfd->doorbell_id_offset);
108+
109+ pr_debug("kfd: doorbell_process_limit == 0x%08lX\n",
110+ doorbell_process_limit);
111+
112+ pr_debug("kfd: doorbell_kernel_offset == 0x%08lX\n",
113+ (uintptr_t)kfd->doorbell_base);
114+
115+ pr_debug("kfd: doorbell aperture size == 0x%08lX\n",
116+ kfd->shared_resources.doorbell_aperture_size);
117+
118+ pr_debug("kfd: doorbell kernel address == 0x%08lX\n",
119+ (uintptr_t)kfd->doorbell_kernel_ptr);
120+}
121+
122+int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma)
123+{
124+ phys_addr_t address;
125+ struct kfd_dev *dev;
126+
127+ /*
128+ * For simplicitly we only allow mapping of the entire doorbell
129+ * allocation of a single device & process.
130+ */
131+ if (vma->vm_end - vma->vm_start != doorbell_process_allocation())
132+ return -EINVAL;
133+
134+ /* Find kfd device according to gpu id */
135+ dev = kfd_device_by_id(vma->vm_pgoff);
136+ if (dev == NULL)
137+ return -EINVAL;
138+
139+ /* Find if pdd exists for combination of process and gpu id */
140+ if (!kfd_get_process_device_data(dev, process, 0))
141+ return -EINVAL;
142+
143+ /* Calculate physical address of doorbell */
144+ address = kfd_get_process_doorbells(dev, process);
145+
146+ vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
147+ VM_DONTDUMP | VM_PFNMAP;
148+
149+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
150+
151+ pr_debug("kfd: mapping doorbell page in kfd_doorbell_mmap\n"
152+ " target user address == 0x%08llX\n"
153+ " physical address == 0x%08llX\n"
154+ " vm_flags == 0x%04lX\n"
155+ " size == 0x%04lX\n",
156+ (unsigned long long) vma->vm_start, address, vma->vm_flags,
157+ doorbell_process_allocation());
158+
159+
160+ return io_remap_pfn_range(vma,
161+ vma->vm_start,
162+ address >> PAGE_SHIFT,
163+ doorbell_process_allocation(),
164+ vma->vm_page_prot);
165+}
166+
167+
168+/* get kernel iomem pointer for a doorbell */
169+u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
170+ unsigned int *doorbell_off)
171+{
172+ u32 inx;
173+
174+ BUG_ON(!kfd || !doorbell_off);
175+
176+ mutex_lock(&doorbell_mutex);
177+ inx = find_first_zero_bit(doorbell_available_index,
178+ KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
179+
180+ __set_bit(inx, doorbell_available_index);
181+ mutex_unlock(&doorbell_mutex);
182+
183+ if (inx >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
184+ return NULL;
185+
186+ /*
187+ * Calculating the kernel doorbell offset using "faked" kernel
188+ * pasid that allocated for kernel queues only
189+ */
190+ *doorbell_off = KERNEL_DOORBELL_PASID * (doorbell_process_allocation() /
191+ sizeof(u32)) + inx;
192+
193+ pr_debug("kfd: get kernel queue doorbell\n"
194+ " doorbell offset == 0x%08d\n"
195+ " kernel address == 0x%08lX\n",
196+ *doorbell_off, (uintptr_t)(kfd->doorbell_kernel_ptr + inx));
197+
198+ return kfd->doorbell_kernel_ptr + inx;
199+}
200+
201+void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr)
202+{
203+ unsigned int inx;
204+
205+ BUG_ON(!kfd || !db_addr);
206+
207+ inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr);
208+
209+ mutex_lock(&doorbell_mutex);
210+ __clear_bit(inx, doorbell_available_index);
211+ mutex_unlock(&doorbell_mutex);
212+}
213+
214+inline void write_kernel_doorbell(u32 __iomem *db, u32 value)
215+{
216+ if (db) {
217+ writel(value, db);
218+ pr_debug("writing %d to doorbell address 0x%p\n", value, db);
219+ }
220+}
221+
222+/*
223+ * queue_ids are in the range [0,MAX_PROCESS_QUEUES) and are mapped 1:1
224+ * to doorbells with the process's doorbell page
225+ */
226+unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd,
227+ struct kfd_process *process,
228+ unsigned int queue_id)
229+{
230+ /*
231+ * doorbell_id_offset accounts for doorbells taken by KGD.
232+ * pasid * doorbell_process_allocation/sizeof(u32) adjusts
233+ * to the process's doorbells
234+ */
235+ return kfd->doorbell_id_offset +
236+ process->pasid * (doorbell_process_allocation()/sizeof(u32)) +
237+ queue_id;
238+}
239+
240+uint64_t kfd_get_number_elems(struct kfd_dev *kfd)
241+{
242+ uint64_t num_of_elems = (kfd->shared_resources.doorbell_aperture_size -
243+ kfd->shared_resources.doorbell_start_offset) /
244+ doorbell_process_allocation() + 1;
245+
246+ return num_of_elems;
247+
248+}
249+
250+phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
251+ struct kfd_process *process)
252+{
253+ return dev->doorbell_base +
254+ process->pasid * doorbell_process_allocation();
255+}
drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c+355−0
@@ -0,0 +1,355 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ *
22+ */
23+
24+#include <linux/device.h>
25+#include <linux/export.h>
26+#include <linux/err.h>
27+#include <linux/fs.h>
28+#include <linux/sched.h>
29+#include <linux/slab.h>
30+#include <linux/uaccess.h>
31+#include <linux/compat.h>
32+#include <uapi/linux/kfd_ioctl.h>
33+#include <linux/time.h>
34+#include "kfd_priv.h"
35+#include <linux/mm.h>
36+#include <uapi/asm-generic/mman-common.h>
37+#include <asm/processor.h>
38+
39+/*
40+ * The primary memory I/O features being added for revisions of gfxip
41+ * beyond 7.0 (Kaveri) are:
42+ *
43+ * Access to ATC/IOMMU mapped memory w/ associated extension of VA to 48b
44+ *
45+ * “Flat” shader memory access – These are new shader vector memory
46+ * operations that do not reference a T#/V# so a “pointer” is what is
47+ * sourced from the vector gprs for direct access to memory.
48+ * This pointer space has the Shared(LDS) and Private(Scratch) memory
49+ * mapped into this pointer space as apertures.
50+ * The hardware then determines how to direct the memory request
51+ * based on what apertures the request falls in.
52+ *
53+ * Unaligned support and alignment check
54+ *
55+ *
56+ * System Unified Address - SUA
57+ *
58+ * The standard usage for GPU virtual addresses are that they are mapped by
59+ * a set of page tables we call GPUVM and these page tables are managed by
60+ * a combination of vidMM/driver software components. The current virtual
61+ * address (VA) range for GPUVM is 40b.
62+ *
63+ * As of gfxip7.1 and beyond we’re adding the ability for compute memory
64+ * clients (CP/RLC, DMA, SHADER(ifetch, scalar, and vector ops)) to access
65+ * the same page tables used by host x86 processors and that are managed by
66+ * the operating system. This is via a technique and hardware called ATC/IOMMU.
67+ * The GPU has the capability of accessing both the GPUVM and ATC address
68+ * spaces for a given VMID (process) simultaneously and we call this feature
69+ * system unified address (SUA).
70+ *
71+ * There are three fundamental address modes of operation for a given VMID
72+ * (process) on the GPU:
73+ *
74+ * HSA64 – 64b pointers and the default address space is ATC
75+ * HSA32 – 32b pointers and the default address space is ATC
76+ * GPUVM – 64b pointers and the default address space is GPUVM (driver
77+ * model mode)
78+ *
79+ *
80+ * HSA64 - ATC/IOMMU 64b
81+ *
82+ * A 64b pointer in the AMD64/IA64 CPU architecture is not fully utilized
83+ * by the CPU so an AMD CPU can only access the high area
84+ * (VA[63:47] == 0x1FFFF) and low area (VA[63:47 == 0) of the address space
85+ * so the actual VA carried to translation is 48b. There is a “hole” in
86+ * the middle of the 64b VA space.
87+ *
88+ * The GPU not only has access to all of the CPU accessible address space via
89+ * ATC/IOMMU, but it also has access to the GPUVM address space. The “system
90+ * unified address” feature (SUA) is the mapping of GPUVM and ATC address
91+ * spaces into a unified pointer space. The method we take for 64b mode is
92+ * to map the full 40b GPUVM address space into the hole of the 64b address
93+ * space.
94+
95+ * The GPUVM_Base/GPUVM_Limit defines the aperture in the 64b space where we
96+ * direct requests to be translated via GPUVM page tables instead of the
97+ * IOMMU path.
98+ *
99+ *
100+ * 64b to 49b Address conversion
101+ *
102+ * Note that there are still significant portions of unused regions (holes)
103+ * in the 64b address space even for the GPU. There are several places in
104+ * the pipeline (sw and hw), we wish to compress the 64b virtual address
105+ * to a 49b address. This 49b address is constituted of an “ATC” bit
106+ * plus a 48b virtual address. This 49b address is what is passed to the
107+ * translation hardware. ATC==0 means the 48b address is a GPUVM address
108+ * (max of 2^40 – 1) intended to be translated via GPUVM page tables.
109+ * ATC==1 means the 48b address is intended to be translated via IOMMU
110+ * page tables.
111+ *
112+ * A 64b pointer is compared to the apertures that are defined (Base/Limit), in
113+ * this case the GPUVM aperture (red) is defined and if a pointer falls in this
114+ * aperture, we subtract the GPUVM_Base address and set the ATC bit to zero
115+ * as part of the 64b to 49b conversion.
116+ *
117+ * Where this 64b to 49b conversion is done is a function of the usage.
118+ * Most GPU memory access is via memory objects where the driver builds
119+ * a descriptor which consists of a base address and a memory access by
120+ * the GPU usually consists of some kind of an offset or Cartesian coordinate
121+ * that references this memory descriptor. This is the case for shader
122+ * instructions that reference the T# or V# constants, or for specified
123+ * locations of assets (ex. the shader program location). In these cases
124+ * the driver is what handles the 64b to 49b conversion and the base
125+ * address in the descriptor (ex. V# or T# or shader program location)
126+ * is defined as a 48b address w/ an ATC bit. For this usage a given
127+ * memory object cannot straddle multiple apertures in the 64b address
128+ * space. For example a shader program cannot jump in/out between ATC
129+ * and GPUVM space.
130+ *
131+ * In some cases we wish to pass a 64b pointer to the GPU hardware and
132+ * the GPU hw does the 64b to 49b conversion before passing memory
133+ * requests to the cache/memory system. This is the case for the
134+ * S_LOAD and FLAT_* shader memory instructions where we have 64b pointers
135+ * in scalar and vector GPRs respectively.
136+ *
137+ * In all cases (no matter where the 64b -> 49b conversion is done), the gfxip
138+ * hardware sends a 48b address along w/ an ATC bit, to the memory controller
139+ * on the memory request interfaces.
140+ *
141+ * <client>_MC_rdreq_atc // read request ATC bit
142+ *
143+ * 0 : <client>_MC_rdreq_addr is a GPUVM VA
144+ *
145+ * 1 : <client>_MC_rdreq_addr is a ATC VA
146+ *
147+ *
148+ * “Spare” aperture (APE1)
149+ *
150+ * We use the GPUVM aperture to differentiate ATC vs. GPUVM, but we also use
151+ * apertures to set the Mtype field for S_LOAD/FLAT_* ops which is input to the
152+ * config tables for setting cache policies. The “spare” (APE1) aperture is
153+ * motivated by getting a different Mtype from the default.
154+ * The default aperture isn’t an actual base/limit aperture; it is just the
155+ * address space that doesn’t hit any defined base/limit apertures.
156+ * The following diagram is a complete picture of the gfxip7.x SUA apertures.
157+ * The APE1 can be placed either below or above
158+ * the hole (cannot be in the hole).
159+ *
160+ *
161+ * General Aperture definitions and rules
162+ *
163+ * An aperture register definition consists of a Base, Limit, Mtype, and
164+ * usually an ATC bit indicating which translation tables that aperture uses.
165+ * In all cases (for SUA and DUA apertures discussed later), aperture base
166+ * and limit definitions are 64KB aligned.
167+ *
168+ * <ape>_Base[63:0] = { <ape>_Base_register[63:16], 0x0000 }
169+ *
170+ * <ape>_Limit[63:0] = { <ape>_Limit_register[63:16], 0xFFFF }
171+ *
172+ * The base and limit are considered inclusive to an aperture so being
173+ * inside an aperture means (address >= Base) AND (address <= Limit).
174+ *
175+ * In no case is a payload that straddles multiple apertures expected to work.
176+ * For example a load_dword_x4 that starts in one aperture and ends in another,
177+ * does not work. For the vector FLAT_* ops we have detection capability in
178+ * the shader for reporting a “memory violation” back to the
179+ * SQ block for use in traps.
180+ * A memory violation results when an op falls into the hole,
181+ * or a payload straddles multiple apertures. The S_LOAD instruction
182+ * does not have this detection.
183+ *
184+ * Apertures cannot overlap.
185+ *
186+ *
187+ *
188+ * HSA32 - ATC/IOMMU 32b
189+ *
190+ * For HSA32 mode, the pointers are interpreted as 32 bits and use a single GPR
191+ * instead of two for the S_LOAD and FLAT_* ops. The entire GPUVM space of 40b
192+ * will not fit so there is only partial visibility to the GPUVM
193+ * space (defined by the aperture) for S_LOAD and FLAT_* ops.
194+ * There is no spare (APE1) aperture for HSA32 mode.
195+ *
196+ *
197+ * GPUVM 64b mode (driver model)
198+ *
199+ * This mode is related to HSA64 in that the difference really is that
200+ * the default aperture is GPUVM (ATC==0) and not ATC space.
201+ * We have gfxip7.x hardware that has FLAT_* and S_LOAD support for
202+ * SUA GPUVM mode, but does not support HSA32/HSA64.
203+ *
204+ *
205+ * Device Unified Address - DUA
206+ *
207+ * Device unified address (DUA) is the name of the feature that maps the
208+ * Shared(LDS) memory and Private(Scratch) memory into the overall address
209+ * space for use by the new FLAT_* vector memory ops. The Shared and
210+ * Private memories are mapped as apertures into the address space,
211+ * and the hardware detects when a FLAT_* memory request is to be redirected
212+ * to the LDS or Scratch memory when it falls into one of these apertures.
213+ * Like the SUA apertures, the Shared/Private apertures are 64KB aligned and
214+ * the base/limit is “in” the aperture. For both HSA64 and GPUVM SUA modes,
215+ * the Shared/Private apertures are always placed in a limited selection of
216+ * options in the hole of the 64b address space. For HSA32 mode, the
217+ * Shared/Private apertures can be placed anywhere in the 32b space
218+ * except at 0.
219+ *
220+ *
221+ * HSA64 Apertures for FLAT_* vector ops
222+ *
223+ * For HSA64 SUA mode, the Shared and Private apertures are always placed
224+ * in the hole w/ a limited selection of possible locations. The requests
225+ * that fall in the private aperture are expanded as a function of the
226+ * work-item id (tid) and redirected to the location of the
227+ * “hidden private memory”. The hidden private can be placed in either GPUVM
228+ * or ATC space. The addresses that fall in the shared aperture are
229+ * re-directed to the on-chip LDS memory hardware.
230+ *
231+ *
232+ * HSA32 Apertures for FLAT_* vector ops
233+ *
234+ * In HSA32 mode, the Private and Shared apertures can be placed anywhere
235+ * in the 32b space except at 0 (Private or Shared Base at zero disables
236+ * the apertures). If the base address of the apertures are non-zero
237+ * (ie apertures exists), the size is always 64KB.
238+ *
239+ *
240+ * GPUVM Apertures for FLAT_* vector ops
241+ *
242+ * In GPUVM mode, the Shared/Private apertures are specified identically
243+ * to HSA64 mode where they are always in the hole at a limited selection
244+ * of locations.
245+ *
246+ *
247+ * Aperture Definitions for SUA and DUA
248+ *
249+ * The interpretation of the aperture register definitions for a given
250+ * VMID is a function of the “SUA Mode” which is one of HSA64, HSA32, or
251+ * GPUVM64 discussed in previous sections. The mode is first decoded, and
252+ * then the remaining register decode is a function of the mode.
253+ *
254+ *
255+ * SUA Mode Decode
256+ *
257+ * For the S_LOAD and FLAT_* shader operations, the SUA mode is decoded from
258+ * the COMPUTE_DISPATCH_INITIATOR:DATA_ATC bit and
259+ * the SH_MEM_CONFIG:PTR32 bits.
260+ *
261+ * COMPUTE_DISPATCH_INITIATOR:DATA_ATC SH_MEM_CONFIG:PTR32 Mode
262+ *
263+ * 1 0 HSA64
264+ *
265+ * 1 1 HSA32
266+ *
267+ * 0 X GPUVM64
268+ *
269+ * In general the hardware will ignore the PTR32 bit and treat
270+ * as “0” whenever DATA_ATC = “0”, but sw should set PTR32=0
271+ * when DATA_ATC=0.
272+ *
273+ * The DATA_ATC bit is only set for compute dispatches.
274+ * All “Draw” dispatches are hardcoded to GPUVM64 mode
275+ * for FLAT_* / S_LOAD operations.
276+ */
277+
278+#define MAKE_GPUVM_APP_BASE(gpu_num) \
279+ (((uint64_t)(gpu_num) << 61) + 0x1000000000000)
280+
281+#define MAKE_GPUVM_APP_LIMIT(base) \
282+ (((uint64_t)(base) & 0xFFFFFF0000000000) | 0xFFFFFFFFFF)
283+
284+#define MAKE_SCRATCH_APP_BASE(gpu_num) \
285+ (((uint64_t)(gpu_num) << 61) + 0x100000000)
286+
287+#define MAKE_SCRATCH_APP_LIMIT(base) \
288+ (((uint64_t)base & 0xFFFFFFFF00000000) | 0xFFFFFFFF)
289+
290+#define MAKE_LDS_APP_BASE(gpu_num) \
291+ (((uint64_t)(gpu_num) << 61) + 0x0)
292+#define MAKE_LDS_APP_LIMIT(base) \
293+ (((uint64_t)(base) & 0xFFFFFFFF00000000) | 0xFFFFFFFF)
294+
295+int kfd_init_apertures(struct kfd_process *process)
296+{
297+ uint8_t id = 0;
298+ struct kfd_dev *dev;
299+ struct kfd_process_device *pdd;
300+
301+ mutex_lock(&process->mutex);
302+
303+ /*Iterating over all devices*/
304+ while ((dev = kfd_topology_enum_kfd_devices(id)) != NULL &&
305+ id < NUM_OF_SUPPORTED_GPUS) {
306+
307+ pdd = kfd_get_process_device_data(dev, process, 1);
308+
309+ /*
310+ * For 64 bit process aperture will be statically reserved in
311+ * the x86_64 non canonical process address space
312+ * amdkfd doesn't currently support apertures for 32 bit process
313+ */
314+ if (process->is_32bit_user_mode) {
315+ pdd->lds_base = pdd->lds_limit = 0;
316+ pdd->gpuvm_base = pdd->gpuvm_limit = 0;
317+ pdd->scratch_base = pdd->scratch_limit = 0;
318+ } else {
319+ /*
320+ * node id couldn't be 0 - the three MSB bits of
321+ * aperture shoudn't be 0
322+ */
323+ pdd->lds_base = MAKE_LDS_APP_BASE(id + 1);
324+
325+ pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);
326+
327+ pdd->gpuvm_base = MAKE_GPUVM_APP_BASE(id + 1);
328+
329+ pdd->gpuvm_limit =
330+ MAKE_GPUVM_APP_LIMIT(pdd->gpuvm_base);
331+
332+ pdd->scratch_base = MAKE_SCRATCH_APP_BASE(id + 1);
333+
334+ pdd->scratch_limit =
335+ MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);
336+ }
337+
338+ dev_dbg(kfd_device, "node id %u\n", id);
339+ dev_dbg(kfd_device, "gpu id %u\n", pdd->dev->id);
340+ dev_dbg(kfd_device, "lds_base %llX\n", pdd->lds_base);
341+ dev_dbg(kfd_device, "lds_limit %llX\n", pdd->lds_limit);
342+ dev_dbg(kfd_device, "gpuvm_base %llX\n", pdd->gpuvm_base);
343+ dev_dbg(kfd_device, "gpuvm_limit %llX\n", pdd->gpuvm_limit);
344+ dev_dbg(kfd_device, "scratch_base %llX\n", pdd->scratch_base);
345+ dev_dbg(kfd_device, "scratch_limit %llX\n", pdd->scratch_limit);
346+
347+ id++;
348+ }
349+
350+ mutex_unlock(&process->mutex);
351+
352+ return 0;
353+}
354+
355+
drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c+176−0
@@ -0,0 +1,176 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+/*
24+ * KFD Interrupts.
25+ *
26+ * AMD GPUs deliver interrupts by pushing an interrupt description onto the
27+ * interrupt ring and then sending an interrupt. KGD receives the interrupt
28+ * in ISR and sends us a pointer to each new entry on the interrupt ring.
29+ *
30+ * We generally can't process interrupt-signaled events from ISR, so we call
31+ * out to each interrupt client module (currently only the scheduler) to ask if
32+ * each interrupt is interesting. If they return true, then it requires further
33+ * processing so we copy it to an internal interrupt ring and call each
34+ * interrupt client again from a work-queue.
35+ *
36+ * There's no acknowledgment for the interrupts we use. The hardware simply
37+ * queues a new interrupt each time without waiting.
38+ *
39+ * The fixed-size internal queue means that it's possible for us to lose
40+ * interrupts because we have no back-pressure to the hardware.
41+ */
42+
43+#include <linux/slab.h>
44+#include <linux/device.h>
45+#include "kfd_priv.h"
46+
47+#define KFD_INTERRUPT_RING_SIZE 256
48+
49+static void interrupt_wq(struct work_struct *);
50+
51+int kfd_interrupt_init(struct kfd_dev *kfd)
52+{
53+ void *interrupt_ring = kmalloc_array(KFD_INTERRUPT_RING_SIZE,
54+ kfd->device_info->ih_ring_entry_size,
55+ GFP_KERNEL);
56+ if (!interrupt_ring)
57+ return -ENOMEM;
58+
59+ kfd->interrupt_ring = interrupt_ring;
60+ kfd->interrupt_ring_size =
61+ KFD_INTERRUPT_RING_SIZE * kfd->device_info->ih_ring_entry_size;
62+ atomic_set(&kfd->interrupt_ring_wptr, 0);
63+ atomic_set(&kfd->interrupt_ring_rptr, 0);
64+
65+ spin_lock_init(&kfd->interrupt_lock);
66+
67+ INIT_WORK(&kfd->interrupt_work, interrupt_wq);
68+
69+ kfd->interrupts_active = true;
70+
71+ /*
72+ * After this function returns, the interrupt will be enabled. This
73+ * barrier ensures that the interrupt running on a different processor
74+ * sees all the above writes.
75+ */
76+ smp_wmb();
77+
78+ return 0;
79+}
80+
81+void kfd_interrupt_exit(struct kfd_dev *kfd)
82+{
83+ /*
84+ * Stop the interrupt handler from writing to the ring and scheduling
85+ * workqueue items. The spinlock ensures that any interrupt running
86+ * after we have unlocked sees interrupts_active = false.
87+ */
88+ unsigned long flags;
89+
90+ spin_lock_irqsave(&kfd->interrupt_lock, flags);
91+ kfd->interrupts_active = false;
92+ spin_unlock_irqrestore(&kfd->interrupt_lock, flags);
93+
94+ /*
95+ * Flush_scheduled_work ensures that there are no outstanding
96+ * work-queue items that will access interrupt_ring. New work items
97+ * can't be created because we stopped interrupt handling above.
98+ */
99+ flush_scheduled_work();
100+
101+ kfree(kfd->interrupt_ring);
102+}
103+
104+/*
105+ * This assumes that it can't be called concurrently with itself
106+ * but only with dequeue_ih_ring_entry.
107+ */
108+bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry)
109+{
110+ unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr);
111+ unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr);
112+
113+ if ((rptr - wptr) % kfd->interrupt_ring_size ==
114+ kfd->device_info->ih_ring_entry_size) {
115+ /* This is very bad, the system is likely to hang. */
116+ dev_err_ratelimited(kfd_chardev(),
117+ "Interrupt ring overflow, dropping interrupt.\n");
118+ return false;
119+ }
120+
121+ memcpy(kfd->interrupt_ring + wptr, ih_ring_entry,
122+ kfd->device_info->ih_ring_entry_size);
123+
124+ wptr = (wptr + kfd->device_info->ih_ring_entry_size) %
125+ kfd->interrupt_ring_size;
126+ smp_wmb(); /* Ensure memcpy'd data is visible before wptr update. */
127+ atomic_set(&kfd->interrupt_ring_wptr, wptr);
128+
129+ return true;
130+}
131+
132+/*
133+ * This assumes that it can't be called concurrently with itself
134+ * but only with enqueue_ih_ring_entry.
135+ */
136+static bool dequeue_ih_ring_entry(struct kfd_dev *kfd, void *ih_ring_entry)
137+{
138+ /*
139+ * Assume that wait queues have an implicit barrier, i.e. anything that
140+ * happened in the ISR before it queued work is visible.
141+ */
142+
143+ unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr);
144+ unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr);
145+
146+ if (rptr == wptr)
147+ return false;
148+
149+ memcpy(ih_ring_entry, kfd->interrupt_ring + rptr,
150+ kfd->device_info->ih_ring_entry_size);
151+
152+ rptr = (rptr + kfd->device_info->ih_ring_entry_size) %
153+ kfd->interrupt_ring_size;
154+
155+ /*
156+ * Ensure the rptr write update is not visible until
157+ * memcpy has finished reading.
158+ */
159+ smp_mb();
160+ atomic_set(&kfd->interrupt_ring_rptr, rptr);
161+
162+ return true;
163+}
164+
165+static void interrupt_wq(struct work_struct *work)
166+{
167+ struct kfd_dev *dev = container_of(work, struct kfd_dev,
168+ interrupt_work);
169+
170+ uint32_t ih_ring_entry[DIV_ROUND_UP(
171+ dev->device_info->ih_ring_entry_size,
172+ sizeof(uint32_t))];
173+
174+ while (dequeue_ih_ring_entry(dev, ih_ring_entry))
175+ ;
176+}
drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c+347−0
@@ -0,0 +1,347 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ *
22+ */
23+
24+#include <linux/types.h>
25+#include <linux/mutex.h>
26+#include <linux/slab.h>
27+#include <linux/printk.h>
28+#include "kfd_kernel_queue.h"
29+#include "kfd_priv.h"
30+#include "kfd_device_queue_manager.h"
31+#include "kfd_pm4_headers.h"
32+#include "kfd_pm4_opcodes.h"
33+
34+#define PM4_COUNT_ZERO (((1 << 15) - 1) << 16)
35+
36+static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
37+ enum kfd_queue_type type, unsigned int queue_size)
38+{
39+ struct queue_properties prop;
40+ int retval;
41+ union PM4_MES_TYPE_3_HEADER nop;
42+
43+ BUG_ON(!kq || !dev);
44+ BUG_ON(type != KFD_QUEUE_TYPE_DIQ && type != KFD_QUEUE_TYPE_HIQ);
45+
46+ pr_debug("kfd: In func %s initializing queue type %d size %d\n",
47+ __func__, KFD_QUEUE_TYPE_HIQ, queue_size);
48+
49+ nop.opcode = IT_NOP;
50+ nop.type = PM4_TYPE_3;
51+ nop.u32all |= PM4_COUNT_ZERO;
52+
53+ kq->dev = dev;
54+ kq->nop_packet = nop.u32all;
55+ switch (type) {
56+ case KFD_QUEUE_TYPE_DIQ:
57+ case KFD_QUEUE_TYPE_HIQ:
58+ kq->mqd = dev->dqm->get_mqd_manager(dev->dqm,
59+ KFD_MQD_TYPE_CIK_HIQ);
60+ break;
61+ default:
62+ BUG();
63+ break;
64+ }
65+
66+ if (kq->mqd == NULL)
67+ return false;
68+
69+ prop.doorbell_ptr =
70+ (uint32_t *)kfd_get_kernel_doorbell(dev, &prop.doorbell_off);
71+
72+ if (prop.doorbell_ptr == NULL)
73+ goto err_get_kernel_doorbell;
74+
75+ retval = kfd2kgd->allocate_mem(dev->kgd,
76+ queue_size,
77+ PAGE_SIZE,
78+ KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
79+ (struct kgd_mem **) &kq->pq);
80+
81+ if (retval != 0)
82+ goto err_pq_allocate_vidmem;
83+
84+ kq->pq_kernel_addr = kq->pq->cpu_ptr;
85+ kq->pq_gpu_addr = kq->pq->gpu_addr;
86+
87+ retval = kfd2kgd->allocate_mem(dev->kgd,
88+ sizeof(*kq->rptr_kernel),
89+ 32,
90+ KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
91+ (struct kgd_mem **) &kq->rptr_mem);
92+
93+ if (retval != 0)
94+ goto err_rptr_allocate_vidmem;
95+
96+ kq->rptr_kernel = kq->rptr_mem->cpu_ptr;
97+ kq->rptr_gpu_addr = kq->rptr_mem->gpu_addr;
98+
99+ retval = kfd2kgd->allocate_mem(dev->kgd,
100+ sizeof(*kq->wptr_kernel),
101+ 32,
102+ KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
103+ (struct kgd_mem **) &kq->wptr_mem);
104+
105+ if (retval != 0)
106+ goto err_wptr_allocate_vidmem;
107+
108+ kq->wptr_kernel = kq->wptr_mem->cpu_ptr;
109+ kq->wptr_gpu_addr = kq->wptr_mem->gpu_addr;
110+
111+ memset(kq->pq_kernel_addr, 0, queue_size);
112+ memset(kq->rptr_kernel, 0, sizeof(*kq->rptr_kernel));
113+ memset(kq->wptr_kernel, 0, sizeof(*kq->wptr_kernel));
114+
115+ prop.queue_size = queue_size;
116+ prop.is_interop = false;
117+ prop.priority = 1;
118+ prop.queue_percent = 100;
119+ prop.type = type;
120+ prop.vmid = 0;
121+ prop.queue_address = kq->pq_gpu_addr;
122+ prop.read_ptr = (uint32_t *) kq->rptr_gpu_addr;
123+ prop.write_ptr = (uint32_t *) kq->wptr_gpu_addr;
124+
125+ if (init_queue(&kq->queue, prop) != 0)
126+ goto err_init_queue;
127+
128+ kq->queue->device = dev;
129+ kq->queue->process = kfd_get_process(current);
130+
131+ retval = kq->mqd->init_mqd(kq->mqd, &kq->queue->mqd,
132+ &kq->queue->mqd_mem_obj,
133+ &kq->queue->gart_mqd_addr,
134+ &kq->queue->properties);
135+ if (retval != 0)
136+ goto err_init_mqd;
137+
138+ /* assign HIQ to HQD */
139+ if (type == KFD_QUEUE_TYPE_HIQ) {
140+ pr_debug("assigning hiq to hqd\n");
141+ kq->queue->pipe = KFD_CIK_HIQ_PIPE;
142+ kq->queue->queue = KFD_CIK_HIQ_QUEUE;
143+ kq->mqd->load_mqd(kq->mqd, kq->queue->mqd, kq->queue->pipe,
144+ kq->queue->queue, NULL);
145+ } else {
146+ /* allocate fence for DIQ */
147+
148+ retval = kfd2kgd->allocate_mem(dev->kgd,
149+ sizeof(uint32_t),
150+ 32,
151+ KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
152+ (struct kgd_mem **) &kq->fence_mem_obj);
153+
154+ if (retval != 0)
155+ goto err_alloc_fence;
156+
157+ kq->fence_kernel_address = kq->fence_mem_obj->cpu_ptr;
158+ kq->fence_gpu_addr = kq->fence_mem_obj->gpu_addr;
159+ }
160+
161+ print_queue(kq->queue);
162+
163+ return true;
164+err_alloc_fence:
165+err_init_mqd:
166+ uninit_queue(kq->queue);
167+err_init_queue:
168+ kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->wptr_mem);
169+err_wptr_allocate_vidmem:
170+ kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->rptr_mem);
171+err_rptr_allocate_vidmem:
172+ kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->pq);
173+err_pq_allocate_vidmem:
174+ pr_err("kfd: error init pq\n");
175+ kfd_release_kernel_doorbell(dev, (u32 *)prop.doorbell_ptr);
176+err_get_kernel_doorbell:
177+ pr_err("kfd: error init doorbell");
178+ return false;
179+
180+}
181+
182+static void uninitialize(struct kernel_queue *kq)
183+{
184+ BUG_ON(!kq);
185+
186+ if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ)
187+ kq->mqd->destroy_mqd(kq->mqd,
188+ NULL,
189+ false,
190+ QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
191+ kq->queue->pipe,
192+ kq->queue->queue);
193+
194+ kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->rptr_mem);
195+ kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->wptr_mem);
196+ kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->pq);
197+ kfd_release_kernel_doorbell(kq->dev,
198+ (u32 *)kq->queue->properties.doorbell_ptr);
199+ uninit_queue(kq->queue);
200+}
201+
202+static int acquire_packet_buffer(struct kernel_queue *kq,
203+ size_t packet_size_in_dwords, unsigned int **buffer_ptr)
204+{
205+ size_t available_size;
206+ size_t queue_size_dwords;
207+ uint32_t wptr, rptr;
208+ unsigned int *queue_address;
209+
210+ BUG_ON(!kq || !buffer_ptr);
211+
212+ rptr = *kq->rptr_kernel;
213+ wptr = *kq->wptr_kernel;
214+ queue_address = (unsigned int *)kq->pq_kernel_addr;
215+ queue_size_dwords = kq->queue->properties.queue_size / sizeof(uint32_t);
216+
217+ pr_debug("kfd: In func %s\nrptr: %d\nwptr: %d\nqueue_address 0x%p\n",
218+ __func__, rptr, wptr, queue_address);
219+
220+ available_size = (rptr - 1 - wptr + queue_size_dwords) %
221+ queue_size_dwords;
222+
223+ if (packet_size_in_dwords >= queue_size_dwords ||
224+ packet_size_in_dwords >= available_size)
225+ return -ENOMEM;
226+
227+ if (wptr + packet_size_in_dwords >= queue_size_dwords) {
228+ while (wptr > 0) {
229+ queue_address[wptr] = kq->nop_packet;
230+ wptr = (wptr + 1) % queue_size_dwords;
231+ }
232+ }
233+
234+ *buffer_ptr = &queue_address[wptr];
235+ kq->pending_wptr = wptr + packet_size_in_dwords;
236+
237+ return 0;
238+}
239+
240+static void submit_packet(struct kernel_queue *kq)
241+{
242+#ifdef DEBUG
243+ int i;
244+#endif
245+
246+ BUG_ON(!kq);
247+
248+#ifdef DEBUG
249+ for (i = *kq->wptr_kernel; i < kq->pending_wptr; i++) {
250+ pr_debug("0x%2X ", kq->pq_kernel_addr[i]);
251+ if (i % 15 == 0)
252+ pr_debug("\n");
253+ }
254+ pr_debug("\n");
255+#endif
256+
257+ *kq->wptr_kernel = kq->pending_wptr;
258+ write_kernel_doorbell((u32 *)kq->queue->properties.doorbell_ptr,
259+ kq->pending_wptr);
260+}
261+
262+static int sync_with_hw(struct kernel_queue *kq, unsigned long timeout_ms)
263+{
264+ unsigned long org_timeout_ms;
265+
266+ BUG_ON(!kq);
267+
268+ org_timeout_ms = timeout_ms;
269+ timeout_ms += jiffies * 1000 / HZ;
270+ while (*kq->wptr_kernel != *kq->rptr_kernel) {
271+ if (time_after(jiffies * 1000 / HZ, timeout_ms)) {
272+ pr_err("kfd: kernel_queue %s timeout expired %lu\n",
273+ __func__, org_timeout_ms);
274+ pr_err("kfd: wptr: %d rptr: %d\n",
275+ *kq->wptr_kernel, *kq->rptr_kernel);
276+ return -ETIME;
277+ }
278+ cpu_relax();
279+ }
280+
281+ return 0;
282+}
283+
284+static void rollback_packet(struct kernel_queue *kq)
285+{
286+ BUG_ON(!kq);
287+ kq->pending_wptr = *kq->queue->properties.write_ptr;
288+}
289+
290+struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
291+ enum kfd_queue_type type)
292+{
293+ struct kernel_queue *kq;
294+
295+ BUG_ON(!dev);
296+
297+ kq = kzalloc(sizeof(struct kernel_queue), GFP_KERNEL);
298+ if (!kq)
299+ return NULL;
300+
301+ kq->initialize = initialize;
302+ kq->uninitialize = uninitialize;
303+ kq->acquire_packet_buffer = acquire_packet_buffer;
304+ kq->submit_packet = submit_packet;
305+ kq->sync_with_hw = sync_with_hw;
306+ kq->rollback_packet = rollback_packet;
307+
308+ if (kq->initialize(kq, dev, type, KFD_KERNEL_QUEUE_SIZE) == false) {
309+ pr_err("kfd: failed to init kernel queue\n");
310+ kfree(kq);
311+ return NULL;
312+ }
313+ return kq;
314+}
315+
316+void kernel_queue_uninit(struct kernel_queue *kq)
317+{
318+ BUG_ON(!kq);
319+
320+ kq->uninitialize(kq);
321+ kfree(kq);
322+}
323+
324+void test_kq(struct kfd_dev *dev)
325+{
326+ struct kernel_queue *kq;
327+ uint32_t *buffer, i;
328+ int retval;
329+
330+ BUG_ON(!dev);
331+
332+ pr_debug("kfd: starting kernel queue test\n");
333+
334+ kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_HIQ);
335+ BUG_ON(!kq);
336+
337+ retval = kq->acquire_packet_buffer(kq, 5, &buffer);
338+ BUG_ON(retval != 0);
339+ for (i = 0; i < 5; i++)
340+ buffer[i] = kq->nop_packet;
341+ kq->submit_packet(kq);
342+ kq->sync_with_hw(kq, 1000);
343+
344+ pr_debug("kfd: ending kernel queue test\n");
345+}
346+
347+
drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h+69−0
@@ -0,0 +1,69 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ *
22+ */
23+
24+#ifndef KFD_KERNEL_QUEUE_H_
25+#define KFD_KERNEL_QUEUE_H_
26+
27+#include <linux/list.h>
28+#include <linux/types.h>
29+#include "kfd_priv.h"
30+
31+struct kernel_queue {
32+ /* interface */
33+ bool (*initialize)(struct kernel_queue *kq, struct kfd_dev *dev,
34+ enum kfd_queue_type type, unsigned int queue_size);
35+ void (*uninitialize)(struct kernel_queue *kq);
36+ int (*acquire_packet_buffer)(struct kernel_queue *kq,
37+ size_t packet_size_in_dwords,
38+ unsigned int **buffer_ptr);
39+
40+ void (*submit_packet)(struct kernel_queue *kq);
41+ int (*sync_with_hw)(struct kernel_queue *kq,
42+ unsigned long timeout_ms);
43+ void (*rollback_packet)(struct kernel_queue *kq);
44+
45+ /* data */
46+ struct kfd_dev *dev;
47+ struct mqd_manager *mqd;
48+ struct queue *queue;
49+ uint32_t pending_wptr;
50+ unsigned int nop_packet;
51+
52+ struct kfd_mem_obj *rptr_mem;
53+ uint32_t *rptr_kernel;
54+ uint64_t rptr_gpu_addr;
55+ struct kfd_mem_obj *wptr_mem;
56+ uint32_t *wptr_kernel;
57+ uint64_t wptr_gpu_addr;
58+ struct kfd_mem_obj *pq;
59+ uint64_t pq_gpu_addr;
60+ uint32_t *pq_kernel_addr;
61+
62+ struct kfd_mem_obj *fence_mem_obj;
63+ uint64_t fence_gpu_addr;
64+ void *fence_kernel_address;
65+
66+ struct list_head list;
67+};
68+
69+#endif /* KFD_KERNEL_QUEUE_H_ */
drivers/gpu/drm/amd/amdkfd/kfd_module.c+159−0
@@ -0,0 +1,159 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+#include <linux/module.h>
24+#include <linux/sched.h>
25+#include <linux/moduleparam.h>
26+#include <linux/device.h>
27+#include "kfd_priv.h"
28+
29+#define KFD_DRIVER_AUTHOR "AMD Inc. and others"
30+
31+#define KFD_DRIVER_DESC "Standalone HSA driver for AMD's GPUs"
32+#define KFD_DRIVER_DATE "20141113"
33+#define KFD_DRIVER_MAJOR 0
34+#define KFD_DRIVER_MINOR 7
35+#define KFD_DRIVER_PATCHLEVEL 0
36+
37+const struct kfd2kgd_calls *kfd2kgd;
38+static const struct kgd2kfd_calls kgd2kfd = {
39+ .exit = kgd2kfd_exit,
40+ .probe = kgd2kfd_probe,
41+ .device_init = kgd2kfd_device_init,
42+ .device_exit = kgd2kfd_device_exit,
43+ .interrupt = kgd2kfd_interrupt,
44+ .suspend = kgd2kfd_suspend,
45+ .resume = kgd2kfd_resume,
46+};
47+
48+int sched_policy = KFD_SCHED_POLICY_HWS;
49+module_param(sched_policy, int, 0444);
50+MODULE_PARM_DESC(sched_policy,
51+ "Kernel cmdline parameter that defines the amdkfd scheduling policy");
52+
53+int max_num_of_processes = KFD_MAX_NUM_OF_PROCESSES_DEFAULT;
54+module_param(max_num_of_processes, int, 0444);
55+MODULE_PARM_DESC(max_num_of_processes,
56+ "Kernel cmdline parameter that defines the amdkfd maximum number of supported processes");
57+
58+int max_num_of_queues_per_process = KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT;
59+module_param(max_num_of_queues_per_process, int, 0444);
60+MODULE_PARM_DESC(max_num_of_queues_per_process,
61+ "Kernel cmdline parameter that defines the amdkfd maximum number of supported queues per process");
62+
63+bool kgd2kfd_init(unsigned interface_version,
64+ const struct kfd2kgd_calls *f2g,
65+ const struct kgd2kfd_calls **g2f)
66+{
67+ /*
68+ * Only one interface version is supported,
69+ * no kfd/kgd version skew allowed.
70+ */
71+ if (interface_version != KFD_INTERFACE_VERSION)
72+ return false;
73+
74+ /* Protection against multiple amd kgd loads */
75+ if (kfd2kgd)
76+ return true;
77+
78+ kfd2kgd = f2g;
79+ *g2f = &kgd2kfd;
80+
81+ return true;
82+}
83+EXPORT_SYMBOL(kgd2kfd_init);
84+
85+void kgd2kfd_exit(void)
86+{
87+}
88+
89+static int __init kfd_module_init(void)
90+{
91+ int err;
92+
93+ kfd2kgd = NULL;
94+
95+ /* Verify module parameters */
96+ if ((sched_policy < KFD_SCHED_POLICY_HWS) ||
97+ (sched_policy > KFD_SCHED_POLICY_NO_HWS)) {
98+ pr_err("kfd: sched_policy has invalid value\n");
99+ return -1;
100+ }
101+
102+ /* Verify module parameters */
103+ if ((max_num_of_processes < 0) ||
104+ (max_num_of_processes > KFD_MAX_NUM_OF_PROCESSES)) {
105+ pr_err("kfd: max_num_of_processes must be between 0 to KFD_MAX_NUM_OF_PROCESSES\n");
106+ return -1;
107+ }
108+
109+ if ((max_num_of_queues_per_process < 0) ||
110+ (max_num_of_queues_per_process >
111+ KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)) {
112+ pr_err("kfd: max_num_of_queues_per_process must be between 0 to KFD_MAX_NUM_OF_QUEUES_PER_PROCESS\n");
113+ return -1;
114+ }
115+
116+ err = kfd_pasid_init();
117+ if (err < 0)
118+ goto err_pasid;
119+
120+ err = kfd_chardev_init();
121+ if (err < 0)
122+ goto err_ioctl;
123+
124+ err = kfd_topology_init();
125+ if (err < 0)
126+ goto err_topology;
127+
128+ kfd_process_create_wq();
129+
130+ dev_info(kfd_device, "Initialized module\n");
131+
132+ return 0;
133+
134+err_topology:
135+ kfd_chardev_exit();
136+err_ioctl:
137+ kfd_pasid_exit();
138+err_pasid:
139+ return err;
140+}
141+
142+static void __exit kfd_module_exit(void)
143+{
144+ kfd_process_destroy_wq();
145+ kfd_topology_shutdown();
146+ kfd_chardev_exit();
147+ kfd_pasid_exit();
148+ dev_info(kfd_device, "Removed module\n");
149+}
150+
151+module_init(kfd_module_init);
152+module_exit(kfd_module_exit);
153+
154+MODULE_AUTHOR(KFD_DRIVER_AUTHOR);
155+MODULE_DESCRIPTION(KFD_DRIVER_DESC);
156+MODULE_LICENSE("GPL and additional rights");
157+MODULE_VERSION(__stringify(KFD_DRIVER_MAJOR) "."
158+ __stringify(KFD_DRIVER_MINOR) "."
159+ __stringify(KFD_DRIVER_PATCHLEVEL));
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c+346−0
@@ -0,0 +1,346 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ *
22+ */
23+
24+#include <linux/printk.h>
25+#include <linux/slab.h>
26+#include "kfd_priv.h"
27+#include "kfd_mqd_manager.h"
28+#include "cik_regs.h"
29+#include "../../radeon/cik_reg.h"
30+
31+inline void busy_wait(unsigned long ms)
32+{
33+ while (time_before(jiffies, ms))
34+ cpu_relax();
35+}
36+
37+static inline struct cik_mqd *get_mqd(void *mqd)
38+{
39+ return (struct cik_mqd *)mqd;
40+}
41+
42+static int init_mqd(struct mqd_manager *mm, void **mqd,
43+ struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
44+ struct queue_properties *q)
45+{
46+ uint64_t addr;
47+ struct cik_mqd *m;
48+ int retval;
49+
50+ BUG_ON(!mm || !q || !mqd);
51+
52+ pr_debug("kfd: In func %s\n", __func__);
53+
54+ retval = kfd2kgd->allocate_mem(mm->dev->kgd,
55+ sizeof(struct cik_mqd),
56+ 256,
57+ KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
58+ (struct kgd_mem **) mqd_mem_obj);
59+
60+ if (retval != 0)
61+ return -ENOMEM;
62+
63+ m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr;
64+ addr = (*mqd_mem_obj)->gpu_addr;
65+
66+ memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
67+
68+ m->header = 0xC0310800;
69+ m->compute_pipelinestat_enable = 1;
70+ m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
71+ m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
72+ m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
73+ m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
74+
75+ /*
76+ * Make sure to use the last queue state saved on mqd when the cp
77+ * reassigns the queue, so when queue is switched on/off (e.g over
78+ * subscription or quantum timeout) the context will be consistent
79+ */
80+ m->cp_hqd_persistent_state =
81+ DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ;
82+
83+ m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN;
84+ m->cp_mqd_base_addr_lo = lower_32_bits(addr);
85+ m->cp_mqd_base_addr_hi = upper_32_bits(addr);
86+
87+ m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN;
88+ /* Although WinKFD writes this, I suspect it should not be necessary */
89+ m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE;
90+
91+ m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
92+ QUANTUM_DURATION(10);
93+
94+ /*
95+ * Pipe Priority
96+ * Identifies the pipe relative priority when this queue is connected
97+ * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
98+ * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
99+ * 0 = CS_LOW (typically below GFX)
100+ * 1 = CS_MEDIUM (typically between HP3D and GFX
101+ * 2 = CS_HIGH (typically above HP3D)
102+ */
103+ m->cp_hqd_pipe_priority = 1;
104+ m->cp_hqd_queue_priority = 15;
105+
106+ *mqd = m;
107+ if (gart_addr != NULL)
108+ *gart_addr = addr;
109+ retval = mm->update_mqd(mm, m, q);
110+
111+ return retval;
112+}
113+
114+static void uninit_mqd(struct mqd_manager *mm, void *mqd,
115+ struct kfd_mem_obj *mqd_mem_obj)
116+{
117+ BUG_ON(!mm || !mqd);
118+ kfd2kgd->free_mem(mm->dev->kgd, (struct kgd_mem *) mqd_mem_obj);
119+}
120+
121+static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
122+ uint32_t queue_id, uint32_t __user *wptr)
123+{
124+ return kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id, wptr);
125+
126+}
127+
128+static int update_mqd(struct mqd_manager *mm, void *mqd,
129+ struct queue_properties *q)
130+{
131+ struct cik_mqd *m;
132+
133+ BUG_ON(!mm || !q || !mqd);
134+
135+ pr_debug("kfd: In func %s\n", __func__);
136+
137+ m = get_mqd(mqd);
138+ m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
139+ DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN;
140+
141+ /*
142+ * Calculating queue size which is log base 2 of actual queue size -1
143+ * dwords and another -1 for ffs
144+ */
145+ m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int))
146+ - 1 - 1;
147+ m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
148+ m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
149+ m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
150+ m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
151+ m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
152+ DOORBELL_OFFSET(q->doorbell_off);
153+
154+ m->cp_hqd_vmid = q->vmid;
155+
156+ if (q->format == KFD_QUEUE_FORMAT_AQL) {
157+ m->cp_hqd_iq_rptr = AQL_ENABLE;
158+ m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
159+ }
160+
161+ m->cp_hqd_active = 0;
162+ q->is_active = false;
163+ if (q->queue_size > 0 &&
164+ q->queue_address != 0 &&
165+ q->queue_percent > 0) {
166+ m->cp_hqd_active = 1;
167+ q->is_active = true;
168+ }
169+
170+ return 0;
171+}
172+
173+static int destroy_mqd(struct mqd_manager *mm, void *mqd,
174+ enum kfd_preempt_type type,
175+ unsigned int timeout, uint32_t pipe_id,
176+ uint32_t queue_id)
177+{
178+ return kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout,
179+ pipe_id, queue_id);
180+}
181+
182+bool is_occupied(struct mqd_manager *mm, void *mqd,
183+ uint64_t queue_address, uint32_t pipe_id,
184+ uint32_t queue_id)
185+{
186+
187+ return kfd2kgd->hqd_is_occupies(mm->dev->kgd, queue_address,
188+ pipe_id, queue_id);
189+
190+}
191+
192+/*
193+ * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation.
194+ * The HIQ queue in Kaveri is using the same MQD structure as all the user mode
195+ * queues but with different initial values.
196+ */
197+
198+static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
199+ struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
200+ struct queue_properties *q)
201+{
202+ uint64_t addr;
203+ struct cik_mqd *m;
204+ int retval;
205+
206+ BUG_ON(!mm || !q || !mqd || !mqd_mem_obj);
207+
208+ pr_debug("kfd: In func %s\n", __func__);
209+
210+ retval = kfd2kgd->allocate_mem(mm->dev->kgd,
211+ sizeof(struct cik_mqd),
212+ 256,
213+ KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
214+ (struct kgd_mem **) mqd_mem_obj);
215+
216+ if (retval != 0)
217+ return -ENOMEM;
218+
219+ m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr;
220+ addr = (*mqd_mem_obj)->gpu_addr;
221+
222+ memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
223+
224+ m->header = 0xC0310800;
225+ m->compute_pipelinestat_enable = 1;
226+ m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
227+ m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
228+ m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
229+ m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
230+
231+ m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE |
232+ PRELOAD_REQ;
233+ m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
234+ QUANTUM_DURATION(10);
235+
236+ m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN;
237+ m->cp_mqd_base_addr_lo = lower_32_bits(addr);
238+ m->cp_mqd_base_addr_hi = upper_32_bits(addr);
239+
240+ m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE;
241+
242+ /*
243+ * Pipe Priority
244+ * Identifies the pipe relative priority when this queue is connected
245+ * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
246+ * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
247+ * 0 = CS_LOW (typically below GFX)
248+ * 1 = CS_MEDIUM (typically between HP3D and GFX
249+ * 2 = CS_HIGH (typically above HP3D)
250+ */
251+ m->cp_hqd_pipe_priority = 1;
252+ m->cp_hqd_queue_priority = 15;
253+
254+ *mqd = m;
255+ if (gart_addr)
256+ *gart_addr = addr;
257+ retval = mm->update_mqd(mm, m, q);
258+
259+ return retval;
260+}
261+
262+static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
263+ struct queue_properties *q)
264+{
265+ struct cik_mqd *m;
266+
267+ BUG_ON(!mm || !q || !mqd);
268+
269+ pr_debug("kfd: In func %s\n", __func__);
270+
271+ m = get_mqd(mqd);
272+ m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
273+ DEFAULT_MIN_AVAIL_SIZE |
274+ PRIV_STATE |
275+ KMD_QUEUE;
276+
277+ /*
278+ * Calculating queue size which is log base 2 of actual queue
279+ * size -1 dwords
280+ */
281+ m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int))
282+ - 1 - 1;
283+ m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
284+ m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
285+ m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
286+ m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
287+ m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
288+ DOORBELL_OFFSET(q->doorbell_off);
289+
290+ m->cp_hqd_vmid = q->vmid;
291+
292+ m->cp_hqd_active = 0;
293+ q->is_active = false;
294+ if (q->queue_size > 0 &&
295+ q->queue_address != 0 &&
296+ q->queue_percent > 0) {
297+ m->cp_hqd_active = 1;
298+ q->is_active = true;
299+ }
300+
301+ return 0;
302+}
303+
304+struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
305+ struct kfd_dev *dev)
306+{
307+ struct mqd_manager *mqd;
308+
309+ BUG_ON(!dev);
310+ BUG_ON(type >= KFD_MQD_TYPE_MAX);
311+
312+ pr_debug("kfd: In func %s\n", __func__);
313+
314+ mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL);
315+ if (!mqd)
316+ return NULL;
317+
318+ mqd->dev = dev;
319+
320+ switch (type) {
321+ case KFD_MQD_TYPE_CIK_CP:
322+ case KFD_MQD_TYPE_CIK_COMPUTE:
323+ mqd->init_mqd = init_mqd;
324+ mqd->uninit_mqd = uninit_mqd;
325+ mqd->load_mqd = load_mqd;
326+ mqd->update_mqd = update_mqd;
327+ mqd->destroy_mqd = destroy_mqd;
328+ mqd->is_occupied = is_occupied;
329+ break;
330+ case KFD_MQD_TYPE_CIK_HIQ:
331+ mqd->init_mqd = init_mqd_hiq;
332+ mqd->uninit_mqd = uninit_mqd;
333+ mqd->load_mqd = load_mqd;
334+ mqd->update_mqd = update_mqd_hiq;
335+ mqd->destroy_mqd = destroy_mqd;
336+ mqd->is_occupied = is_occupied;
337+ break;
338+ default:
339+ kfree(mqd);
340+ return NULL;
341+ }
342+
343+ return mqd;
344+}
345+
346+/* SDMA queues should be implemented here when the cp will supports them */
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h+91−0
@@ -0,0 +1,91 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ *
22+ */
23+
24+#ifndef KFD_MQD_MANAGER_H_
25+#define KFD_MQD_MANAGER_H_
26+
27+#include "kfd_priv.h"
28+
29+/**
30+ * struct mqd_manager
31+ *
32+ * @init_mqd: Allocates the mqd buffer on local gpu memory and initialize it.
33+ *
34+ * @load_mqd: Loads the mqd to a concrete hqd slot. Used only for no cp
35+ * scheduling mode.
36+ *
37+ * @update_mqd: Handles a update call for the MQD
38+ *
39+ * @destroy_mqd: Destroys the HQD slot and by that preempt the relevant queue.
40+ * Used only for no cp scheduling.
41+ *
42+ * @uninit_mqd: Releases the mqd buffer from local gpu memory.
43+ *
44+ * @is_occupied: Checks if the relevant HQD slot is occupied.
45+ *
46+ * @mqd_mutex: Mqd manager mutex.
47+ *
48+ * @dev: The kfd device structure coupled with this module.
49+ *
50+ * MQD stands for Memory Queue Descriptor which represents the current queue
51+ * state in the memory and initiate the HQD (Hardware Queue Descriptor) state.
52+ * This structure is actually a base class for the different types of MQDs
53+ * structures for the variant ASICs that should be supported in the future.
54+ * This base class is also contains all the MQD specific operations.
55+ * Another important thing to mention is that each queue has a MQD that keeps
56+ * his state (or context) after each preemption or reassignment.
57+ * Basically there are a instances of the mqd manager class per MQD type per
58+ * ASIC. Currently the kfd driver supports only Kaveri so there are instances
59+ * per KFD_MQD_TYPE for each device.
60+ *
61+ */
62+
63+struct mqd_manager {
64+ int (*init_mqd)(struct mqd_manager *mm, void **mqd,
65+ struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
66+ struct queue_properties *q);
67+
68+ int (*load_mqd)(struct mqd_manager *mm, void *mqd,
69+ uint32_t pipe_id, uint32_t queue_id,
70+ uint32_t __user *wptr);
71+
72+ int (*update_mqd)(struct mqd_manager *mm, void *mqd,
73+ struct queue_properties *q);
74+
75+ int (*destroy_mqd)(struct mqd_manager *mm, void *mqd,
76+ enum kfd_preempt_type type,
77+ unsigned int timeout, uint32_t pipe_id,
78+ uint32_t queue_id);
79+
80+ void (*uninit_mqd)(struct mqd_manager *mm, void *mqd,
81+ struct kfd_mem_obj *mqd_mem_obj);
82+
83+ bool (*is_occupied)(struct mqd_manager *mm, void *mqd,
84+ uint64_t queue_address, uint32_t pipe_id,
85+ uint32_t queue_id);
86+
87+ struct mutex mqd_mutex;
88+ struct kfd_dev *dev;
89+};
90+
91+#endif /* KFD_MQD_MANAGER_H_ */
drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c+565−0
@@ -0,0 +1,565 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ *
22+ */
23+
24+#include <linux/slab.h>
25+#include <linux/mutex.h>
26+#include "kfd_device_queue_manager.h"
27+#include "kfd_kernel_queue.h"
28+#include "kfd_priv.h"
29+#include "kfd_pm4_headers.h"
30+#include "kfd_pm4_opcodes.h"
31+
32+static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes,
33+ unsigned int buffer_size_bytes)
34+{
35+ unsigned int temp = *wptr + increment_bytes / sizeof(uint32_t);
36+
37+ BUG_ON((temp * sizeof(uint32_t)) > buffer_size_bytes);
38+ *wptr = temp;
39+}
40+
41+static unsigned int build_pm4_header(unsigned int opcode, size_t packet_size)
42+{
43+ union PM4_MES_TYPE_3_HEADER header;
44+
45+ header.u32all = 0;
46+ header.opcode = opcode;
47+ header.count = packet_size/sizeof(uint32_t) - 2;
48+ header.type = PM4_TYPE_3;
49+
50+ return header.u32all;
51+}
52+
53+static void pm_calc_rlib_size(struct packet_manager *pm,
54+ unsigned int *rlib_size,
55+ bool *over_subscription)
56+{
57+ unsigned int process_count, queue_count;
58+
59+ BUG_ON(!pm || !rlib_size || !over_subscription);
60+
61+ process_count = pm->dqm->processes_count;
62+ queue_count = pm->dqm->queue_count;
63+
64+ /* check if there is over subscription*/
65+ *over_subscription = false;
66+ if ((process_count > 1) ||
67+ queue_count > PIPE_PER_ME_CP_SCHEDULING * QUEUES_PER_PIPE) {
68+ *over_subscription = true;
69+ pr_debug("kfd: over subscribed runlist\n");
70+ }
71+
72+ /* calculate run list ib allocation size */
73+ *rlib_size = process_count * sizeof(struct pm4_map_process) +
74+ queue_count * sizeof(struct pm4_map_queues);
75+
76+ /*
77+ * Increase the allocation size in case we need a chained run list
78+ * when over subscription
79+ */
80+ if (*over_subscription)
81+ *rlib_size += sizeof(struct pm4_runlist);
82+
83+ pr_debug("kfd: runlist ib size %d\n", *rlib_size);
84+}
85+
86+static int pm_allocate_runlist_ib(struct packet_manager *pm,
87+ unsigned int **rl_buffer,
88+ uint64_t *rl_gpu_buffer,
89+ unsigned int *rl_buffer_size,
90+ bool *is_over_subscription)
91+{
92+ int retval;
93+
94+ BUG_ON(!pm);
95+ BUG_ON(pm->allocated == true);
96+ BUG_ON(is_over_subscription == NULL);
97+
98+ pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription);
99+
100+ retval = kfd2kgd->allocate_mem(pm->dqm->dev->kgd,
101+ *rl_buffer_size,
102+ PAGE_SIZE,
103+ KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
104+ (struct kgd_mem **) &pm->ib_buffer_obj);
105+
106+ if (retval != 0) {
107+ pr_err("kfd: failed to allocate runlist IB\n");
108+ return retval;
109+ }
110+
111+ *(void **)rl_buffer = pm->ib_buffer_obj->cpu_ptr;
112+ *rl_gpu_buffer = pm->ib_buffer_obj->gpu_addr;
113+
114+ memset(*rl_buffer, 0, *rl_buffer_size);
115+ pm->allocated = true;
116+ return retval;
117+}
118+
119+static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer,
120+ uint64_t ib, size_t ib_size_in_dwords, bool chain)
121+{
122+ struct pm4_runlist *packet;
123+
124+ BUG_ON(!pm || !buffer || !ib);
125+
126+ packet = (struct pm4_runlist *)buffer;
127+
128+ memset(buffer, 0, sizeof(struct pm4_runlist));
129+ packet->header.u32all = build_pm4_header(IT_RUN_LIST,
130+ sizeof(struct pm4_runlist));
131+
132+ packet->bitfields4.ib_size = ib_size_in_dwords;
133+ packet->bitfields4.chain = chain ? 1 : 0;
134+ packet->bitfields4.offload_polling = 0;
135+ packet->bitfields4.valid = 1;
136+ packet->ordinal2 = lower_32_bits(ib);
137+ packet->bitfields3.ib_base_hi = upper_32_bits(ib);
138+
139+ return 0;
140+}
141+
142+static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer,
143+ struct qcm_process_device *qpd)
144+{
145+ struct pm4_map_process *packet;
146+ struct queue *cur;
147+ uint32_t num_queues;
148+
149+ BUG_ON(!pm || !buffer || !qpd);
150+
151+ packet = (struct pm4_map_process *)buffer;
152+
153+ pr_debug("kfd: In func %s\n", __func__);
154+
155+ memset(buffer, 0, sizeof(struct pm4_map_process));
156+
157+ packet->header.u32all = build_pm4_header(IT_MAP_PROCESS,
158+ sizeof(struct pm4_map_process));
159+ packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
160+ packet->bitfields2.process_quantum = 1;
161+ packet->bitfields2.pasid = qpd->pqm->process->pasid;
162+ packet->bitfields3.page_table_base = qpd->page_table_base;
163+ packet->bitfields10.gds_size = qpd->gds_size;
164+ packet->bitfields10.num_gws = qpd->num_gws;
165+ packet->bitfields10.num_oac = qpd->num_oac;
166+ num_queues = 0;
167+ list_for_each_entry(cur, &qpd->queues_list, list)
168+ num_queues++;
169+ packet->bitfields10.num_queues = num_queues;
170+
171+ packet->sh_mem_config = qpd->sh_mem_config;
172+ packet->sh_mem_bases = qpd->sh_mem_bases;
173+ packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base;
174+ packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit;
175+
176+ packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
177+ packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
178+
179+ return 0;
180+}
181+
182+static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer,
183+ struct queue *q)
184+{
185+ struct pm4_map_queues *packet;
186+
187+ BUG_ON(!pm || !buffer || !q);
188+
189+ pr_debug("kfd: In func %s\n", __func__);
190+
191+ packet = (struct pm4_map_queues *)buffer;
192+ memset(buffer, 0, sizeof(struct pm4_map_queues));
193+
194+ packet->header.u32all = build_pm4_header(IT_MAP_QUEUES,
195+ sizeof(struct pm4_map_queues));
196+ packet->bitfields2.alloc_format =
197+ alloc_format__mes_map_queues__one_per_pipe;
198+ packet->bitfields2.num_queues = 1;
199+ packet->bitfields2.queue_sel =
200+ queue_sel__mes_map_queues__map_to_hws_determined_queue_slots;
201+
202+ packet->bitfields2.vidmem = (q->properties.is_interop) ?
203+ vidmem__mes_map_queues__uses_video_memory :
204+ vidmem__mes_map_queues__uses_no_video_memory;
205+
206+ switch (q->properties.type) {
207+ case KFD_QUEUE_TYPE_COMPUTE:
208+ case KFD_QUEUE_TYPE_DIQ:
209+ packet->bitfields2.engine_sel =
210+ engine_sel__mes_map_queues__compute;
211+ break;
212+ case KFD_QUEUE_TYPE_SDMA:
213+ packet->bitfields2.engine_sel =
214+ engine_sel__mes_map_queues__sdma0;
215+ break;
216+ default:
217+ BUG();
218+ break;
219+ }
220+
221+ packet->mes_map_queues_ordinals[0].bitfields3.doorbell_offset =
222+ q->properties.doorbell_off;
223+
224+ packet->mes_map_queues_ordinals[0].mqd_addr_lo =
225+ lower_32_bits(q->gart_mqd_addr);
226+
227+ packet->mes_map_queues_ordinals[0].mqd_addr_hi =
228+ upper_32_bits(q->gart_mqd_addr);
229+
230+ packet->mes_map_queues_ordinals[0].wptr_addr_lo =
231+ lower_32_bits((uint64_t)q->properties.write_ptr);
232+
233+ packet->mes_map_queues_ordinals[0].wptr_addr_hi =
234+ upper_32_bits((uint64_t)q->properties.write_ptr);
235+
236+ return 0;
237+}
238+
239+static int pm_create_runlist_ib(struct packet_manager *pm,
240+ struct list_head *queues,
241+ uint64_t *rl_gpu_addr,
242+ size_t *rl_size_bytes)
243+{
244+ unsigned int alloc_size_bytes;
245+ unsigned int *rl_buffer, rl_wptr, i;
246+ int retval, proccesses_mapped;
247+ struct device_process_node *cur;
248+ struct qcm_process_device *qpd;
249+ struct queue *q;
250+ struct kernel_queue *kq;
251+ bool is_over_subscription;
252+
253+ BUG_ON(!pm || !queues || !rl_size_bytes || !rl_gpu_addr);
254+
255+ rl_wptr = retval = proccesses_mapped = 0;
256+
257+ retval = pm_allocate_runlist_ib(pm, &rl_buffer, rl_gpu_addr,
258+ &alloc_size_bytes, &is_over_subscription);
259+ if (retval != 0)
260+ return retval;
261+
262+ *rl_size_bytes = alloc_size_bytes;
263+
264+ pr_debug("kfd: In func %s\n", __func__);
265+ pr_debug("kfd: building runlist ib process count: %d queues count %d\n",
266+ pm->dqm->processes_count, pm->dqm->queue_count);
267+
268+ /* build the run list ib packet */
269+ list_for_each_entry(cur, queues, list) {
270+ qpd = cur->qpd;
271+ /* build map process packet */
272+ if (proccesses_mapped >= pm->dqm->processes_count) {
273+ pr_debug("kfd: not enough space left in runlist IB\n");
274+ pm_release_ib(pm);
275+ return -ENOMEM;
276+ }
277+ retval = pm_create_map_process(pm, &rl_buffer[rl_wptr], qpd);
278+ if (retval != 0)
279+ return retval;
280+ proccesses_mapped++;
281+ inc_wptr(&rl_wptr, sizeof(struct pm4_map_process),
282+ alloc_size_bytes);
283+
284+ list_for_each_entry(kq, &qpd->priv_queue_list, list) {
285+ if (kq->queue->properties.is_active != true)
286+ continue;
287+ retval = pm_create_map_queue(pm, &rl_buffer[rl_wptr],
288+ kq->queue);
289+ if (retval != 0)
290+ return retval;
291+ inc_wptr(&rl_wptr, sizeof(struct pm4_map_queues),
292+ alloc_size_bytes);
293+ }
294+
295+ list_for_each_entry(q, &qpd->queues_list, list) {
296+ if (q->properties.is_active != true)
297+ continue;
298+ retval = pm_create_map_queue(pm,
299+ &rl_buffer[rl_wptr], q);
300+ if (retval != 0)
301+ return retval;
302+ inc_wptr(&rl_wptr, sizeof(struct pm4_map_queues),
303+ alloc_size_bytes);
304+ }
305+ }
306+
307+ pr_debug("kfd: finished map process and queues to runlist\n");
308+
309+ if (is_over_subscription)
310+ pm_create_runlist(pm, &rl_buffer[rl_wptr], *rl_gpu_addr,
311+ alloc_size_bytes / sizeof(uint32_t), true);
312+
313+ for (i = 0; i < alloc_size_bytes / sizeof(uint32_t); i++)
314+ pr_debug("0x%2X ", rl_buffer[i]);
315+ pr_debug("\n");
316+
317+ return 0;
318+}
319+
320+int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)
321+{
322+ BUG_ON(!dqm);
323+
324+ pm->dqm = dqm;
325+ mutex_init(&pm->lock);
326+ pm->priv_queue = kernel_queue_init(dqm->dev, KFD_QUEUE_TYPE_HIQ);
327+ if (pm->priv_queue == NULL) {
328+ mutex_destroy(&pm->lock);
329+ return -ENOMEM;
330+ }
331+ pm->allocated = false;
332+
333+ return 0;
334+}
335+
336+void pm_uninit(struct packet_manager *pm)
337+{
338+ BUG_ON(!pm);
339+
340+ mutex_destroy(&pm->lock);
341+ kernel_queue_uninit(pm->priv_queue);
342+}
343+
344+int pm_send_set_resources(struct packet_manager *pm,
345+ struct scheduling_resources *res)
346+{
347+ struct pm4_set_resources *packet;
348+
349+ BUG_ON(!pm || !res);
350+
351+ pr_debug("kfd: In func %s\n", __func__);
352+
353+ mutex_lock(&pm->lock);
354+ pm->priv_queue->acquire_packet_buffer(pm->priv_queue,
355+ sizeof(*packet) / sizeof(uint32_t),
356+ (unsigned int **)&packet);
357+ if (packet == NULL) {
358+ mutex_unlock(&pm->lock);
359+ pr_err("kfd: failed to allocate buffer on kernel queue\n");
360+ return -ENOMEM;
361+ }
362+
363+ memset(packet, 0, sizeof(struct pm4_set_resources));
364+ packet->header.u32all = build_pm4_header(IT_SET_RESOURCES,
365+ sizeof(struct pm4_set_resources));
366+
367+ packet->bitfields2.queue_type =
368+ queue_type__mes_set_resources__hsa_interface_queue_hiq;
369+ packet->bitfields2.vmid_mask = res->vmid_mask;
370+ packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY;
371+ packet->bitfields7.oac_mask = res->oac_mask;
372+ packet->bitfields8.gds_heap_base = res->gds_heap_base;
373+ packet->bitfields8.gds_heap_size = res->gds_heap_size;
374+
375+ packet->gws_mask_lo = lower_32_bits(res->gws_mask);
376+ packet->gws_mask_hi = upper_32_bits(res->gws_mask);
377+
378+ packet->queue_mask_lo = lower_32_bits(res->queue_mask);
379+ packet->queue_mask_hi = upper_32_bits(res->queue_mask);
380+
381+ pm->priv_queue->submit_packet(pm->priv_queue);
382+ pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT);
383+
384+ mutex_unlock(&pm->lock);
385+
386+ return 0;
387+}
388+
389+int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues)
390+{
391+ uint64_t rl_gpu_ib_addr;
392+ uint32_t *rl_buffer;
393+ size_t rl_ib_size, packet_size_dwords;
394+ int retval;
395+
396+ BUG_ON(!pm || !dqm_queues);
397+
398+ retval = pm_create_runlist_ib(pm, dqm_queues, &rl_gpu_ib_addr,
399+ &rl_ib_size);
400+ if (retval != 0)
401+ goto fail_create_runlist_ib;
402+
403+ pr_debug("kfd: runlist IB address: 0x%llX\n", rl_gpu_ib_addr);
404+
405+ packet_size_dwords = sizeof(struct pm4_runlist) / sizeof(uint32_t);
406+ mutex_lock(&pm->lock);
407+
408+ retval = pm->priv_queue->acquire_packet_buffer(pm->priv_queue,
409+ packet_size_dwords, &rl_buffer);
410+ if (retval != 0)
411+ goto fail_acquire_packet_buffer;
412+
413+ retval = pm_create_runlist(pm, rl_buffer, rl_gpu_ib_addr,
414+ rl_ib_size / sizeof(uint32_t), false);
415+ if (retval != 0)
416+ goto fail_create_runlist;
417+
418+ pm->priv_queue->submit_packet(pm->priv_queue);
419+ pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT);
420+
421+ mutex_unlock(&pm->lock);
422+
423+ return retval;
424+
425+fail_create_runlist:
426+ pm->priv_queue->rollback_packet(pm->priv_queue);
427+fail_acquire_packet_buffer:
428+ mutex_unlock(&pm->lock);
429+fail_create_runlist_ib:
430+ if (pm->allocated == true)
431+ pm_release_ib(pm);
432+ return retval;
433+}
434+
435+int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
436+ uint32_t fence_value)
437+{
438+ int retval;
439+ struct pm4_query_status *packet;
440+
441+ BUG_ON(!pm || !fence_address);
442+
443+ mutex_lock(&pm->lock);
444+ retval = pm->priv_queue->acquire_packet_buffer(
445+ pm->priv_queue,
446+ sizeof(struct pm4_query_status) / sizeof(uint32_t),
447+ (unsigned int **)&packet);
448+ if (retval != 0)
449+ goto fail_acquire_packet_buffer;
450+
451+ packet->header.u32all = build_pm4_header(IT_QUERY_STATUS,
452+ sizeof(struct pm4_query_status));
453+
454+ packet->bitfields2.context_id = 0;
455+ packet->bitfields2.interrupt_sel =
456+ interrupt_sel__mes_query_status__completion_status;
457+ packet->bitfields2.command =
458+ command__mes_query_status__fence_only_after_write_ack;
459+
460+ packet->addr_hi = upper_32_bits((uint64_t)fence_address);
461+ packet->addr_lo = lower_32_bits((uint64_t)fence_address);
462+ packet->data_hi = upper_32_bits((uint64_t)fence_value);
463+ packet->data_lo = lower_32_bits((uint64_t)fence_value);
464+
465+ pm->priv_queue->submit_packet(pm->priv_queue);
466+ pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT);
467+ mutex_unlock(&pm->lock);
468+
469+ return 0;
470+
471+fail_acquire_packet_buffer:
472+ mutex_unlock(&pm->lock);
473+ return retval;
474+}
475+
476+int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
477+ enum kfd_preempt_type_filter mode,
478+ uint32_t filter_param, bool reset,
479+ unsigned int sdma_engine)
480+{
481+ int retval;
482+ uint32_t *buffer;
483+ struct pm4_unmap_queues *packet;
484+
485+ BUG_ON(!pm);
486+
487+ mutex_lock(&pm->lock);
488+ retval = pm->priv_queue->acquire_packet_buffer(
489+ pm->priv_queue,
490+ sizeof(struct pm4_unmap_queues) / sizeof(uint32_t),
491+ &buffer);
492+ if (retval != 0)
493+ goto err_acquire_packet_buffer;
494+
495+ packet = (struct pm4_unmap_queues *)buffer;
496+ memset(buffer, 0, sizeof(struct pm4_unmap_queues));
497+
498+ packet->header.u32all = build_pm4_header(IT_UNMAP_QUEUES,
499+ sizeof(struct pm4_unmap_queues));
500+ switch (type) {
501+ case KFD_QUEUE_TYPE_COMPUTE:
502+ case KFD_QUEUE_TYPE_DIQ:
503+ packet->bitfields2.engine_sel =
504+ engine_sel__mes_unmap_queues__compute;
505+ break;
506+ case KFD_QUEUE_TYPE_SDMA:
507+ packet->bitfields2.engine_sel =
508+ engine_sel__mes_unmap_queues__sdma0 + sdma_engine;
509+ break;
510+ default:
511+ BUG();
512+ break;
513+ }
514+
515+ if (reset)
516+ packet->bitfields2.action =
517+ action__mes_unmap_queues__reset_queues;
518+ else
519+ packet->bitfields2.action =
520+ action__mes_unmap_queues__preempt_queues;
521+
522+ switch (mode) {
523+ case KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE:
524+ packet->bitfields2.queue_sel =
525+ queue_sel__mes_unmap_queues__perform_request_on_specified_queues;
526+ packet->bitfields2.num_queues = 1;
527+ packet->bitfields3b.doorbell_offset0 = filter_param;
528+ break;
529+ case KFD_PREEMPT_TYPE_FILTER_BY_PASID:
530+ packet->bitfields2.queue_sel =
531+ queue_sel__mes_unmap_queues__perform_request_on_pasid_queues;
532+ packet->bitfields3a.pasid = filter_param;
533+ break;
534+ case KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES:
535+ packet->bitfields2.queue_sel =
536+ queue_sel__mes_unmap_queues__perform_request_on_all_active_queues;
537+ break;
538+ default:
539+ BUG();
540+ break;
541+ };
542+
543+ pm->priv_queue->submit_packet(pm->priv_queue);
544+ pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT);
545+
546+ mutex_unlock(&pm->lock);
547+ return 0;
548+
549+err_acquire_packet_buffer:
550+ mutex_unlock(&pm->lock);
551+ return retval;
552+}
553+
554+void pm_release_ib(struct packet_manager *pm)
555+{
556+ BUG_ON(!pm);
557+
558+ mutex_lock(&pm->lock);
559+ if (pm->allocated) {
560+ kfd2kgd->free_mem(pm->dqm->dev->kgd,
561+ (struct kgd_mem *) pm->ib_buffer_obj);
562+ pm->allocated = false;
563+ }
564+ mutex_unlock(&pm->lock);
565+}
drivers/gpu/drm/amd/amdkfd/kfd_pasid.c+97−0
@@ -0,0 +1,97 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+#include <linux/slab.h>
24+#include <linux/types.h>
25+#include "kfd_priv.h"
26+
27+static unsigned long *pasid_bitmap;
28+static unsigned int pasid_limit;
29+static DEFINE_MUTEX(pasid_mutex);
30+
31+int kfd_pasid_init(void)
32+{
33+ pasid_limit = max_num_of_processes;
34+
35+ pasid_bitmap = kzalloc(DIV_ROUND_UP(pasid_limit, BITS_PER_BYTE),
36+ GFP_KERNEL);
37+ if (!pasid_bitmap)
38+ return -ENOMEM;
39+
40+ set_bit(0, pasid_bitmap); /* PASID 0 is reserved. */
41+
42+ return 0;
43+}
44+
45+void kfd_pasid_exit(void)
46+{
47+ kfree(pasid_bitmap);
48+}
49+
50+bool kfd_set_pasid_limit(unsigned int new_limit)
51+{
52+ if (new_limit < pasid_limit) {
53+ bool ok;
54+
55+ mutex_lock(&pasid_mutex);
56+
57+ /* ensure that no pasids >= new_limit are in-use */
58+ ok = (find_next_bit(pasid_bitmap, pasid_limit, new_limit) ==
59+ pasid_limit);
60+ if (ok)
61+ pasid_limit = new_limit;
62+
63+ mutex_unlock(&pasid_mutex);
64+
65+ return ok;
66+ }
67+
68+ return true;
69+}
70+
71+inline unsigned int kfd_get_pasid_limit(void)
72+{
73+ return pasid_limit;
74+}
75+
76+unsigned int kfd_pasid_alloc(void)
77+{
78+ unsigned int found;
79+
80+ mutex_lock(&pasid_mutex);
81+
82+ found = find_first_zero_bit(pasid_bitmap, pasid_limit);
83+ if (found == pasid_limit)
84+ found = 0;
85+ else
86+ set_bit(found, pasid_bitmap);
87+
88+ mutex_unlock(&pasid_mutex);
89+
90+ return found;
91+}
92+
93+void kfd_pasid_free(unsigned int pasid)
94+{
95+ BUG_ON(pasid == 0 || pasid >= pasid_limit);
96+ clear_bit(pasid, pasid_bitmap);
97+}
drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h+405−0
@@ -0,0 +1,405 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ *
22+ */
23+
24+#ifndef KFD_PM4_HEADERS_H_
25+#define KFD_PM4_HEADERS_H_
26+
27+#ifndef PM4_MES_HEADER_DEFINED
28+#define PM4_MES_HEADER_DEFINED
29+union PM4_MES_TYPE_3_HEADER {
30+ struct {
31+ uint32_t reserved1:8; /* < reserved */
32+ uint32_t opcode:8; /* < IT opcode */
33+ uint32_t count:14; /* < number of DWORDs - 1
34+ * in the information body.
35+ */
36+ uint32_t type:2; /* < packet identifier.
37+ * It should be 3 for type 3 packets
38+ */
39+ };
40+ uint32_t u32all;
41+};
42+#endif /* PM4_MES_HEADER_DEFINED */
43+
44+/* --------------------MES_SET_RESOURCES-------------------- */
45+
46+#ifndef PM4_MES_SET_RESOURCES_DEFINED
47+#define PM4_MES_SET_RESOURCES_DEFINED
48+enum set_resources_queue_type_enum {
49+ queue_type__mes_set_resources__kernel_interface_queue_kiq = 0,
50+ queue_type__mes_set_resources__hsa_interface_queue_hiq = 1,
51+ queue_type__mes_set_resources__hsa_debug_interface_queue = 4
52+};
53+
54+struct pm4_set_resources {
55+ union {
56+ union PM4_MES_TYPE_3_HEADER header; /* header */
57+ uint32_t ordinal1;
58+ };
59+
60+ union {
61+ struct {
62+ uint32_t vmid_mask:16;
63+ uint32_t unmap_latency:8;
64+ uint32_t reserved1:5;
65+ enum set_resources_queue_type_enum queue_type:3;
66+ } bitfields2;
67+ uint32_t ordinal2;
68+ };
69+
70+ uint32_t queue_mask_lo;
71+ uint32_t queue_mask_hi;
72+ uint32_t gws_mask_lo;
73+ uint32_t gws_mask_hi;
74+
75+ union {
76+ struct {
77+ uint32_t oac_mask:16;
78+ uint32_t reserved2:16;
79+ } bitfields7;
80+ uint32_t ordinal7;
81+ };
82+
83+ union {
84+ struct {
85+ uint32_t gds_heap_base:6;
86+ uint32_t reserved3:5;
87+ uint32_t gds_heap_size:6;
88+ uint32_t reserved4:15;
89+ } bitfields8;
90+ uint32_t ordinal8;
91+ };
92+
93+};
94+#endif
95+
96+/*--------------------MES_RUN_LIST-------------------- */
97+
98+#ifndef PM4_MES_RUN_LIST_DEFINED
99+#define PM4_MES_RUN_LIST_DEFINED
100+
101+struct pm4_runlist {
102+ union {
103+ union PM4_MES_TYPE_3_HEADER header; /* header */
104+ uint32_t ordinal1;
105+ };
106+
107+ union {
108+ struct {
109+ uint32_t reserved1:2;
110+ uint32_t ib_base_lo:30;
111+ } bitfields2;
112+ uint32_t ordinal2;
113+ };
114+
115+ union {
116+ struct {
117+ uint32_t ib_base_hi:16;
118+ uint32_t reserved2:16;
119+ } bitfields3;
120+ uint32_t ordinal3;
121+ };
122+
123+ union {
124+ struct {
125+ uint32_t ib_size:20;
126+ uint32_t chain:1;
127+ uint32_t offload_polling:1;
128+ uint32_t reserved3:1;
129+ uint32_t valid:1;
130+ uint32_t reserved4:8;
131+ } bitfields4;
132+ uint32_t ordinal4;
133+ };
134+
135+};
136+#endif
137+
138+/*--------------------MES_MAP_PROCESS-------------------- */
139+
140+#ifndef PM4_MES_MAP_PROCESS_DEFINED
141+#define PM4_MES_MAP_PROCESS_DEFINED
142+
143+struct pm4_map_process {
144+ union {
145+ union PM4_MES_TYPE_3_HEADER header; /* header */
146+ uint32_t ordinal1;
147+ };
148+
149+ union {
150+ struct {
151+ uint32_t pasid:16;
152+ uint32_t reserved1:8;
153+ uint32_t diq_enable:1;
154+ uint32_t process_quantum:7;
155+ } bitfields2;
156+ uint32_t ordinal2;
157+ };
158+
159+ union {
160+ struct {
161+ uint32_t page_table_base:28;
162+ uint32_t reserved3:4;
163+ } bitfields3;
164+ uint32_t ordinal3;
165+ };
166+
167+ uint32_t sh_mem_bases;
168+ uint32_t sh_mem_ape1_base;
169+ uint32_t sh_mem_ape1_limit;
170+ uint32_t sh_mem_config;
171+ uint32_t gds_addr_lo;
172+ uint32_t gds_addr_hi;
173+
174+ union {
175+ struct {
176+ uint32_t num_gws:6;
177+ uint32_t reserved4:2;
178+ uint32_t num_oac:4;
179+ uint32_t reserved5:4;
180+ uint32_t gds_size:6;
181+ uint32_t num_queues:10;
182+ } bitfields10;
183+ uint32_t ordinal10;
184+ };
185+
186+};
187+#endif
188+
189+/*--------------------MES_MAP_QUEUES--------------------*/
190+
191+#ifndef PM4_MES_MAP_QUEUES_DEFINED
192+#define PM4_MES_MAP_QUEUES_DEFINED
193+enum map_queues_queue_sel_enum {
194+ queue_sel__mes_map_queues__map_to_specified_queue_slots = 0,
195+ queue_sel__mes_map_queues__map_to_hws_determined_queue_slots = 1,
196+ queue_sel__mes_map_queues__enable_process_queues = 2
197+};
198+
199+enum map_queues_vidmem_enum {
200+ vidmem__mes_map_queues__uses_no_video_memory = 0,
201+ vidmem__mes_map_queues__uses_video_memory = 1
202+};
203+
204+enum map_queues_alloc_format_enum {
205+ alloc_format__mes_map_queues__one_per_pipe = 0,
206+ alloc_format__mes_map_queues__all_on_one_pipe = 1
207+};
208+
209+enum map_queues_engine_sel_enum {
210+ engine_sel__mes_map_queues__compute = 0,
211+ engine_sel__mes_map_queues__sdma0 = 2,
212+ engine_sel__mes_map_queues__sdma1 = 3
213+};
214+
215+struct pm4_map_queues {
216+ union {
217+ union PM4_MES_TYPE_3_HEADER header; /* header */
218+ uint32_t ordinal1;
219+ };
220+
221+ union {
222+ struct {
223+ uint32_t reserved1:4;
224+ enum map_queues_queue_sel_enum queue_sel:2;
225+ uint32_t reserved2:2;
226+ uint32_t vmid:4;
227+ uint32_t reserved3:4;
228+ enum map_queues_vidmem_enum vidmem:2;
229+ uint32_t reserved4:6;
230+ enum map_queues_alloc_format_enum alloc_format:2;
231+ enum map_queues_engine_sel_enum engine_sel:3;
232+ uint32_t num_queues:3;
233+ } bitfields2;
234+ uint32_t ordinal2;
235+ };
236+
237+ struct {
238+ union {
239+ struct {
240+ uint32_t reserved5:2;
241+ uint32_t doorbell_offset:21;
242+ uint32_t reserved6:3;
243+ uint32_t queue:6;
244+ } bitfields3;
245+ uint32_t ordinal3;
246+ };
247+
248+ uint32_t mqd_addr_lo;
249+ uint32_t mqd_addr_hi;
250+ uint32_t wptr_addr_lo;
251+ uint32_t wptr_addr_hi;
252+
253+ } mes_map_queues_ordinals[1]; /* 1..N of these ordinal groups */
254+
255+};
256+#endif
257+
258+/*--------------------MES_QUERY_STATUS--------------------*/
259+
260+#ifndef PM4_MES_QUERY_STATUS_DEFINED
261+#define PM4_MES_QUERY_STATUS_DEFINED
262+enum query_status_interrupt_sel_enum {
263+ interrupt_sel__mes_query_status__completion_status = 0,
264+ interrupt_sel__mes_query_status__process_status = 1,
265+ interrupt_sel__mes_query_status__queue_status = 2
266+};
267+
268+enum query_status_command_enum {
269+ command__mes_query_status__interrupt_only = 0,
270+ command__mes_query_status__fence_only_immediate = 1,
271+ command__mes_query_status__fence_only_after_write_ack = 2,
272+ command__mes_query_status__fence_wait_for_write_ack_send_interrupt = 3
273+};
274+
275+enum query_status_engine_sel_enum {
276+ engine_sel__mes_query_status__compute = 0,
277+ engine_sel__mes_query_status__sdma0_queue = 2,
278+ engine_sel__mes_query_status__sdma1_queue = 3
279+};
280+
281+struct pm4_query_status {
282+ union {
283+ union PM4_MES_TYPE_3_HEADER header; /* header */
284+ uint32_t ordinal1;
285+ };
286+
287+ union {
288+ struct {
289+ uint32_t context_id:28;
290+ enum query_status_interrupt_sel_enum interrupt_sel:2;
291+ enum query_status_command_enum command:2;
292+ } bitfields2;
293+ uint32_t ordinal2;
294+ };
295+
296+ union {
297+ struct {
298+ uint32_t pasid:16;
299+ uint32_t reserved1:16;
300+ } bitfields3a;
301+ struct {
302+ uint32_t reserved2:2;
303+ uint32_t doorbell_offset:21;
304+ uint32_t reserved3:3;
305+ enum query_status_engine_sel_enum engine_sel:3;
306+ uint32_t reserved4:3;
307+ } bitfields3b;
308+ uint32_t ordinal3;
309+ };
310+
311+ uint32_t addr_lo;
312+ uint32_t addr_hi;
313+ uint32_t data_lo;
314+ uint32_t data_hi;
315+};
316+#endif
317+
318+/*--------------------MES_UNMAP_QUEUES--------------------*/
319+
320+#ifndef PM4_MES_UNMAP_QUEUES_DEFINED
321+#define PM4_MES_UNMAP_QUEUES_DEFINED
322+enum unmap_queues_action_enum {
323+ action__mes_unmap_queues__preempt_queues = 0,
324+ action__mes_unmap_queues__reset_queues = 1,
325+ action__mes_unmap_queues__disable_process_queues = 2
326+};
327+
328+enum unmap_queues_queue_sel_enum {
329+ queue_sel__mes_unmap_queues__perform_request_on_specified_queues = 0,
330+ queue_sel__mes_unmap_queues__perform_request_on_pasid_queues = 1,
331+ queue_sel__mes_unmap_queues__perform_request_on_all_active_queues = 2
332+};
333+
334+enum unmap_queues_engine_sel_enum {
335+ engine_sel__mes_unmap_queues__compute = 0,
336+ engine_sel__mes_unmap_queues__sdma0 = 2,
337+ engine_sel__mes_unmap_queues__sdma1 = 3
338+};
339+
340+struct pm4_unmap_queues {
341+ union {
342+ union PM4_MES_TYPE_3_HEADER header; /* header */
343+ uint32_t ordinal1;
344+ };
345+
346+ union {
347+ struct {
348+ enum unmap_queues_action_enum action:2;
349+ uint32_t reserved1:2;
350+ enum unmap_queues_queue_sel_enum queue_sel:2;
351+ uint32_t reserved2:20;
352+ enum unmap_queues_engine_sel_enum engine_sel:3;
353+ uint32_t num_queues:3;
354+ } bitfields2;
355+ uint32_t ordinal2;
356+ };
357+
358+ union {
359+ struct {
360+ uint32_t pasid:16;
361+ uint32_t reserved3:16;
362+ } bitfields3a;
363+ struct {
364+ uint32_t reserved4:2;
365+ uint32_t doorbell_offset0:21;
366+ uint32_t reserved5:9;
367+ } bitfields3b;
368+ uint32_t ordinal3;
369+ };
370+
371+ union {
372+ struct {
373+ uint32_t reserved6:2;
374+ uint32_t doorbell_offset1:21;
375+ uint32_t reserved7:9;
376+ } bitfields4;
377+ uint32_t ordinal4;
378+ };
379+
380+ union {
381+ struct {
382+ uint32_t reserved8:2;
383+ uint32_t doorbell_offset2:21;
384+ uint32_t reserved9:9;
385+ } bitfields5;
386+ uint32_t ordinal5;
387+ };
388+
389+ union {
390+ struct {
391+ uint32_t reserved10:2;
392+ uint32_t doorbell_offset3:21;
393+ uint32_t reserved11:9;
394+ } bitfields6;
395+ uint32_t ordinal6;
396+ };
397+
398+};
399+#endif
400+
401+enum {
402+ CACHE_FLUSH_AND_INV_TS_EVENT = 0x00000014
403+};
404+
405+#endif /* KFD_PM4_HEADERS_H_ */
drivers/gpu/drm/amd/amdkfd/kfd_pm4_opcodes.h+107−0
@@ -0,0 +1,107 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ *
22+ */
23+
24+
25+#ifndef KFD_PM4_OPCODES_H
26+#define KFD_PM4_OPCODES_H
27+
28+enum it_opcode_type {
29+ IT_NOP = 0x10,
30+ IT_SET_BASE = 0x11,
31+ IT_CLEAR_STATE = 0x12,
32+ IT_INDEX_BUFFER_SIZE = 0x13,
33+ IT_DISPATCH_DIRECT = 0x15,
34+ IT_DISPATCH_INDIRECT = 0x16,
35+ IT_ATOMIC_GDS = 0x1D,
36+ IT_OCCLUSION_QUERY = 0x1F,
37+ IT_SET_PREDICATION = 0x20,
38+ IT_REG_RMW = 0x21,
39+ IT_COND_EXEC = 0x22,
40+ IT_PRED_EXEC = 0x23,
41+ IT_DRAW_INDIRECT = 0x24,
42+ IT_DRAW_INDEX_INDIRECT = 0x25,
43+ IT_INDEX_BASE = 0x26,
44+ IT_DRAW_INDEX_2 = 0x27,
45+ IT_CONTEXT_CONTROL = 0x28,
46+ IT_INDEX_TYPE = 0x2A,
47+ IT_DRAW_INDIRECT_MULTI = 0x2C,
48+ IT_DRAW_INDEX_AUTO = 0x2D,
49+ IT_NUM_INSTANCES = 0x2F,
50+ IT_DRAW_INDEX_MULTI_AUTO = 0x30,
51+ IT_INDIRECT_BUFFER_CNST = 0x33,
52+ IT_STRMOUT_BUFFER_UPDATE = 0x34,
53+ IT_DRAW_INDEX_OFFSET_2 = 0x35,
54+ IT_DRAW_PREAMBLE = 0x36,
55+ IT_WRITE_DATA = 0x37,
56+ IT_DRAW_INDEX_INDIRECT_MULTI = 0x38,
57+ IT_MEM_SEMAPHORE = 0x39,
58+ IT_COPY_DW = 0x3B,
59+ IT_WAIT_REG_MEM = 0x3C,
60+ IT_INDIRECT_BUFFER = 0x3F,
61+ IT_COPY_DATA = 0x40,
62+ IT_PFP_SYNC_ME = 0x42,
63+ IT_SURFACE_SYNC = 0x43,
64+ IT_COND_WRITE = 0x45,
65+ IT_EVENT_WRITE = 0x46,
66+ IT_EVENT_WRITE_EOP = 0x47,
67+ IT_EVENT_WRITE_EOS = 0x48,
68+ IT_RELEASE_MEM = 0x49,
69+ IT_PREAMBLE_CNTL = 0x4A,
70+ IT_DMA_DATA = 0x50,
71+ IT_ACQUIRE_MEM = 0x58,
72+ IT_REWIND = 0x59,
73+ IT_LOAD_UCONFIG_REG = 0x5E,
74+ IT_LOAD_SH_REG = 0x5F,
75+ IT_LOAD_CONFIG_REG = 0x60,
76+ IT_LOAD_CONTEXT_REG = 0x61,
77+ IT_SET_CONFIG_REG = 0x68,
78+ IT_SET_CONTEXT_REG = 0x69,
79+ IT_SET_CONTEXT_REG_INDIRECT = 0x73,
80+ IT_SET_SH_REG = 0x76,
81+ IT_SET_SH_REG_OFFSET = 0x77,
82+ IT_SET_QUEUE_REG = 0x78,
83+ IT_SET_UCONFIG_REG = 0x79,
84+ IT_SCRATCH_RAM_WRITE = 0x7D,
85+ IT_SCRATCH_RAM_READ = 0x7E,
86+ IT_LOAD_CONST_RAM = 0x80,
87+ IT_WRITE_CONST_RAM = 0x81,
88+ IT_DUMP_CONST_RAM = 0x83,
89+ IT_INCREMENT_CE_COUNTER = 0x84,
90+ IT_INCREMENT_DE_COUNTER = 0x85,
91+ IT_WAIT_ON_CE_COUNTER = 0x86,
92+ IT_WAIT_ON_DE_COUNTER_DIFF = 0x88,
93+ IT_SWITCH_BUFFER = 0x8B,
94+ IT_SET_RESOURCES = 0xA0,
95+ IT_MAP_PROCESS = 0xA1,
96+ IT_MAP_QUEUES = 0xA2,
97+ IT_UNMAP_QUEUES = 0xA3,
98+ IT_QUERY_STATUS = 0xA4,
99+ IT_RUN_LIST = 0xA5,
100+};
101+
102+#define PM4_TYPE_0 0
103+#define PM4_TYPE_2 2
104+#define PM4_TYPE_3 3
105+
106+#endif /* KFD_PM4_OPCODES_H */
107+
drivers/gpu/drm/amd/amdkfd/kfd_priv.h+598−0
@@ -0,0 +1,598 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+#ifndef KFD_PRIV_H_INCLUDED
24+#define KFD_PRIV_H_INCLUDED
25+
26+#include <linux/hashtable.h>
27+#include <linux/mmu_notifier.h>
28+#include <linux/mutex.h>
29+#include <linux/types.h>
30+#include <linux/atomic.h>
31+#include <linux/workqueue.h>
32+#include <linux/spinlock.h>
33+#include <linux/kfd_ioctl.h>
34+#include <kgd_kfd_interface.h>
35+
36+#define KFD_SYSFS_FILE_MODE 0444
37+
38+/*
39+ * When working with cp scheduler we should assign the HIQ manually or via
40+ * the radeon driver to a fixed hqd slot, here are the fixed HIQ hqd slot
41+ * definitions for Kaveri. In Kaveri only the first ME queues participates
42+ * in the cp scheduling taking that in mind we set the HIQ slot in the
43+ * second ME.
44+ */
45+#define KFD_CIK_HIQ_PIPE 4
46+#define KFD_CIK_HIQ_QUEUE 0
47+
48+/* GPU ID hash width in bits */
49+#define KFD_GPU_ID_HASH_WIDTH 16
50+
51+/* Macro for allocating structures */
52+#define kfd_alloc_struct(ptr_to_struct) \
53+ ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))
54+
55+/* Kernel module parameter to specify maximum number of supported processes */
56+extern int max_num_of_processes;
57+
58+#define KFD_MAX_NUM_OF_PROCESSES_DEFAULT 32
59+#define KFD_MAX_NUM_OF_PROCESSES 512
60+
61+/*
62+ * Kernel module parameter to specify maximum number of supported queues
63+ * per process
64+ */
65+extern int max_num_of_queues_per_process;
66+
67+#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT 128
68+#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
69+
70+#define KFD_KERNEL_QUEUE_SIZE 2048
71+
72+/* Kernel module parameter to specify the scheduling policy */
73+extern int sched_policy;
74+
75+/**
76+ * enum kfd_sched_policy
77+ *
78+ * @KFD_SCHED_POLICY_HWS: H/W scheduling policy known as command processor (cp)
79+ * scheduling. In this scheduling mode we're using the firmware code to
80+ * schedule the user mode queues and kernel queues such as HIQ and DIQ.
81+ * the HIQ queue is used as a special queue that dispatches the configuration
82+ * to the cp and the user mode queues list that are currently running.
83+ * the DIQ queue is a debugging queue that dispatches debugging commands to the
84+ * firmware.
85+ * in this scheduling mode user mode queues over subscription feature is
86+ * enabled.
87+ *
88+ * @KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: The same as above but the over
89+ * subscription feature disabled.
90+ *
91+ * @KFD_SCHED_POLICY_NO_HWS: no H/W scheduling policy is a mode which directly
92+ * set the command processor registers and sets the queues "manually". This
93+ * mode is used *ONLY* for debugging proposes.
94+ *
95+ */
96+enum kfd_sched_policy {
97+ KFD_SCHED_POLICY_HWS = 0,
98+ KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION,
99+ KFD_SCHED_POLICY_NO_HWS
100+};
101+
102+enum cache_policy {
103+ cache_policy_coherent,
104+ cache_policy_noncoherent
105+};
106+
107+struct kfd_device_info {
108+ unsigned int max_pasid_bits;
109+ size_t ih_ring_entry_size;
110+ uint16_t mqd_size_aligned;
111+};
112+
113+struct kfd_dev {
114+ struct kgd_dev *kgd;
115+
116+ const struct kfd_device_info *device_info;
117+ struct pci_dev *pdev;
118+
119+ unsigned int id; /* topology stub index */
120+
121+ phys_addr_t doorbell_base; /* Start of actual doorbells used by
122+ * KFD. It is aligned for mapping
123+ * into user mode
124+ */
125+ size_t doorbell_id_offset; /* Doorbell offset (from KFD doorbell
126+ * to HW doorbell, GFX reserved some
127+ * at the start)
128+ */
129+ size_t doorbell_process_limit; /* Number of processes we have doorbell
130+ * space for.
131+ */
132+ u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells
133+ * page used by kernel queue
134+ */
135+
136+ struct kgd2kfd_shared_resources shared_resources;
137+
138+ void *interrupt_ring;
139+ size_t interrupt_ring_size;
140+ atomic_t interrupt_ring_rptr;
141+ atomic_t interrupt_ring_wptr;
142+ struct work_struct interrupt_work;
143+ spinlock_t interrupt_lock;
144+
145+ /* QCM Device instance */
146+ struct device_queue_manager *dqm;
147+
148+ bool init_complete;
149+ /*
150+ * Interrupts of interest to KFD are copied
151+ * from the HW ring into a SW ring.
152+ */
153+ bool interrupts_active;
154+};
155+
156+/* KGD2KFD callbacks */
157+void kgd2kfd_exit(void);
158+struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev);
159+bool kgd2kfd_device_init(struct kfd_dev *kfd,
160+ const struct kgd2kfd_shared_resources *gpu_resources);
161+void kgd2kfd_device_exit(struct kfd_dev *kfd);
162+
163+extern const struct kfd2kgd_calls *kfd2kgd;
164+
165+struct kfd_mem_obj {
166+ void *bo;
167+ uint64_t gpu_addr;
168+ uint32_t *cpu_ptr;
169+};
170+
171+enum kfd_mempool {
172+ KFD_MEMPOOL_SYSTEM_CACHEABLE = 1,
173+ KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2,
174+ KFD_MEMPOOL_FRAMEBUFFER = 3,
175+};
176+
177+/* Character device interface */
178+int kfd_chardev_init(void);
179+void kfd_chardev_exit(void);
180+struct device *kfd_chardev(void);
181+
182+/**
183+ * enum kfd_preempt_type_filter
184+ *
185+ * @KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE: Preempts single queue.
186+ *
187+ * @KFD_PRERMPT_TYPE_FILTER_ALL_QUEUES: Preempts all queues in the
188+ * running queues list.
189+ *
190+ * @KFD_PRERMPT_TYPE_FILTER_BY_PASID: Preempts queues that belongs to
191+ * specific process.
192+ *
193+ */
194+enum kfd_preempt_type_filter {
195+ KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE,
196+ KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES,
197+ KFD_PREEMPT_TYPE_FILTER_BY_PASID
198+};
199+
200+enum kfd_preempt_type {
201+ KFD_PREEMPT_TYPE_WAVEFRONT,
202+ KFD_PREEMPT_TYPE_WAVEFRONT_RESET
203+};
204+
205+/**
206+ * enum kfd_queue_type
207+ *
208+ * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type.
209+ *
210+ * @KFD_QUEUE_TYPE_SDMA: Sdma user mode queue type.
211+ *
212+ * @KFD_QUEUE_TYPE_HIQ: HIQ queue type.
213+ *
214+ * @KFD_QUEUE_TYPE_DIQ: DIQ queue type.
215+ */
216+enum kfd_queue_type {
217+ KFD_QUEUE_TYPE_COMPUTE,
218+ KFD_QUEUE_TYPE_SDMA,
219+ KFD_QUEUE_TYPE_HIQ,
220+ KFD_QUEUE_TYPE_DIQ
221+};
222+
223+enum kfd_queue_format {
224+ KFD_QUEUE_FORMAT_PM4,
225+ KFD_QUEUE_FORMAT_AQL
226+};
227+
228+/**
229+ * struct queue_properties
230+ *
231+ * @type: The queue type.
232+ *
233+ * @queue_id: Queue identifier.
234+ *
235+ * @queue_address: Queue ring buffer address.
236+ *
237+ * @queue_size: Queue ring buffer size.
238+ *
239+ * @priority: Defines the queue priority relative to other queues in the
240+ * process.
241+ * This is just an indication and HW scheduling may override the priority as
242+ * necessary while keeping the relative prioritization.
243+ * the priority granularity is from 0 to f which f is the highest priority.
244+ * currently all queues are initialized with the highest priority.
245+ *
246+ * @queue_percent: This field is partially implemented and currently a zero in
247+ * this field defines that the queue is non active.
248+ *
249+ * @read_ptr: User space address which points to the number of dwords the
250+ * cp read from the ring buffer. This field updates automatically by the H/W.
251+ *
252+ * @write_ptr: Defines the number of dwords written to the ring buffer.
253+ *
254+ * @doorbell_ptr: This field aim is to notify the H/W of new packet written to
255+ * the queue ring buffer. This field should be similar to write_ptr and the user
256+ * should update this field after he updated the write_ptr.
257+ *
258+ * @doorbell_off: The doorbell offset in the doorbell pci-bar.
259+ *
260+ * @is_interop: Defines if this is a interop queue. Interop queue means that the
261+ * queue can access both graphics and compute resources.
262+ *
263+ * @is_active: Defines if the queue is active or not.
264+ *
265+ * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid
266+ * of the queue.
267+ *
268+ * This structure represents the queue properties for each queue no matter if
269+ * it's user mode or kernel mode queue.
270+ *
271+ */
272+struct queue_properties {
273+ enum kfd_queue_type type;
274+ enum kfd_queue_format format;
275+ unsigned int queue_id;
276+ uint64_t queue_address;
277+ uint64_t queue_size;
278+ uint32_t priority;
279+ uint32_t queue_percent;
280+ uint32_t *read_ptr;
281+ uint32_t *write_ptr;
282+ uint32_t *doorbell_ptr;
283+ uint32_t doorbell_off;
284+ bool is_interop;
285+ bool is_active;
286+ /* Not relevant for user mode queues in cp scheduling */
287+ unsigned int vmid;
288+};
289+
290+/**
291+ * struct queue
292+ *
293+ * @list: Queue linked list.
294+ *
295+ * @mqd: The queue MQD.
296+ *
297+ * @mqd_mem_obj: The MQD local gpu memory object.
298+ *
299+ * @gart_mqd_addr: The MQD gart mc address.
300+ *
301+ * @properties: The queue properties.
302+ *
303+ * @mec: Used only in no cp scheduling mode and identifies to micro engine id
304+ * that the queue should be execute on.
305+ *
306+ * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe id.
307+ *
308+ * @queue: Used only in no cp scheduliong mode and identifies the queue's slot.
309+ *
310+ * @process: The kfd process that created this queue.
311+ *
312+ * @device: The kfd device that created this queue.
313+ *
314+ * This structure represents user mode compute queues.
315+ * It contains all the necessary data to handle such queues.
316+ *
317+ */
318+
319+struct queue {
320+ struct list_head list;
321+ void *mqd;
322+ struct kfd_mem_obj *mqd_mem_obj;
323+ uint64_t gart_mqd_addr;
324+ struct queue_properties properties;
325+
326+ uint32_t mec;
327+ uint32_t pipe;
328+ uint32_t queue;
329+
330+ struct kfd_process *process;
331+ struct kfd_dev *device;
332+};
333+
334+/*
335+ * Please read the kfd_mqd_manager.h description.
336+ */
337+enum KFD_MQD_TYPE {
338+ KFD_MQD_TYPE_CIK_COMPUTE = 0, /* for no cp scheduling */
339+ KFD_MQD_TYPE_CIK_HIQ, /* for hiq */
340+ KFD_MQD_TYPE_CIK_CP, /* for cp queues and diq */
341+ KFD_MQD_TYPE_CIK_SDMA, /* for sdma queues */
342+ KFD_MQD_TYPE_MAX
343+};
344+
345+struct scheduling_resources {
346+ unsigned int vmid_mask;
347+ enum kfd_queue_type type;
348+ uint64_t queue_mask;
349+ uint64_t gws_mask;
350+ uint32_t oac_mask;
351+ uint32_t gds_heap_base;
352+ uint32_t gds_heap_size;
353+};
354+
355+struct process_queue_manager {
356+ /* data */
357+ struct kfd_process *process;
358+ unsigned int num_concurrent_processes;
359+ struct list_head queues;
360+ unsigned long *queue_slot_bitmap;
361+};
362+
363+struct qcm_process_device {
364+ /* The Device Queue Manager that owns this data */
365+ struct device_queue_manager *dqm;
366+ struct process_queue_manager *pqm;
367+ /* Device Queue Manager lock */
368+ struct mutex *lock;
369+ /* Queues list */
370+ struct list_head queues_list;
371+ struct list_head priv_queue_list;
372+
373+ unsigned int queue_count;
374+ unsigned int vmid;
375+ bool is_debug;
376+ /*
377+ * All the memory management data should be here too
378+ */
379+ uint64_t gds_context_area;
380+ uint32_t sh_mem_config;
381+ uint32_t sh_mem_bases;
382+ uint32_t sh_mem_ape1_base;
383+ uint32_t sh_mem_ape1_limit;
384+ uint32_t page_table_base;
385+ uint32_t gds_size;
386+ uint32_t num_gws;
387+ uint32_t num_oac;
388+};
389+
390+/* Data that is per-process-per device. */
391+struct kfd_process_device {
392+ /*
393+ * List of all per-device data for a process.
394+ * Starts from kfd_process.per_device_data.
395+ */
396+ struct list_head per_device_list;
397+
398+ /* The device that owns this data. */
399+ struct kfd_dev *dev;
400+
401+
402+ /* per-process-per device QCM data structure */
403+ struct qcm_process_device qpd;
404+
405+ /*Apertures*/
406+ uint64_t lds_base;
407+ uint64_t lds_limit;
408+ uint64_t gpuvm_base;
409+ uint64_t gpuvm_limit;
410+ uint64_t scratch_base;
411+ uint64_t scratch_limit;
412+
413+ /* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */
414+ bool bound;
415+};
416+
417+/* Process data */
418+struct kfd_process {
419+ /*
420+ * kfd_process are stored in an mm_struct*->kfd_process*
421+ * hash table (kfd_processes in kfd_process.c)
422+ */
423+ struct hlist_node kfd_processes;
424+
425+ struct mm_struct *mm;
426+
427+ struct mutex mutex;
428+
429+ /*
430+ * In any process, the thread that started main() is the lead
431+ * thread and outlives the rest.
432+ * It is here because amd_iommu_bind_pasid wants a task_struct.
433+ */
434+ struct task_struct *lead_thread;
435+
436+ /* We want to receive a notification when the mm_struct is destroyed */
437+ struct mmu_notifier mmu_notifier;
438+
439+ /* Use for delayed freeing of kfd_process structure */
440+ struct rcu_head rcu;
441+
442+ unsigned int pasid;
443+
444+ /*
445+ * List of kfd_process_device structures,
446+ * one for each device the process is using.
447+ */
448+ struct list_head per_device_data;
449+
450+ struct process_queue_manager pqm;
451+
452+ /* The process's queues. */
453+ size_t queue_array_size;
454+
455+ /* Size is queue_array_size, up to MAX_PROCESS_QUEUES. */
456+ struct kfd_queue **queues;
457+
458+ unsigned long allocated_queue_bitmap[DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)];
459+
460+ /*Is the user space process 32 bit?*/
461+ bool is_32bit_user_mode;
462+};
463+
464+void kfd_process_create_wq(void);
465+void kfd_process_destroy_wq(void);
466+struct kfd_process *kfd_create_process(const struct task_struct *);
467+struct kfd_process *kfd_get_process(const struct task_struct *);
468+
469+struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
470+ struct kfd_process *p);
471+void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid);
472+struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
473+ struct kfd_process *p,
474+ int create_pdd);
475+
476+/* Process device data iterator */
477+struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p);
478+struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
479+ struct kfd_process_device *pdd);
480+bool kfd_has_process_device_data(struct kfd_process *p);
481+
482+/* PASIDs */
483+int kfd_pasid_init(void);
484+void kfd_pasid_exit(void);
485+bool kfd_set_pasid_limit(unsigned int new_limit);
486+unsigned int kfd_get_pasid_limit(void);
487+unsigned int kfd_pasid_alloc(void);
488+void kfd_pasid_free(unsigned int pasid);
489+
490+/* Doorbells */
491+void kfd_doorbell_init(struct kfd_dev *kfd);
492+int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma);
493+u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
494+ unsigned int *doorbell_off);
495+void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
496+u32 read_kernel_doorbell(u32 __iomem *db);
497+void write_kernel_doorbell(u32 __iomem *db, u32 value);
498+unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd,
499+ struct kfd_process *process,
500+ unsigned int queue_id);
501+
502+extern struct device *kfd_device;
503+
504+/* Topology */
505+int kfd_topology_init(void);
506+void kfd_topology_shutdown(void);
507+int kfd_topology_add_device(struct kfd_dev *gpu);
508+int kfd_topology_remove_device(struct kfd_dev *gpu);
509+struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
510+struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
511+struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx);
512+
513+/* Interrupts */
514+int kfd_interrupt_init(struct kfd_dev *dev);
515+void kfd_interrupt_exit(struct kfd_dev *dev);
516+void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);
517+bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry);
518+
519+/* Power Management */
520+void kgd2kfd_suspend(struct kfd_dev *kfd);
521+int kgd2kfd_resume(struct kfd_dev *kfd);
522+
523+/* amdkfd Apertures */
524+int kfd_init_apertures(struct kfd_process *process);
525+
526+/* Queue Context Management */
527+inline uint32_t lower_32(uint64_t x);
528+inline uint32_t upper_32(uint64_t x);
529+
530+int init_queue(struct queue **q, struct queue_properties properties);
531+void uninit_queue(struct queue *q);
532+void print_queue_properties(struct queue_properties *q);
533+void print_queue(struct queue *q);
534+
535+struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
536+ struct kfd_dev *dev);
537+struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
538+void device_queue_manager_uninit(struct device_queue_manager *dqm);
539+struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
540+ enum kfd_queue_type type);
541+void kernel_queue_uninit(struct kernel_queue *kq);
542+
543+/* Process Queue Manager */
544+struct process_queue_node {
545+ struct queue *q;
546+ struct kernel_queue *kq;
547+ struct list_head process_queue_list;
548+};
549+
550+int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
551+void pqm_uninit(struct process_queue_manager *pqm);
552+int pqm_create_queue(struct process_queue_manager *pqm,
553+ struct kfd_dev *dev,
554+ struct file *f,
555+ struct queue_properties *properties,
556+ unsigned int flags,
557+ enum kfd_queue_type type,
558+ unsigned int *qid);
559+int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
560+int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
561+ struct queue_properties *p);
562+
563+/* Packet Manager */
564+
565+#define KFD_HIQ_TIMEOUT (500)
566+
567+#define KFD_FENCE_COMPLETED (100)
568+#define KFD_FENCE_INIT (10)
569+#define KFD_UNMAP_LATENCY (150)
570+
571+struct packet_manager {
572+ struct device_queue_manager *dqm;
573+ struct kernel_queue *priv_queue;
574+ struct mutex lock;
575+ bool allocated;
576+ struct kfd_mem_obj *ib_buffer_obj;
577+};
578+
579+int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
580+void pm_uninit(struct packet_manager *pm);
581+int pm_send_set_resources(struct packet_manager *pm,
582+ struct scheduling_resources *res);
583+int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues);
584+int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
585+ uint32_t fence_value);
586+
587+int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
588+ enum kfd_preempt_type_filter mode,
589+ uint32_t filter_param, bool reset,
590+ unsigned int sdma_engine);
591+
592+void pm_release_ib(struct packet_manager *pm);
593+
594+uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
595+phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
596+ struct kfd_process *process);
597+
598+#endif
drivers/gpu/drm/amd/amdkfd/kfd_process.c+415−0
@@ -0,0 +1,415 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+#include <linux/mutex.h>
24+#include <linux/log2.h>
25+#include <linux/sched.h>
26+#include <linux/slab.h>
27+#include <linux/amd-iommu.h>
28+#include <linux/notifier.h>
29+struct mm_struct;
30+
31+#include "kfd_priv.h"
32+
33+/*
34+ * Initial size for the array of queues.
35+ * The allocated size is doubled each time
36+ * it is exceeded up to MAX_PROCESS_QUEUES.
37+ */
38+#define INITIAL_QUEUE_ARRAY_SIZE 16
39+
40+/*
41+ * List of struct kfd_process (field kfd_process).
42+ * Unique/indexed by mm_struct*
43+ */
44+#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
45+static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
46+static DEFINE_MUTEX(kfd_processes_mutex);
47+
48+DEFINE_STATIC_SRCU(kfd_processes_srcu);
49+
50+static struct workqueue_struct *kfd_process_wq;
51+
52+struct kfd_process_release_work {
53+ struct work_struct kfd_work;
54+ struct kfd_process *p;
55+};
56+
57+static struct kfd_process *find_process(const struct task_struct *thread);
58+static struct kfd_process *create_process(const struct task_struct *thread);
59+
60+void kfd_process_create_wq(void)
61+{
62+ if (!kfd_process_wq)
63+ kfd_process_wq = create_workqueue("kfd_process_wq");
64+}
65+
66+void kfd_process_destroy_wq(void)
67+{
68+ if (kfd_process_wq) {
69+ flush_workqueue(kfd_process_wq);
70+ destroy_workqueue(kfd_process_wq);
71+ kfd_process_wq = NULL;
72+ }
73+}
74+
75+struct kfd_process *kfd_create_process(const struct task_struct *thread)
76+{
77+ struct kfd_process *process;
78+
79+ BUG_ON(!kfd_process_wq);
80+
81+ if (thread->mm == NULL)
82+ return ERR_PTR(-EINVAL);
83+
84+ /* Only the pthreads threading model is supported. */
85+ if (thread->group_leader->mm != thread->mm)
86+ return ERR_PTR(-EINVAL);
87+
88+ /* Take mmap_sem because we call __mmu_notifier_register inside */
89+ down_write(&thread->mm->mmap_sem);
90+
91+ /*
92+ * take kfd processes mutex before starting of process creation
93+ * so there won't be a case where two threads of the same process
94+ * create two kfd_process structures
95+ */
96+ mutex_lock(&kfd_processes_mutex);
97+
98+ /* A prior open of /dev/kfd could have already created the process. */
99+ process = find_process(thread);
100+ if (process)
101+ pr_debug("kfd: process already found\n");
102+
103+ if (!process)
104+ process = create_process(thread);
105+
106+ mutex_unlock(&kfd_processes_mutex);
107+
108+ up_write(&thread->mm->mmap_sem);
109+
110+ return process;
111+}
112+
113+struct kfd_process *kfd_get_process(const struct task_struct *thread)
114+{
115+ struct kfd_process *process;
116+
117+ if (thread->mm == NULL)
118+ return ERR_PTR(-EINVAL);
119+
120+ /* Only the pthreads threading model is supported. */
121+ if (thread->group_leader->mm != thread->mm)
122+ return ERR_PTR(-EINVAL);
123+
124+ process = find_process(thread);
125+
126+ return process;
127+}
128+
129+static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
130+{
131+ struct kfd_process *process;
132+
133+ hash_for_each_possible_rcu(kfd_processes_table, process,
134+ kfd_processes, (uintptr_t)mm)
135+ if (process->mm == mm)
136+ return process;
137+
138+ return NULL;
139+}
140+
141+static struct kfd_process *find_process(const struct task_struct *thread)
142+{
143+ struct kfd_process *p;
144+ int idx;
145+
146+ idx = srcu_read_lock(&kfd_processes_srcu);
147+ p = find_process_by_mm(thread->mm);
148+ srcu_read_unlock(&kfd_processes_srcu, idx);
149+
150+ return p;
151+}
152+
153+static void kfd_process_wq_release(struct work_struct *work)
154+{
155+ struct kfd_process_release_work *my_work;
156+ struct kfd_process_device *pdd, *temp;
157+ struct kfd_process *p;
158+
159+ my_work = (struct kfd_process_release_work *) work;
160+
161+ p = my_work->p;
162+
163+ mutex_lock(&p->mutex);
164+
165+ list_for_each_entry_safe(pdd, temp, &p->per_device_data,
166+ per_device_list) {
167+ amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
168+ list_del(&pdd->per_device_list);
169+
170+ kfree(pdd);
171+ }
172+
173+ kfd_pasid_free(p->pasid);
174+
175+ mutex_unlock(&p->mutex);
176+
177+ mutex_destroy(&p->mutex);
178+
179+ kfree(p->queues);
180+
181+ kfree(p);
182+
183+ kfree((void *)work);
184+}
185+
186+static void kfd_process_destroy_delayed(struct rcu_head *rcu)
187+{
188+ struct kfd_process_release_work *work;
189+ struct kfd_process *p;
190+
191+ BUG_ON(!kfd_process_wq);
192+
193+ p = container_of(rcu, struct kfd_process, rcu);
194+ BUG_ON(atomic_read(&p->mm->mm_count) <= 0);
195+
196+ mmdrop(p->mm);
197+
198+ work = (struct kfd_process_release_work *)
199+ kmalloc(sizeof(struct kfd_process_release_work), GFP_KERNEL);
200+
201+ if (work) {
202+ INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
203+ work->p = p;
204+ queue_work(kfd_process_wq, (struct work_struct *) work);
205+ }
206+}
207+
208+static void kfd_process_notifier_release(struct mmu_notifier *mn,
209+ struct mm_struct *mm)
210+{
211+ struct kfd_process *p;
212+
213+ /*
214+ * The kfd_process structure can not be free because the
215+ * mmu_notifier srcu is read locked
216+ */
217+ p = container_of(mn, struct kfd_process, mmu_notifier);
218+ BUG_ON(p->mm != mm);
219+
220+ mutex_lock(&kfd_processes_mutex);
221+ hash_del_rcu(&p->kfd_processes);
222+ mutex_unlock(&kfd_processes_mutex);
223+ synchronize_srcu(&kfd_processes_srcu);
224+
225+ mutex_lock(&p->mutex);
226+
227+ /* In case our notifier is called before IOMMU notifier */
228+ pqm_uninit(&p->pqm);
229+
230+ mutex_unlock(&p->mutex);
231+
232+ /*
233+ * Because we drop mm_count inside kfd_process_destroy_delayed
234+ * and because the mmu_notifier_unregister function also drop
235+ * mm_count we need to take an extra count here.
236+ */
237+ atomic_inc(&p->mm->mm_count);
238+ mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
239+ mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
240+}
241+
242+static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
243+ .release = kfd_process_notifier_release,
244+};
245+
246+static struct kfd_process *create_process(const struct task_struct *thread)
247+{
248+ struct kfd_process *process;
249+ int err = -ENOMEM;
250+
251+ process = kzalloc(sizeof(*process), GFP_KERNEL);
252+
253+ if (!process)
254+ goto err_alloc_process;
255+
256+ process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
257+ sizeof(process->queues[0]), GFP_KERNEL);
258+ if (!process->queues)
259+ goto err_alloc_queues;
260+
261+ process->pasid = kfd_pasid_alloc();
262+ if (process->pasid == 0)
263+ goto err_alloc_pasid;
264+
265+ mutex_init(&process->mutex);
266+
267+ process->mm = thread->mm;
268+
269+ /* register notifier */
270+ process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
271+ err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
272+ if (err)
273+ goto err_mmu_notifier;
274+
275+ hash_add_rcu(kfd_processes_table, &process->kfd_processes,
276+ (uintptr_t)process->mm);
277+
278+ process->lead_thread = thread->group_leader;
279+
280+ process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
281+
282+ INIT_LIST_HEAD(&process->per_device_data);
283+
284+ err = pqm_init(&process->pqm, process);
285+ if (err != 0)
286+ goto err_process_pqm_init;
287+
288+ return process;
289+
290+err_process_pqm_init:
291+ hash_del_rcu(&process->kfd_processes);
292+ synchronize_rcu();
293+ mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
294+err_mmu_notifier:
295+ kfd_pasid_free(process->pasid);
296+err_alloc_pasid:
297+ kfree(process->queues);
298+err_alloc_queues:
299+ kfree(process);
300+err_alloc_process:
301+ return ERR_PTR(err);
302+}
303+
304+struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
305+ struct kfd_process *p,
306+ int create_pdd)
307+{
308+ struct kfd_process_device *pdd = NULL;
309+
310+ list_for_each_entry(pdd, &p->per_device_data, per_device_list)
311+ if (pdd->dev == dev)
312+ return pdd;
313+
314+ if (create_pdd) {
315+ pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
316+ if (pdd != NULL) {
317+ pdd->dev = dev;
318+ INIT_LIST_HEAD(&pdd->qpd.queues_list);
319+ INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
320+ pdd->qpd.dqm = dev->dqm;
321+ list_add(&pdd->per_device_list, &p->per_device_data);
322+ }
323+ }
324+
325+ return pdd;
326+}
327+
328+/*
329+ * Direct the IOMMU to bind the process (specifically the pasid->mm)
330+ * to the device.
331+ * Unbinding occurs when the process dies or the device is removed.
332+ *
333+ * Assumes that the process lock is held.
334+ */
335+struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
336+ struct kfd_process *p)
337+{
338+ struct kfd_process_device *pdd = kfd_get_process_device_data(dev, p, 1);
339+ int err;
340+
341+ if (pdd == NULL)
342+ return ERR_PTR(-ENOMEM);
343+
344+ if (pdd->bound)
345+ return pdd;
346+
347+ err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
348+ if (err < 0)
349+ return ERR_PTR(err);
350+
351+ if (err < 0) {
352+ amd_iommu_unbind_pasid(dev->pdev, p->pasid);
353+ return ERR_PTR(err);
354+ }
355+
356+ pdd->bound = true;
357+
358+ return pdd;
359+}
360+
361+void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
362+{
363+ struct kfd_process *p;
364+ struct kfd_process_device *pdd;
365+ int idx, i;
366+
367+ BUG_ON(dev == NULL);
368+
369+ idx = srcu_read_lock(&kfd_processes_srcu);
370+
371+ hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes)
372+ if (p->pasid == pasid)
373+ break;
374+
375+ srcu_read_unlock(&kfd_processes_srcu, idx);
376+
377+ BUG_ON(p->pasid != pasid);
378+
379+ mutex_lock(&p->mutex);
380+
381+ pqm_uninit(&p->pqm);
382+
383+ pdd = kfd_get_process_device_data(dev, p, 0);
384+
385+ /*
386+ * Just mark pdd as unbound, because we still need it to call
387+ * amd_iommu_unbind_pasid() in when the process exits.
388+ * We don't call amd_iommu_unbind_pasid() here
389+ * because the IOMMU called us.
390+ */
391+ if (pdd)
392+ pdd->bound = false;
393+
394+ mutex_unlock(&p->mutex);
395+}
396+
397+struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
398+{
399+ return list_first_entry(&p->per_device_data,
400+ struct kfd_process_device,
401+ per_device_list);
402+}
403+
404+struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
405+ struct kfd_process_device *pdd)
406+{
407+ if (list_is_last(&pdd->per_device_list, &p->per_device_data))
408+ return NULL;
409+ return list_next_entry(pdd, per_device_list);
410+}
411+
412+bool kfd_has_process_device_data(struct kfd_process *p)
413+{
414+ return !(list_empty(&p->per_device_data));
415+}
drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c+342−0
@@ -0,0 +1,342 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ *
22+ */
23+
24+#include <linux/slab.h>
25+#include <linux/list.h>
26+#include "kfd_device_queue_manager.h"
27+#include "kfd_priv.h"
28+#include "kfd_kernel_queue.h"
29+
30+static inline struct process_queue_node *get_queue_by_qid(
31+ struct process_queue_manager *pqm, unsigned int qid)
32+{
33+ struct process_queue_node *pqn;
34+
35+ BUG_ON(!pqm);
36+
37+ list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
38+ if (pqn->q && pqn->q->properties.queue_id == qid)
39+ return pqn;
40+ if (pqn->kq && pqn->kq->queue->properties.queue_id == qid)
41+ return pqn;
42+ }
43+
44+ return NULL;
45+}
46+
47+static int find_available_queue_slot(struct process_queue_manager *pqm,
48+ unsigned int *qid)
49+{
50+ unsigned long found;
51+
52+ BUG_ON(!pqm || !qid);
53+
54+ pr_debug("kfd: in %s\n", __func__);
55+
56+ found = find_first_zero_bit(pqm->queue_slot_bitmap,
57+ max_num_of_queues_per_process);
58+
59+ pr_debug("kfd: the new slot id %lu\n", found);
60+
61+ if (found >= max_num_of_queues_per_process) {
62+ pr_info("amdkfd: Can not open more queues for process with pasid %d\n",
63+ pqm->process->pasid);
64+ return -ENOMEM;
65+ }
66+
67+ set_bit(found, pqm->queue_slot_bitmap);
68+ *qid = found;
69+
70+ return 0;
71+}
72+
73+int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
74+{
75+ BUG_ON(!pqm);
76+
77+ INIT_LIST_HEAD(&pqm->queues);
78+ pqm->queue_slot_bitmap =
79+ kzalloc(DIV_ROUND_UP(max_num_of_queues_per_process,
80+ BITS_PER_BYTE), GFP_KERNEL);
81+ if (pqm->queue_slot_bitmap == NULL)
82+ return -ENOMEM;
83+ pqm->process = p;
84+
85+ return 0;
86+}
87+
88+void pqm_uninit(struct process_queue_manager *pqm)
89+{
90+ int retval;
91+ struct process_queue_node *pqn, *next;
92+
93+ BUG_ON(!pqm);
94+
95+ pr_debug("In func %s\n", __func__);
96+
97+ list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
98+ retval = pqm_destroy_queue(
99+ pqm,
100+ (pqn->q != NULL) ?
101+ pqn->q->properties.queue_id :
102+ pqn->kq->queue->properties.queue_id);
103+
104+ if (retval != 0) {
105+ pr_err("kfd: failed to destroy queue\n");
106+ return;
107+ }
108+ }
109+ kfree(pqm->queue_slot_bitmap);
110+ pqm->queue_slot_bitmap = NULL;
111+}
112+
113+static int create_cp_queue(struct process_queue_manager *pqm,
114+ struct kfd_dev *dev, struct queue **q,
115+ struct queue_properties *q_properties,
116+ struct file *f, unsigned int qid)
117+{
118+ int retval;
119+
120+ retval = 0;
121+
122+ /* Doorbell initialized in user space*/
123+ q_properties->doorbell_ptr = NULL;
124+
125+ q_properties->doorbell_off =
126+ kfd_queue_id_to_doorbell(dev, pqm->process, qid);
127+
128+ /* let DQM handle it*/
129+ q_properties->vmid = 0;
130+ q_properties->queue_id = qid;
131+ q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
132+
133+ retval = init_queue(q, *q_properties);
134+ if (retval != 0)
135+ goto err_init_queue;
136+
137+ (*q)->device = dev;
138+ (*q)->process = pqm->process;
139+
140+ pr_debug("kfd: PQM After init queue");
141+
142+ return retval;
143+
144+err_init_queue:
145+ return retval;
146+}
147+
148+int pqm_create_queue(struct process_queue_manager *pqm,
149+ struct kfd_dev *dev,
150+ struct file *f,
151+ struct queue_properties *properties,
152+ unsigned int flags,
153+ enum kfd_queue_type type,
154+ unsigned int *qid)
155+{
156+ int retval;
157+ struct kfd_process_device *pdd;
158+ struct queue_properties q_properties;
159+ struct queue *q;
160+ struct process_queue_node *pqn;
161+ struct kernel_queue *kq;
162+
163+ BUG_ON(!pqm || !dev || !properties || !qid);
164+
165+ memset(&q_properties, 0, sizeof(struct queue_properties));
166+ memcpy(&q_properties, properties, sizeof(struct queue_properties));
167+ q = NULL;
168+ kq = NULL;
169+
170+ pdd = kfd_get_process_device_data(dev, pqm->process, 1);
171+ BUG_ON(!pdd);
172+
173+ retval = find_available_queue_slot(pqm, qid);
174+ if (retval != 0)
175+ return retval;
176+
177+ if (list_empty(&pqm->queues)) {
178+ pdd->qpd.pqm = pqm;
179+ dev->dqm->register_process(dev->dqm, &pdd->qpd);
180+ }
181+
182+ pqn = kzalloc(sizeof(struct process_queue_node), GFP_KERNEL);
183+ if (!pqn) {
184+ retval = -ENOMEM;
185+ goto err_allocate_pqn;
186+ }
187+
188+ switch (type) {
189+ case KFD_QUEUE_TYPE_COMPUTE:
190+ /* check if there is over subscription */
191+ if ((sched_policy == KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
192+ ((dev->dqm->processes_count >= VMID_PER_DEVICE) ||
193+ (dev->dqm->queue_count >= PIPE_PER_ME_CP_SCHEDULING * QUEUES_PER_PIPE))) {
194+ pr_err("kfd: over-subscription is not allowed in radeon_kfd.sched_policy == 1\n");
195+ retval = -EPERM;
196+ goto err_create_queue;
197+ }
198+
199+ retval = create_cp_queue(pqm, dev, &q, &q_properties, f, *qid);
200+ if (retval != 0)
201+ goto err_create_queue;
202+ pqn->q = q;
203+ pqn->kq = NULL;
204+ retval = dev->dqm->create_queue(dev->dqm, q, &pdd->qpd,
205+ &q->properties.vmid);
206+ print_queue(q);
207+ break;
208+ case KFD_QUEUE_TYPE_DIQ:
209+ kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ);
210+ if (kq == NULL) {
211+ kernel_queue_uninit(kq);
212+ goto err_create_queue;
213+ }
214+ kq->queue->properties.queue_id = *qid;
215+ pqn->kq = kq;
216+ pqn->q = NULL;
217+ retval = dev->dqm->create_kernel_queue(dev->dqm, kq, &pdd->qpd);
218+ break;
219+ default:
220+ BUG();
221+ break;
222+ }
223+
224+ if (retval != 0) {
225+ pr_err("kfd: error dqm create queue\n");
226+ goto err_create_queue;
227+ }
228+
229+ pr_debug("kfd: PQM After DQM create queue\n");
230+
231+ list_add(&pqn->process_queue_list, &pqm->queues);
232+
233+ if (q) {
234+ *properties = q->properties;
235+ pr_debug("kfd: PQM done creating queue\n");
236+ print_queue_properties(properties);
237+ }
238+
239+ return retval;
240+
241+err_create_queue:
242+ kfree(pqn);
243+err_allocate_pqn:
244+ clear_bit(*qid, pqm->queue_slot_bitmap);
245+ return retval;
246+}
247+
248+int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
249+{
250+ struct process_queue_node *pqn;
251+ struct kfd_process_device *pdd;
252+ struct device_queue_manager *dqm;
253+ struct kfd_dev *dev;
254+ int retval;
255+
256+ dqm = NULL;
257+
258+ BUG_ON(!pqm);
259+ retval = 0;
260+
261+ pr_debug("kfd: In Func %s\n", __func__);
262+
263+ pqn = get_queue_by_qid(pqm, qid);
264+ if (pqn == NULL) {
265+ pr_err("kfd: queue id does not match any known queue\n");
266+ return -EINVAL;
267+ }
268+
269+ dev = NULL;
270+ if (pqn->kq)
271+ dev = pqn->kq->dev;
272+ if (pqn->q)
273+ dev = pqn->q->device;
274+ BUG_ON(!dev);
275+
276+ pdd = kfd_get_process_device_data(dev, pqm->process, 1);
277+ BUG_ON(!pdd);
278+
279+ if (pqn->kq) {
280+ /* destroy kernel queue (DIQ) */
281+ dqm = pqn->kq->dev->dqm;
282+ dqm->destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);
283+ kernel_queue_uninit(pqn->kq);
284+ }
285+
286+ if (pqn->q) {
287+ dqm = pqn->q->device->dqm;
288+ retval = dqm->destroy_queue(dqm, &pdd->qpd, pqn->q);
289+ if (retval != 0)
290+ return retval;
291+
292+ uninit_queue(pqn->q);
293+ }
294+
295+ list_del(&pqn->process_queue_list);
296+ kfree(pqn);
297+ clear_bit(qid, pqm->queue_slot_bitmap);
298+
299+ if (list_empty(&pqm->queues))
300+ dqm->unregister_process(dqm, &pdd->qpd);
301+
302+ return retval;
303+}
304+
305+int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
306+ struct queue_properties *p)
307+{
308+ int retval;
309+ struct process_queue_node *pqn;
310+
311+ BUG_ON(!pqm);
312+
313+ pqn = get_queue_by_qid(pqm, qid);
314+ BUG_ON(!pqn);
315+
316+ pqn->q->properties.queue_address = p->queue_address;
317+ pqn->q->properties.queue_size = p->queue_size;
318+ pqn->q->properties.queue_percent = p->queue_percent;
319+ pqn->q->properties.priority = p->priority;
320+
321+ retval = pqn->q->device->dqm->update_queue(pqn->q->device->dqm, pqn->q);
322+ if (retval != 0)
323+ return retval;
324+
325+ return 0;
326+}
327+
328+struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
329+ unsigned int qid)
330+{
331+ struct process_queue_node *pqn;
332+
333+ BUG_ON(!pqm);
334+
335+ pqn = get_queue_by_qid(pqm, qid);
336+ if (pqn && pqn->kq)
337+ return pqn->kq;
338+
339+ return NULL;
340+}
341+
342+
drivers/gpu/drm/amd/amdkfd/kfd_queue.c+85−0
@@ -0,0 +1,85 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ *
22+ */
23+
24+#include <linux/slab.h>
25+#include "kfd_priv.h"
26+
27+void print_queue_properties(struct queue_properties *q)
28+{
29+ if (!q)
30+ return;
31+
32+ pr_debug("Printing queue properties:\n");
33+ pr_debug("Queue Type: %u\n", q->type);
34+ pr_debug("Queue Size: %llu\n", q->queue_size);
35+ pr_debug("Queue percent: %u\n", q->queue_percent);
36+ pr_debug("Queue Address: 0x%llX\n", q->queue_address);
37+ pr_debug("Queue Id: %u\n", q->queue_id);
38+ pr_debug("Queue Process Vmid: %u\n", q->vmid);
39+ pr_debug("Queue Read Pointer: 0x%p\n", q->read_ptr);
40+ pr_debug("Queue Write Pointer: 0x%p\n", q->write_ptr);
41+ pr_debug("Queue Doorbell Pointer: 0x%p\n", q->doorbell_ptr);
42+ pr_debug("Queue Doorbell Offset: %u\n", q->doorbell_off);
43+}
44+
45+void print_queue(struct queue *q)
46+{
47+ if (!q)
48+ return;
49+ pr_debug("Printing queue:\n");
50+ pr_debug("Queue Type: %u\n", q->properties.type);
51+ pr_debug("Queue Size: %llu\n", q->properties.queue_size);
52+ pr_debug("Queue percent: %u\n", q->properties.queue_percent);
53+ pr_debug("Queue Address: 0x%llX\n", q->properties.queue_address);
54+ pr_debug("Queue Id: %u\n", q->properties.queue_id);
55+ pr_debug("Queue Process Vmid: %u\n", q->properties.vmid);
56+ pr_debug("Queue Read Pointer: 0x%p\n", q->properties.read_ptr);
57+ pr_debug("Queue Write Pointer: 0x%p\n", q->properties.write_ptr);
58+ pr_debug("Queue Doorbell Pointer: 0x%p\n", q->properties.doorbell_ptr);
59+ pr_debug("Queue Doorbell Offset: %u\n", q->properties.doorbell_off);
60+ pr_debug("Queue MQD Address: 0x%p\n", q->mqd);
61+ pr_debug("Queue MQD Gart: 0x%llX\n", q->gart_mqd_addr);
62+ pr_debug("Queue Process Address: 0x%p\n", q->process);
63+ pr_debug("Queue Device Address: 0x%p\n", q->device);
64+}
65+
66+int init_queue(struct queue **q, struct queue_properties properties)
67+{
68+ struct queue *tmp;
69+
70+ BUG_ON(!q);
71+
72+ tmp = kzalloc(sizeof(struct queue), GFP_KERNEL);
73+ if (!tmp)
74+ return -ENOMEM;
75+
76+ memcpy(&tmp->properties, &properties, sizeof(struct queue_properties));
77+
78+ *q = tmp;
79+ return 0;
80+}
81+
82+void uninit_queue(struct queue *q)
83+{
84+ kfree(q);
85+}
drivers/gpu/drm/amd/amdkfd/kfd_topology.c+1235−0
@@ -0,0 +1,1235 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+#include <linux/types.h>
24+#include <linux/kernel.h>
25+#include <linux/pci.h>
26+#include <linux/errno.h>
27+#include <linux/acpi.h>
28+#include <linux/hash.h>
29+#include <linux/cpufreq.h>
30+
31+#include "kfd_priv.h"
32+#include "kfd_crat.h"
33+#include "kfd_topology.h"
34+
35+static struct list_head topology_device_list;
36+static int topology_crat_parsed;
37+static struct kfd_system_properties sys_props;
38+
39+static DECLARE_RWSEM(topology_lock);
40+
41+struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
42+{
43+ struct kfd_topology_device *top_dev;
44+ struct kfd_dev *device = NULL;
45+
46+ down_read(&topology_lock);
47+
48+ list_for_each_entry(top_dev, &topology_device_list, list)
49+ if (top_dev->gpu_id == gpu_id) {
50+ device = top_dev->gpu;
51+ break;
52+ }
53+
54+ up_read(&topology_lock);
55+
56+ return device;
57+}
58+
59+struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
60+{
61+ struct kfd_topology_device *top_dev;
62+ struct kfd_dev *device = NULL;
63+
64+ down_read(&topology_lock);
65+
66+ list_for_each_entry(top_dev, &topology_device_list, list)
67+ if (top_dev->gpu->pdev == pdev) {
68+ device = top_dev->gpu;
69+ break;
70+ }
71+
72+ up_read(&topology_lock);
73+
74+ return device;
75+}
76+
77+static int kfd_topology_get_crat_acpi(void *crat_image, size_t *size)
78+{
79+ struct acpi_table_header *crat_table;
80+ acpi_status status;
81+
82+ if (!size)
83+ return -EINVAL;
84+
85+ /*
86+ * Fetch the CRAT table from ACPI
87+ */
88+ status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
89+ if (status == AE_NOT_FOUND) {
90+ pr_warn("CRAT table not found\n");
91+ return -ENODATA;
92+ } else if (ACPI_FAILURE(status)) {
93+ const char *err = acpi_format_exception(status);
94+
95+ pr_err("CRAT table error: %s\n", err);
96+ return -EINVAL;
97+ }
98+
99+ if (*size >= crat_table->length && crat_image != 0)
100+ memcpy(crat_image, crat_table, crat_table->length);
101+
102+ *size = crat_table->length;
103+
104+ return 0;
105+}
106+
107+static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
108+ struct crat_subtype_computeunit *cu)
109+{
110+ BUG_ON(!dev);
111+ BUG_ON(!cu);
112+
113+ dev->node_props.cpu_cores_count = cu->num_cpu_cores;
114+ dev->node_props.cpu_core_id_base = cu->processor_id_low;
115+ if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
116+ dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
117+
118+ pr_info("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores,
119+ cu->processor_id_low);
120+}
121+
122+static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
123+ struct crat_subtype_computeunit *cu)
124+{
125+ BUG_ON(!dev);
126+ BUG_ON(!cu);
127+
128+ dev->node_props.simd_id_base = cu->processor_id_low;
129+ dev->node_props.simd_count = cu->num_simd_cores;
130+ dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
131+ dev->node_props.max_waves_per_simd = cu->max_waves_simd;
132+ dev->node_props.wave_front_size = cu->wave_front_size;
133+ dev->node_props.mem_banks_count = cu->num_banks;
134+ dev->node_props.array_count = cu->num_arrays;
135+ dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
136+ dev->node_props.simd_per_cu = cu->num_simd_per_cu;
137+ dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
138+ if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE)
139+ dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE;
140+ pr_info("CU GPU: simds=%d id_base=%d\n", cu->num_simd_cores,
141+ cu->processor_id_low);
142+}
143+
144+/* kfd_parse_subtype_cu is called when the topology mutex is already acquired */
145+static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu)
146+{
147+ struct kfd_topology_device *dev;
148+ int i = 0;
149+
150+ BUG_ON(!cu);
151+
152+ pr_info("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
153+ cu->proximity_domain, cu->hsa_capability);
154+ list_for_each_entry(dev, &topology_device_list, list) {
155+ if (cu->proximity_domain == i) {
156+ if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT)
157+ kfd_populated_cu_info_cpu(dev, cu);
158+
159+ if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT)
160+ kfd_populated_cu_info_gpu(dev, cu);
161+ break;
162+ }
163+ i++;
164+ }
165+
166+ return 0;
167+}
168+
169+/*
170+ * kfd_parse_subtype_mem is called when the topology mutex is
171+ * already acquired
172+ */
173+static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem)
174+{
175+ struct kfd_mem_properties *props;
176+ struct kfd_topology_device *dev;
177+ int i = 0;
178+
179+ BUG_ON(!mem);
180+
181+ pr_info("Found memory entry in CRAT table with proximity_domain=%d\n",
182+ mem->promixity_domain);
183+ list_for_each_entry(dev, &topology_device_list, list) {
184+ if (mem->promixity_domain == i) {
185+ props = kfd_alloc_struct(props);
186+ if (props == 0)
187+ return -ENOMEM;
188+
189+ if (dev->node_props.cpu_cores_count == 0)
190+ props->heap_type = HSA_MEM_HEAP_TYPE_FB_PRIVATE;
191+ else
192+ props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
193+
194+ if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
195+ props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
196+ if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
197+ props->flags |= HSA_MEM_FLAGS_NON_VOLATILE;
198+
199+ props->size_in_bytes =
200+ ((uint64_t)mem->length_high << 32) +
201+ mem->length_low;
202+ props->width = mem->width;
203+
204+ dev->mem_bank_count++;
205+ list_add_tail(&props->list, &dev->mem_props);
206+
207+ break;
208+ }
209+ i++;
210+ }
211+
212+ return 0;
213+}
214+
215+/*
216+ * kfd_parse_subtype_cache is called when the topology mutex
217+ * is already acquired
218+ */
219+static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache)
220+{
221+ struct kfd_cache_properties *props;
222+ struct kfd_topology_device *dev;
223+ uint32_t id;
224+
225+ BUG_ON(!cache);
226+
227+ id = cache->processor_id_low;
228+
229+ pr_info("Found cache entry in CRAT table with processor_id=%d\n", id);
230+ list_for_each_entry(dev, &topology_device_list, list)
231+ if (id == dev->node_props.cpu_core_id_base ||
232+ id == dev->node_props.simd_id_base) {
233+ props = kfd_alloc_struct(props);
234+ if (props == 0)
235+ return -ENOMEM;
236+
237+ props->processor_id_low = id;
238+ props->cache_level = cache->cache_level;
239+ props->cache_size = cache->cache_size;
240+ props->cacheline_size = cache->cache_line_size;
241+ props->cachelines_per_tag = cache->lines_per_tag;
242+ props->cache_assoc = cache->associativity;
243+ props->cache_latency = cache->cache_latency;
244+
245+ if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
246+ props->cache_type |= HSA_CACHE_TYPE_DATA;
247+ if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE)
248+ props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
249+ if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE)
250+ props->cache_type |= HSA_CACHE_TYPE_CPU;
251+ if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
252+ props->cache_type |= HSA_CACHE_TYPE_HSACU;
253+
254+ dev->cache_count++;
255+ dev->node_props.caches_count++;
256+ list_add_tail(&props->list, &dev->cache_props);
257+
258+ break;
259+ }
260+
261+ return 0;
262+}
263+
264+/*
265+ * kfd_parse_subtype_iolink is called when the topology mutex
266+ * is already acquired
267+ */
268+static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink)
269+{
270+ struct kfd_iolink_properties *props;
271+ struct kfd_topology_device *dev;
272+ uint32_t i = 0;
273+ uint32_t id_from;
274+ uint32_t id_to;
275+
276+ BUG_ON(!iolink);
277+
278+ id_from = iolink->proximity_domain_from;
279+ id_to = iolink->proximity_domain_to;
280+
281+ pr_info("Found IO link entry in CRAT table with id_from=%d\n", id_from);
282+ list_for_each_entry(dev, &topology_device_list, list) {
283+ if (id_from == i) {
284+ props = kfd_alloc_struct(props);
285+ if (props == 0)
286+ return -ENOMEM;
287+
288+ props->node_from = id_from;
289+ props->node_to = id_to;
290+ props->ver_maj = iolink->version_major;
291+ props->ver_min = iolink->version_minor;
292+
293+ /*
294+ * weight factor (derived from CDIR), currently always 1
295+ */
296+ props->weight = 1;
297+
298+ props->min_latency = iolink->minimum_latency;
299+ props->max_latency = iolink->maximum_latency;
300+ props->min_bandwidth = iolink->minimum_bandwidth_mbs;
301+ props->max_bandwidth = iolink->maximum_bandwidth_mbs;
302+ props->rec_transfer_size =
303+ iolink->recommended_transfer_size;
304+
305+ dev->io_link_count++;
306+ dev->node_props.io_links_count++;
307+ list_add_tail(&props->list, &dev->io_link_props);
308+
309+ break;
310+ }
311+ i++;
312+ }
313+
314+ return 0;
315+}
316+
317+static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr)
318+{
319+ struct crat_subtype_computeunit *cu;
320+ struct crat_subtype_memory *mem;
321+ struct crat_subtype_cache *cache;
322+ struct crat_subtype_iolink *iolink;
323+ int ret = 0;
324+
325+ BUG_ON(!sub_type_hdr);
326+
327+ switch (sub_type_hdr->type) {
328+ case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
329+ cu = (struct crat_subtype_computeunit *)sub_type_hdr;
330+ ret = kfd_parse_subtype_cu(cu);
331+ break;
332+ case CRAT_SUBTYPE_MEMORY_AFFINITY:
333+ mem = (struct crat_subtype_memory *)sub_type_hdr;
334+ ret = kfd_parse_subtype_mem(mem);
335+ break;
336+ case CRAT_SUBTYPE_CACHE_AFFINITY:
337+ cache = (struct crat_subtype_cache *)sub_type_hdr;
338+ ret = kfd_parse_subtype_cache(cache);
339+ break;
340+ case CRAT_SUBTYPE_TLB_AFFINITY:
341+ /*
342+ * For now, nothing to do here
343+ */
344+ pr_info("Found TLB entry in CRAT table (not processing)\n");
345+ break;
346+ case CRAT_SUBTYPE_CCOMPUTE_AFFINITY:
347+ /*
348+ * For now, nothing to do here
349+ */
350+ pr_info("Found CCOMPUTE entry in CRAT table (not processing)\n");
351+ break;
352+ case CRAT_SUBTYPE_IOLINK_AFFINITY:
353+ iolink = (struct crat_subtype_iolink *)sub_type_hdr;
354+ ret = kfd_parse_subtype_iolink(iolink);
355+ break;
356+ default:
357+ pr_warn("Unknown subtype (%d) in CRAT\n",
358+ sub_type_hdr->type);
359+ }
360+
361+ return ret;
362+}
363+
364+static void kfd_release_topology_device(struct kfd_topology_device *dev)
365+{
366+ struct kfd_mem_properties *mem;
367+ struct kfd_cache_properties *cache;
368+ struct kfd_iolink_properties *iolink;
369+
370+ BUG_ON(!dev);
371+
372+ list_del(&dev->list);
373+
374+ while (dev->mem_props.next != &dev->mem_props) {
375+ mem = container_of(dev->mem_props.next,
376+ struct kfd_mem_properties, list);
377+ list_del(&mem->list);
378+ kfree(mem);
379+ }
380+
381+ while (dev->cache_props.next != &dev->cache_props) {
382+ cache = container_of(dev->cache_props.next,
383+ struct kfd_cache_properties, list);
384+ list_del(&cache->list);
385+ kfree(cache);
386+ }
387+
388+ while (dev->io_link_props.next != &dev->io_link_props) {
389+ iolink = container_of(dev->io_link_props.next,
390+ struct kfd_iolink_properties, list);
391+ list_del(&iolink->list);
392+ kfree(iolink);
393+ }
394+
395+ kfree(dev);
396+
397+ sys_props.num_devices--;
398+}
399+
400+static void kfd_release_live_view(void)
401+{
402+ struct kfd_topology_device *dev;
403+
404+ while (topology_device_list.next != &topology_device_list) {
405+ dev = container_of(topology_device_list.next,
406+ struct kfd_topology_device, list);
407+ kfd_release_topology_device(dev);
408+}
409+
410+ memset(&sys_props, 0, sizeof(sys_props));
411+}
412+
413+static struct kfd_topology_device *kfd_create_topology_device(void)
414+{
415+ struct kfd_topology_device *dev;
416+
417+ dev = kfd_alloc_struct(dev);
418+ if (dev == 0) {
419+ pr_err("No memory to allocate a topology device");
420+ return 0;
421+ }
422+
423+ INIT_LIST_HEAD(&dev->mem_props);
424+ INIT_LIST_HEAD(&dev->cache_props);
425+ INIT_LIST_HEAD(&dev->io_link_props);
426+
427+ list_add_tail(&dev->list, &topology_device_list);
428+ sys_props.num_devices++;
429+
430+ return dev;
431+ }
432+
433+static int kfd_parse_crat_table(void *crat_image)
434+{
435+ struct kfd_topology_device *top_dev;
436+ struct crat_subtype_generic *sub_type_hdr;
437+ uint16_t node_id;
438+ int ret;
439+ struct crat_header *crat_table = (struct crat_header *)crat_image;
440+ uint16_t num_nodes;
441+ uint32_t image_len;
442+
443+ if (!crat_image)
444+ return -EINVAL;
445+
446+ num_nodes = crat_table->num_domains;
447+ image_len = crat_table->length;
448+
449+ pr_info("Parsing CRAT table with %d nodes\n", num_nodes);
450+
451+ for (node_id = 0; node_id < num_nodes; node_id++) {
452+ top_dev = kfd_create_topology_device();
453+ if (!top_dev) {
454+ kfd_release_live_view();
455+ return -ENOMEM;
456+ }
457+ }
458+
459+ sys_props.platform_id =
460+ (*((uint64_t *)crat_table->oem_id)) & CRAT_OEMID_64BIT_MASK;
461+ sys_props.platform_oem = *((uint64_t *)crat_table->oem_table_id);
462+ sys_props.platform_rev = crat_table->revision;
463+
464+ sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
465+ while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) <
466+ ((char *)crat_image) + image_len) {
467+ if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
468+ ret = kfd_parse_subtype(sub_type_hdr);
469+ if (ret != 0) {
470+ kfd_release_live_view();
471+ return ret;
472+ }
473+ }
474+
475+ sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
476+ sub_type_hdr->length);
477+ }
478+
479+ sys_props.generation_count++;
480+ topology_crat_parsed = 1;
481+
482+ return 0;
483+}
484+
485+
486+#define sysfs_show_gen_prop(buffer, fmt, ...) \
487+ snprintf(buffer, PAGE_SIZE, "%s"fmt, buffer, __VA_ARGS__)
488+#define sysfs_show_32bit_prop(buffer, name, value) \
489+ sysfs_show_gen_prop(buffer, "%s %u\n", name, value)
490+#define sysfs_show_64bit_prop(buffer, name, value) \
491+ sysfs_show_gen_prop(buffer, "%s %llu\n", name, value)
492+#define sysfs_show_32bit_val(buffer, value) \
493+ sysfs_show_gen_prop(buffer, "%u\n", value)
494+#define sysfs_show_str_val(buffer, value) \
495+ sysfs_show_gen_prop(buffer, "%s\n", value)
496+
497+static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr,
498+ char *buffer)
499+{
500+ ssize_t ret;
501+
502+ /* Making sure that the buffer is an empty string */
503+ buffer[0] = 0;
504+
505+ if (attr == &sys_props.attr_genid) {
506+ ret = sysfs_show_32bit_val(buffer, sys_props.generation_count);
507+ } else if (attr == &sys_props.attr_props) {
508+ sysfs_show_64bit_prop(buffer, "platform_oem",
509+ sys_props.platform_oem);
510+ sysfs_show_64bit_prop(buffer, "platform_id",
511+ sys_props.platform_id);
512+ ret = sysfs_show_64bit_prop(buffer, "platform_rev",
513+ sys_props.platform_rev);
514+ } else {
515+ ret = -EINVAL;
516+ }
517+
518+ return ret;
519+}
520+
521+static const struct sysfs_ops sysprops_ops = {
522+ .show = sysprops_show,
523+};
524+
525+static struct kobj_type sysprops_type = {
526+ .sysfs_ops = &sysprops_ops,
527+};
528+
529+static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr,
530+ char *buffer)
531+{
532+ ssize_t ret;
533+ struct kfd_iolink_properties *iolink;
534+
535+ /* Making sure that the buffer is an empty string */
536+ buffer[0] = 0;
537+
538+ iolink = container_of(attr, struct kfd_iolink_properties, attr);
539+ sysfs_show_32bit_prop(buffer, "type", iolink->iolink_type);
540+ sysfs_show_32bit_prop(buffer, "version_major", iolink->ver_maj);
541+ sysfs_show_32bit_prop(buffer, "version_minor", iolink->ver_min);
542+ sysfs_show_32bit_prop(buffer, "node_from", iolink->node_from);
543+ sysfs_show_32bit_prop(buffer, "node_to", iolink->node_to);
544+ sysfs_show_32bit_prop(buffer, "weight", iolink->weight);
545+ sysfs_show_32bit_prop(buffer, "min_latency", iolink->min_latency);
546+ sysfs_show_32bit_prop(buffer, "max_latency", iolink->max_latency);
547+ sysfs_show_32bit_prop(buffer, "min_bandwidth", iolink->min_bandwidth);
548+ sysfs_show_32bit_prop(buffer, "max_bandwidth", iolink->max_bandwidth);
549+ sysfs_show_32bit_prop(buffer, "recommended_transfer_size",
550+ iolink->rec_transfer_size);
551+ ret = sysfs_show_32bit_prop(buffer, "flags", iolink->flags);
552+
553+ return ret;
554+}
555+
556+static const struct sysfs_ops iolink_ops = {
557+ .show = iolink_show,
558+};
559+
560+static struct kobj_type iolink_type = {
561+ .sysfs_ops = &iolink_ops,
562+};
563+
564+static ssize_t mem_show(struct kobject *kobj, struct attribute *attr,
565+ char *buffer)
566+{
567+ ssize_t ret;
568+ struct kfd_mem_properties *mem;
569+
570+ /* Making sure that the buffer is an empty string */
571+ buffer[0] = 0;
572+
573+ mem = container_of(attr, struct kfd_mem_properties, attr);
574+ sysfs_show_32bit_prop(buffer, "heap_type", mem->heap_type);
575+ sysfs_show_64bit_prop(buffer, "size_in_bytes", mem->size_in_bytes);
576+ sysfs_show_32bit_prop(buffer, "flags", mem->flags);
577+ sysfs_show_32bit_prop(buffer, "width", mem->width);
578+ ret = sysfs_show_32bit_prop(buffer, "mem_clk_max", mem->mem_clk_max);
579+
580+ return ret;
581+}
582+
583+static const struct sysfs_ops mem_ops = {
584+ .show = mem_show,
585+};
586+
587+static struct kobj_type mem_type = {
588+ .sysfs_ops = &mem_ops,
589+};
590+
591+static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr,
592+ char *buffer)
593+{
594+ ssize_t ret;
595+ uint32_t i;
596+ struct kfd_cache_properties *cache;
597+
598+ /* Making sure that the buffer is an empty string */
599+ buffer[0] = 0;
600+
601+ cache = container_of(attr, struct kfd_cache_properties, attr);
602+ sysfs_show_32bit_prop(buffer, "processor_id_low",
603+ cache->processor_id_low);
604+ sysfs_show_32bit_prop(buffer, "level", cache->cache_level);
605+ sysfs_show_32bit_prop(buffer, "size", cache->cache_size);
606+ sysfs_show_32bit_prop(buffer, "cache_line_size", cache->cacheline_size);
607+ sysfs_show_32bit_prop(buffer, "cache_lines_per_tag",
608+ cache->cachelines_per_tag);
609+ sysfs_show_32bit_prop(buffer, "association", cache->cache_assoc);
610+ sysfs_show_32bit_prop(buffer, "latency", cache->cache_latency);
611+ sysfs_show_32bit_prop(buffer, "type", cache->cache_type);
612+ snprintf(buffer, PAGE_SIZE, "%ssibling_map ", buffer);
613+ for (i = 0; i < KFD_TOPOLOGY_CPU_SIBLINGS; i++)
614+ ret = snprintf(buffer, PAGE_SIZE, "%s%d%s",
615+ buffer, cache->sibling_map[i],
616+ (i == KFD_TOPOLOGY_CPU_SIBLINGS-1) ?
617+ "\n" : ",");
618+
619+ return ret;
620+}
621+
622+static const struct sysfs_ops cache_ops = {
623+ .show = kfd_cache_show,
624+};
625+
626+static struct kobj_type cache_type = {
627+ .sysfs_ops = &cache_ops,
628+};
629+
630+static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
631+ char *buffer)
632+{
633+ ssize_t ret;
634+ struct kfd_topology_device *dev;
635+ char public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE];
636+ uint32_t i;
637+
638+ /* Making sure that the buffer is an empty string */
639+ buffer[0] = 0;
640+
641+ if (strcmp(attr->name, "gpu_id") == 0) {
642+ dev = container_of(attr, struct kfd_topology_device,
643+ attr_gpuid);
644+ ret = sysfs_show_32bit_val(buffer, dev->gpu_id);
645+ } else if (strcmp(attr->name, "name") == 0) {
646+ dev = container_of(attr, struct kfd_topology_device,
647+ attr_name);
648+ for (i = 0; i < KFD_TOPOLOGY_PUBLIC_NAME_SIZE; i++) {
649+ public_name[i] =
650+ (char)dev->node_props.marketing_name[i];
651+ if (dev->node_props.marketing_name[i] == 0)
652+ break;
653+ }
654+ public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE-1] = 0x0;
655+ ret = sysfs_show_str_val(buffer, public_name);
656+ } else {
657+ dev = container_of(attr, struct kfd_topology_device,
658+ attr_props);
659+ sysfs_show_32bit_prop(buffer, "cpu_cores_count",
660+ dev->node_props.cpu_cores_count);
661+ sysfs_show_32bit_prop(buffer, "simd_count",
662+ dev->node_props.simd_count);
663+
664+ if (dev->mem_bank_count < dev->node_props.mem_banks_count) {
665+ pr_warn("kfd: mem_banks_count truncated from %d to %d\n",
666+ dev->node_props.mem_banks_count,
667+ dev->mem_bank_count);
668+ sysfs_show_32bit_prop(buffer, "mem_banks_count",
669+ dev->mem_bank_count);
670+ } else {
671+ sysfs_show_32bit_prop(buffer, "mem_banks_count",
672+ dev->node_props.mem_banks_count);
673+ }
674+
675+ sysfs_show_32bit_prop(buffer, "caches_count",
676+ dev->node_props.caches_count);
677+ sysfs_show_32bit_prop(buffer, "io_links_count",
678+ dev->node_props.io_links_count);
679+ sysfs_show_32bit_prop(buffer, "cpu_core_id_base",
680+ dev->node_props.cpu_core_id_base);
681+ sysfs_show_32bit_prop(buffer, "simd_id_base",
682+ dev->node_props.simd_id_base);
683+ sysfs_show_32bit_prop(buffer, "capability",
684+ dev->node_props.capability);
685+ sysfs_show_32bit_prop(buffer, "max_waves_per_simd",
686+ dev->node_props.max_waves_per_simd);
687+ sysfs_show_32bit_prop(buffer, "lds_size_in_kb",
688+ dev->node_props.lds_size_in_kb);
689+ sysfs_show_32bit_prop(buffer, "gds_size_in_kb",
690+ dev->node_props.gds_size_in_kb);
691+ sysfs_show_32bit_prop(buffer, "wave_front_size",
692+ dev->node_props.wave_front_size);
693+ sysfs_show_32bit_prop(buffer, "array_count",
694+ dev->node_props.array_count);
695+ sysfs_show_32bit_prop(buffer, "simd_arrays_per_engine",
696+ dev->node_props.simd_arrays_per_engine);
697+ sysfs_show_32bit_prop(buffer, "cu_per_simd_array",
698+ dev->node_props.cu_per_simd_array);
699+ sysfs_show_32bit_prop(buffer, "simd_per_cu",
700+ dev->node_props.simd_per_cu);
701+ sysfs_show_32bit_prop(buffer, "max_slots_scratch_cu",
702+ dev->node_props.max_slots_scratch_cu);
703+ sysfs_show_32bit_prop(buffer, "engine_id",
704+ dev->node_props.engine_id);
705+ sysfs_show_32bit_prop(buffer, "vendor_id",
706+ dev->node_props.vendor_id);
707+ sysfs_show_32bit_prop(buffer, "device_id",
708+ dev->node_props.device_id);
709+ sysfs_show_32bit_prop(buffer, "location_id",
710+ dev->node_props.location_id);
711+
712+ if (dev->gpu) {
713+ sysfs_show_32bit_prop(buffer, "max_engine_clk_fcompute",
714+ kfd2kgd->get_max_engine_clock_in_mhz(
715+ dev->gpu->kgd));
716+ sysfs_show_64bit_prop(buffer, "local_mem_size",
717+ kfd2kgd->get_vmem_size(dev->gpu->kgd));
718+ }
719+
720+ ret = sysfs_show_32bit_prop(buffer, "max_engine_clk_ccompute",
721+ cpufreq_quick_get_max(0)/1000);
722+ }
723+
724+ return ret;
725+}
726+
727+static const struct sysfs_ops node_ops = {
728+ .show = node_show,
729+};
730+
731+static struct kobj_type node_type = {
732+ .sysfs_ops = &node_ops,
733+};
734+
735+static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr)
736+{
737+ sysfs_remove_file(kobj, attr);
738+ kobject_del(kobj);
739+ kobject_put(kobj);
740+}
741+
742+static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
743+{
744+ struct kfd_iolink_properties *iolink;
745+ struct kfd_cache_properties *cache;
746+ struct kfd_mem_properties *mem;
747+
748+ BUG_ON(!dev);
749+
750+ if (dev->kobj_iolink) {
751+ list_for_each_entry(iolink, &dev->io_link_props, list)
752+ if (iolink->kobj) {
753+ kfd_remove_sysfs_file(iolink->kobj,
754+ &iolink->attr);
755+ iolink->kobj = 0;
756+ }
757+ kobject_del(dev->kobj_iolink);
758+ kobject_put(dev->kobj_iolink);
759+ dev->kobj_iolink = 0;
760+ }
761+
762+ if (dev->kobj_cache) {
763+ list_for_each_entry(cache, &dev->cache_props, list)
764+ if (cache->kobj) {
765+ kfd_remove_sysfs_file(cache->kobj,
766+ &cache->attr);
767+ cache->kobj = 0;
768+ }
769+ kobject_del(dev->kobj_cache);
770+ kobject_put(dev->kobj_cache);
771+ dev->kobj_cache = 0;
772+ }
773+
774+ if (dev->kobj_mem) {
775+ list_for_each_entry(mem, &dev->mem_props, list)
776+ if (mem->kobj) {
777+ kfd_remove_sysfs_file(mem->kobj, &mem->attr);
778+ mem->kobj = 0;
779+ }
780+ kobject_del(dev->kobj_mem);
781+ kobject_put(dev->kobj_mem);
782+ dev->kobj_mem = 0;
783+ }
784+
785+ if (dev->kobj_node) {
786+ sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid);
787+ sysfs_remove_file(dev->kobj_node, &dev->attr_name);
788+ sysfs_remove_file(dev->kobj_node, &dev->attr_props);
789+ kobject_del(dev->kobj_node);
790+ kobject_put(dev->kobj_node);
791+ dev->kobj_node = 0;
792+ }
793+}
794+
795+static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
796+ uint32_t id)
797+{
798+ struct kfd_iolink_properties *iolink;
799+ struct kfd_cache_properties *cache;
800+ struct kfd_mem_properties *mem;
801+ int ret;
802+ uint32_t i;
803+
804+ BUG_ON(!dev);
805+
806+ /*
807+ * Creating the sysfs folders
808+ */
809+ BUG_ON(dev->kobj_node);
810+ dev->kobj_node = kfd_alloc_struct(dev->kobj_node);
811+ if (!dev->kobj_node)
812+ return -ENOMEM;
813+
814+ ret = kobject_init_and_add(dev->kobj_node, &node_type,
815+ sys_props.kobj_nodes, "%d", id);
816+ if (ret < 0)
817+ return ret;
818+
819+ dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node);
820+ if (!dev->kobj_mem)
821+ return -ENOMEM;
822+
823+ dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node);
824+ if (!dev->kobj_cache)
825+ return -ENOMEM;
826+
827+ dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node);
828+ if (!dev->kobj_iolink)
829+ return -ENOMEM;
830+
831+ /*
832+ * Creating sysfs files for node properties
833+ */
834+ dev->attr_gpuid.name = "gpu_id";
835+ dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE;
836+ sysfs_attr_init(&dev->attr_gpuid);
837+ dev->attr_name.name = "name";
838+ dev->attr_name.mode = KFD_SYSFS_FILE_MODE;
839+ sysfs_attr_init(&dev->attr_name);
840+ dev->attr_props.name = "properties";
841+ dev->attr_props.mode = KFD_SYSFS_FILE_MODE;
842+ sysfs_attr_init(&dev->attr_props);
843+ ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid);
844+ if (ret < 0)
845+ return ret;
846+ ret = sysfs_create_file(dev->kobj_node, &dev->attr_name);
847+ if (ret < 0)
848+ return ret;
849+ ret = sysfs_create_file(dev->kobj_node, &dev->attr_props);
850+ if (ret < 0)
851+ return ret;
852+
853+ i = 0;
854+ list_for_each_entry(mem, &dev->mem_props, list) {
855+ mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
856+ if (!mem->kobj)
857+ return -ENOMEM;
858+ ret = kobject_init_and_add(mem->kobj, &mem_type,
859+ dev->kobj_mem, "%d", i);
860+ if (ret < 0)
861+ return ret;
862+
863+ mem->attr.name = "properties";
864+ mem->attr.mode = KFD_SYSFS_FILE_MODE;
865+ sysfs_attr_init(&mem->attr);
866+ ret = sysfs_create_file(mem->kobj, &mem->attr);
867+ if (ret < 0)
868+ return ret;
869+ i++;
870+ }
871+
872+ i = 0;
873+ list_for_each_entry(cache, &dev->cache_props, list) {
874+ cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
875+ if (!cache->kobj)
876+ return -ENOMEM;
877+ ret = kobject_init_and_add(cache->kobj, &cache_type,
878+ dev->kobj_cache, "%d", i);
879+ if (ret < 0)
880+ return ret;
881+
882+ cache->attr.name = "properties";
883+ cache->attr.mode = KFD_SYSFS_FILE_MODE;
884+ sysfs_attr_init(&cache->attr);
885+ ret = sysfs_create_file(cache->kobj, &cache->attr);
886+ if (ret < 0)
887+ return ret;
888+ i++;
889+ }
890+
891+ i = 0;
892+ list_for_each_entry(iolink, &dev->io_link_props, list) {
893+ iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
894+ if (!iolink->kobj)
895+ return -ENOMEM;
896+ ret = kobject_init_and_add(iolink->kobj, &iolink_type,
897+ dev->kobj_iolink, "%d", i);
898+ if (ret < 0)
899+ return ret;
900+
901+ iolink->attr.name = "properties";
902+ iolink->attr.mode = KFD_SYSFS_FILE_MODE;
903+ sysfs_attr_init(&iolink->attr);
904+ ret = sysfs_create_file(iolink->kobj, &iolink->attr);
905+ if (ret < 0)
906+ return ret;
907+ i++;
908+}
909+
910+ return 0;
911+}
912+
913+static int kfd_build_sysfs_node_tree(void)
914+{
915+ struct kfd_topology_device *dev;
916+ int ret;
917+ uint32_t i = 0;
918+
919+ list_for_each_entry(dev, &topology_device_list, list) {
920+ ret = kfd_build_sysfs_node_entry(dev, 0);
921+ if (ret < 0)
922+ return ret;
923+ i++;
924+ }
925+
926+ return 0;
927+}
928+
929+static void kfd_remove_sysfs_node_tree(void)
930+{
931+ struct kfd_topology_device *dev;
932+
933+ list_for_each_entry(dev, &topology_device_list, list)
934+ kfd_remove_sysfs_node_entry(dev);
935+}
936+
937+static int kfd_topology_update_sysfs(void)
938+{
939+ int ret;
940+
941+ pr_info("Creating topology SYSFS entries\n");
942+ if (sys_props.kobj_topology == 0) {
943+ sys_props.kobj_topology =
944+ kfd_alloc_struct(sys_props.kobj_topology);
945+ if (!sys_props.kobj_topology)
946+ return -ENOMEM;
947+
948+ ret = kobject_init_and_add(sys_props.kobj_topology,
949+ &sysprops_type, &kfd_device->kobj,
950+ "topology");
951+ if (ret < 0)
952+ return ret;
953+
954+ sys_props.kobj_nodes = kobject_create_and_add("nodes",
955+ sys_props.kobj_topology);
956+ if (!sys_props.kobj_nodes)
957+ return -ENOMEM;
958+
959+ sys_props.attr_genid.name = "generation_id";
960+ sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE;
961+ sysfs_attr_init(&sys_props.attr_genid);
962+ ret = sysfs_create_file(sys_props.kobj_topology,
963+ &sys_props.attr_genid);
964+ if (ret < 0)
965+ return ret;
966+
967+ sys_props.attr_props.name = "system_properties";
968+ sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE;
969+ sysfs_attr_init(&sys_props.attr_props);
970+ ret = sysfs_create_file(sys_props.kobj_topology,
971+ &sys_props.attr_props);
972+ if (ret < 0)
973+ return ret;
974+ }
975+
976+ kfd_remove_sysfs_node_tree();
977+
978+ return kfd_build_sysfs_node_tree();
979+}
980+
981+static void kfd_topology_release_sysfs(void)
982+{
983+ kfd_remove_sysfs_node_tree();
984+ if (sys_props.kobj_topology) {
985+ sysfs_remove_file(sys_props.kobj_topology,
986+ &sys_props.attr_genid);
987+ sysfs_remove_file(sys_props.kobj_topology,
988+ &sys_props.attr_props);
989+ if (sys_props.kobj_nodes) {
990+ kobject_del(sys_props.kobj_nodes);
991+ kobject_put(sys_props.kobj_nodes);
992+ sys_props.kobj_nodes = 0;
993+ }
994+ kobject_del(sys_props.kobj_topology);
995+ kobject_put(sys_props.kobj_topology);
996+ sys_props.kobj_topology = 0;
997+ }
998+}
999+
1000+int kfd_topology_init(void)
1001+{
1002+ void *crat_image = 0;
1003+ size_t image_size = 0;
1004+ int ret;
1005+
1006+ /*
1007+ * Initialize the head for the topology device list
1008+ */
1009+ INIT_LIST_HEAD(&topology_device_list);
1010+ init_rwsem(&topology_lock);
1011+ topology_crat_parsed = 0;
1012+
1013+ memset(&sys_props, 0, sizeof(sys_props));
1014+
1015+ /*
1016+ * Get the CRAT image from the ACPI
1017+ */
1018+ ret = kfd_topology_get_crat_acpi(crat_image, &image_size);
1019+ if (ret == 0 && image_size > 0) {
1020+ pr_info("Found CRAT image with size=%zd\n", image_size);
1021+ crat_image = kmalloc(image_size, GFP_KERNEL);
1022+ if (!crat_image) {
1023+ ret = -ENOMEM;
1024+ pr_err("No memory for allocating CRAT image\n");
1025+ goto err;
1026+ }
1027+ ret = kfd_topology_get_crat_acpi(crat_image, &image_size);
1028+
1029+ if (ret == 0) {
1030+ down_write(&topology_lock);
1031+ ret = kfd_parse_crat_table(crat_image);
1032+ if (ret == 0)
1033+ ret = kfd_topology_update_sysfs();
1034+ up_write(&topology_lock);
1035+ } else {
1036+ pr_err("Couldn't get CRAT table size from ACPI\n");
1037+ }
1038+ kfree(crat_image);
1039+ } else if (ret == -ENODATA) {
1040+ ret = 0;
1041+ } else {
1042+ pr_err("Couldn't get CRAT table size from ACPI\n");
1043+ }
1044+
1045+err:
1046+ pr_info("Finished initializing topology ret=%d\n", ret);
1047+ return ret;
1048+}
1049+
1050+void kfd_topology_shutdown(void)
1051+{
1052+ kfd_topology_release_sysfs();
1053+ kfd_release_live_view();
1054+}
1055+
1056+static void kfd_debug_print_topology(void)
1057+{
1058+ struct kfd_topology_device *dev;
1059+ uint32_t i = 0;
1060+
1061+ pr_info("DEBUG PRINT OF TOPOLOGY:");
1062+ list_for_each_entry(dev, &topology_device_list, list) {
1063+ pr_info("Node: %d\n", i);
1064+ pr_info("\tGPU assigned: %s\n", (dev->gpu ? "yes" : "no"));
1065+ pr_info("\tCPU count: %d\n", dev->node_props.cpu_cores_count);
1066+ pr_info("\tSIMD count: %d", dev->node_props.simd_count);
1067+ i++;
1068+ }
1069+}
1070+
1071+static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
1072+{
1073+ uint32_t hashout;
1074+ uint32_t buf[7];
1075+ int i;
1076+
1077+ if (!gpu)
1078+ return 0;
1079+
1080+ buf[0] = gpu->pdev->devfn;
1081+ buf[1] = gpu->pdev->subsystem_vendor;
1082+ buf[2] = gpu->pdev->subsystem_device;
1083+ buf[3] = gpu->pdev->device;
1084+ buf[4] = gpu->pdev->bus->number;
1085+ buf[5] = (uint32_t)(kfd2kgd->get_vmem_size(gpu->kgd) & 0xffffffff);
1086+ buf[6] = (uint32_t)(kfd2kgd->get_vmem_size(gpu->kgd) >> 32);
1087+
1088+ for (i = 0, hashout = 0; i < 7; i++)
1089+ hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH);
1090+
1091+ return hashout;
1092+}
1093+
1094+static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu)
1095+{
1096+ struct kfd_topology_device *dev;
1097+ struct kfd_topology_device *out_dev = 0;
1098+
1099+ BUG_ON(!gpu);
1100+
1101+ list_for_each_entry(dev, &topology_device_list, list)
1102+ if (dev->gpu == 0 && dev->node_props.simd_count > 0) {
1103+ dev->gpu = gpu;
1104+ out_dev = dev;
1105+ break;
1106+ }
1107+
1108+ return out_dev;
1109+}
1110+
1111+static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival)
1112+{
1113+ /*
1114+ * TODO: Generate an event for thunk about the arrival/removal
1115+ * of the GPU
1116+ */
1117+}
1118+
1119+int kfd_topology_add_device(struct kfd_dev *gpu)
1120+{
1121+ uint32_t gpu_id;
1122+ struct kfd_topology_device *dev;
1123+ int res;
1124+
1125+ BUG_ON(!gpu);
1126+
1127+ gpu_id = kfd_generate_gpu_id(gpu);
1128+
1129+ pr_debug("kfd: Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
1130+
1131+ down_write(&topology_lock);
1132+ /*
1133+ * Try to assign the GPU to existing topology device (generated from
1134+ * CRAT table
1135+ */
1136+ dev = kfd_assign_gpu(gpu);
1137+ if (!dev) {
1138+ pr_info("GPU was not found in the current topology. Extending.\n");
1139+ kfd_debug_print_topology();
1140+ dev = kfd_create_topology_device();
1141+ if (!dev) {
1142+ res = -ENOMEM;
1143+ goto err;
1144+ }
1145+ dev->gpu = gpu;
1146+
1147+ /*
1148+ * TODO: Make a call to retrieve topology information from the
1149+ * GPU vBIOS
1150+ */
1151+
1152+ /*
1153+ * Update the SYSFS tree, since we added another topology device
1154+ */
1155+ if (kfd_topology_update_sysfs() < 0)
1156+ kfd_topology_release_sysfs();
1157+
1158+ }
1159+
1160+ dev->gpu_id = gpu_id;
1161+ gpu->id = gpu_id;
1162+ dev->node_props.vendor_id = gpu->pdev->vendor;
1163+ dev->node_props.device_id = gpu->pdev->device;
1164+ dev->node_props.location_id = (gpu->pdev->bus->number << 24) +
1165+ (gpu->pdev->devfn & 0xffffff);
1166+ /*
1167+ * TODO: Retrieve max engine clock values from KGD
1168+ */
1169+
1170+ res = 0;
1171+
1172+err:
1173+ up_write(&topology_lock);
1174+
1175+ if (res == 0)
1176+ kfd_notify_gpu_change(gpu_id, 1);
1177+
1178+ return res;
1179+}
1180+
1181+int kfd_topology_remove_device(struct kfd_dev *gpu)
1182+{
1183+ struct kfd_topology_device *dev;
1184+ uint32_t gpu_id;
1185+ int res = -ENODEV;
1186+
1187+ BUG_ON(!gpu);
1188+
1189+ down_write(&topology_lock);
1190+
1191+ list_for_each_entry(dev, &topology_device_list, list)
1192+ if (dev->gpu == gpu) {
1193+ gpu_id = dev->gpu_id;
1194+ kfd_remove_sysfs_node_entry(dev);
1195+ kfd_release_topology_device(dev);
1196+ res = 0;
1197+ if (kfd_topology_update_sysfs() < 0)
1198+ kfd_topology_release_sysfs();
1199+ break;
1200+ }
1201+
1202+ up_write(&topology_lock);
1203+
1204+ if (res == 0)
1205+ kfd_notify_gpu_change(gpu_id, 0);
1206+
1207+ return res;
1208+}
1209+
1210+/*
1211+ * When idx is out of bounds, the function will return NULL
1212+ */
1213+struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx)
1214+{
1215+
1216+ struct kfd_topology_device *top_dev;
1217+ struct kfd_dev *device = NULL;
1218+ uint8_t device_idx = 0;
1219+
1220+ down_read(&topology_lock);
1221+
1222+ list_for_each_entry(top_dev, &topology_device_list, list) {
1223+ if (device_idx == idx) {
1224+ device = top_dev->gpu;
1225+ break;
1226+ }
1227+
1228+ device_idx++;
1229+ }
1230+
1231+ up_read(&topology_lock);
1232+
1233+ return device;
1234+
1235+}
drivers/gpu/drm/amd/amdkfd/kfd_topology.h+168−0
@@ -0,0 +1,168 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+#ifndef __KFD_TOPOLOGY_H__
24+#define __KFD_TOPOLOGY_H__
25+
26+#include <linux/types.h>
27+#include <linux/list.h>
28+#include "kfd_priv.h"
29+
30+#define KFD_TOPOLOGY_PUBLIC_NAME_SIZE 128
31+
32+#define HSA_CAP_HOT_PLUGGABLE 0x00000001
33+#define HSA_CAP_ATS_PRESENT 0x00000002
34+#define HSA_CAP_SHARED_WITH_GRAPHICS 0x00000004
35+#define HSA_CAP_QUEUE_SIZE_POW2 0x00000008
36+#define HSA_CAP_QUEUE_SIZE_32BIT 0x00000010
37+#define HSA_CAP_QUEUE_IDLE_EVENT 0x00000020
38+#define HSA_CAP_VA_LIMIT 0x00000040
39+#define HSA_CAP_WATCH_POINTS_SUPPORTED 0x00000080
40+#define HSA_CAP_WATCH_POINTS_TOTALBITS_MASK 0x00000f00
41+#define HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT 8
42+#define HSA_CAP_RESERVED 0xfffff000
43+
44+struct kfd_node_properties {
45+ uint32_t cpu_cores_count;
46+ uint32_t simd_count;
47+ uint32_t mem_banks_count;
48+ uint32_t caches_count;
49+ uint32_t io_links_count;
50+ uint32_t cpu_core_id_base;
51+ uint32_t simd_id_base;
52+ uint32_t capability;
53+ uint32_t max_waves_per_simd;
54+ uint32_t lds_size_in_kb;
55+ uint32_t gds_size_in_kb;
56+ uint32_t wave_front_size;
57+ uint32_t array_count;
58+ uint32_t simd_arrays_per_engine;
59+ uint32_t cu_per_simd_array;
60+ uint32_t simd_per_cu;
61+ uint32_t max_slots_scratch_cu;
62+ uint32_t engine_id;
63+ uint32_t vendor_id;
64+ uint32_t device_id;
65+ uint32_t location_id;
66+ uint32_t max_engine_clk_fcompute;
67+ uint32_t max_engine_clk_ccompute;
68+ uint16_t marketing_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE];
69+};
70+
71+#define HSA_MEM_HEAP_TYPE_SYSTEM 0
72+#define HSA_MEM_HEAP_TYPE_FB_PUBLIC 1
73+#define HSA_MEM_HEAP_TYPE_FB_PRIVATE 2
74+#define HSA_MEM_HEAP_TYPE_GPU_GDS 3
75+#define HSA_MEM_HEAP_TYPE_GPU_LDS 4
76+#define HSA_MEM_HEAP_TYPE_GPU_SCRATCH 5
77+
78+#define HSA_MEM_FLAGS_HOT_PLUGGABLE 0x00000001
79+#define HSA_MEM_FLAGS_NON_VOLATILE 0x00000002
80+#define HSA_MEM_FLAGS_RESERVED 0xfffffffc
81+
82+struct kfd_mem_properties {
83+ struct list_head list;
84+ uint32_t heap_type;
85+ uint64_t size_in_bytes;
86+ uint32_t flags;
87+ uint32_t width;
88+ uint32_t mem_clk_max;
89+ struct kobject *kobj;
90+ struct attribute attr;
91+};
92+
93+#define KFD_TOPOLOGY_CPU_SIBLINGS 256
94+
95+#define HSA_CACHE_TYPE_DATA 0x00000001
96+#define HSA_CACHE_TYPE_INSTRUCTION 0x00000002
97+#define HSA_CACHE_TYPE_CPU 0x00000004
98+#define HSA_CACHE_TYPE_HSACU 0x00000008
99+#define HSA_CACHE_TYPE_RESERVED 0xfffffff0
100+
101+struct kfd_cache_properties {
102+ struct list_head list;
103+ uint32_t processor_id_low;
104+ uint32_t cache_level;
105+ uint32_t cache_size;
106+ uint32_t cacheline_size;
107+ uint32_t cachelines_per_tag;
108+ uint32_t cache_assoc;
109+ uint32_t cache_latency;
110+ uint32_t cache_type;
111+ uint8_t sibling_map[KFD_TOPOLOGY_CPU_SIBLINGS];
112+ struct kobject *kobj;
113+ struct attribute attr;
114+};
115+
116+struct kfd_iolink_properties {
117+ struct list_head list;
118+ uint32_t iolink_type;
119+ uint32_t ver_maj;
120+ uint32_t ver_min;
121+ uint32_t node_from;
122+ uint32_t node_to;
123+ uint32_t weight;
124+ uint32_t min_latency;
125+ uint32_t max_latency;
126+ uint32_t min_bandwidth;
127+ uint32_t max_bandwidth;
128+ uint32_t rec_transfer_size;
129+ uint32_t flags;
130+ struct kobject *kobj;
131+ struct attribute attr;
132+};
133+
134+struct kfd_topology_device {
135+ struct list_head list;
136+ uint32_t gpu_id;
137+ struct kfd_node_properties node_props;
138+ uint32_t mem_bank_count;
139+ struct list_head mem_props;
140+ uint32_t cache_count;
141+ struct list_head cache_props;
142+ uint32_t io_link_count;
143+ struct list_head io_link_props;
144+ struct kfd_dev *gpu;
145+ struct kobject *kobj_node;
146+ struct kobject *kobj_mem;
147+ struct kobject *kobj_cache;
148+ struct kobject *kobj_iolink;
149+ struct attribute attr_gpuid;
150+ struct attribute attr_name;
151+ struct attribute attr_props;
152+};
153+
154+struct kfd_system_properties {
155+ uint32_t num_devices; /* Number of H-NUMA nodes */
156+ uint32_t generation_count;
157+ uint64_t platform_oem;
158+ uint64_t platform_id;
159+ uint64_t platform_rev;
160+ struct kobject *kobj_topology;
161+ struct kobject *kobj_nodes;
162+ struct attribute attr_genid;
163+ struct attribute attr_props;
164+};
165+
166+
167+
168+#endif /* __KFD_TOPOLOGY_H__ */
drivers/gpu/drm/amd/include/kgd_kfd_interface.h+185−0
@@ -0,0 +1,185 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+/*
24+ * This file defines the private interface between the
25+ * AMD kernel graphics drivers and the AMD KFD.
26+ */
27+
28+#ifndef KGD_KFD_INTERFACE_H_INCLUDED
29+#define KGD_KFD_INTERFACE_H_INCLUDED
30+
31+#include <linux/types.h>
32+
33+struct pci_dev;
34+
35+#define KFD_INTERFACE_VERSION 1
36+
37+struct kfd_dev;
38+struct kgd_dev;
39+
40+struct kgd_mem;
41+
42+enum kgd_memory_pool {
43+ KGD_POOL_SYSTEM_CACHEABLE = 1,
44+ KGD_POOL_SYSTEM_WRITECOMBINE = 2,
45+ KGD_POOL_FRAMEBUFFER = 3,
46+};
47+
48+struct kgd2kfd_shared_resources {
49+ /* Bit n == 1 means VMID n is available for KFD. */
50+ unsigned int compute_vmid_bitmap;
51+
52+ /* Compute pipes are counted starting from MEC0/pipe0 as 0. */
53+ unsigned int first_compute_pipe;
54+
55+ /* Number of MEC pipes available for KFD. */
56+ unsigned int compute_pipe_count;
57+
58+ /* Base address of doorbell aperture. */
59+ phys_addr_t doorbell_physical_address;
60+
61+ /* Size in bytes of doorbell aperture. */
62+ size_t doorbell_aperture_size;
63+
64+ /* Number of bytes at start of aperture reserved for KGD. */
65+ size_t doorbell_start_offset;
66+};
67+
68+/**
69+ * struct kgd2kfd_calls
70+ *
71+ * @exit: Notifies amdkfd that kgd module is unloaded
72+ *
73+ * @probe: Notifies amdkfd about a probe done on a device in the kgd driver.
74+ *
75+ * @device_init: Initialize the newly probed device (if it is a device that
76+ * amdkfd supports)
77+ *
78+ * @device_exit: Notifies amdkfd about a removal of a kgd device
79+ *
80+ * @suspend: Notifies amdkfd about a suspend action done to a kgd device
81+ *
82+ * @resume: Notifies amdkfd about a resume action done to a kgd device
83+ *
84+ * This structure contains function callback pointers so the kgd driver
85+ * will notify to the amdkfd about certain status changes.
86+ *
87+ */
88+struct kgd2kfd_calls {
89+ void (*exit)(void);
90+ struct kfd_dev* (*probe)(struct kgd_dev *kgd, struct pci_dev *pdev);
91+ bool (*device_init)(struct kfd_dev *kfd,
92+ const struct kgd2kfd_shared_resources *gpu_resources);
93+ void (*device_exit)(struct kfd_dev *kfd);
94+ void (*interrupt)(struct kfd_dev *kfd, const void *ih_ring_entry);
95+ void (*suspend)(struct kfd_dev *kfd);
96+ int (*resume)(struct kfd_dev *kfd);
97+};
98+
99+/**
100+ * struct kfd2kgd_calls
101+ *
102+ * @init_sa_manager: Initialize an instance of the sa manager, used by
103+ * amdkfd for all system memory allocations that are mapped to the GART
104+ * address space
105+ *
106+ * @fini_sa_manager: Releases all memory allocations for amdkfd that are
107+ * handled by kgd sa manager
108+ *
109+ * @allocate_mem: Allocate a buffer from amdkfd's sa manager. The buffer can
110+ * be used for mqds, hpds, kernel queue, fence and runlists
111+ *
112+ * @free_mem: Frees a buffer that was allocated by amdkfd's sa manager
113+ *
114+ * @get_vmem_size: Retrieves (physical) size of VRAM
115+ *
116+ * @get_gpu_clock_counter: Retrieves GPU clock counter
117+ *
118+ * @get_max_engine_clock_in_mhz: Retrieves maximum GPU clock in MHz
119+ *
120+ * @program_sh_mem_settings: A function that should initiate the memory
121+ * properties such as main aperture memory type (cache / non cached) and
122+ * secondary aperture base address, size and memory type.
123+ * This function is used only for no cp scheduling mode.
124+ *
125+ * @set_pasid_vmid_mapping: Exposes pasid/vmid pair to the H/W for no cp
126+ * scheduling mode. Only used for no cp scheduling mode.
127+ *
128+ * @init_memory: Initializes memory apertures to fixed base/limit address
129+ * and non cached memory types.
130+ *
131+ * @init_pipeline: Initialized the compute pipelines.
132+ *
133+ * @hqd_load: Loads the mqd structure to a H/W hqd slot. used only for no cp
134+ * sceduling mode.
135+ *
136+ * @hqd_is_occupies: Checks if a hqd slot is occupied.
137+ *
138+ * @hqd_destroy: Destructs and preempts the queue assigned to that hqd slot.
139+ *
140+ * This structure contains function pointers to services that the kgd driver
141+ * provides to amdkfd driver.
142+ *
143+ */
144+struct kfd2kgd_calls {
145+ /* Memory management. */
146+ int (*init_sa_manager)(struct kgd_dev *kgd, unsigned int size);
147+ void (*fini_sa_manager)(struct kgd_dev *kgd);
148+ int (*allocate_mem)(struct kgd_dev *kgd, size_t size, size_t alignment,
149+ enum kgd_memory_pool pool, struct kgd_mem **mem);
150+
151+ void (*free_mem)(struct kgd_dev *kgd, struct kgd_mem *mem);
152+
153+ uint64_t (*get_vmem_size)(struct kgd_dev *kgd);
154+ uint64_t (*get_gpu_clock_counter)(struct kgd_dev *kgd);
155+
156+ uint32_t (*get_max_engine_clock_in_mhz)(struct kgd_dev *kgd);
157+
158+ /* Register access functions */
159+ void (*program_sh_mem_settings)(struct kgd_dev *kgd, uint32_t vmid,
160+ uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
161+ uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
162+
163+ int (*set_pasid_vmid_mapping)(struct kgd_dev *kgd, unsigned int pasid,
164+ unsigned int vmid);
165+
166+ int (*init_memory)(struct kgd_dev *kgd);
167+ int (*init_pipeline)(struct kgd_dev *kgd, uint32_t pipe_id,
168+ uint32_t hpd_size, uint64_t hpd_gpu_addr);
169+
170+ int (*hqd_load)(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
171+ uint32_t queue_id, uint32_t __user *wptr);
172+
173+ bool (*hqd_is_occupies)(struct kgd_dev *kgd, uint64_t queue_address,
174+ uint32_t pipe_id, uint32_t queue_id);
175+
176+ int (*hqd_destroy)(struct kgd_dev *kgd, uint32_t reset_type,
177+ unsigned int timeout, uint32_t pipe_id,
178+ uint32_t queue_id);
179+};
180+
181+bool kgd2kfd_init(unsigned interface_version,
182+ const struct kfd2kgd_calls *f2g,
183+ const struct kgd2kfd_calls **g2f);
184+
185+#endif /* KGD_KFD_INTERFACE_H_INCLUDED */
drivers/gpu/drm/radeon/Makefile+1−0
@@ -104,6 +104,7 @@ radeon-y += \ · 103 unmodified lines
104104 radeon_vce.o \
105105 vce_v1_0.o \
106106 vce_v2_0.o \
107+ radeon_kfd.o
107108
108109 radeon-$(CONFIG_COMPAT) += radeon_ioc32.o
109110 radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o
drivers/gpu/drm/radeon/cik.c+61−94
@@ -32,6 +32,7 @@ · 31 unmodified lines
3232 #include "cik_blit_shaders.h"
3333 #include "radeon_ucode.h"
3434 #include "clearstate_ci.h"
35+#include "radeon_kfd.h"
3536
3637 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
3738 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
@@ -1563,6 +1564,8 @@ static const u32 godavari_golden_registers[] = · 1525 unmodified lines
15631564
15641565 static void cik_init_golden_registers(struct radeon_device *rdev)
15651566 {
1567+ /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1568+ mutex_lock(&rdev->grbm_idx_mutex);
15661569 switch (rdev->family) {
15671570 case CHIP_BONAIRE:
15681571 radeon_program_register_sequence(rdev,
@@ -1637,6 +1640,7 @@ static void cik_init_golden_registers(struct radeon_device *rdev) · 68 unmodified lines
16371640 default:
16381641 break;
16391642 }
1643+ mutex_unlock(&rdev->grbm_idx_mutex);
16401644 }
16411645
16421646 /**
@@ -3428,6 +3432,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 1785 unmodified lines
34283432 u32 disabled_rbs = 0;
34293433 u32 enabled_rbs = 0;
34303434
3435+ mutex_lock(&rdev->grbm_idx_mutex);
34313436 for (i = 0; i < se_num; i++) {
34323437 for (j = 0; j < sh_per_se; j++) {
34333438 cik_select_se_sh(rdev, i, j);
@@ -3439,6 +3444,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 5 unmodified lines
34393444 }
34403445 }
34413446 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3447+ mutex_unlock(&rdev->grbm_idx_mutex);
34423448
34433449 mask = 1;
34443450 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
@@ -3449,6 +3455,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 4 unmodified lines
34493455
34503456 rdev->config.cik.backend_enable_mask = enabled_rbs;
34513457
3458+ mutex_lock(&rdev->grbm_idx_mutex);
34523459 for (i = 0; i < se_num; i++) {
34533460 cik_select_se_sh(rdev, i, 0xffffffff);
34543461 data = 0;
@@ -3476,6 +3483,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 21 unmodified lines
34763483 WREG32(PA_SC_RASTER_CONFIG, data);
34773484 }
34783485 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3486+ mutex_unlock(&rdev->grbm_idx_mutex);
34793487 }
34803488
34813489 /**
@@ -3693,6 +3701,12 @@ static void cik_gpu_init(struct radeon_device *rdev) · 211 unmodified lines
36933701 /* set HW defaults for 3D engine */
36943702 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
36953703
3704+ mutex_lock(&rdev->grbm_idx_mutex);
3705+ /*
3706+ * making sure that the following register writes will be broadcasted
3707+ * to all the shaders
3708+ */
3709+ cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
36963710 WREG32(SX_DEBUG_1, 0x20);
36973711
36983712 WREG32(TA_CNTL_AUX, 0x00010000);
@@ -3748,6 +3762,7 @@ static void cik_gpu_init(struct radeon_device *rdev) · 49 unmodified lines
37483762
37493763 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
37503764 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3765+ mutex_unlock(&rdev->grbm_idx_mutex);
37513766
37523767 udelay(50);
37533768 }
@@ -4684,12 +4699,11 @@ static int cik_mec_init(struct radeon_device *rdev) · 930 unmodified lines
46844699 /*
46854700 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
46864701 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4702+ * Nonetheless, we assign only 1 pipe because all other pipes will
4703+ * be handled by KFD
46874704 */
4688− if (rdev->family == CHIP_KAVERI)
4689− rdev->mec.num_mec = 2;
4690− else
4691− rdev->mec.num_mec = 1;
4692− rdev->mec.num_pipe = 4;
4705+ rdev->mec.num_mec = 1;
4706+ rdev->mec.num_pipe = 1;
46934707 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
46944708
46954709 if (rdev->mec.hpd_eop_obj == NULL) {
@@ -4831,28 +4845,24 @@ static int cik_cp_compute_resume(struct radeon_device *rdev) · 135 unmodified lines
48314845
48324846 /* init the pipes */
48334847 mutex_lock(&rdev->srbm_mutex);
4834− for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4835− int me = (i < 4) ? 1 : 2;
4836− int pipe = (i < 4) ? i : (i - 4);
48374848
4838− eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4849+ eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;
48394850
4840− cik_srbm_select(rdev, me, pipe, 0, 0);
4851+ cik_srbm_select(rdev, 0, 0, 0, 0);
48414852
4842− /* write the EOP addr */
4843− WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4844− WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4853+ /* write the EOP addr */
4854+ WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4855+ WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
48454856
4846− /* set the VMID assigned */
4847− WREG32(CP_HPD_EOP_VMID, 0);
4857+ /* set the VMID assigned */
4858+ WREG32(CP_HPD_EOP_VMID, 0);
4859+
4860+ /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4861+ tmp = RREG32(CP_HPD_EOP_CONTROL);
4862+ tmp &= ~EOP_SIZE_MASK;
4863+ tmp |= order_base_2(MEC_HPD_SIZE / 8);
4864+ WREG32(CP_HPD_EOP_CONTROL, tmp);
48484865
4849− /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4850− tmp = RREG32(CP_HPD_EOP_CONTROL);
4851− tmp &= ~EOP_SIZE_MASK;
4852− tmp |= order_base_2(MEC_HPD_SIZE / 8);
4853− WREG32(CP_HPD_EOP_CONTROL, tmp);
4854− }
4855− cik_srbm_select(rdev, 0, 0, 0, 0);
48564866 mutex_unlock(&rdev->srbm_mutex);
48574867
48584868 /* init the queues. Just two for now. */
@@ -5906,8 +5916,13 @@ int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) · 1047 unmodified lines
59065916 */
59075917 int cik_vm_init(struct radeon_device *rdev)
59085918 {
5909− /* number of VMs */
5910− rdev->vm_manager.nvm = 16;
5919+ /*
5920+ * number of VMs
5921+ * VMID 0 is reserved for System
5922+ * radeon graphics/compute will use VMIDs 1-7
5923+ * amdkfd will use VMIDs 8-15
5924+ */
5925+ rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;
59115926 /* base offset of vram pages */
59125927 if (rdev->flags & RADEON_IS_IGP) {
59135928 u64 tmp = RREG32(MC_VM_FB_OFFSET);
@@ -6068,6 +6083,7 @@ static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) · 154 unmodified lines
60686083 u32 i, j, k;
60696084 u32 mask;
60706085
6086+ mutex_lock(&rdev->grbm_idx_mutex);
60716087 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
60726088 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
60736089 cik_select_se_sh(rdev, i, j);
@@ -6079,6 +6095,7 @@ static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) · 5 unmodified lines
60796095 }
60806096 }
60816097 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6098+ mutex_unlock(&rdev->grbm_idx_mutex);
60826099
60836100 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
60846101 for (k = 0; k < rdev->usec_timeout; k++) {
@@ -6213,10 +6230,12 @@ static int cik_rlc_resume(struct radeon_device *rdev) · 128 unmodified lines
62136230 WREG32(RLC_LB_CNTR_INIT, 0);
62146231 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
62156232
6233+ mutex_lock(&rdev->grbm_idx_mutex);
62166234 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
62176235 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
62186236 WREG32(RLC_LB_PARAMS, 0x00600408);
62196237 WREG32(RLC_LB_CNTL, 0x80000004);
6238+ mutex_unlock(&rdev->grbm_idx_mutex);
62206239
62216240 WREG32(RLC_MC_CNTL, 0);
62226241 WREG32(RLC_UCODE_CNTL, 0);
@@ -6283,11 +6302,13 @@ static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) · 60 unmodified lines
62836302
62846303 tmp = cik_halt_rlc(rdev);
62856304
6305+ mutex_lock(&rdev->grbm_idx_mutex);
62866306 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
62876307 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
62886308 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
62896309 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
62906310 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6311+ mutex_unlock(&rdev->grbm_idx_mutex);
62916312
62926313 cik_update_rlc(rdev, tmp);
62936314
@@ -6329,11 +6350,13 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) · 35 unmodified lines
63296350
63306351 tmp = cik_halt_rlc(rdev);
63316352
6353+ mutex_lock(&rdev->grbm_idx_mutex);
63326354 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
63336355 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
63346356 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
63356357 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
63366358 WREG32(RLC_SERDES_WR_CTRL, data);
6359+ mutex_unlock(&rdev->grbm_idx_mutex);
63376360
63386361 cik_update_rlc(rdev, tmp);
63396362
@@ -6377,11 +6400,13 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) · 37 unmodified lines
63776400
63786401 tmp = cik_halt_rlc(rdev);
63796402
6403+ mutex_lock(&rdev->grbm_idx_mutex);
63806404 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
63816405 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
63826406 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
63836407 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
63846408 WREG32(RLC_SERDES_WR_CTRL, data);
6409+ mutex_unlock(&rdev->grbm_idx_mutex);
63856410
63866411 cik_update_rlc(rdev, tmp);
63876412 }
@@ -6810,10 +6835,12 @@ static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) · 422 unmodified lines
68106835 u32 mask = 0, tmp, tmp1;
68116836 int i;
68126837
6838+ mutex_lock(&rdev->grbm_idx_mutex);
68136839 cik_select_se_sh(rdev, se, sh);
68146840 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
68156841 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
68166842 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6843+ mutex_unlock(&rdev->grbm_idx_mutex);
68176844
68186845 tmp &= 0xffff0000;
68196846
@@ -7297,8 +7324,7 @@ static int cik_irq_init(struct radeon_device *rdev) · 477 unmodified lines
72977324 int cik_irq_set(struct radeon_device *rdev)
72987325 {
72997326 u32 cp_int_cntl;
7300− u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
7301− u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
7327+ u32 cp_m1p0;
73027328 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
73037329 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
73047330 u32 grbm_int_cntl = 0;
@@ -7332,13 +7358,6 @@ int cik_irq_set(struct radeon_device *rdev) · 27 unmodified lines
73327358 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
73337359
73347360 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7335− cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7336− cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7337− cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7338− cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7339− cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7340− cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7341− cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
73427361
73437362 if (rdev->flags & RADEON_IS_IGP)
73447363 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
@@ -7360,33 +7379,6 @@ int cik_irq_set(struct radeon_device *rdev) · 15 unmodified lines
73607379 case 0:
73617380 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
73627381 break;
7363− case 1:
7364− cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7365− break;
7366− case 2:
7367− cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7368− break;
7369− case 3:
7370− cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7371− break;
7372− default:
7373− DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7374− break;
7375− }
7376− } else if (ring->me == 2) {
7377− switch (ring->pipe) {
7378− case 0:
7379− cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7380− break;
7381− case 1:
7382− cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7383− break;
7384− case 2:
7385− cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7386− break;
7387− case 3:
7388− cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7389− break;
73907382 default:
73917383 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
73927384 break;
@@ -7403,33 +7395,6 @@ int cik_irq_set(struct radeon_device *rdev) · 10 unmodified lines
74037395 case 0:
74047396 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
74057397 break;
7406− case 1:
7407− cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7408− break;
7409− case 2:
7410− cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7411− break;
7412− case 3:
7413− cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7414− break;
7415− default:
7416− DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7417− break;
7418− }
7419− } else if (ring->me == 2) {
7420− switch (ring->pipe) {
7421− case 0:
7422− cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7423− break;
7424− case 1:
7425− cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7426− break;
7427− case 2:
7428− cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7429− break;
7430− case 3:
7431− cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7432− break;
74337398 default:
74347399 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
74357400 break;
@@ -7518,13 +7483,6 @@ int cik_irq_set(struct radeon_device *rdev) · 82 unmodified lines
75187483 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
75197484
75207485 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7521− WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7522− WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7523− WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7524− WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7525− WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7526− WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7527− WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
75287486
75297487 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
75307488
@@ -7841,6 +7799,10 @@ restart_ih: · 310 unmodified lines
78417799 while (rptr != wptr) {
78427800 /* wptr/rptr are in bytes! */
78437801 ring_index = rptr / 4;
7802+
7803+ radeon_kfd_interrupt(rdev,
7804+ (const void *) &rdev->ih.ring[ring_index]);
7805+
78447806 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
78457807 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
78467808 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
@@ -8530,6 +8492,10 @@ static int cik_startup(struct radeon_device *rdev) · 683 unmodified lines
85308492 if (r)
85318493 return r;
85328494
8495+ r = radeon_kfd_resume(rdev);
8496+ if (r)
8497+ return r;
8498+
85338499 return 0;
85348500 }
85358501
@@ -8578,6 +8544,7 @@ int cik_resume(struct radeon_device *rdev) · 42 unmodified lines
85788544 */
85798545 int cik_suspend(struct radeon_device *rdev)
85808546 {
8547+ radeon_kfd_suspend(rdev);
85818548 radeon_pm_suspend(rdev);
85828549 dce6_audio_fini(rdev);
85838550 radeon_vm_manager_fini(rdev);
drivers/gpu/drm/radeon/cik_reg.h+136−0
@@ -147,4 +147,140 @@ · 146 unmodified lines
147147
148148 #define CIK_LB_DESKTOP_HEIGHT 0x6b0c
149149
150+#define CP_HQD_IQ_RPTR 0xC970u
151+#define AQL_ENABLE (1U << 0)
152+
153+#define IDLE (1 << 2)
154+
155+struct cik_mqd {
156+ uint32_t header;
157+ uint32_t compute_dispatch_initiator;
158+ uint32_t compute_dim_x;
159+ uint32_t compute_dim_y;
160+ uint32_t compute_dim_z;
161+ uint32_t compute_start_x;
162+ uint32_t compute_start_y;
163+ uint32_t compute_start_z;
164+ uint32_t compute_num_thread_x;
165+ uint32_t compute_num_thread_y;
166+ uint32_t compute_num_thread_z;
167+ uint32_t compute_pipelinestat_enable;
168+ uint32_t compute_perfcount_enable;
169+ uint32_t compute_pgm_lo;
170+ uint32_t compute_pgm_hi;
171+ uint32_t compute_tba_lo;
172+ uint32_t compute_tba_hi;
173+ uint32_t compute_tma_lo;
174+ uint32_t compute_tma_hi;
175+ uint32_t compute_pgm_rsrc1;
176+ uint32_t compute_pgm_rsrc2;
177+ uint32_t compute_vmid;
178+ uint32_t compute_resource_limits;
179+ uint32_t compute_static_thread_mgmt_se0;
180+ uint32_t compute_static_thread_mgmt_se1;
181+ uint32_t compute_tmpring_size;
182+ uint32_t compute_static_thread_mgmt_se2;
183+ uint32_t compute_static_thread_mgmt_se3;
184+ uint32_t compute_restart_x;
185+ uint32_t compute_restart_y;
186+ uint32_t compute_restart_z;
187+ uint32_t compute_thread_trace_enable;
188+ uint32_t compute_misc_reserved;
189+ uint32_t compute_user_data_0;
190+ uint32_t compute_user_data_1;
191+ uint32_t compute_user_data_2;
192+ uint32_t compute_user_data_3;
193+ uint32_t compute_user_data_4;
194+ uint32_t compute_user_data_5;
195+ uint32_t compute_user_data_6;
196+ uint32_t compute_user_data_7;
197+ uint32_t compute_user_data_8;
198+ uint32_t compute_user_data_9;
199+ uint32_t compute_user_data_10;
200+ uint32_t compute_user_data_11;
201+ uint32_t compute_user_data_12;
202+ uint32_t compute_user_data_13;
203+ uint32_t compute_user_data_14;
204+ uint32_t compute_user_data_15;
205+ uint32_t cp_compute_csinvoc_count_lo;
206+ uint32_t cp_compute_csinvoc_count_hi;
207+ uint32_t cp_mqd_base_addr_lo;
208+ uint32_t cp_mqd_base_addr_hi;
209+ uint32_t cp_hqd_active;
210+ uint32_t cp_hqd_vmid;
211+ uint32_t cp_hqd_persistent_state;
212+ uint32_t cp_hqd_pipe_priority;
213+ uint32_t cp_hqd_queue_priority;
214+ uint32_t cp_hqd_quantum;
215+ uint32_t cp_hqd_pq_base_lo;
216+ uint32_t cp_hqd_pq_base_hi;
217+ uint32_t cp_hqd_pq_rptr;
218+ uint32_t cp_hqd_pq_rptr_report_addr_lo;
219+ uint32_t cp_hqd_pq_rptr_report_addr_hi;
220+ uint32_t cp_hqd_pq_wptr_poll_addr_lo;
221+ uint32_t cp_hqd_pq_wptr_poll_addr_hi;
222+ uint32_t cp_hqd_pq_doorbell_control;
223+ uint32_t cp_hqd_pq_wptr;
224+ uint32_t cp_hqd_pq_control;
225+ uint32_t cp_hqd_ib_base_addr_lo;
226+ uint32_t cp_hqd_ib_base_addr_hi;
227+ uint32_t cp_hqd_ib_rptr;
228+ uint32_t cp_hqd_ib_control;
229+ uint32_t cp_hqd_iq_timer;
230+ uint32_t cp_hqd_iq_rptr;
231+ uint32_t cp_hqd_dequeue_request;
232+ uint32_t cp_hqd_dma_offload;
233+ uint32_t cp_hqd_sema_cmd;
234+ uint32_t cp_hqd_msg_type;
235+ uint32_t cp_hqd_atomic0_preop_lo;
236+ uint32_t cp_hqd_atomic0_preop_hi;
237+ uint32_t cp_hqd_atomic1_preop_lo;
238+ uint32_t cp_hqd_atomic1_preop_hi;
239+ uint32_t cp_hqd_hq_status0;
240+ uint32_t cp_hqd_hq_control0;
241+ uint32_t cp_mqd_control;
242+ uint32_t cp_mqd_query_time_lo;
243+ uint32_t cp_mqd_query_time_hi;
244+ uint32_t cp_mqd_connect_start_time_lo;
245+ uint32_t cp_mqd_connect_start_time_hi;
246+ uint32_t cp_mqd_connect_end_time_lo;
247+ uint32_t cp_mqd_connect_end_time_hi;
248+ uint32_t cp_mqd_connect_end_wf_count;
249+ uint32_t cp_mqd_connect_end_pq_rptr;
250+ uint32_t cp_mqd_connect_end_pq_wptr;
251+ uint32_t cp_mqd_connect_end_ib_rptr;
252+ uint32_t reserved_96;
253+ uint32_t reserved_97;
254+ uint32_t reserved_98;
255+ uint32_t reserved_99;
256+ uint32_t iqtimer_pkt_header;
257+ uint32_t iqtimer_pkt_dw0;
258+ uint32_t iqtimer_pkt_dw1;
259+ uint32_t iqtimer_pkt_dw2;
260+ uint32_t iqtimer_pkt_dw3;
261+ uint32_t iqtimer_pkt_dw4;
262+ uint32_t iqtimer_pkt_dw5;
263+ uint32_t iqtimer_pkt_dw6;
264+ uint32_t reserved_108;
265+ uint32_t reserved_109;
266+ uint32_t reserved_110;
267+ uint32_t reserved_111;
268+ uint32_t queue_doorbell_id0;
269+ uint32_t queue_doorbell_id1;
270+ uint32_t queue_doorbell_id2;
271+ uint32_t queue_doorbell_id3;
272+ uint32_t queue_doorbell_id4;
273+ uint32_t queue_doorbell_id5;
274+ uint32_t queue_doorbell_id6;
275+ uint32_t queue_doorbell_id7;
276+ uint32_t queue_doorbell_id8;
277+ uint32_t queue_doorbell_id9;
278+ uint32_t queue_doorbell_id10;
279+ uint32_t queue_doorbell_id11;
280+ uint32_t queue_doorbell_id12;
281+ uint32_t queue_doorbell_id13;
282+ uint32_t queue_doorbell_id14;
283+ uint32_t queue_doorbell_id15;
284+};
285+
150286 #endif
drivers/gpu/drm/radeon/cikd.h+52−1
@@ -30,6 +30,8 @@ · 29 unmodified lines
3030 #define CIK_RB_BITMAP_WIDTH_PER_SH 2
3131 #define HAWAII_RB_BITMAP_WIDTH_PER_SH 4
3232
33+#define RADEON_NUM_OF_VMIDS 8
34+
3335 /* DIDT IND registers */
3436 #define DIDT_SQ_CTRL0 0x0
3537 # define DIDT_CTRL_EN (1 << 0)
@@ -1137,6 +1139,9 @@ · 1101 unmodified lines
11371139 #define SH_MEM_ALIGNMENT_MODE_UNALIGNED 3
11381140 #define DEFAULT_MTYPE(x) ((x) << 4)
11391141 #define APE1_MTYPE(x) ((x) << 7)
1142+/* valid for both DEFAULT_MTYPE and APE1_MTYPE */
1143+#define MTYPE_CACHED 0
1144+#define MTYPE_NONCACHED 3
11401145
11411146 #define SX_DEBUG_1 0x9060
11421147
@@ -1447,6 +1452,16 @@ · 304 unmodified lines
14471452 #define CP_HQD_ACTIVE 0xC91C
14481453 #define CP_HQD_VMID 0xC920
14491454
1455+#define CP_HQD_PERSISTENT_STATE 0xC924u
1456+#define DEFAULT_CP_HQD_PERSISTENT_STATE (0x33U << 8)
1457+
1458+#define CP_HQD_PIPE_PRIORITY 0xC928u
1459+#define CP_HQD_QUEUE_PRIORITY 0xC92Cu
1460+#define CP_HQD_QUANTUM 0xC930u
1461+#define QUANTUM_EN 1U
1462+#define QUANTUM_SCALE_1MS (1U << 4)
1463+#define QUANTUM_DURATION(x) ((x) << 8)
1464+
14501465 #define CP_HQD_PQ_BASE 0xC934
14511466 #define CP_HQD_PQ_BASE_HI 0xC938
14521467 #define CP_HQD_PQ_RPTR 0xC93C
@@ -1474,12 +1489,32 @@ · 21 unmodified lines
14741489 #define PRIV_STATE (1 << 30)
14751490 #define KMD_QUEUE (1 << 31)
14761491
1477−#define CP_HQD_DEQUEUE_REQUEST 0xC974
1492+#define CP_HQD_IB_BASE_ADDR 0xC95Cu
1493+#define CP_HQD_IB_BASE_ADDR_HI 0xC960u
1494+#define CP_HQD_IB_RPTR 0xC964u
1495+#define CP_HQD_IB_CONTROL 0xC968u
1496+#define IB_ATC_EN (1U << 23)
1497+#define DEFAULT_MIN_IB_AVAIL_SIZE (3U << 20)
1498+
1499+#define CP_HQD_DEQUEUE_REQUEST 0xC974
1500+#define DEQUEUE_REQUEST_DRAIN 1
1501+#define DEQUEUE_REQUEST_RESET 2
14781502
14791503 #define CP_MQD_CONTROL 0xC99C
14801504 #define MQD_VMID(x) ((x) << 0)
14811505 #define MQD_VMID_MASK (0xf << 0)
14821506
1507+#define CP_HQD_SEMA_CMD 0xC97Cu
1508+#define CP_HQD_MSG_TYPE 0xC980u
1509+#define CP_HQD_ATOMIC0_PREOP_LO 0xC984u
1510+#define CP_HQD_ATOMIC0_PREOP_HI 0xC988u
1511+#define CP_HQD_ATOMIC1_PREOP_LO 0xC98Cu
1512+#define CP_HQD_ATOMIC1_PREOP_HI 0xC990u
1513+#define CP_HQD_HQ_SCHEDULER0 0xC994u
1514+#define CP_HQD_HQ_SCHEDULER1 0xC998u
1515+
1516+#define SH_STATIC_MEM_CONFIG 0x9604u
1517+
14831518 #define DB_RENDER_CONTROL 0x28000
14841519
14851520 #define PA_SC_RASTER_CONFIG 0x28350
@@ -2069,4 +2104,20 @@ · 583 unmodified lines
20692104 #define VCE_CMD_IB_AUTO 0x00000005
20702105 #define VCE_CMD_SEMAPHORE 0x00000006
20712106
2107+#define ATC_VMID0_PASID_MAPPING 0x339Cu
2108+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS 0x3398u
2109+#define ATC_VMID_PASID_MAPPING_VALID (1U << 31)
2110+
2111+#define ATC_VM_APERTURE0_CNTL 0x3310u
2112+#define ATS_ACCESS_MODE_NEVER 0
2113+#define ATS_ACCESS_MODE_ALWAYS 1
2114+
2115+#define ATC_VM_APERTURE0_CNTL2 0x3318u
2116+#define ATC_VM_APERTURE0_HIGH_ADDR 0x3308u
2117+#define ATC_VM_APERTURE0_LOW_ADDR 0x3300u
2118+#define ATC_VM_APERTURE1_CNTL 0x3314u
2119+#define ATC_VM_APERTURE1_CNTL2 0x331Cu
2120+#define ATC_VM_APERTURE1_HIGH_ADDR 0x330Cu
2121+#define ATC_VM_APERTURE1_LOW_ADDR 0x3304u
2122+
20722123 #endif
drivers/gpu/drm/radeon/radeon.h+10−0
@@ -701,6 +701,10 @@ struct radeon_doorbell { · 700 unmodified lines
701701
702702 int radeon_doorbell_get(struct radeon_device *rdev, u32 *page);
703703 void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell);
704+void radeon_doorbell_get_kfd_info(struct radeon_device *rdev,
705+ phys_addr_t *aperture_base,
706+ size_t *aperture_size,
707+ size_t *start_offset);
704708
705709 /*
706710 * IRQS.
@@ -2393,6 +2397,8 @@ struct radeon_device { · 1686 unmodified lines
23932397 struct radeon_atcs atcs;
23942398 /* srbm instance registers */
23952399 struct mutex srbm_mutex;
2400+ /* GRBM index mutex. Protects concurrents access to GRBM index */
2401+ struct mutex grbm_idx_mutex;
23962402 /* clock, powergating flags */
23972403 u32 cg_flags;
23982404 u32 pg_flags;
@@ -2405,6 +2411,10 @@ struct radeon_device { · 6 unmodified lines
24052411 u64 vram_pin_size;
24062412 u64 gart_pin_size;
24072413
2414+ /* amdkfd interface */
2415+ struct kfd_dev *kfd;
2416+ struct radeon_sa_manager kfd_bo;
2417+
24082418 struct mutex mn_lock;
24092419 DECLARE_HASHTABLE(mn_hash, 7);
24102420 };
drivers/gpu/drm/radeon/radeon_device.c+32−0
@@ -377,6 +377,37 @@ void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell) · 376 unmodified lines
377377 __clear_bit(doorbell, rdev->doorbell.used);
378378 }
379379
380+/**
381+ * radeon_doorbell_get_kfd_info - Report doorbell configuration required to
382+ * setup KFD
383+ *
384+ * @rdev: radeon_device pointer
385+ * @aperture_base: output returning doorbell aperture base physical address
386+ * @aperture_size: output returning doorbell aperture size in bytes
387+ * @start_offset: output returning # of doorbell bytes reserved for radeon.
388+ *
389+ * Radeon and the KFD share the doorbell aperture. Radeon sets it up,
390+ * takes doorbells required for its own rings and reports the setup to KFD.
391+ * Radeon reserved doorbells are at the start of the doorbell aperture.
392+ */
393+void radeon_doorbell_get_kfd_info(struct radeon_device *rdev,
394+ phys_addr_t *aperture_base,
395+ size_t *aperture_size,
396+ size_t *start_offset)
397+{
398+ /* The first num_doorbells are used by radeon.
399+ * KFD takes whatever's left in the aperture. */
400+ if (rdev->doorbell.size > rdev->doorbell.num_doorbells * sizeof(u32)) {
401+ *aperture_base = rdev->doorbell.base;
402+ *aperture_size = rdev->doorbell.size;
403+ *start_offset = rdev->doorbell.num_doorbells * sizeof(u32);
404+ } else {
405+ *aperture_base = 0;
406+ *aperture_size = 0;
407+ *start_offset = 0;
408+ }
409+}
410+
380411 /*
381412 * radeon_wb_*()
382413 * Writeback is the the method by which the the GPU updates special pages
@@ -1272,6 +1303,7 @@ int radeon_device_init(struct radeon_device *rdev, · 889 unmodified lines
12721303 mutex_init(&rdev->pm.mutex);
12731304 mutex_init(&rdev->gpu_clock_mutex);
12741305 mutex_init(&rdev->srbm_mutex);
1306+ mutex_init(&rdev->grbm_idx_mutex);
12751307 init_rwsem(&rdev->pm.mclk_lock);
12761308 init_rwsem(&rdev->exclusive_lock);
12771309 init_waitqueue_head(&rdev->irq.vblank_queue);
drivers/gpu/drm/radeon/radeon_drv.c+5−0
@@ -41,6 +41,8 @@ · 40 unmodified lines
4141 #include <drm/drm_gem.h>
4242
4343 #include "drm_crtc_helper.h"
44+#include "radeon_kfd.h"
45+
4446 /*
4547 * KMS wrapper.
4648 * - 2.0.0 - initial interface
@@ -654,12 +656,15 @@ static int __init radeon_init(void) · 607 unmodified lines
654656 #endif
655657 }
656658
659+ radeon_kfd_init();
660+
657661 /* let modprobe override vga console setting */
658662 return drm_pci_init(driver, pdriver);
659663 }
660664
661665 static void __exit radeon_exit(void)
662666 {
667+ radeon_kfd_fini();
663668 drm_pci_exit(driver, pdriver);
664669 radeon_unregister_atpx_handler();
665670 }
drivers/gpu/drm/radeon/radeon_kfd.c+563−0
@@ -0,0 +1,563 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+#include <linux/module.h>
24+#include <linux/fdtable.h>
25+#include <linux/uaccess.h>
26+#include <drm/drmP.h>
27+#include "radeon.h"
28+#include "cikd.h"
29+#include "cik_reg.h"
30+#include "radeon_kfd.h"
31+
32+#define CIK_PIPE_PER_MEC (4)
33+
34+struct kgd_mem {
35+ struct radeon_sa_bo *sa_bo;
36+ uint64_t gpu_addr;
37+ void *ptr;
38+};
39+
40+static int init_sa_manager(struct kgd_dev *kgd, unsigned int size);
41+static void fini_sa_manager(struct kgd_dev *kgd);
42+
43+static int allocate_mem(struct kgd_dev *kgd, size_t size, size_t alignment,
44+ enum kgd_memory_pool pool, struct kgd_mem **mem);
45+
46+static void free_mem(struct kgd_dev *kgd, struct kgd_mem *mem);
47+
48+static uint64_t get_vmem_size(struct kgd_dev *kgd);
49+static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
50+
51+static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
52+
53+/*
54+ * Register access functions
55+ */
56+
57+static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
58+ uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
59+ uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
60+
61+static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
62+ unsigned int vmid);
63+
64+static int kgd_init_memory(struct kgd_dev *kgd);
65+
66+static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
67+ uint32_t hpd_size, uint64_t hpd_gpu_addr);
68+
69+static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
70+ uint32_t queue_id, uint32_t __user *wptr);
71+
72+static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address,
73+ uint32_t pipe_id, uint32_t queue_id);
74+
75+static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
76+ unsigned int timeout, uint32_t pipe_id,
77+ uint32_t queue_id);
78+
79+static const struct kfd2kgd_calls kfd2kgd = {
80+ .init_sa_manager = init_sa_manager,
81+ .fini_sa_manager = fini_sa_manager,
82+ .allocate_mem = allocate_mem,
83+ .free_mem = free_mem,
84+ .get_vmem_size = get_vmem_size,
85+ .get_gpu_clock_counter = get_gpu_clock_counter,
86+ .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
87+ .program_sh_mem_settings = kgd_program_sh_mem_settings,
88+ .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
89+ .init_memory = kgd_init_memory,
90+ .init_pipeline = kgd_init_pipeline,
91+ .hqd_load = kgd_hqd_load,
92+ .hqd_is_occupies = kgd_hqd_is_occupies,
93+ .hqd_destroy = kgd_hqd_destroy,
94+};
95+
96+static const struct kgd2kfd_calls *kgd2kfd;
97+
98+bool radeon_kfd_init(void)
99+{
100+ bool (*kgd2kfd_init_p)(unsigned, const struct kfd2kgd_calls*,
101+ const struct kgd2kfd_calls**);
102+
103+ kgd2kfd_init_p = symbol_request(kgd2kfd_init);
104+
105+ if (kgd2kfd_init_p == NULL)
106+ return false;
107+
108+ if (!kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kfd2kgd, &kgd2kfd)) {
109+ symbol_put(kgd2kfd_init);
110+ kgd2kfd = NULL;
111+
112+ return false;
113+ }
114+
115+ return true;
116+}
117+
118+void radeon_kfd_fini(void)
119+{
120+ if (kgd2kfd) {
121+ kgd2kfd->exit();
122+ symbol_put(kgd2kfd_init);
123+ }
124+}
125+
126+void radeon_kfd_device_probe(struct radeon_device *rdev)
127+{
128+ if (kgd2kfd)
129+ rdev->kfd = kgd2kfd->probe((struct kgd_dev *)rdev, rdev->pdev);
130+}
131+
132+void radeon_kfd_device_init(struct radeon_device *rdev)
133+{
134+ if (rdev->kfd) {
135+ struct kgd2kfd_shared_resources gpu_resources = {
136+ .compute_vmid_bitmap = 0xFF00,
137+
138+ .first_compute_pipe = 1,
139+ .compute_pipe_count = 8 - 1,
140+ };
141+
142+ radeon_doorbell_get_kfd_info(rdev,
143+ &gpu_resources.doorbell_physical_address,
144+ &gpu_resources.doorbell_aperture_size,
145+ &gpu_resources.doorbell_start_offset);
146+
147+ kgd2kfd->device_init(rdev->kfd, &gpu_resources);
148+ }
149+}
150+
151+void radeon_kfd_device_fini(struct radeon_device *rdev)
152+{
153+ if (rdev->kfd) {
154+ kgd2kfd->device_exit(rdev->kfd);
155+ rdev->kfd = NULL;
156+ }
157+}
158+
159+void radeon_kfd_interrupt(struct radeon_device *rdev, const void *ih_ring_entry)
160+{
161+ if (rdev->kfd)
162+ kgd2kfd->interrupt(rdev->kfd, ih_ring_entry);
163+}
164+
165+void radeon_kfd_suspend(struct radeon_device *rdev)
166+{
167+ if (rdev->kfd)
168+ kgd2kfd->suspend(rdev->kfd);
169+}
170+
171+int radeon_kfd_resume(struct radeon_device *rdev)
172+{
173+ int r = 0;
174+
175+ if (rdev->kfd)
176+ r = kgd2kfd->resume(rdev->kfd);
177+
178+ return r;
179+}
180+
181+static u32 pool_to_domain(enum kgd_memory_pool p)
182+{
183+ switch (p) {
184+ case KGD_POOL_FRAMEBUFFER: return RADEON_GEM_DOMAIN_VRAM;
185+ default: return RADEON_GEM_DOMAIN_GTT;
186+ }
187+}
188+
189+static int init_sa_manager(struct kgd_dev *kgd, unsigned int size)
190+{
191+ struct radeon_device *rdev = (struct radeon_device *)kgd;
192+ int r;
193+
194+ BUG_ON(kgd == NULL);
195+
196+ r = radeon_sa_bo_manager_init(rdev, &rdev->kfd_bo,
197+ size,
198+ RADEON_GPU_PAGE_SIZE,
199+ RADEON_GEM_DOMAIN_GTT,
200+ RADEON_GEM_GTT_WC);
201+
202+ if (r)
203+ return r;
204+
205+ r = radeon_sa_bo_manager_start(rdev, &rdev->kfd_bo);
206+ if (r)
207+ radeon_sa_bo_manager_fini(rdev, &rdev->kfd_bo);
208+
209+ return r;
210+}
211+
212+static void fini_sa_manager(struct kgd_dev *kgd)
213+{
214+ struct radeon_device *rdev = (struct radeon_device *)kgd;
215+
216+ BUG_ON(kgd == NULL);
217+
218+ radeon_sa_bo_manager_suspend(rdev, &rdev->kfd_bo);
219+ radeon_sa_bo_manager_fini(rdev, &rdev->kfd_bo);
220+}
221+
222+static int allocate_mem(struct kgd_dev *kgd, size_t size, size_t alignment,
223+ enum kgd_memory_pool pool, struct kgd_mem **mem)
224+{
225+ struct radeon_device *rdev = (struct radeon_device *)kgd;
226+ u32 domain;
227+ int r;
228+
229+ BUG_ON(kgd == NULL);
230+
231+ domain = pool_to_domain(pool);
232+ if (domain != RADEON_GEM_DOMAIN_GTT) {
233+ dev_err(rdev->dev,
234+ "Only allowed to allocate gart memory for kfd\n");
235+ return -EINVAL;
236+ }
237+
238+ *mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL);
239+ if ((*mem) == NULL)
240+ return -ENOMEM;
241+
242+ r = radeon_sa_bo_new(rdev, &rdev->kfd_bo, &(*mem)->sa_bo, size,
243+ alignment);
244+ if (r) {
245+ dev_err(rdev->dev, "failed to get memory for kfd (%d)\n", r);
246+ return r;
247+ }
248+
249+ (*mem)->ptr = radeon_sa_bo_cpu_addr((*mem)->sa_bo);
250+ (*mem)->gpu_addr = radeon_sa_bo_gpu_addr((*mem)->sa_bo);
251+
252+ return 0;
253+}
254+
255+static void free_mem(struct kgd_dev *kgd, struct kgd_mem *mem)
256+{
257+ struct radeon_device *rdev = (struct radeon_device *)kgd;
258+
259+ BUG_ON(kgd == NULL);
260+
261+ radeon_sa_bo_free(rdev, &mem->sa_bo, NULL);
262+ kfree(mem);
263+}
264+
265+static uint64_t get_vmem_size(struct kgd_dev *kgd)
266+{
267+ struct radeon_device *rdev = (struct radeon_device *)kgd;
268+
269+ BUG_ON(kgd == NULL);
270+
271+ return rdev->mc.real_vram_size;
272+}
273+
274+static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
275+{
276+ struct radeon_device *rdev = (struct radeon_device *)kgd;
277+
278+ return rdev->asic->get_gpu_clock_counter(rdev);
279+}
280+
281+static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
282+{
283+ struct radeon_device *rdev = (struct radeon_device *)kgd;
284+
285+ /* The sclk is in quantas of 10kHz */
286+ return rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100;
287+}
288+
289+static inline struct radeon_device *get_radeon_device(struct kgd_dev *kgd)
290+{
291+ return (struct radeon_device *)kgd;
292+}
293+
294+static void write_register(struct kgd_dev *kgd, uint32_t offset, uint32_t value)
295+{
296+ struct radeon_device *rdev = get_radeon_device(kgd);
297+
298+ writel(value, (void __iomem *)(rdev->rmmio + offset));
299+}
300+
301+static uint32_t read_register(struct kgd_dev *kgd, uint32_t offset)
302+{
303+ struct radeon_device *rdev = get_radeon_device(kgd);
304+
305+ return readl((void __iomem *)(rdev->rmmio + offset));
306+}
307+
308+static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
309+ uint32_t queue, uint32_t vmid)
310+{
311+ struct radeon_device *rdev = get_radeon_device(kgd);
312+ uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue);
313+
314+ mutex_lock(&rdev->srbm_mutex);
315+ write_register(kgd, SRBM_GFX_CNTL, value);
316+}
317+
318+static void unlock_srbm(struct kgd_dev *kgd)
319+{
320+ struct radeon_device *rdev = get_radeon_device(kgd);
321+
322+ write_register(kgd, SRBM_GFX_CNTL, 0);
323+ mutex_unlock(&rdev->srbm_mutex);
324+}
325+
326+static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
327+ uint32_t queue_id)
328+{
329+ uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1;
330+ uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
331+
332+ lock_srbm(kgd, mec, pipe, queue_id, 0);
333+}
334+
335+static void release_queue(struct kgd_dev *kgd)
336+{
337+ unlock_srbm(kgd);
338+}
339+
340+static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
341+ uint32_t sh_mem_config,
342+ uint32_t sh_mem_ape1_base,
343+ uint32_t sh_mem_ape1_limit,
344+ uint32_t sh_mem_bases)
345+{
346+ lock_srbm(kgd, 0, 0, 0, vmid);
347+
348+ write_register(kgd, SH_MEM_CONFIG, sh_mem_config);
349+ write_register(kgd, SH_MEM_APE1_BASE, sh_mem_ape1_base);
350+ write_register(kgd, SH_MEM_APE1_LIMIT, sh_mem_ape1_limit);
351+ write_register(kgd, SH_MEM_BASES, sh_mem_bases);
352+
353+ unlock_srbm(kgd);
354+}
355+
356+static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
357+ unsigned int vmid)
358+{
359+ /*
360+ * We have to assume that there is no outstanding mapping.
361+ * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0
362+ * because a mapping is in progress or because a mapping finished and
363+ * the SW cleared it.
364+ * So the protocol is to always wait & clear.
365+ */
366+ uint32_t pasid_mapping = (pasid == 0) ? 0 :
367+ (uint32_t)pasid | ATC_VMID_PASID_MAPPING_VALID;
368+
369+ write_register(kgd, ATC_VMID0_PASID_MAPPING + vmid*sizeof(uint32_t),
370+ pasid_mapping);
371+
372+ while (!(read_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS) &
373+ (1U << vmid)))
374+ cpu_relax();
375+ write_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid);
376+
377+ return 0;
378+}
379+
380+static int kgd_init_memory(struct kgd_dev *kgd)
381+{
382+ /*
383+ * Configure apertures:
384+ * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
385+ * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
386+ * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
387+ */
388+ int i;
389+ uint32_t sh_mem_bases = PRIVATE_BASE(0x6000) | SHARED_BASE(0x6000);
390+
391+ for (i = 8; i < 16; i++) {
392+ uint32_t sh_mem_config;
393+
394+ lock_srbm(kgd, 0, 0, 0, i);
395+
396+ sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
397+ sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
398+
399+ write_register(kgd, SH_MEM_CONFIG, sh_mem_config);
400+
401+ write_register(kgd, SH_MEM_BASES, sh_mem_bases);
402+
403+ /* Scratch aperture is not supported for now. */
404+ write_register(kgd, SH_STATIC_MEM_CONFIG, 0);
405+
406+ /* APE1 disabled for now. */
407+ write_register(kgd, SH_MEM_APE1_BASE, 1);
408+ write_register(kgd, SH_MEM_APE1_LIMIT, 0);
409+
410+ unlock_srbm(kgd);
411+ }
412+
413+ return 0;
414+}
415+
416+static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
417+ uint32_t hpd_size, uint64_t hpd_gpu_addr)
418+{
419+ uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1;
420+ uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
421+
422+ lock_srbm(kgd, mec, pipe, 0, 0);
423+ write_register(kgd, CP_HPD_EOP_BASE_ADDR,
424+ lower_32_bits(hpd_gpu_addr >> 8));
425+ write_register(kgd, CP_HPD_EOP_BASE_ADDR_HI,
426+ upper_32_bits(hpd_gpu_addr >> 8));
427+ write_register(kgd, CP_HPD_EOP_VMID, 0);
428+ write_register(kgd, CP_HPD_EOP_CONTROL, hpd_size);
429+ unlock_srbm(kgd);
430+
431+ return 0;
432+}
433+
434+static inline struct cik_mqd *get_mqd(void *mqd)
435+{
436+ return (struct cik_mqd *)mqd;
437+}
438+
439+static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
440+ uint32_t queue_id, uint32_t __user *wptr)
441+{
442+ uint32_t wptr_shadow, is_wptr_shadow_valid;
443+ struct cik_mqd *m;
444+
445+ m = get_mqd(mqd);
446+
447+ is_wptr_shadow_valid = !get_user(wptr_shadow, wptr);
448+
449+ acquire_queue(kgd, pipe_id, queue_id);
450+ write_register(kgd, CP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo);
451+ write_register(kgd, CP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi);
452+ write_register(kgd, CP_MQD_CONTROL, m->cp_mqd_control);
453+
454+ write_register(kgd, CP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo);
455+ write_register(kgd, CP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi);
456+ write_register(kgd, CP_HQD_PQ_CONTROL, m->cp_hqd_pq_control);
457+
458+ write_register(kgd, CP_HQD_IB_CONTROL, m->cp_hqd_ib_control);
459+ write_register(kgd, CP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo);
460+ write_register(kgd, CP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi);
461+
462+ write_register(kgd, CP_HQD_IB_RPTR, m->cp_hqd_ib_rptr);
463+
464+ write_register(kgd, CP_HQD_PERSISTENT_STATE,
465+ m->cp_hqd_persistent_state);
466+ write_register(kgd, CP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd);
467+ write_register(kgd, CP_HQD_MSG_TYPE, m->cp_hqd_msg_type);
468+
469+ write_register(kgd, CP_HQD_ATOMIC0_PREOP_LO,
470+ m->cp_hqd_atomic0_preop_lo);
471+
472+ write_register(kgd, CP_HQD_ATOMIC0_PREOP_HI,
473+ m->cp_hqd_atomic0_preop_hi);
474+
475+ write_register(kgd, CP_HQD_ATOMIC1_PREOP_LO,
476+ m->cp_hqd_atomic1_preop_lo);
477+
478+ write_register(kgd, CP_HQD_ATOMIC1_PREOP_HI,
479+ m->cp_hqd_atomic1_preop_hi);
480+
481+ write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR,
482+ m->cp_hqd_pq_rptr_report_addr_lo);
483+
484+ write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
485+ m->cp_hqd_pq_rptr_report_addr_hi);
486+
487+ write_register(kgd, CP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr);
488+
489+ write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR,
490+ m->cp_hqd_pq_wptr_poll_addr_lo);
491+
492+ write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR_HI,
493+ m->cp_hqd_pq_wptr_poll_addr_hi);
494+
495+ write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL,
496+ m->cp_hqd_pq_doorbell_control);
497+
498+ write_register(kgd, CP_HQD_VMID, m->cp_hqd_vmid);
499+
500+ write_register(kgd, CP_HQD_QUANTUM, m->cp_hqd_quantum);
501+
502+ write_register(kgd, CP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority);
503+ write_register(kgd, CP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority);
504+
505+ write_register(kgd, CP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr);
506+
507+ if (is_wptr_shadow_valid)
508+ write_register(kgd, CP_HQD_PQ_WPTR, wptr_shadow);
509+
510+ write_register(kgd, CP_HQD_ACTIVE, m->cp_hqd_active);
511+ release_queue(kgd);
512+
513+ return 0;
514+}
515+
516+static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address,
517+ uint32_t pipe_id, uint32_t queue_id)
518+{
519+ uint32_t act;
520+ bool retval = false;
521+ uint32_t low, high;
522+
523+ acquire_queue(kgd, pipe_id, queue_id);
524+ act = read_register(kgd, CP_HQD_ACTIVE);
525+ if (act) {
526+ low = lower_32_bits(queue_address >> 8);
527+ high = upper_32_bits(queue_address >> 8);
528+
529+ if (low == read_register(kgd, CP_HQD_PQ_BASE) &&
530+ high == read_register(kgd, CP_HQD_PQ_BASE_HI))
531+ retval = true;
532+ }
533+ release_queue(kgd);
534+ return retval;
535+}
536+
537+static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
538+ unsigned int timeout, uint32_t pipe_id,
539+ uint32_t queue_id)
540+{
541+ uint32_t temp;
542+
543+ acquire_queue(kgd, pipe_id, queue_id);
544+ write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, 0);
545+
546+ write_register(kgd, CP_HQD_DEQUEUE_REQUEST, reset_type);
547+
548+ while (true) {
549+ temp = read_register(kgd, CP_HQD_ACTIVE);
550+ if (temp & 0x1)
551+ break;
552+ if (timeout == 0) {
553+ pr_err("kfd: cp queue preemption time out (%dms)\n",
554+ temp);
555+ return -ETIME;
556+ }
557+ msleep(20);
558+ timeout -= 20;
559+ }
560+
561+ release_queue(kgd);
562+ return 0;
563+}
drivers/gpu/drm/radeon/radeon_kfd.h+47−0
@@ -0,0 +1,47 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+/*
24+ * radeon_kfd.h defines the private interface between the
25+ * AMD kernel graphics drivers and the AMD KFD.
26+ */
27+
28+#ifndef RADEON_KFD_H_INCLUDED
29+#define RADEON_KFD_H_INCLUDED
30+
31+#include <linux/types.h>
32+#include "../amd/include/kgd_kfd_interface.h"
33+
34+struct radeon_device;
35+
36+bool radeon_kfd_init(void);
37+void radeon_kfd_fini(void);
38+
39+void radeon_kfd_suspend(struct radeon_device *rdev);
40+int radeon_kfd_resume(struct radeon_device *rdev);
41+void radeon_kfd_interrupt(struct radeon_device *rdev,
42+ const void *ih_ring_entry);
43+void radeon_kfd_device_probe(struct radeon_device *rdev);
44+void radeon_kfd_device_init(struct radeon_device *rdev);
45+void radeon_kfd_device_fini(struct radeon_device *rdev);
46+
47+#endif /* RADEON_KFD_H_INCLUDED */
drivers/gpu/drm/radeon/radeon_kms.c+7−0
@@ -34,6 +34,8 @@ · 33 unmodified lines
3434 #include <linux/slab.h>
3535 #include <linux/pm_runtime.h>
3636
37+#include "radeon_kfd.h"
38+
3739 #if defined(CONFIG_VGA_SWITCHEROO)
3840 bool radeon_has_atpx(void);
3941 #else
@@ -63,6 +65,8 @@ int radeon_driver_unload_kms(struct drm_device *dev) · 23 unmodified lines
6365
6466 pm_runtime_get_sync(dev->dev);
6567
68+ radeon_kfd_device_fini(rdev);
69+
6670 radeon_acpi_fini(rdev);
6771
6872 radeon_modeset_fini(rdev);
@@ -142,6 +146,9 @@ int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags) · 73 unmodified lines
142146 "Error during ACPI methods call\n");
143147 }
144148
149+ radeon_kfd_device_probe(rdev);
150+ radeon_kfd_device_init(rdev);
151+
145152 if (radeon_is_px(dev)) {
146153 pm_runtime_use_autosuspend(dev->dev);
147154 pm_runtime_set_autosuspend_delay(dev->dev, 5000);
drivers/iommu/amd_iommu_v2.c+11−58
@@ -92,13 +92,6 @@ static spinlock_t state_lock; · 91 unmodified lines
9292
9393 static struct workqueue_struct *iommu_wq;
9494
95−/*
96− * Empty page table - Used between
97− * mmu_notifier_invalidate_range_start and
98− * mmu_notifier_invalidate_range_end
99− */
100−static u64 *empty_page_table;
101−
10295 static void free_pasid_states(struct device_state *dev_state);
10396
10497 static u16 device_id(struct pci_dev *pdev)
@@ -279,10 +272,8 @@ static void free_pasid_state(struct pasid_state *pasid_state) · 174 unmodified lines
279272
280273 static void put_pasid_state(struct pasid_state *pasid_state)
281274 {
282− if (atomic_dec_and_test(&pasid_state->count)) {
283− put_device_state(pasid_state->device_state);
275+ if (atomic_dec_and_test(&pasid_state->count))
284276 wake_up(&pasid_state->wq);
285− }
286277 }
287278
288279 static void put_pasid_state_wait(struct pasid_state *pasid_state)
@@ -291,9 +282,7 @@ static void put_pasid_state_wait(struct pasid_state *pasid_state) · 2 unmodified lines
291282
292283 prepare_to_wait(&pasid_state->wq, &wait, TASK_UNINTERRUPTIBLE);
293284
294− if (atomic_dec_and_test(&pasid_state->count))
295− put_device_state(pasid_state->device_state);
296− else
285+ if (!atomic_dec_and_test(&pasid_state->count))
297286 schedule();
298287
299288 finish_wait(&pasid_state->wq, &wait);
@@ -418,46 +407,21 @@ static void mn_invalidate_page(struct mmu_notifier *mn, · 118 unmodified lines
418407 __mn_flush_page(mn, address);
419408 }
420409
421−static void mn_invalidate_range_start(struct mmu_notifier *mn,
422− struct mm_struct *mm,
423− unsigned long start, unsigned long end)
424−{
425− struct pasid_state *pasid_state;
426− struct device_state *dev_state;
427− unsigned long flags;
428−
429− pasid_state = mn_to_state(mn);
430− dev_state = pasid_state->device_state;
431−
432− spin_lock_irqsave(&pasid_state->lock, flags);
433− if (pasid_state->mmu_notifier_count == 0) {
434− amd_iommu_domain_set_gcr3(dev_state->domain,
435− pasid_state->pasid,
436− __pa(empty_page_table));
437− }
438− pasid_state->mmu_notifier_count += 1;
439− spin_unlock_irqrestore(&pasid_state->lock, flags);
440−}
441−
442−static void mn_invalidate_range_end(struct mmu_notifier *mn,
443− struct mm_struct *mm,
444− unsigned long start, unsigned long end)
410+static void mn_invalidate_range(struct mmu_notifier *mn,
411+ struct mm_struct *mm,
412+ unsigned long start, unsigned long end)
445413 {
446414 struct pasid_state *pasid_state;
447415 struct device_state *dev_state;
448− unsigned long flags;
449416
450417 pasid_state = mn_to_state(mn);
451418 dev_state = pasid_state->device_state;
452419
453− spin_lock_irqsave(&pasid_state->lock, flags);
454− pasid_state->mmu_notifier_count -= 1;
455− if (pasid_state->mmu_notifier_count == 0) {
456− amd_iommu_domain_set_gcr3(dev_state->domain,
457− pasid_state->pasid,
458− __pa(pasid_state->mm->pgd));
459− }
460− spin_unlock_irqrestore(&pasid_state->lock, flags);
420+ if ((start ^ (end - 1)) < PAGE_SIZE)
421+ amd_iommu_flush_page(dev_state->domain, pasid_state->pasid,
422+ start);
423+ else
424+ amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid);
461425 }
462426
463427 static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
@@ -482,8 +446,7 @@ static struct mmu_notifier_ops iommu_mn = { · 18 unmodified lines
482446 .release = mn_release,
483447 .clear_flush_young = mn_clear_flush_young,
484448 .invalidate_page = mn_invalidate_page,
485− .invalidate_range_start = mn_invalidate_range_start,
486− .invalidate_range_end = mn_invalidate_range_end,
449+ .invalidate_range = mn_invalidate_range,
487450 };
488451
489452 static void set_pri_tag_status(struct pasid_state *pasid_state,
@@ -954,18 +917,10 @@ static int __init amd_iommu_v2_init(void) · 464 unmodified lines
954917 if (iommu_wq == NULL)
955918 goto out;
956919
957− ret = -ENOMEM;
958− empty_page_table = (u64 *)get_zeroed_page(GFP_KERNEL);
959− if (empty_page_table == NULL)
960− goto out_destroy_wq;
961−
962920 amd_iommu_register_ppr_notifier(&ppr_nb);
963921
964922 return 0;
965923
966−out_destroy_wq:
967− destroy_workqueue(iommu_wq);
968−
969924 out:
970925 return ret;
971926 }
@@ -999,8 +954,6 @@ static void __exit amd_iommu_v2_exit(void) · 27 unmodified lines
999954 }
1000955
1001956 destroy_workqueue(iommu_wq);
1002−
1003− free_page((unsigned long)empty_page_table);
1004957 }
1005958
1006959 module_init(amd_iommu_v2_init);
include/linux/mmu_notifier.h+83−5
@@ -98,11 +98,11 @@ struct mmu_notifier_ops { · 97 unmodified lines
9898 /*
9999 * invalidate_range_start() and invalidate_range_end() must be
100100 * paired and are called only when the mmap_sem and/or the
101− * locks protecting the reverse maps are held. The subsystem
102− * must guarantee that no additional references are taken to
103− * the pages in the range established between the call to
104− * invalidate_range_start() and the matching call to
105− * invalidate_range_end().
101+ * locks protecting the reverse maps are held. If the subsystem
102+ * can't guarantee that no additional references are taken to
103+ * the pages in the range, it has to implement the
104+ * invalidate_range() notifier to remove any references taken
105+ * after invalidate_range_start().
106106 *
107107 * Invalidation of multiple concurrent ranges may be
108108 * optionally permitted by the driver. Either way the
@@ -144,6 +144,29 @@ struct mmu_notifier_ops { · 35 unmodified lines
144144 void (*invalidate_range_end)(struct mmu_notifier *mn,
145145 struct mm_struct *mm,
146146 unsigned long start, unsigned long end);
147+
148+ /*
149+ * invalidate_range() is either called between
150+ * invalidate_range_start() and invalidate_range_end() when the
151+ * VM has to free pages that where unmapped, but before the
152+ * pages are actually freed, or outside of _start()/_end() when
153+ * a (remote) TLB is necessary.
154+ *
155+ * If invalidate_range() is used to manage a non-CPU TLB with
156+ * shared page-tables, it not necessary to implement the
157+ * invalidate_range_start()/end() notifiers, as
158+ * invalidate_range() alread catches the points in time when an
159+ * external TLB range needs to be flushed.
160+ *
161+ * The invalidate_range() function is called under the ptl
162+ * spin-lock and not allowed to sleep.
163+ *
164+ * Note that this function might be called with just a sub-range
165+ * of what was passed to invalidate_range_start()/end(), if
166+ * called between those functions.
167+ */
168+ void (*invalidate_range)(struct mmu_notifier *mn, struct mm_struct *mm,
169+ unsigned long start, unsigned long end);
147170 };
148171
149172 /*
@@ -190,6 +213,8 @@ extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm, · 40 unmodified lines
190213 unsigned long start, unsigned long end);
191214 extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
192215 unsigned long start, unsigned long end);
216+extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,
217+ unsigned long start, unsigned long end);
193218
194219 static inline void mmu_notifier_release(struct mm_struct *mm)
195220 {
@@ -242,6 +267,13 @@ static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 46 unmodified lines
242267 __mmu_notifier_invalidate_range_end(mm, start, end);
243268 }
244269
270+static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
271+ unsigned long start, unsigned long end)
272+{
273+ if (mm_has_notifiers(mm))
274+ __mmu_notifier_invalidate_range(mm, start, end);
275+}
276+
245277 static inline void mmu_notifier_mm_init(struct mm_struct *mm)
246278 {
247279 mm->mmu_notifier_mm = NULL;
@@ -279,6 +311,44 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) · 31 unmodified lines
279311 __young; \
280312 })
281313
314+#define ptep_clear_flush_notify(__vma, __address, __ptep) \
315+({ \
316+ unsigned long ___addr = __address & PAGE_MASK; \
317+ struct mm_struct *___mm = (__vma)->vm_mm; \
318+ pte_t ___pte; \
319+ \
320+ ___pte = ptep_clear_flush(__vma, __address, __ptep); \
321+ mmu_notifier_invalidate_range(___mm, ___addr, \
322+ ___addr + PAGE_SIZE); \
323+ \
324+ ___pte; \
325+})
326+
327+#define pmdp_clear_flush_notify(__vma, __haddr, __pmd) \
328+({ \
329+ unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \
330+ struct mm_struct *___mm = (__vma)->vm_mm; \
331+ pmd_t ___pmd; \
332+ \
333+ ___pmd = pmdp_clear_flush(__vma, __haddr, __pmd); \
334+ mmu_notifier_invalidate_range(___mm, ___haddr, \
335+ ___haddr + HPAGE_PMD_SIZE); \
336+ \
337+ ___pmd; \
338+})
339+
340+#define pmdp_get_and_clear_notify(__mm, __haddr, __pmd) \
341+({ \
342+ unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \
343+ pmd_t ___pmd; \
344+ \
345+ ___pmd = pmdp_get_and_clear(__mm, __haddr, __pmd); \
346+ mmu_notifier_invalidate_range(__mm, ___haddr, \
347+ ___haddr + HPAGE_PMD_SIZE); \
348+ \
349+ ___pmd; \
350+})
351+
282352 /*
283353 * set_pte_at_notify() sets the pte _after_ running the notifier.
284354 * This is safe to start by updating the secondary MMUs, because the primary MMU
@@ -342,6 +412,11 @@ static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 57 unmodified lines
342412 {
343413 }
344414
415+static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
416+ unsigned long start, unsigned long end)
417+{
418+}
419+
345420 static inline void mmu_notifier_mm_init(struct mm_struct *mm)
346421 {
347422 }
@@ -352,6 +427,9 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) · 4 unmodified lines
352427
353428 #define ptep_clear_flush_young_notify ptep_clear_flush_young
354429 #define pmdp_clear_flush_young_notify pmdp_clear_flush_young
430+#define ptep_clear_flush_notify ptep_clear_flush
431+#define pmdp_clear_flush_notify pmdp_clear_flush
432+#define pmdp_get_and_clear_notify pmdp_get_and_clear
355433 #define set_pte_at_notify set_pte_at
356434
357435 #endif /* CONFIG_MMU_NOTIFIER */
include/uapi/linux/kfd_ioctl.h+154−0
@@ -0,0 +1,154 @@
1+/*
2+ * Copyright 2014 Advanced Micro Devices, Inc.
3+ *
4+ * Permission is hereby granted, free of charge, to any person obtaining a
5+ * copy of this software and associated documentation files (the "Software"),
6+ * to deal in the Software without restriction, including without limitation
7+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+ * and/or sell copies of the Software, and to permit persons to whom the
9+ * Software is furnished to do so, subject to the following conditions:
10+ *
11+ * The above copyright notice and this permission notice shall be included in
12+ * all copies or substantial portions of the Software.
13+ *
14+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+ * OTHER DEALINGS IN THE SOFTWARE.
21+ */
22+
23+#ifndef KFD_IOCTL_H_INCLUDED
24+#define KFD_IOCTL_H_INCLUDED
25+
26+#include <linux/types.h>
27+#include <linux/ioctl.h>
28+
29+#define KFD_IOCTL_MAJOR_VERSION 1
30+#define KFD_IOCTL_MINOR_VERSION 0
31+
32+struct kfd_ioctl_get_version_args {
33+ uint32_t major_version; /* from KFD */
34+ uint32_t minor_version; /* from KFD */
35+};
36+
37+/* For kfd_ioctl_create_queue_args.queue_type. */
38+#define KFD_IOC_QUEUE_TYPE_COMPUTE 0
39+#define KFD_IOC_QUEUE_TYPE_SDMA 1
40+#define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 2
41+
42+#define KFD_MAX_QUEUE_PERCENTAGE 100
43+#define KFD_MAX_QUEUE_PRIORITY 15
44+
45+struct kfd_ioctl_create_queue_args {
46+ uint64_t ring_base_address; /* to KFD */
47+ uint64_t write_pointer_address; /* from KFD */
48+ uint64_t read_pointer_address; /* from KFD */
49+ uint64_t doorbell_offset; /* from KFD */
50+
51+ uint32_t ring_size; /* to KFD */
52+ uint32_t gpu_id; /* to KFD */
53+ uint32_t queue_type; /* to KFD */
54+ uint32_t queue_percentage; /* to KFD */
55+ uint32_t queue_priority; /* to KFD */
56+ uint32_t queue_id; /* from KFD */
57+
58+ uint64_t eop_buffer_address; /* to KFD */
59+ uint64_t eop_buffer_size; /* to KFD */
60+ uint64_t ctx_save_restore_address; /* to KFD */
61+ uint64_t ctx_save_restore_size; /* to KFD */
62+};
63+
64+struct kfd_ioctl_destroy_queue_args {
65+ uint32_t queue_id; /* to KFD */
66+ uint32_t pad;
67+};
68+
69+struct kfd_ioctl_update_queue_args {
70+ uint64_t ring_base_address; /* to KFD */
71+
72+ uint32_t queue_id; /* to KFD */
73+ uint32_t ring_size; /* to KFD */
74+ uint32_t queue_percentage; /* to KFD */
75+ uint32_t queue_priority; /* to KFD */
76+};
77+
78+/* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */
79+#define KFD_IOC_CACHE_POLICY_COHERENT 0
80+#define KFD_IOC_CACHE_POLICY_NONCOHERENT 1
81+
82+struct kfd_ioctl_set_memory_policy_args {
83+ uint64_t alternate_aperture_base; /* to KFD */
84+ uint64_t alternate_aperture_size; /* to KFD */
85+
86+ uint32_t gpu_id; /* to KFD */
87+ uint32_t default_policy; /* to KFD */
88+ uint32_t alternate_policy; /* to KFD */
89+ uint32_t pad;
90+};
91+
92+/*
93+ * All counters are monotonic. They are used for profiling of compute jobs.
94+ * The profiling is done by userspace.
95+ *
96+ * In case of GPU reset, the counter should not be affected.
97+ */
98+
99+struct kfd_ioctl_get_clock_counters_args {
100+ uint64_t gpu_clock_counter; /* from KFD */
101+ uint64_t cpu_clock_counter; /* from KFD */
102+ uint64_t system_clock_counter; /* from KFD */
103+ uint64_t system_clock_freq; /* from KFD */
104+
105+ uint32_t gpu_id; /* to KFD */
106+ uint32_t pad;
107+};
108+
109+#define NUM_OF_SUPPORTED_GPUS 7
110+
111+struct kfd_process_device_apertures {
112+ uint64_t lds_base; /* from KFD */
113+ uint64_t lds_limit; /* from KFD */
114+ uint64_t scratch_base; /* from KFD */
115+ uint64_t scratch_limit; /* from KFD */
116+ uint64_t gpuvm_base; /* from KFD */
117+ uint64_t gpuvm_limit; /* from KFD */
118+ uint32_t gpu_id; /* from KFD */
119+ uint32_t pad;
120+};
121+
122+struct kfd_ioctl_get_process_apertures_args {
123+ struct kfd_process_device_apertures
124+ process_apertures[NUM_OF_SUPPORTED_GPUS];/* from KFD */
125+
126+ /* from KFD, should be in the range [1 - NUM_OF_SUPPORTED_GPUS] */
127+ uint32_t num_of_nodes;
128+ uint32_t pad;
129+};
130+
131+#define KFD_IOC_MAGIC 'K'
132+
133+#define KFD_IOC_GET_VERSION \
134+ _IOR(KFD_IOC_MAGIC, 1, struct kfd_ioctl_get_version_args)
135+
136+#define KFD_IOC_CREATE_QUEUE \
137+ _IOWR(KFD_IOC_MAGIC, 2, struct kfd_ioctl_create_queue_args)
138+
139+#define KFD_IOC_DESTROY_QUEUE \
140+ _IOWR(KFD_IOC_MAGIC, 3, struct kfd_ioctl_destroy_queue_args)
141+
142+#define KFD_IOC_SET_MEMORY_POLICY \
143+ _IOW(KFD_IOC_MAGIC, 4, struct kfd_ioctl_set_memory_policy_args)
144+
145+#define KFD_IOC_GET_CLOCK_COUNTERS \
146+ _IOWR(KFD_IOC_MAGIC, 5, struct kfd_ioctl_get_clock_counters_args)
147+
148+#define KFD_IOC_GET_PROCESS_APERTURES \
149+ _IOR(KFD_IOC_MAGIC, 6, struct kfd_ioctl_get_process_apertures_args)
150+
151+#define KFD_IOC_UPDATE_QUEUE \
152+ _IOW(KFD_IOC_MAGIC, 7, struct kfd_ioctl_update_queue_args)
153+
154+#endif
kernel/events/uprobes.c+1−1
@@ -193,7 +193,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, · 192 unmodified lines
193193 }
194194
195195 flush_cache_page(vma, addr, pte_pfn(*ptep));
196− ptep_clear_flush(vma, addr, ptep);
196+ ptep_clear_flush_notify(vma, addr, ptep);
197197 set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
198198
199199 page_remove_rmap(page);
mm/fremap.c+1−1
@@ -37,7 +37,7 @@ static void zap_pte(struct mm_struct *mm, struct vm_area_struct *vma, · 36 unmodified lines
3737
3838 if (pte_present(pte)) {
3939 flush_cache_page(vma, addr, pte_pfn(pte));
40− pte = ptep_clear_flush(vma, addr, ptep);
40+ pte = ptep_clear_flush_notify(vma, addr, ptep);
4141 page = vm_normal_page(vma, addr, pte);
4242 if (page) {
4343 if (pte_dirty(pte))
mm/huge_memory.c+5−4
@@ -1036,7 +1036,7 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, · 1035 unmodified lines
10361036 goto out_free_pages;
10371037 VM_BUG_ON_PAGE(!PageHead(page), page);
10381038
1039− pmdp_clear_flush(vma, haddr, pmd);
1039+ pmdp_clear_flush_notify(vma, haddr, pmd);
10401040 /* leave pmd empty until pte is filled */
10411041
10421042 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
@@ -1179,7 +1179,7 @@ alloc: · 136 unmodified lines
11791179 pmd_t entry;
11801180 entry = mk_huge_pmd(new_page, vma->vm_page_prot);
11811181 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1182− pmdp_clear_flush(vma, haddr, pmd);
1182+ pmdp_clear_flush_notify(vma, haddr, pmd);
11831183 page_add_new_anon_rmap(new_page, vma, haddr);
11841184 mem_cgroup_commit_charge(new_page, memcg, false);
11851185 lru_cache_add_active_or_unevictable(new_page, vma);
@@ -1512,7 +1512,7 @@ int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, · 326 unmodified lines
15121512 pmd_t entry;
15131513 ret = 1;
15141514 if (!prot_numa) {
1515− entry = pmdp_get_and_clear(mm, addr, pmd);
1515+ entry = pmdp_get_and_clear_notify(mm, addr, pmd);
15161516 if (pmd_numa(entry))
15171517 entry = pmd_mknonnuma(entry);
15181518 entry = pmd_modify(entry, newprot);
@@ -1644,6 +1644,7 @@ static int __split_huge_page_splitting(struct page *page, · 125 unmodified lines
16441644 * serialize against split_huge_page*.
16451645 */
16461646 pmdp_splitting_flush(vma, address, pmd);
1647+
16471648 ret = 1;
16481649 spin_unlock(ptl);
16491650 }
@@ -2834,7 +2835,7 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, · 1184 unmodified lines
28342835 pmd_t _pmd;
28352836 int i;
28362837
2837− pmdp_clear_flush(vma, haddr, pmd);
2838+ pmdp_clear_flush_notify(vma, haddr, pmd);
28382839 /* leave pmd empty until pte is filled */
28392840
28402841 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
mm/hugetlb.c+6−1
@@ -2598,8 +2598,11 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, · 2597 unmodified lines
25982598 }
25992599 set_huge_pte_at(dst, addr, dst_pte, entry);
26002600 } else {
2601− if (cow)
2601+ if (cow) {
26022602 huge_ptep_set_wrprotect(src, addr, src_pte);
2603+ mmu_notifier_invalidate_range(src, mmun_start,
2604+ mmun_end);
2605+ }
26032606 entry = huge_ptep_get(src_pte);
26042607 ptepage = pte_page(entry);
26052608 get_page(ptepage);
@@ -2899,6 +2902,7 @@ retry_avoidcopy: · 293 unmodified lines
28992902
29002903 /* Break COW */
29012904 huge_ptep_clear_flush(vma, address, ptep);
2905+ mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
29022906 set_huge_pte_at(mm, address, ptep,
29032907 make_huge_pte(vma, new_page, 1));
29042908 page_remove_rmap(old_page);
@@ -3374,6 +3378,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, · 469 unmodified lines
33743378 * and that page table be reused and filled with junk.
33753379 */
33763380 flush_tlb_range(vma, start, end);
3381+ mmu_notifier_invalidate_range(mm, start, end);
33773382 mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
33783383 mmu_notifier_invalidate_range_end(mm, start, end);
33793384
mm/ksm.c+2−2
@@ -892,7 +892,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page, · 891 unmodified lines
892892 * this assure us that no O_DIRECT can happen after the check
893893 * or in the middle of the check.
894894 */
895− entry = ptep_clear_flush(vma, addr, ptep);
895+ entry = ptep_clear_flush_notify(vma, addr, ptep);
896896 /*
897897 * Check that no O_DIRECT or similar I/O is in progress on the
898898 * page
@@ -960,7 +960,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, · 61 unmodified lines
960960 page_add_anon_rmap(kpage, vma, addr);
961961
962962 flush_cache_page(vma, addr, pte_pfn(*ptep));
963− ptep_clear_flush(vma, addr, ptep);
963+ ptep_clear_flush_notify(vma, addr, ptep);
964964 set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
965965
966966 page_remove_rmap(page);
mm/memory.c+2−1
@@ -238,6 +238,7 @@ static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) · 237 unmodified lines
238238 {
239239 tlb->need_flush = 0;
240240 tlb_flush(tlb);
241+ mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end);
241242 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
242243 tlb_table_flush(tlb);
243244 #endif
@@ -2234,7 +2235,7 @@ gotten: · 1990 unmodified lines
22342235 * seen in the presence of one thread doing SMC and another
22352236 * thread doing COW.
22362237 */
2237− ptep_clear_flush(vma, address, page_table);
2238+ ptep_clear_flush_notify(vma, address, page_table);
22382239 page_add_new_anon_rmap(new_page, vma, address);
22392240 mem_cgroup_commit_charge(new_page, memcg, false);
22402241 lru_cache_add_active_or_unevictable(new_page, vma);
mm/migrate.c+2−1
@@ -1854,7 +1854,7 @@ fail_putback: · 1853 unmodified lines
18541854 */
18551855 flush_cache_range(vma, mmun_start, mmun_end);
18561856 page_add_anon_rmap(new_page, vma, mmun_start);
1857− pmdp_clear_flush(vma, mmun_start, pmd);
1857+ pmdp_clear_flush_notify(vma, mmun_start, pmd);
18581858 set_pmd_at(mm, mmun_start, pmd, entry);
18591859 flush_tlb_range(vma, mmun_start, mmun_end);
18601860 update_mmu_cache_pmd(vma, address, &entry);
@@ -1862,6 +1862,7 @@ fail_putback: · 1 unmodified line
18621862 if (page_count(page) != 2) {
18631863 set_pmd_at(mm, mmun_start, pmd, orig_entry);
18641864 flush_tlb_range(vma, mmun_start, mmun_end);
1865+ mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
18651866 update_mmu_cache_pmd(vma, address, &entry);
18661867 page_remove_rmap(new_page);
18671868 goto fail_putback;
mm/mmu_notifier.c+25−0
@@ -193,6 +193,16 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 192 unmodified lines
193193
194194 id = srcu_read_lock(&srcu);
195195 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
196+ /*
197+ * Call invalidate_range here too to avoid the need for the
198+ * subsystem of having to register an invalidate_range_end
199+ * call-back when there is invalidate_range already. Usually a
200+ * subsystem registers either invalidate_range_start()/end() or
201+ * invalidate_range(), so this will be no additional overhead
202+ * (besides the pointer check).
203+ */
204+ if (mn->ops->invalidate_range)
205+ mn->ops->invalidate_range(mn, mm, start, end);
196206 if (mn->ops->invalidate_range_end)
197207 mn->ops->invalidate_range_end(mn, mm, start, end);
198208 }
@@ -200,6 +210,21 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 1 unmodified line
200210 }
201211 EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_end);
202212
213+void __mmu_notifier_invalidate_range(struct mm_struct *mm,
214+ unsigned long start, unsigned long end)
215+{
216+ struct mmu_notifier *mn;
217+ int id;
218+
219+ id = srcu_read_lock(&srcu);
220+ hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
221+ if (mn->ops->invalidate_range)
222+ mn->ops->invalidate_range(mn, mm, start, end);
223+ }
224+ srcu_read_unlock(&srcu, id);
225+}
226+EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range);
227+
203228 static int do_mmu_notifier_register(struct mmu_notifier *mn,
204229 struct mm_struct *mm,
205230 int take_mmap_sem)
mm/rmap.c+1−1
@@ -1378,7 +1378,7 @@ static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount, · 1377 unmodified lines
13781378
13791379 /* Nuke the page table entry. */
13801380 flush_cache_page(vma, address, pte_pfn(*pte));
1381− pteval = ptep_clear_flush(vma, address, pte);
1381+ pteval = ptep_clear_flush_notify(vma, address, pte);
13821382
13831383 /* If nonlinear, store the file page offset in the pte. */
13841384 if (page->index != linear_page_index(vma, address)) {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t