History

Merge branch 'amdkfd-v6' of git://people.freedesktop.org/~gabbayo/linux into drm-next

Merge AMDKFD it seems clean enough. * 'amdkfd-v6' of git://people.freedesktop.org/~gabbayo/linux: (29 commits) amdkfd: Implement the Get Version IOCTL amdkfd: Implement the Get Process Aperture IOCTL amdkfd: Implement the Get Clock Counters IOCTL amdkfd: Implement the Set Memory Policy IOCTL amdkfd: Implement the create/destroy/update queue IOCTLs amdkfd: Add interrupt handling module amdkfd: Add device queue manager module amdkfd: Add process queue manager module amdkfd: Add packet manager module amdkfd: Add module parameter of scheduling policy amdkfd: Add kernel queue module amdkfd: Add mqd_manager module amdkfd: Add queue module amdkfd: Add binding/unbinding calls to amd_iommu driver amdkfd: Add basic modules to amdkfd amdkfd: Add topology module to amdkfd amdkfd: Add amdkfd skeleton driver amdkfd: Add IOCTL set definitions of amdkfd Update MAINTAINERS and CREDITS files with amdkfd info drm/radeon: Add radeon <--> amdkfd interface ...

Dave Airlie committed 11 years ago

Commit cc5ac1c2 parents 8975626 ecd5c9853 files · +9853 −170Browse files

Unified · Split

53 files
  1. CREDITS +7 −0
  2. MAINTAINERS +10 −0
  3. drivers/gpu/drm/Kconfig +2 −0
  4. drivers/gpu/drm/Makefile +1 −0
  5. drivers/gpu/drm/amd/amdkfd/Kconfig +9 −0
  6. drivers/gpu/drm/amd/amdkfd/Makefile +14 −0
  7. drivers/gpu/drm/amd/amdkfd/cik_regs.h +221 −0
  8. drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +576 −0
  9. drivers/gpu/drm/amd/amdkfd/kfd_crat.h +294 −0
  10. drivers/gpu/drm/amd/amdkfd/kfd_device.c +307 −0
  11. drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +1059 −0
  12. drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +146 −0
  13. drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c +255 −0
  14. drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c +355 −0
  15. drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c +176 −0
  16. drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c +347 −0
  17. drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h +69 −0
  18. drivers/gpu/drm/amd/amdkfd/kfd_module.c +159 −0
  19. drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c +346 −0
  20. drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h +91 −0
  21. drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c +565 −0
  22. drivers/gpu/drm/amd/amdkfd/kfd_pasid.c +97 −0
  23. drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h +405 −0
  24. drivers/gpu/drm/amd/amdkfd/kfd_pm4_opcodes.h +107 −0
  25. drivers/gpu/drm/amd/amdkfd/kfd_priv.h +598 −0
  26. drivers/gpu/drm/amd/amdkfd/kfd_process.c +415 −0
  27. drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +342 −0
  28. drivers/gpu/drm/amd/amdkfd/kfd_queue.c +85 −0
  29. drivers/gpu/drm/amd/amdkfd/kfd_topology.c +1235 −0
  30. drivers/gpu/drm/amd/amdkfd/kfd_topology.h +168 −0
  31. drivers/gpu/drm/amd/include/kgd_kfd_interface.h +185 −0
  32. drivers/gpu/drm/radeon/Makefile +1 −0
  33. drivers/gpu/drm/radeon/cik.c +61 −94
  34. drivers/gpu/drm/radeon/cik_reg.h +136 −0
  35. drivers/gpu/drm/radeon/cikd.h +52 −1
  36. drivers/gpu/drm/radeon/radeon.h +10 −0
  37. drivers/gpu/drm/radeon/radeon_device.c +32 −0
  38. drivers/gpu/drm/radeon/radeon_drv.c +5 −0
  39. drivers/gpu/drm/radeon/radeon_kfd.c +563 −0
  40. drivers/gpu/drm/radeon/radeon_kfd.h +47 −0
  41. drivers/gpu/drm/radeon/radeon_kms.c +7 −0
  42. drivers/iommu/amd_iommu_v2.c +11 −58
  43. include/linux/mmu_notifier.h +83 −5
  44. include/uapi/linux/kfd_ioctl.h +154 −0
  45. kernel/events/uprobes.c +1 −1
  46. mm/fremap.c +1 −1
  47. mm/huge_memory.c +5 −4
  48. mm/hugetlb.c +6 −1
  49. mm/ksm.c +2 −2
  50. mm/memory.c +2 −1
  51. mm/migrate.c +2 −1
  52. mm/mmu_notifier.c +25 −0
  53. mm/rmap.c +1 −1
CREDITS+7−0
@@ -1197,6 +1197,13 @@ S: R. Tocantins, 89 - Cristo Rei · 1196 unmodified lines
1197S: 80050-430 - Curitiba - Paraná1197S: 80050-430 - Curitiba - Paraná
1198S: Brazil1198S: Brazil
1199 1199
1200N: Oded Gabbay
1201E: oded.gabbay@gmail.com
1202D: AMD KFD maintainer
1203S: 12 Shraga Raphaeli
1204S: Petah-Tikva, 4906418
1205S: Israel
1206
1200N: Kumar Gala1207N: Kumar Gala
1201E: galak@kernel.crashing.org1208E: galak@kernel.crashing.org
1202D: Embedded PowerPC 6xx/7xx/74xx/82xx/83xx/85xx support1209D: Embedded PowerPC 6xx/7xx/74xx/82xx/83xx/85xx support
MAINTAINERS+10−0
@@ -618,6 +618,16 @@ S: Maintained · 617 unmodified lines
618F: drivers/iommu/amd_iommu*.[ch]618F: drivers/iommu/amd_iommu*.[ch]
619F: include/linux/amd-iommu.h619F: include/linux/amd-iommu.h
620 620
621AMD KFD
622M: Oded Gabbay <oded.gabbay@amd.com>
623L: dri-devel@lists.freedesktop.org
624T: git git://people.freedesktop.org/~gabbayo/linux.git
625S: Supported
626F: drivers/gpu/drm/amd/amdkfd/
627F: drivers/gpu/drm/radeon/radeon_kfd.c
628F: drivers/gpu/drm/radeon/radeon_kfd.h
629F: include/uapi/linux/kfd_ioctl.h
630
621AMD MICROCODE UPDATE SUPPORT631AMD MICROCODE UPDATE SUPPORT
622M: Andreas Herrmann <herrmann.der.user@googlemail.com>632M: Andreas Herrmann <herrmann.der.user@googlemail.com>
623L: amd64-microcode@amd64.org633L: amd64-microcode@amd64.org
drivers/gpu/drm/Kconfig+2−0
@@ -200,3 +200,5 @@ source "drivers/gpu/drm/tegra/Kconfig" · 199 unmodified lines
200source "drivers/gpu/drm/panel/Kconfig"200source "drivers/gpu/drm/panel/Kconfig"
201 201
202source "drivers/gpu/drm/sti/Kconfig"202source "drivers/gpu/drm/sti/Kconfig"
203
204source "drivers/gpu/drm/amd/amdkfd/Kconfig"
drivers/gpu/drm/Makefile+1−0
@@ -65,3 +65,4 @@ obj-$(CONFIG_DRM_STI) += sti/ · 64 unmodified lines
65obj-y += i2c/65obj-y += i2c/
66obj-y += panel/66obj-y += panel/
67obj-y += bridge/67obj-y += bridge/
68obj-$(CONFIG_HSA_AMD) += amd/amdkfd/
drivers/gpu/drm/amd/amdkfd/Kconfig+9−0
@@ -0,0 +1,9 @@
1#
2# Heterogenous system architecture configuration
3#
4
5config HSA_AMD
6 tristate "HSA kernel driver for AMD GPU devices"
7 depends on (DRM_RADEON || DRM_AMDGPU) && AMD_IOMMU_V2 && X86_64
8 help
9 Enable this if you want to use HSA features on AMD GPU devices.
drivers/gpu/drm/amd/amdkfd/Makefile+14−0
@@ -0,0 +1,14 @@
1#
2# Makefile for Heterogenous System Architecture support for AMD GPU devices
3#
4
5ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/amd/include/
6
7amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \
8 kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \
9 kfd_process.o kfd_queue.o kfd_mqd_manager.o \
10 kfd_kernel_queue.o kfd_packet_manager.o \
11 kfd_process_queue_manager.o kfd_device_queue_manager.o \
12 kfd_interrupt.o
13
14obj-$(CONFIG_HSA_AMD) += amdkfd.o
drivers/gpu/drm/amd/amdkfd/cik_regs.h+221−0
@@ -0,0 +1,221 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef CIK_REGS_H
24#define CIK_REGS_H
25
26#define IH_VMID_0_LUT 0x3D40u
27
28#define BIF_DOORBELL_CNTL 0x530Cu
29
30#define SRBM_GFX_CNTL 0xE44
31#define PIPEID(x) ((x) << 0)
32#define MEID(x) ((x) << 2)
33#define VMID(x) ((x) << 4)
34#define QUEUEID(x) ((x) << 8)
35
36#define SQ_CONFIG 0x8C00
37
38#define SH_MEM_BASES 0x8C28
39/* if PTR32, these are the bases for scratch and lds */
40#define PRIVATE_BASE(x) ((x) << 0) /* scratch */
41#define SHARED_BASE(x) ((x) << 16) /* LDS */
42#define SH_MEM_APE1_BASE 0x8C2C
43/* if PTR32, this is the base location of GPUVM */
44#define SH_MEM_APE1_LIMIT 0x8C30
45/* if PTR32, this is the upper limit of GPUVM */
46#define SH_MEM_CONFIG 0x8C34
47#define PTR32 (1 << 0)
48#define PRIVATE_ATC (1 << 1)
49#define ALIGNMENT_MODE(x) ((x) << 2)
50#define SH_MEM_ALIGNMENT_MODE_DWORD 0
51#define SH_MEM_ALIGNMENT_MODE_DWORD_STRICT 1
52#define SH_MEM_ALIGNMENT_MODE_STRICT 2
53#define SH_MEM_ALIGNMENT_MODE_UNALIGNED 3
54#define DEFAULT_MTYPE(x) ((x) << 4)
55#define APE1_MTYPE(x) ((x) << 7)
56
57/* valid for both DEFAULT_MTYPE and APE1_MTYPE */
58#define MTYPE_CACHED 0
59#define MTYPE_NONCACHED 3
60
61
62#define SH_STATIC_MEM_CONFIG 0x9604u
63
64#define TC_CFG_L1_LOAD_POLICY0 0xAC68
65#define TC_CFG_L1_LOAD_POLICY1 0xAC6C
66#define TC_CFG_L1_STORE_POLICY 0xAC70
67#define TC_CFG_L2_LOAD_POLICY0 0xAC74
68#define TC_CFG_L2_LOAD_POLICY1 0xAC78
69#define TC_CFG_L2_STORE_POLICY0 0xAC7C
70#define TC_CFG_L2_STORE_POLICY1 0xAC80
71#define TC_CFG_L2_ATOMIC_POLICY 0xAC84
72#define TC_CFG_L1_VOLATILE 0xAC88
73#define TC_CFG_L2_VOLATILE 0xAC8C
74
75#define CP_PQ_WPTR_POLL_CNTL 0xC20C
76#define WPTR_POLL_EN (1 << 31)
77
78#define CPC_INT_CNTL 0xC2D0
79#define CP_ME1_PIPE0_INT_CNTL 0xC214
80#define CP_ME1_PIPE1_INT_CNTL 0xC218
81#define CP_ME1_PIPE2_INT_CNTL 0xC21C
82#define CP_ME1_PIPE3_INT_CNTL 0xC220
83#define CP_ME2_PIPE0_INT_CNTL 0xC224
84#define CP_ME2_PIPE1_INT_CNTL 0xC228
85#define CP_ME2_PIPE2_INT_CNTL 0xC22C
86#define CP_ME2_PIPE3_INT_CNTL 0xC230
87#define DEQUEUE_REQUEST_INT_ENABLE (1 << 13)
88#define WRM_POLL_TIMEOUT_INT_ENABLE (1 << 17)
89#define PRIV_REG_INT_ENABLE (1 << 23)
90#define TIME_STAMP_INT_ENABLE (1 << 26)
91#define GENERIC2_INT_ENABLE (1 << 29)
92#define GENERIC1_INT_ENABLE (1 << 30)
93#define GENERIC0_INT_ENABLE (1 << 31)
94#define CP_ME1_PIPE0_INT_STATUS 0xC214
95#define CP_ME1_PIPE1_INT_STATUS 0xC218
96#define CP_ME1_PIPE2_INT_STATUS 0xC21C
97#define CP_ME1_PIPE3_INT_STATUS 0xC220
98#define CP_ME2_PIPE0_INT_STATUS 0xC224
99#define CP_ME2_PIPE1_INT_STATUS 0xC228
100#define CP_ME2_PIPE2_INT_STATUS 0xC22C
101#define CP_ME2_PIPE3_INT_STATUS 0xC230
102#define DEQUEUE_REQUEST_INT_STATUS (1 << 13)
103#define WRM_POLL_TIMEOUT_INT_STATUS (1 << 17)
104#define PRIV_REG_INT_STATUS (1 << 23)
105#define TIME_STAMP_INT_STATUS (1 << 26)
106#define GENERIC2_INT_STATUS (1 << 29)
107#define GENERIC1_INT_STATUS (1 << 30)
108#define GENERIC0_INT_STATUS (1 << 31)
109
110#define CP_HPD_EOP_BASE_ADDR 0xC904
111#define CP_HPD_EOP_BASE_ADDR_HI 0xC908
112#define CP_HPD_EOP_VMID 0xC90C
113#define CP_HPD_EOP_CONTROL 0xC910
114#define EOP_SIZE(x) ((x) << 0)
115#define EOP_SIZE_MASK (0x3f << 0)
116#define CP_MQD_BASE_ADDR 0xC914
117#define CP_MQD_BASE_ADDR_HI 0xC918
118#define CP_HQD_ACTIVE 0xC91C
119#define CP_HQD_VMID 0xC920
120
121#define CP_HQD_PERSISTENT_STATE 0xC924u
122#define DEFAULT_CP_HQD_PERSISTENT_STATE (0x33U << 8)
123#define PRELOAD_REQ (1 << 0)
124
125#define CP_HQD_PIPE_PRIORITY 0xC928u
126#define CP_HQD_QUEUE_PRIORITY 0xC92Cu
127#define CP_HQD_QUANTUM 0xC930u
128#define QUANTUM_EN 1U
129#define QUANTUM_SCALE_1MS (1U << 4)
130#define QUANTUM_DURATION(x) ((x) << 8)
131
132#define CP_HQD_PQ_BASE 0xC934
133#define CP_HQD_PQ_BASE_HI 0xC938
134#define CP_HQD_PQ_RPTR 0xC93C
135#define CP_HQD_PQ_RPTR_REPORT_ADDR 0xC940
136#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI 0xC944
137#define CP_HQD_PQ_WPTR_POLL_ADDR 0xC948
138#define CP_HQD_PQ_WPTR_POLL_ADDR_HI 0xC94C
139#define CP_HQD_PQ_DOORBELL_CONTROL 0xC950
140#define DOORBELL_OFFSET(x) ((x) << 2)
141#define DOORBELL_OFFSET_MASK (0x1fffff << 2)
142#define DOORBELL_SOURCE (1 << 28)
143#define DOORBELL_SCHD_HIT (1 << 29)
144#define DOORBELL_EN (1 << 30)
145#define DOORBELL_HIT (1 << 31)
146#define CP_HQD_PQ_WPTR 0xC954
147#define CP_HQD_PQ_CONTROL 0xC958
148#define QUEUE_SIZE(x) ((x) << 0)
149#define QUEUE_SIZE_MASK (0x3f << 0)
150#define RPTR_BLOCK_SIZE(x) ((x) << 8)
151#define RPTR_BLOCK_SIZE_MASK (0x3f << 8)
152#define MIN_AVAIL_SIZE(x) ((x) << 20)
153#define PQ_ATC_EN (1 << 23)
154#define PQ_VOLATILE (1 << 26)
155#define NO_UPDATE_RPTR (1 << 27)
156#define UNORD_DISPATCH (1 << 28)
157#define ROQ_PQ_IB_FLIP (1 << 29)
158#define PRIV_STATE (1 << 30)
159#define KMD_QUEUE (1 << 31)
160
161#define DEFAULT_RPTR_BLOCK_SIZE RPTR_BLOCK_SIZE(5)
162#define DEFAULT_MIN_AVAIL_SIZE MIN_AVAIL_SIZE(3)
163
164#define CP_HQD_IB_BASE_ADDR 0xC95Cu
165#define CP_HQD_IB_BASE_ADDR_HI 0xC960u
166#define CP_HQD_IB_RPTR 0xC964u
167#define CP_HQD_IB_CONTROL 0xC968u
168#define IB_ATC_EN (1U << 23)
169#define DEFAULT_MIN_IB_AVAIL_SIZE (3U << 20)
170
171#define CP_HQD_DEQUEUE_REQUEST 0xC974
172#define DEQUEUE_REQUEST_DRAIN 1
173#define DEQUEUE_REQUEST_RESET 2
174#define DEQUEUE_INT (1U << 8)
175
176#define CP_HQD_SEMA_CMD 0xC97Cu
177#define CP_HQD_MSG_TYPE 0xC980u
178#define CP_HQD_ATOMIC0_PREOP_LO 0xC984u
179#define CP_HQD_ATOMIC0_PREOP_HI 0xC988u
180#define CP_HQD_ATOMIC1_PREOP_LO 0xC98Cu
181#define CP_HQD_ATOMIC1_PREOP_HI 0xC990u
182#define CP_HQD_HQ_SCHEDULER0 0xC994u
183#define CP_HQD_HQ_SCHEDULER1 0xC998u
184
185
186#define CP_MQD_CONTROL 0xC99C
187#define MQD_VMID(x) ((x) << 0)
188#define MQD_VMID_MASK (0xf << 0)
189#define MQD_CONTROL_PRIV_STATE_EN (1U << 8)
190
191#define GRBM_GFX_INDEX 0x30800
192#define INSTANCE_INDEX(x) ((x) << 0)
193#define SH_INDEX(x) ((x) << 8)
194#define SE_INDEX(x) ((x) << 16)
195#define SH_BROADCAST_WRITES (1 << 29)
196#define INSTANCE_BROADCAST_WRITES (1 << 30)
197#define SE_BROADCAST_WRITES (1 << 31)
198
199#define SQC_CACHES 0x30d20
200#define SQC_POLICY 0x8C38u
201#define SQC_VOLATILE 0x8C3Cu
202
203#define CP_PERFMON_CNTL 0x36020
204
205#define ATC_VMID0_PASID_MAPPING 0x339Cu
206#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS 0x3398u
207#define ATC_VMID_PASID_MAPPING_VALID (1U << 31)
208
209#define ATC_VM_APERTURE0_CNTL 0x3310u
210#define ATS_ACCESS_MODE_NEVER 0
211#define ATS_ACCESS_MODE_ALWAYS 1
212
213#define ATC_VM_APERTURE0_CNTL2 0x3318u
214#define ATC_VM_APERTURE0_HIGH_ADDR 0x3308u
215#define ATC_VM_APERTURE0_LOW_ADDR 0x3300u
216#define ATC_VM_APERTURE1_CNTL 0x3314u
217#define ATC_VM_APERTURE1_CNTL2 0x331Cu
218#define ATC_VM_APERTURE1_HIGH_ADDR 0x330Cu
219#define ATC_VM_APERTURE1_LOW_ADDR 0x3304u
220
221#endif
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c+576−0
@@ -0,0 +1,576 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/device.h>
24#include <linux/export.h>
25#include <linux/err.h>
26#include <linux/fs.h>
27#include <linux/sched.h>
28#include <linux/slab.h>
29#include <linux/uaccess.h>
30#include <linux/compat.h>
31#include <uapi/linux/kfd_ioctl.h>
32#include <linux/time.h>
33#include <linux/mm.h>
34#include <linux/uaccess.h>
35#include <uapi/asm-generic/mman-common.h>
36#include <asm/processor.h>
37#include "kfd_priv.h"
38#include "kfd_device_queue_manager.h"
39
40static long kfd_ioctl(struct file *, unsigned int, unsigned long);
41static int kfd_open(struct inode *, struct file *);
42static int kfd_mmap(struct file *, struct vm_area_struct *);
43
44static const char kfd_dev_name[] = "kfd";
45
46static const struct file_operations kfd_fops = {
47 .owner = THIS_MODULE,
48 .unlocked_ioctl = kfd_ioctl,
49 .compat_ioctl = kfd_ioctl,
50 .open = kfd_open,
51 .mmap = kfd_mmap,
52};
53
54static int kfd_char_dev_major = -1;
55static struct class *kfd_class;
56struct device *kfd_device;
57
58int kfd_chardev_init(void)
59{
60 int err = 0;
61
62 kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops);
63 err = kfd_char_dev_major;
64 if (err < 0)
65 goto err_register_chrdev;
66
67 kfd_class = class_create(THIS_MODULE, kfd_dev_name);
68 err = PTR_ERR(kfd_class);
69 if (IS_ERR(kfd_class))
70 goto err_class_create;
71
72 kfd_device = device_create(kfd_class, NULL,
73 MKDEV(kfd_char_dev_major, 0),
74 NULL, kfd_dev_name);
75 err = PTR_ERR(kfd_device);
76 if (IS_ERR(kfd_device))
77 goto err_device_create;
78
79 return 0;
80
81err_device_create:
82 class_destroy(kfd_class);
83err_class_create:
84 unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
85err_register_chrdev:
86 return err;
87}
88
89void kfd_chardev_exit(void)
90{
91 device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0));
92 class_destroy(kfd_class);
93 unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
94}
95
96struct device *kfd_chardev(void)
97{
98 return kfd_device;
99}
100
101
102static int kfd_open(struct inode *inode, struct file *filep)
103{
104 struct kfd_process *process;
105
106 if (iminor(inode) != 0)
107 return -ENODEV;
108
109 process = kfd_create_process(current);
110 if (IS_ERR(process))
111 return PTR_ERR(process);
112
113 process->is_32bit_user_mode = is_compat_task();
114
115 dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
116 process->pasid, process->is_32bit_user_mode);
117
118 kfd_init_apertures(process);
119
120 return 0;
121}
122
123static long kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
124 void __user *arg)
125{
126 struct kfd_ioctl_get_version_args args;
127 int err = 0;
128
129 args.major_version = KFD_IOCTL_MAJOR_VERSION;
130 args.minor_version = KFD_IOCTL_MINOR_VERSION;
131
132 if (copy_to_user(arg, &args, sizeof(args)))
133 err = -EFAULT;
134
135 return err;
136}
137
138static int set_queue_properties_from_user(struct queue_properties *q_properties,
139 struct kfd_ioctl_create_queue_args *args)
140{
141 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
142 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
143 return -EINVAL;
144 }
145
146 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
147 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
148 return -EINVAL;
149 }
150
151 if ((args->ring_base_address) &&
152 (!access_ok(VERIFY_WRITE, args->ring_base_address, sizeof(uint64_t)))) {
153 pr_err("kfd: can't access ring base address\n");
154 return -EFAULT;
155 }
156
157 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
158 pr_err("kfd: ring size must be a power of 2 or 0\n");
159 return -EINVAL;
160 }
161
162 if (!access_ok(VERIFY_WRITE, args->read_pointer_address, sizeof(uint32_t))) {
163 pr_err("kfd: can't access read pointer\n");
164 return -EFAULT;
165 }
166
167 if (!access_ok(VERIFY_WRITE, args->write_pointer_address, sizeof(uint32_t))) {
168 pr_err("kfd: can't access write pointer\n");
169 return -EFAULT;
170 }
171
172 q_properties->is_interop = false;
173 q_properties->queue_percent = args->queue_percentage;
174 q_properties->priority = args->queue_priority;
175 q_properties->queue_address = args->ring_base_address;
176 q_properties->queue_size = args->ring_size;
177 q_properties->read_ptr = (uint32_t *) args->read_pointer_address;
178 q_properties->write_ptr = (uint32_t *) args->write_pointer_address;
179 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
180 args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
181 q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
182 else
183 return -ENOTSUPP;
184
185 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
186 q_properties->format = KFD_QUEUE_FORMAT_AQL;
187 else
188 q_properties->format = KFD_QUEUE_FORMAT_PM4;
189
190 pr_debug("Queue Percentage (%d, %d)\n",
191 q_properties->queue_percent, args->queue_percentage);
192
193 pr_debug("Queue Priority (%d, %d)\n",
194 q_properties->priority, args->queue_priority);
195
196 pr_debug("Queue Address (0x%llX, 0x%llX)\n",
197 q_properties->queue_address, args->ring_base_address);
198
199 pr_debug("Queue Size (0x%llX, %u)\n",
200 q_properties->queue_size, args->ring_size);
201
202 pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n",
203 (uint64_t) q_properties->read_ptr,
204 (uint64_t) q_properties->write_ptr);
205
206 pr_debug("Queue Format (%d)\n", q_properties->format);
207
208 return 0;
209}
210
211static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
212 void __user *arg)
213{
214 struct kfd_ioctl_create_queue_args args;
215 struct kfd_dev *dev;
216 int err = 0;
217 unsigned int queue_id;
218 struct kfd_process_device *pdd;
219 struct queue_properties q_properties;
220
221 memset(&q_properties, 0, sizeof(struct queue_properties));
222
223 if (copy_from_user(&args, arg, sizeof(args)))
224 return -EFAULT;
225
226 pr_debug("kfd: creating queue ioctl\n");
227
228 err = set_queue_properties_from_user(&q_properties, &args);
229 if (err)
230 return err;
231
232 dev = kfd_device_by_id(args.gpu_id);
233 if (dev == NULL)
234 return -EINVAL;
235
236 mutex_lock(&p->mutex);
237
238 pdd = kfd_bind_process_to_device(dev, p);
239 if (IS_ERR(pdd) < 0) {
240 err = PTR_ERR(pdd);
241 goto err_bind_process;
242 }
243
244 pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n",
245 p->pasid,
246 dev->id);
247
248 err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, 0,
249 KFD_QUEUE_TYPE_COMPUTE, &queue_id);
250 if (err != 0)
251 goto err_create_queue;
252
253 args.queue_id = queue_id;
254
255 /* Return gpu_id as doorbell offset for mmap usage */
256 args.doorbell_offset = args.gpu_id << PAGE_SHIFT;
257
258 if (copy_to_user(arg, &args, sizeof(args))) {
259 err = -EFAULT;
260 goto err_copy_args_out;
261 }
262
263 mutex_unlock(&p->mutex);
264
265 pr_debug("kfd: queue id %d was created successfully\n", args.queue_id);
266
267 pr_debug("ring buffer address == 0x%016llX\n",
268 args.ring_base_address);
269
270 pr_debug("read ptr address == 0x%016llX\n",
271 args.read_pointer_address);
272
273 pr_debug("write ptr address == 0x%016llX\n",
274 args.write_pointer_address);
275
276 return 0;
277
278err_copy_args_out:
279 pqm_destroy_queue(&p->pqm, queue_id);
280err_create_queue:
281err_bind_process:
282 mutex_unlock(&p->mutex);
283 return err;
284}
285
286static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
287 void __user *arg)
288{
289 int retval;
290 struct kfd_ioctl_destroy_queue_args args;
291
292 if (copy_from_user(&args, arg, sizeof(args)))
293 return -EFAULT;
294
295 pr_debug("kfd: destroying queue id %d for PASID %d\n",
296 args.queue_id,
297 p->pasid);
298
299 mutex_lock(&p->mutex);
300
301 retval = pqm_destroy_queue(&p->pqm, args.queue_id);
302
303 mutex_unlock(&p->mutex);
304 return retval;
305}
306
307static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
308 void __user *arg)
309{
310 int retval;
311 struct kfd_ioctl_update_queue_args args;
312 struct queue_properties properties;
313
314 if (copy_from_user(&args, arg, sizeof(args)))
315 return -EFAULT;
316
317 if (args.queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
318 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
319 return -EINVAL;
320 }
321
322 if (args.queue_priority > KFD_MAX_QUEUE_PRIORITY) {
323 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
324 return -EINVAL;
325 }
326
327 if ((args.ring_base_address) &&
328 (!access_ok(VERIFY_WRITE, args.ring_base_address, sizeof(uint64_t)))) {
329 pr_err("kfd: can't access ring base address\n");
330 return -EFAULT;
331 }
332
333 if (!is_power_of_2(args.ring_size) && (args.ring_size != 0)) {
334 pr_err("kfd: ring size must be a power of 2 or 0\n");
335 return -EINVAL;
336 }
337
338 properties.queue_address = args.ring_base_address;
339 properties.queue_size = args.ring_size;
340 properties.queue_percent = args.queue_percentage;
341 properties.priority = args.queue_priority;
342
343 pr_debug("kfd: updating queue id %d for PASID %d\n",
344 args.queue_id, p->pasid);
345
346 mutex_lock(&p->mutex);
347
348 retval = pqm_update_queue(&p->pqm, args.queue_id, &properties);
349
350 mutex_unlock(&p->mutex);
351
352 return retval;
353}
354
355static long kfd_ioctl_set_memory_policy(struct file *filep,
356 struct kfd_process *p, void __user *arg)
357{
358 struct kfd_ioctl_set_memory_policy_args args;
359 struct kfd_dev *dev;
360 int err = 0;
361 struct kfd_process_device *pdd;
362 enum cache_policy default_policy, alternate_policy;
363
364 if (copy_from_user(&args, arg, sizeof(args)))
365 return -EFAULT;
366
367 if (args.default_policy != KFD_IOC_CACHE_POLICY_COHERENT
368 && args.default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
369 return -EINVAL;
370 }
371
372 if (args.alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
373 && args.alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
374 return -EINVAL;
375 }
376
377 dev = kfd_device_by_id(args.gpu_id);
378 if (dev == NULL)
379 return -EINVAL;
380
381 mutex_lock(&p->mutex);
382
383 pdd = kfd_bind_process_to_device(dev, p);
384 if (IS_ERR(pdd) < 0) {
385 err = PTR_ERR(pdd);
386 goto out;
387 }
388
389 default_policy = (args.default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
390 ? cache_policy_coherent : cache_policy_noncoherent;
391
392 alternate_policy =
393 (args.alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
394 ? cache_policy_coherent : cache_policy_noncoherent;
395
396 if (!dev->dqm->set_cache_memory_policy(dev->dqm,
397 &pdd->qpd,
398 default_policy,
399 alternate_policy,
400 (void __user *)args.alternate_aperture_base,
401 args.alternate_aperture_size))
402 err = -EINVAL;
403
404out:
405 mutex_unlock(&p->mutex);
406
407 return err;
408}
409
410static long kfd_ioctl_get_clock_counters(struct file *filep,
411 struct kfd_process *p, void __user *arg)
412{
413 struct kfd_ioctl_get_clock_counters_args args;
414 struct kfd_dev *dev;
415 struct timespec time;
416
417 if (copy_from_user(&args, arg, sizeof(args)))
418 return -EFAULT;
419
420 dev = kfd_device_by_id(args.gpu_id);
421 if (dev == NULL)
422 return -EINVAL;
423
424 /* Reading GPU clock counter from KGD */
425 args.gpu_clock_counter = kfd2kgd->get_gpu_clock_counter(dev->kgd);
426
427 /* No access to rdtsc. Using raw monotonic time */
428 getrawmonotonic(&time);
429 args.cpu_clock_counter = (uint64_t)timespec_to_ns(&time);
430
431 get_monotonic_boottime(&time);
432 args.system_clock_counter = (uint64_t)timespec_to_ns(&time);
433
434 /* Since the counter is in nano-seconds we use 1GHz frequency */
435 args.system_clock_freq = 1000000000;
436
437 if (copy_to_user(arg, &args, sizeof(args)))
438 return -EFAULT;
439
440 return 0;
441}
442
443
444static int kfd_ioctl_get_process_apertures(struct file *filp,
445 struct kfd_process *p, void __user *arg)
446{
447 struct kfd_ioctl_get_process_apertures_args args;
448 struct kfd_process_device_apertures *pAperture;
449 struct kfd_process_device *pdd;
450
451 dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid);
452
453 if (copy_from_user(&args, arg, sizeof(args)))
454 return -EFAULT;
455
456 args.num_of_nodes = 0;
457
458 mutex_lock(&p->mutex);
459
460 /*if the process-device list isn't empty*/
461 if (kfd_has_process_device_data(p)) {
462 /* Run over all pdd of the process */
463 pdd = kfd_get_first_process_device_data(p);
464 do {
465 pAperture = &args.process_apertures[args.num_of_nodes];
466 pAperture->gpu_id = pdd->dev->id;
467 pAperture->lds_base = pdd->lds_base;
468 pAperture->lds_limit = pdd->lds_limit;
469 pAperture->gpuvm_base = pdd->gpuvm_base;
470 pAperture->gpuvm_limit = pdd->gpuvm_limit;
471 pAperture->scratch_base = pdd->scratch_base;
472 pAperture->scratch_limit = pdd->scratch_limit;
473
474 dev_dbg(kfd_device,
475 "node id %u\n", args.num_of_nodes);
476 dev_dbg(kfd_device,
477 "gpu id %u\n", pdd->dev->id);
478 dev_dbg(kfd_device,
479 "lds_base %llX\n", pdd->lds_base);
480 dev_dbg(kfd_device,
481 "lds_limit %llX\n", pdd->lds_limit);
482 dev_dbg(kfd_device,
483 "gpuvm_base %llX\n", pdd->gpuvm_base);
484 dev_dbg(kfd_device,
485 "gpuvm_limit %llX\n", pdd->gpuvm_limit);
486 dev_dbg(kfd_device,
487 "scratch_base %llX\n", pdd->scratch_base);
488 dev_dbg(kfd_device,
489 "scratch_limit %llX\n", pdd->scratch_limit);
490
491 args.num_of_nodes++;
492 } while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL &&
493 (args.num_of_nodes < NUM_OF_SUPPORTED_GPUS));
494 }
495
496 mutex_unlock(&p->mutex);
497
498 if (copy_to_user(arg, &args, sizeof(args)))
499 return -EFAULT;
500
501 return 0;
502}
503
504static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
505{
506 struct kfd_process *process;
507 long err = -EINVAL;
508
509 dev_dbg(kfd_device,
510 "ioctl cmd 0x%x (#%d), arg 0x%lx\n",
511 cmd, _IOC_NR(cmd), arg);
512
513 process = kfd_get_process(current);
514 if (IS_ERR(process))
515 return PTR_ERR(process);
516
517 switch (cmd) {
518 case KFD_IOC_GET_VERSION:
519 err = kfd_ioctl_get_version(filep, process, (void __user *)arg);
520 break;
521 case KFD_IOC_CREATE_QUEUE:
522 err = kfd_ioctl_create_queue(filep, process,
523 (void __user *)arg);
524 break;
525
526 case KFD_IOC_DESTROY_QUEUE:
527 err = kfd_ioctl_destroy_queue(filep, process,
528 (void __user *)arg);
529 break;
530
531 case KFD_IOC_SET_MEMORY_POLICY:
532 err = kfd_ioctl_set_memory_policy(filep, process,
533 (void __user *)arg);
534 break;
535
536 case KFD_IOC_GET_CLOCK_COUNTERS:
537 err = kfd_ioctl_get_clock_counters(filep, process,
538 (void __user *)arg);
539 break;
540
541 case KFD_IOC_GET_PROCESS_APERTURES:
542 err = kfd_ioctl_get_process_apertures(filep, process,
543 (void __user *)arg);
544 break;
545
546 case KFD_IOC_UPDATE_QUEUE:
547 err = kfd_ioctl_update_queue(filep, process,
548 (void __user *)arg);
549 break;
550
551 default:
552 dev_err(kfd_device,
553 "unknown ioctl cmd 0x%x, arg 0x%lx)\n",
554 cmd, arg);
555 err = -EINVAL;
556 break;
557 }
558
559 if (err < 0)
560 dev_err(kfd_device,
561 "ioctl error %ld for ioctl cmd 0x%x (#%d)\n",
562 err, cmd, _IOC_NR(cmd));
563
564 return err;
565}
566
567static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
568{
569 struct kfd_process *process;
570
571 process = kfd_get_process(current);
572 if (IS_ERR(process))
573 return PTR_ERR(process);
574
575 return kfd_doorbell_mmap(process, vma);
576}
drivers/gpu/drm/amd/amdkfd/kfd_crat.h+294−0
@@ -0,0 +1,294 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef KFD_CRAT_H_INCLUDED
24#define KFD_CRAT_H_INCLUDED
25
26#include <linux/types.h>
27
28#pragma pack(1)
29
30/*
31 * 4CC signature values for the CRAT and CDIT ACPI tables
32 */
33
34#define CRAT_SIGNATURE "CRAT"
35#define CDIT_SIGNATURE "CDIT"
36
37/*
38 * Component Resource Association Table (CRAT)
39 */
40
41#define CRAT_OEMID_LENGTH 6
42#define CRAT_OEMTABLEID_LENGTH 8
43#define CRAT_RESERVED_LENGTH 6
44
45#define CRAT_OEMID_64BIT_MASK ((1ULL << (CRAT_OEMID_LENGTH * 8)) - 1)
46
47struct crat_header {
48 uint32_t signature;
49 uint32_t length;
50 uint8_t revision;
51 uint8_t checksum;
52 uint8_t oem_id[CRAT_OEMID_LENGTH];
53 uint8_t oem_table_id[CRAT_OEMTABLEID_LENGTH];
54 uint32_t oem_revision;
55 uint32_t creator_id;
56 uint32_t creator_revision;
57 uint32_t total_entries;
58 uint16_t num_domains;
59 uint8_t reserved[CRAT_RESERVED_LENGTH];
60};
61
62/*
63 * The header structure is immediately followed by total_entries of the
64 * data definitions
65 */
66
67/*
68 * The currently defined subtype entries in the CRAT
69 */
70#define CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY 0
71#define CRAT_SUBTYPE_MEMORY_AFFINITY 1
72#define CRAT_SUBTYPE_CACHE_AFFINITY 2
73#define CRAT_SUBTYPE_TLB_AFFINITY 3
74#define CRAT_SUBTYPE_CCOMPUTE_AFFINITY 4
75#define CRAT_SUBTYPE_IOLINK_AFFINITY 5
76#define CRAT_SUBTYPE_MAX 6
77
78#define CRAT_SIBLINGMAP_SIZE 32
79
80/*
81 * ComputeUnit Affinity structure and definitions
82 */
83#define CRAT_CU_FLAGS_ENABLED 0x00000001
84#define CRAT_CU_FLAGS_HOT_PLUGGABLE 0x00000002
85#define CRAT_CU_FLAGS_CPU_PRESENT 0x00000004
86#define CRAT_CU_FLAGS_GPU_PRESENT 0x00000008
87#define CRAT_CU_FLAGS_IOMMU_PRESENT 0x00000010
88#define CRAT_CU_FLAGS_RESERVED 0xffffffe0
89
90#define CRAT_COMPUTEUNIT_RESERVED_LENGTH 4
91
92struct crat_subtype_computeunit {
93 uint8_t type;
94 uint8_t length;
95 uint16_t reserved;
96 uint32_t flags;
97 uint32_t proximity_domain;
98 uint32_t processor_id_low;
99 uint16_t num_cpu_cores;
100 uint16_t num_simd_cores;
101 uint16_t max_waves_simd;
102 uint16_t io_count;
103 uint16_t hsa_capability;
104 uint16_t lds_size_in_kb;
105 uint8_t wave_front_size;
106 uint8_t num_banks;
107 uint16_t micro_engine_id;
108 uint8_t num_arrays;
109 uint8_t num_cu_per_array;
110 uint8_t num_simd_per_cu;
111 uint8_t max_slots_scatch_cu;
112 uint8_t reserved2[CRAT_COMPUTEUNIT_RESERVED_LENGTH];
113};
114
115/*
116 * HSA Memory Affinity structure and definitions
117 */
118#define CRAT_MEM_FLAGS_ENABLED 0x00000001
119#define CRAT_MEM_FLAGS_HOT_PLUGGABLE 0x00000002
120#define CRAT_MEM_FLAGS_NON_VOLATILE 0x00000004
121#define CRAT_MEM_FLAGS_RESERVED 0xfffffff8
122
123#define CRAT_MEMORY_RESERVED_LENGTH 8
124
125struct crat_subtype_memory {
126 uint8_t type;
127 uint8_t length;
128 uint16_t reserved;
129 uint32_t flags;
130 uint32_t promixity_domain;
131 uint32_t base_addr_low;
132 uint32_t base_addr_high;
133 uint32_t length_low;
134 uint32_t length_high;
135 uint32_t width;
136 uint8_t reserved2[CRAT_MEMORY_RESERVED_LENGTH];
137};
138
139/*
140 * HSA Cache Affinity structure and definitions
141 */
142#define CRAT_CACHE_FLAGS_ENABLED 0x00000001
143#define CRAT_CACHE_FLAGS_DATA_CACHE 0x00000002
144#define CRAT_CACHE_FLAGS_INST_CACHE 0x00000004
145#define CRAT_CACHE_FLAGS_CPU_CACHE 0x00000008
146#define CRAT_CACHE_FLAGS_SIMD_CACHE 0x00000010
147#define CRAT_CACHE_FLAGS_RESERVED 0xffffffe0
148
149#define CRAT_CACHE_RESERVED_LENGTH 8
150
151struct crat_subtype_cache {
152 uint8_t type;
153 uint8_t length;
154 uint16_t reserved;
155 uint32_t flags;
156 uint32_t processor_id_low;
157 uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE];
158 uint32_t cache_size;
159 uint8_t cache_level;
160 uint8_t lines_per_tag;
161 uint16_t cache_line_size;
162 uint8_t associativity;
163 uint8_t cache_properties;
164 uint16_t cache_latency;
165 uint8_t reserved2[CRAT_CACHE_RESERVED_LENGTH];
166};
167
168/*
169 * HSA TLB Affinity structure and definitions
170 */
171#define CRAT_TLB_FLAGS_ENABLED 0x00000001
172#define CRAT_TLB_FLAGS_DATA_TLB 0x00000002
173#define CRAT_TLB_FLAGS_INST_TLB 0x00000004
174#define CRAT_TLB_FLAGS_CPU_TLB 0x00000008
175#define CRAT_TLB_FLAGS_SIMD_TLB 0x00000010
176#define CRAT_TLB_FLAGS_RESERVED 0xffffffe0
177
178#define CRAT_TLB_RESERVED_LENGTH 4
179
180struct crat_subtype_tlb {
181 uint8_t type;
182 uint8_t length;
183 uint16_t reserved;
184 uint32_t flags;
185 uint32_t processor_id_low;
186 uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE];
187 uint32_t tlb_level;
188 uint8_t data_tlb_associativity_2mb;
189 uint8_t data_tlb_size_2mb;
190 uint8_t instruction_tlb_associativity_2mb;
191 uint8_t instruction_tlb_size_2mb;
192 uint8_t data_tlb_associativity_4k;
193 uint8_t data_tlb_size_4k;
194 uint8_t instruction_tlb_associativity_4k;
195 uint8_t instruction_tlb_size_4k;
196 uint8_t data_tlb_associativity_1gb;
197 uint8_t data_tlb_size_1gb;
198 uint8_t instruction_tlb_associativity_1gb;
199 uint8_t instruction_tlb_size_1gb;
200 uint8_t reserved2[CRAT_TLB_RESERVED_LENGTH];
201};
202
203/*
204 * HSA CCompute/APU Affinity structure and definitions
205 */
206#define CRAT_CCOMPUTE_FLAGS_ENABLED 0x00000001
207#define CRAT_CCOMPUTE_FLAGS_RESERVED 0xfffffffe
208
209#define CRAT_CCOMPUTE_RESERVED_LENGTH 16
210
211struct crat_subtype_ccompute {
212 uint8_t type;
213 uint8_t length;
214 uint16_t reserved;
215 uint32_t flags;
216 uint32_t processor_id_low;
217 uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE];
218 uint32_t apu_size;
219 uint8_t reserved2[CRAT_CCOMPUTE_RESERVED_LENGTH];
220};
221
222/*
223 * HSA IO Link Affinity structure and definitions
224 */
225#define CRAT_IOLINK_FLAGS_ENABLED 0x00000001
226#define CRAT_IOLINK_FLAGS_COHERENCY 0x00000002
227#define CRAT_IOLINK_FLAGS_RESERVED 0xfffffffc
228
229/*
230 * IO interface types
231 */
232#define CRAT_IOLINK_TYPE_UNDEFINED 0
233#define CRAT_IOLINK_TYPE_HYPERTRANSPORT 1
234#define CRAT_IOLINK_TYPE_PCIEXPRESS 2
235#define CRAT_IOLINK_TYPE_OTHER 3
236#define CRAT_IOLINK_TYPE_MAX 255
237
238#define CRAT_IOLINK_RESERVED_LENGTH 24
239
240struct crat_subtype_iolink {
241 uint8_t type;
242 uint8_t length;
243 uint16_t reserved;
244 uint32_t flags;
245 uint32_t proximity_domain_from;
246 uint32_t proximity_domain_to;
247 uint8_t io_interface_type;
248 uint8_t version_major;
249 uint16_t version_minor;
250 uint32_t minimum_latency;
251 uint32_t maximum_latency;
252 uint32_t minimum_bandwidth_mbs;
253 uint32_t maximum_bandwidth_mbs;
254 uint32_t recommended_transfer_size;
255 uint8_t reserved2[CRAT_IOLINK_RESERVED_LENGTH];
256};
257
258/*
259 * HSA generic sub-type header
260 */
261
262#define CRAT_SUBTYPE_FLAGS_ENABLED 0x00000001
263
264struct crat_subtype_generic {
265 uint8_t type;
266 uint8_t length;
267 uint16_t reserved;
268 uint32_t flags;
269};
270
271/*
272 * Component Locality Distance Information Table (CDIT)
273 */
274#define CDIT_OEMID_LENGTH 6
275#define CDIT_OEMTABLEID_LENGTH 8
276
277struct cdit_header {
278 uint32_t signature;
279 uint32_t length;
280 uint8_t revision;
281 uint8_t checksum;
282 uint8_t oem_id[CDIT_OEMID_LENGTH];
283 uint8_t oem_table_id[CDIT_OEMTABLEID_LENGTH];
284 uint32_t oem_revision;
285 uint32_t creator_id;
286 uint32_t creator_revision;
287 uint32_t total_entries;
288 uint16_t num_domains;
289 uint8_t entry[1];
290};
291
292#pragma pack()
293
294#endif /* KFD_CRAT_H_INCLUDED */
drivers/gpu/drm/amd/amdkfd/kfd_device.c+307−0
@@ -0,0 +1,307 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/amd-iommu.h>
24#include <linux/bsearch.h>
25#include <linux/pci.h>
26#include <linux/slab.h>
27#include "kfd_priv.h"
28#include "kfd_device_queue_manager.h"
29
30#define MQD_SIZE_ALIGNED 768
31
32static const struct kfd_device_info kaveri_device_info = {
33 .max_pasid_bits = 16,
34 .ih_ring_entry_size = 4 * sizeof(uint32_t),
35 .mqd_size_aligned = MQD_SIZE_ALIGNED
36};
37
38struct kfd_deviceid {
39 unsigned short did;
40 const struct kfd_device_info *device_info;
41};
42
43/* Please keep this sorted by increasing device id. */
44static const struct kfd_deviceid supported_devices[] = {
45 { 0x1304, &kaveri_device_info }, /* Kaveri */
46 { 0x1305, &kaveri_device_info }, /* Kaveri */
47 { 0x1306, &kaveri_device_info }, /* Kaveri */
48 { 0x1307, &kaveri_device_info }, /* Kaveri */
49 { 0x1309, &kaveri_device_info }, /* Kaveri */
50 { 0x130A, &kaveri_device_info }, /* Kaveri */
51 { 0x130B, &kaveri_device_info }, /* Kaveri */
52 { 0x130C, &kaveri_device_info }, /* Kaveri */
53 { 0x130D, &kaveri_device_info }, /* Kaveri */
54 { 0x130E, &kaveri_device_info }, /* Kaveri */
55 { 0x130F, &kaveri_device_info }, /* Kaveri */
56 { 0x1310, &kaveri_device_info }, /* Kaveri */
57 { 0x1311, &kaveri_device_info }, /* Kaveri */
58 { 0x1312, &kaveri_device_info }, /* Kaveri */
59 { 0x1313, &kaveri_device_info }, /* Kaveri */
60 { 0x1315, &kaveri_device_info }, /* Kaveri */
61 { 0x1316, &kaveri_device_info }, /* Kaveri */
62 { 0x1317, &kaveri_device_info }, /* Kaveri */
63 { 0x1318, &kaveri_device_info }, /* Kaveri */
64 { 0x131B, &kaveri_device_info }, /* Kaveri */
65 { 0x131C, &kaveri_device_info }, /* Kaveri */
66 { 0x131D, &kaveri_device_info }, /* Kaveri */
67};
68
69static const struct kfd_device_info *lookup_device_info(unsigned short did)
70{
71 size_t i;
72
73 for (i = 0; i < ARRAY_SIZE(supported_devices); i++) {
74 if (supported_devices[i].did == did) {
75 BUG_ON(supported_devices[i].device_info == NULL);
76 return supported_devices[i].device_info;
77 }
78 }
79
80 return NULL;
81}
82
83struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev)
84{
85 struct kfd_dev *kfd;
86
87 const struct kfd_device_info *device_info =
88 lookup_device_info(pdev->device);
89
90 if (!device_info)
91 return NULL;
92
93 kfd = kzalloc(sizeof(*kfd), GFP_KERNEL);
94 if (!kfd)
95 return NULL;
96
97 kfd->kgd = kgd;
98 kfd->device_info = device_info;
99 kfd->pdev = pdev;
100 kfd->init_complete = false;
101
102 return kfd;
103}
104
105static bool device_iommu_pasid_init(struct kfd_dev *kfd)
106{
107 const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
108 AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
109 AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
110
111 struct amd_iommu_device_info iommu_info;
112 unsigned int pasid_limit;
113 int err;
114
115 err = amd_iommu_device_info(kfd->pdev, &iommu_info);
116 if (err < 0) {
117 dev_err(kfd_device,
118 "error getting iommu info. is the iommu enabled?\n");
119 return false;
120 }
121
122 if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
123 dev_err(kfd_device, "error required iommu flags ats(%i), pri(%i), pasid(%i)\n",
124 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
125 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
126 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) != 0);
127 return false;
128 }
129
130 pasid_limit = min_t(unsigned int,
131 (unsigned int)1 << kfd->device_info->max_pasid_bits,
132 iommu_info.max_pasids);
133 /*
134 * last pasid is used for kernel queues doorbells
135 * in the future the last pasid might be used for a kernel thread.
136 */
137 pasid_limit = min_t(unsigned int,
138 pasid_limit,
139 kfd->doorbell_process_limit - 1);
140
141 err = amd_iommu_init_device(kfd->pdev, pasid_limit);
142 if (err < 0) {
143 dev_err(kfd_device, "error initializing iommu device\n");
144 return false;
145 }
146
147 if (!kfd_set_pasid_limit(pasid_limit)) {
148 dev_err(kfd_device, "error setting pasid limit\n");
149 amd_iommu_free_device(kfd->pdev);
150 return false;
151 }
152
153 return true;
154}
155
156static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int pasid)
157{
158 struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
159
160 if (dev)
161 kfd_unbind_process_from_device(dev, pasid);
162}
163
164bool kgd2kfd_device_init(struct kfd_dev *kfd,
165 const struct kgd2kfd_shared_resources *gpu_resources)
166{
167 unsigned int size;
168
169 kfd->shared_resources = *gpu_resources;
170
171 /* calculate max size of mqds needed for queues */
172 size = max_num_of_processes *
173 max_num_of_queues_per_process *
174 kfd->device_info->mqd_size_aligned;
175
176 /* add another 512KB for all other allocations on gart */
177 size += 512 * 1024;
178
179 if (kfd2kgd->init_sa_manager(kfd->kgd, size)) {
180 dev_err(kfd_device,
181 "Error initializing sa manager for device (%x:%x)\n",
182 kfd->pdev->vendor, kfd->pdev->device);
183 goto out;
184 }
185
186 kfd_doorbell_init(kfd);
187
188 if (kfd_topology_add_device(kfd) != 0) {
189 dev_err(kfd_device,
190 "Error adding device (%x:%x) to topology\n",
191 kfd->pdev->vendor, kfd->pdev->device);
192 goto kfd_topology_add_device_error;
193 }
194
195 if (kfd_interrupt_init(kfd)) {
196 dev_err(kfd_device,
197 "Error initializing interrupts for device (%x:%x)\n",
198 kfd->pdev->vendor, kfd->pdev->device);
199 goto kfd_interrupt_error;
200 }
201
202 if (!device_iommu_pasid_init(kfd)) {
203 dev_err(kfd_device,
204 "Error initializing iommuv2 for device (%x:%x)\n",
205 kfd->pdev->vendor, kfd->pdev->device);
206 goto device_iommu_pasid_error;
207 }
208 amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
209 iommu_pasid_shutdown_callback);
210
211 kfd->dqm = device_queue_manager_init(kfd);
212 if (!kfd->dqm) {
213 dev_err(kfd_device,
214 "Error initializing queue manager for device (%x:%x)\n",
215 kfd->pdev->vendor, kfd->pdev->device);
216 goto device_queue_manager_error;
217 }
218
219 if (kfd->dqm->start(kfd->dqm) != 0) {
220 dev_err(kfd_device,
221 "Error starting queuen manager for device (%x:%x)\n",
222 kfd->pdev->vendor, kfd->pdev->device);
223 goto dqm_start_error;
224 }
225
226 kfd->init_complete = true;
227 dev_info(kfd_device, "added device (%x:%x)\n", kfd->pdev->vendor,
228 kfd->pdev->device);
229
230 pr_debug("kfd: Starting kfd with the following scheduling policy %d\n",
231 sched_policy);
232
233 goto out;
234
235dqm_start_error:
236 device_queue_manager_uninit(kfd->dqm);
237device_queue_manager_error:
238 amd_iommu_free_device(kfd->pdev);
239device_iommu_pasid_error:
240 kfd_interrupt_exit(kfd);
241kfd_interrupt_error:
242 kfd_topology_remove_device(kfd);
243kfd_topology_add_device_error:
244 kfd2kgd->fini_sa_manager(kfd->kgd);
245 dev_err(kfd_device,
246 "device (%x:%x) NOT added due to errors\n",
247 kfd->pdev->vendor, kfd->pdev->device);
248out:
249 return kfd->init_complete;
250}
251
252void kgd2kfd_device_exit(struct kfd_dev *kfd)
253{
254 if (kfd->init_complete) {
255 device_queue_manager_uninit(kfd->dqm);
256 amd_iommu_free_device(kfd->pdev);
257 kfd_interrupt_exit(kfd);
258 kfd_topology_remove_device(kfd);
259 }
260
261 kfree(kfd);
262}
263
264void kgd2kfd_suspend(struct kfd_dev *kfd)
265{
266 BUG_ON(kfd == NULL);
267
268 if (kfd->init_complete) {
269 kfd->dqm->stop(kfd->dqm);
270 amd_iommu_free_device(kfd->pdev);
271 }
272}
273
274int kgd2kfd_resume(struct kfd_dev *kfd)
275{
276 unsigned int pasid_limit;
277 int err;
278
279 BUG_ON(kfd == NULL);
280
281 pasid_limit = kfd_get_pasid_limit();
282
283 if (kfd->init_complete) {
284 err = amd_iommu_init_device(kfd->pdev, pasid_limit);
285 if (err < 0)
286 return -ENXIO;
287 amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
288 iommu_pasid_shutdown_callback);
289 kfd->dqm->start(kfd->dqm);
290 }
291
292 return 0;
293}
294
295/* This is called directly from KGD at ISR. */
296void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
297{
298 if (kfd->init_complete) {
299 spin_lock(&kfd->interrupt_lock);
300
301 if (kfd->interrupts_active
302 && enqueue_ih_ring_entry(kfd, ih_ring_entry))
303 schedule_work(&kfd->interrupt_work);
304
305 spin_unlock(&kfd->interrupt_lock);
306 }
307}
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c+1059−0
@@ -0,0 +1,1059 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/slab.h>
25#include <linux/list.h>
26#include <linux/types.h>
27#include <linux/printk.h>
28#include <linux/bitops.h>
29#include "kfd_priv.h"
30#include "kfd_device_queue_manager.h"
31#include "kfd_mqd_manager.h"
32#include "cik_regs.h"
33#include "kfd_kernel_queue.h"
34#include "../../radeon/cik_reg.h"
35
36/* Size of the per-pipe EOP queue */
37#define CIK_HPD_EOP_BYTES_LOG2 11
38#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
39
40static bool is_mem_initialized;
41
42static int init_memory(struct device_queue_manager *dqm);
43static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
44 unsigned int pasid, unsigned int vmid);
45
46static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
47 struct queue *q,
48 struct qcm_process_device *qpd);
49static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock);
50static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock);
51
52
53static inline unsigned int get_pipes_num(struct device_queue_manager *dqm)
54{
55 BUG_ON(!dqm || !dqm->dev);
56 return dqm->dev->shared_resources.compute_pipe_count;
57}
58
59static inline unsigned int get_first_pipe(struct device_queue_manager *dqm)
60{
61 BUG_ON(!dqm);
62 return dqm->dev->shared_resources.first_compute_pipe;
63}
64
65static inline unsigned int get_pipes_num_cpsch(void)
66{
67 return PIPE_PER_ME_CP_SCHEDULING;
68}
69
70static unsigned int get_sh_mem_bases_nybble_64(struct kfd_process *process,
71 struct kfd_dev *dev)
72{
73 struct kfd_process_device *pdd;
74 uint32_t nybble;
75
76 pdd = kfd_get_process_device_data(dev, process, 1);
77 nybble = (pdd->lds_base >> 60) & 0x0E;
78
79 return nybble;
80
81}
82
83static unsigned int get_sh_mem_bases_32(struct kfd_process *process,
84 struct kfd_dev *dev)
85{
86 struct kfd_process_device *pdd;
87 unsigned int shared_base;
88
89 pdd = kfd_get_process_device_data(dev, process, 1);
90 shared_base = (pdd->lds_base >> 16) & 0xFF;
91
92 return shared_base;
93}
94
95static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble);
96static void init_process_memory(struct device_queue_manager *dqm,
97 struct qcm_process_device *qpd)
98{
99 unsigned int temp;
100
101 BUG_ON(!dqm || !qpd);
102
103 /* check if sh_mem_config register already configured */
104 if (qpd->sh_mem_config == 0) {
105 qpd->sh_mem_config =
106 ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) |
107 DEFAULT_MTYPE(MTYPE_NONCACHED) |
108 APE1_MTYPE(MTYPE_NONCACHED);
109 qpd->sh_mem_ape1_limit = 0;
110 qpd->sh_mem_ape1_base = 0;
111 }
112
113 if (qpd->pqm->process->is_32bit_user_mode) {
114 temp = get_sh_mem_bases_32(qpd->pqm->process, dqm->dev);
115 qpd->sh_mem_bases = SHARED_BASE(temp);
116 qpd->sh_mem_config |= PTR32;
117 } else {
118 temp = get_sh_mem_bases_nybble_64(qpd->pqm->process, dqm->dev);
119 qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
120 }
121
122 pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
123 qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
124}
125
126static void program_sh_mem_settings(struct device_queue_manager *dqm,
127 struct qcm_process_device *qpd)
128{
129 return kfd2kgd->program_sh_mem_settings(dqm->dev->kgd, qpd->vmid,
130 qpd->sh_mem_config,
131 qpd->sh_mem_ape1_base,
132 qpd->sh_mem_ape1_limit,
133 qpd->sh_mem_bases);
134}
135
136static int allocate_vmid(struct device_queue_manager *dqm,
137 struct qcm_process_device *qpd,
138 struct queue *q)
139{
140 int bit, allocated_vmid;
141
142 if (dqm->vmid_bitmap == 0)
143 return -ENOMEM;
144
145 bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM);
146 clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
147
148 /* Kaveri kfd vmid's starts from vmid 8 */
149 allocated_vmid = bit + KFD_VMID_START_OFFSET;
150 pr_debug("kfd: vmid allocation %d\n", allocated_vmid);
151 qpd->vmid = allocated_vmid;
152 q->properties.vmid = allocated_vmid;
153
154 set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
155 program_sh_mem_settings(dqm, qpd);
156
157 return 0;
158}
159
160static void deallocate_vmid(struct device_queue_manager *dqm,
161 struct qcm_process_device *qpd,
162 struct queue *q)
163{
164 int bit = qpd->vmid - KFD_VMID_START_OFFSET;
165
166 set_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
167 qpd->vmid = 0;
168 q->properties.vmid = 0;
169}
170
171static int create_queue_nocpsch(struct device_queue_manager *dqm,
172 struct queue *q,
173 struct qcm_process_device *qpd,
174 int *allocated_vmid)
175{
176 int retval;
177
178 BUG_ON(!dqm || !q || !qpd || !allocated_vmid);
179
180 pr_debug("kfd: In func %s\n", __func__);
181 print_queue(q);
182
183 mutex_lock(&dqm->lock);
184
185 if (list_empty(&qpd->queues_list)) {
186 retval = allocate_vmid(dqm, qpd, q);
187 if (retval != 0) {
188 mutex_unlock(&dqm->lock);
189 return retval;
190 }
191 }
192 *allocated_vmid = qpd->vmid;
193 q->properties.vmid = qpd->vmid;
194
195 retval = create_compute_queue_nocpsch(dqm, q, qpd);
196
197 if (retval != 0) {
198 if (list_empty(&qpd->queues_list)) {
199 deallocate_vmid(dqm, qpd, q);
200 *allocated_vmid = 0;
201 }
202 mutex_unlock(&dqm->lock);
203 return retval;
204 }
205
206 list_add(&q->list, &qpd->queues_list);
207 dqm->queue_count++;
208
209 mutex_unlock(&dqm->lock);
210 return 0;
211}
212
213static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
214{
215 bool set;
216 int pipe, bit;
217
218 set = false;
219
220 for (pipe = dqm->next_pipe_to_allocate; pipe < get_pipes_num(dqm);
221 pipe = (pipe + 1) % get_pipes_num(dqm)) {
222 if (dqm->allocated_queues[pipe] != 0) {
223 bit = find_first_bit(
224 (unsigned long *)&dqm->allocated_queues[pipe],
225 QUEUES_PER_PIPE);
226
227 clear_bit(bit,
228 (unsigned long *)&dqm->allocated_queues[pipe]);
229 q->pipe = pipe;
230 q->queue = bit;
231 set = true;
232 break;
233 }
234 }
235
236 if (set == false)
237 return -EBUSY;
238
239 pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n",
240 __func__, q->pipe, q->queue);
241 /* horizontal hqd allocation */
242 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm);
243
244 return 0;
245}
246
247static inline void deallocate_hqd(struct device_queue_manager *dqm,
248 struct queue *q)
249{
250 set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]);
251}
252
253static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
254 struct queue *q,
255 struct qcm_process_device *qpd)
256{
257 int retval;
258 struct mqd_manager *mqd;
259
260 BUG_ON(!dqm || !q || !qpd);
261
262 mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
263 if (mqd == NULL)
264 return -ENOMEM;
265
266 retval = allocate_hqd(dqm, q);
267 if (retval != 0)
268 return retval;
269
270 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
271 &q->gart_mqd_addr, &q->properties);
272 if (retval != 0) {
273 deallocate_hqd(dqm, q);
274 return retval;
275 }
276
277 return 0;
278}
279
280static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
281 struct qcm_process_device *qpd,
282 struct queue *q)
283{
284 int retval;
285 struct mqd_manager *mqd;
286
287 BUG_ON(!dqm || !q || !q->mqd || !qpd);
288
289 retval = 0;
290
291 pr_debug("kfd: In Func %s\n", __func__);
292
293 mutex_lock(&dqm->lock);
294 mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
295 if (mqd == NULL) {
296 retval = -ENOMEM;
297 goto out;
298 }
299
300 retval = mqd->destroy_mqd(mqd, q->mqd,
301 KFD_PREEMPT_TYPE_WAVEFRONT,
302 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
303 q->pipe, q->queue);
304
305 if (retval != 0)
306 goto out;
307
308 deallocate_hqd(dqm, q);
309
310 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
311
312 list_del(&q->list);
313 if (list_empty(&qpd->queues_list))
314 deallocate_vmid(dqm, qpd, q);
315 dqm->queue_count--;
316out:
317 mutex_unlock(&dqm->lock);
318 return retval;
319}
320
321static int update_queue(struct device_queue_manager *dqm, struct queue *q)
322{
323 int retval;
324 struct mqd_manager *mqd;
325
326 BUG_ON(!dqm || !q || !q->mqd);
327
328 mutex_lock(&dqm->lock);
329 mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
330 if (mqd == NULL) {
331 mutex_unlock(&dqm->lock);
332 return -ENOMEM;
333 }
334
335 retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
336 if (q->properties.is_active == true)
337 dqm->queue_count++;
338 else
339 dqm->queue_count--;
340
341 if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
342 retval = execute_queues_cpsch(dqm, false);
343
344 mutex_unlock(&dqm->lock);
345 return retval;
346}
347
348static struct mqd_manager *get_mqd_manager_nocpsch(
349 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
350{
351 struct mqd_manager *mqd;
352
353 BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX);
354
355 pr_debug("kfd: In func %s mqd type %d\n", __func__, type);
356
357 mqd = dqm->mqds[type];
358 if (!mqd) {
359 mqd = mqd_manager_init(type, dqm->dev);
360 if (mqd == NULL)
361 pr_err("kfd: mqd manager is NULL");
362 dqm->mqds[type] = mqd;
363 }
364
365 return mqd;
366}
367
368static int register_process_nocpsch(struct device_queue_manager *dqm,
369 struct qcm_process_device *qpd)
370{
371 struct device_process_node *n;
372
373 BUG_ON(!dqm || !qpd);
374
375 pr_debug("kfd: In func %s\n", __func__);
376
377 n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL);
378 if (!n)
379 return -ENOMEM;
380
381 n->qpd = qpd;
382
383 mutex_lock(&dqm->lock);
384 list_add(&n->list, &dqm->queues);
385
386 init_process_memory(dqm, qpd);
387 dqm->processes_count++;
388
389 mutex_unlock(&dqm->lock);
390
391 return 0;
392}
393
394static int unregister_process_nocpsch(struct device_queue_manager *dqm,
395 struct qcm_process_device *qpd)
396{
397 int retval;
398 struct device_process_node *cur, *next;
399
400 BUG_ON(!dqm || !qpd);
401
402 BUG_ON(!list_empty(&qpd->queues_list));
403
404 pr_debug("kfd: In func %s\n", __func__);
405
406 retval = 0;
407 mutex_lock(&dqm->lock);
408
409 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
410 if (qpd == cur->qpd) {
411 list_del(&cur->list);
412 dqm->processes_count--;
413 goto out;
414 }
415 }
416 /* qpd not found in dqm list */
417 retval = 1;
418out:
419 mutex_unlock(&dqm->lock);
420 return retval;
421}
422
423static int
424set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
425 unsigned int vmid)
426{
427 uint32_t pasid_mapping;
428
429 pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
430 ATC_VMID_PASID_MAPPING_VALID;
431 return kfd2kgd->set_pasid_vmid_mapping(dqm->dev->kgd, pasid_mapping,
432 vmid);
433}
434
435static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
436{
437 /* In 64-bit mode, we can only control the top 3 bits of the LDS,
438 * scratch and GPUVM apertures.
439 * The hardware fills in the remaining 59 bits according to the
440 * following pattern:
441 * LDS: X0000000'00000000 - X0000001'00000000 (4GB)
442 * Scratch: X0000001'00000000 - X0000002'00000000 (4GB)
443 * GPUVM: Y0010000'00000000 - Y0020000'00000000 (1TB)
444 *
445 * (where X/Y is the configurable nybble with the low-bit 0)
446 *
447 * LDS and scratch will have the same top nybble programmed in the
448 * top 3 bits of SH_MEM_BASES.PRIVATE_BASE.
449 * GPUVM can have a different top nybble programmed in the
450 * top 3 bits of SH_MEM_BASES.SHARED_BASE.
451 * We don't bother to support different top nybbles
452 * for LDS/Scratch and GPUVM.
453 */
454
455 BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
456 top_address_nybble == 0);
457
458 return PRIVATE_BASE(top_address_nybble << 12) |
459 SHARED_BASE(top_address_nybble << 12);
460}
461
462static int init_memory(struct device_queue_manager *dqm)
463{
464 int i, retval;
465
466 for (i = 8; i < 16; i++)
467 set_pasid_vmid_mapping(dqm, 0, i);
468
469 retval = kfd2kgd->init_memory(dqm->dev->kgd);
470 if (retval == 0)
471 is_mem_initialized = true;
472 return retval;
473}
474
475
476static int init_pipelines(struct device_queue_manager *dqm,
477 unsigned int pipes_num, unsigned int first_pipe)
478{
479 void *hpdptr;
480 struct mqd_manager *mqd;
481 unsigned int i, err, inx;
482 uint64_t pipe_hpd_addr;
483
484 BUG_ON(!dqm || !dqm->dev);
485
486 pr_debug("kfd: In func %s\n", __func__);
487
488 /*
489 * Allocate memory for the HPDs. This is hardware-owned per-pipe data.
490 * The driver never accesses this memory after zeroing it.
491 * It doesn't even have to be saved/restored on suspend/resume
492 * because it contains no data when there are no active queues.
493 */
494
495 err = kfd2kgd->allocate_mem(dqm->dev->kgd,
496 CIK_HPD_EOP_BYTES * pipes_num,
497 PAGE_SIZE,
498 KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
499 (struct kgd_mem **) &dqm->pipeline_mem);
500
501 if (err) {
502 pr_err("kfd: error allocate vidmem num pipes: %d\n",
503 pipes_num);
504 return -ENOMEM;
505 }
506
507 hpdptr = dqm->pipeline_mem->cpu_ptr;
508 dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr;
509
510 memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num);
511
512 mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
513 if (mqd == NULL) {
514 kfd2kgd->free_mem(dqm->dev->kgd,
515 (struct kgd_mem *) dqm->pipeline_mem);
516 return -ENOMEM;
517 }
518
519 for (i = 0; i < pipes_num; i++) {
520 inx = i + first_pipe;
521 pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES;
522 pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr);
523 /* = log2(bytes/4)-1 */
524 kfd2kgd->init_pipeline(dqm->dev->kgd, i,
525 CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr);
526 }
527
528 return 0;
529}
530
531
532static int init_scheduler(struct device_queue_manager *dqm)
533{
534 int retval;
535
536 BUG_ON(!dqm);
537
538 pr_debug("kfd: In %s\n", __func__);
539
540 retval = init_pipelines(dqm, get_pipes_num(dqm), KFD_DQM_FIRST_PIPE);
541 if (retval != 0)
542 return retval;
543
544 retval = init_memory(dqm);
545
546 return retval;
547}
548
549static int initialize_nocpsch(struct device_queue_manager *dqm)
550{
551 int i;
552
553 BUG_ON(!dqm);
554
555 pr_debug("kfd: In func %s num of pipes: %d\n",
556 __func__, get_pipes_num(dqm));
557
558 mutex_init(&dqm->lock);
559 INIT_LIST_HEAD(&dqm->queues);
560 dqm->queue_count = dqm->next_pipe_to_allocate = 0;
561 dqm->allocated_queues = kcalloc(get_pipes_num(dqm),
562 sizeof(unsigned int), GFP_KERNEL);
563 if (!dqm->allocated_queues) {
564 mutex_destroy(&dqm->lock);
565 return -ENOMEM;
566 }
567
568 for (i = 0; i < get_pipes_num(dqm); i++)
569 dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1;
570
571 dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1;
572
573 init_scheduler(dqm);
574 return 0;
575}
576
577static void uninitialize_nocpsch(struct device_queue_manager *dqm)
578{
579 BUG_ON(!dqm);
580
581 BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
582
583 kfree(dqm->allocated_queues);
584 mutex_destroy(&dqm->lock);
585 kfd2kgd->free_mem(dqm->dev->kgd,
586 (struct kgd_mem *) dqm->pipeline_mem);
587}
588
589static int start_nocpsch(struct device_queue_manager *dqm)
590{
591 return 0;
592}
593
594static int stop_nocpsch(struct device_queue_manager *dqm)
595{
596 return 0;
597}
598
599/*
600 * Device Queue Manager implementation for cp scheduler
601 */
602
603static int set_sched_resources(struct device_queue_manager *dqm)
604{
605 struct scheduling_resources res;
606 unsigned int queue_num, queue_mask;
607
608 BUG_ON(!dqm);
609
610 pr_debug("kfd: In func %s\n", __func__);
611
612 queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE;
613 queue_mask = (1 << queue_num) - 1;
614 res.vmid_mask = (1 << VMID_PER_DEVICE) - 1;
615 res.vmid_mask <<= KFD_VMID_START_OFFSET;
616 res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE);
617 res.gws_mask = res.oac_mask = res.gds_heap_base =
618 res.gds_heap_size = 0;
619
620 pr_debug("kfd: scheduling resources:\n"
621 " vmid mask: 0x%8X\n"
622 " queue mask: 0x%8llX\n",
623 res.vmid_mask, res.queue_mask);
624
625 return pm_send_set_resources(&dqm->packets, &res);
626}
627
628static int initialize_cpsch(struct device_queue_manager *dqm)
629{
630 int retval;
631
632 BUG_ON(!dqm);
633
634 pr_debug("kfd: In func %s num of pipes: %d\n",
635 __func__, get_pipes_num_cpsch());
636
637 mutex_init(&dqm->lock);
638 INIT_LIST_HEAD(&dqm->queues);
639 dqm->queue_count = dqm->processes_count = 0;
640 dqm->active_runlist = false;
641 retval = init_pipelines(dqm, get_pipes_num(dqm), 0);
642 if (retval != 0)
643 goto fail_init_pipelines;
644
645 return 0;
646
647fail_init_pipelines:
648 mutex_destroy(&dqm->lock);
649 return retval;
650}
651
652static int start_cpsch(struct device_queue_manager *dqm)
653{
654 struct device_process_node *node;
655 int retval;
656
657 BUG_ON(!dqm);
658
659 retval = 0;
660
661 retval = pm_init(&dqm->packets, dqm);
662 if (retval != 0)
663 goto fail_packet_manager_init;
664
665 retval = set_sched_resources(dqm);
666 if (retval != 0)
667 goto fail_set_sched_resources;
668
669 pr_debug("kfd: allocating fence memory\n");
670
671 /* allocate fence memory on the gart */
672 retval = kfd2kgd->allocate_mem(dqm->dev->kgd,
673 sizeof(*dqm->fence_addr),
674 32,
675 KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
676 (struct kgd_mem **) &dqm->fence_mem);
677
678 if (retval != 0)
679 goto fail_allocate_vidmem;
680
681 dqm->fence_addr = dqm->fence_mem->cpu_ptr;
682 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
683
684 list_for_each_entry(node, &dqm->queues, list)
685 if (node->qpd->pqm->process && dqm->dev)
686 kfd_bind_process_to_device(dqm->dev,
687 node->qpd->pqm->process);
688
689 execute_queues_cpsch(dqm, true);
690
691 return 0;
692fail_allocate_vidmem:
693fail_set_sched_resources:
694 pm_uninit(&dqm->packets);
695fail_packet_manager_init:
696 return retval;
697}
698
699static int stop_cpsch(struct device_queue_manager *dqm)
700{
701 struct device_process_node *node;
702 struct kfd_process_device *pdd;
703
704 BUG_ON(!dqm);
705
706 destroy_queues_cpsch(dqm, true);
707
708 list_for_each_entry(node, &dqm->queues, list) {
709 pdd = kfd_get_process_device_data(dqm->dev,
710 node->qpd->pqm->process, 1);
711 pdd->bound = false;
712 }
713 kfd2kgd->free_mem(dqm->dev->kgd,
714 (struct kgd_mem *) dqm->fence_mem);
715 pm_uninit(&dqm->packets);
716
717 return 0;
718}
719
720static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
721 struct kernel_queue *kq,
722 struct qcm_process_device *qpd)
723{
724 BUG_ON(!dqm || !kq || !qpd);
725
726 pr_debug("kfd: In func %s\n", __func__);
727
728 mutex_lock(&dqm->lock);
729 list_add(&kq->list, &qpd->priv_queue_list);
730 dqm->queue_count++;
731 qpd->is_debug = true;
732 execute_queues_cpsch(dqm, false);
733 mutex_unlock(&dqm->lock);
734
735 return 0;
736}
737
738static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
739 struct kernel_queue *kq,
740 struct qcm_process_device *qpd)
741{
742 BUG_ON(!dqm || !kq);
743
744 pr_debug("kfd: In %s\n", __func__);
745
746 mutex_lock(&dqm->lock);
747 destroy_queues_cpsch(dqm, false);
748 list_del(&kq->list);
749 dqm->queue_count--;
750 qpd->is_debug = false;
751 execute_queues_cpsch(dqm, false);
752 mutex_unlock(&dqm->lock);
753}
754
755static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
756 struct qcm_process_device *qpd, int *allocate_vmid)
757{
758 int retval;
759 struct mqd_manager *mqd;
760
761 BUG_ON(!dqm || !q || !qpd);
762
763 retval = 0;
764
765 if (allocate_vmid)
766 *allocate_vmid = 0;
767
768 mutex_lock(&dqm->lock);
769
770 mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_CP);
771 if (mqd == NULL) {
772 mutex_unlock(&dqm->lock);
773 return -ENOMEM;
774 }
775
776 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
777 &q->gart_mqd_addr, &q->properties);
778 if (retval != 0)
779 goto out;
780
781 list_add(&q->list, &qpd->queues_list);
782 if (q->properties.is_active) {
783 dqm->queue_count++;
784 retval = execute_queues_cpsch(dqm, false);
785 }
786
787out:
788 mutex_unlock(&dqm->lock);
789 return retval;
790}
791
792int fence_wait_timeout(unsigned int *fence_addr, unsigned int fence_value,
793 unsigned long timeout)
794{
795 BUG_ON(!fence_addr);
796 timeout += jiffies;
797
798 while (*fence_addr != fence_value) {
799 if (time_after(jiffies, timeout)) {
800 pr_err("kfd: qcm fence wait loop timeout expired\n");
801 return -ETIME;
802 }
803 cpu_relax();
804 }
805
806 return 0;
807}
808
809static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock)
810{
811 int retval;
812
813 BUG_ON(!dqm);
814
815 retval = 0;
816
817 if (lock)
818 mutex_lock(&dqm->lock);
819 if (dqm->active_runlist == false)
820 goto out;
821 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
822 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0);
823 if (retval != 0)
824 goto out;
825
826 *dqm->fence_addr = KFD_FENCE_INIT;
827 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
828 KFD_FENCE_COMPLETED);
829 /* should be timed out */
830 fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
831 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
832 pm_release_ib(&dqm->packets);
833 dqm->active_runlist = false;
834
835out:
836 if (lock)
837 mutex_unlock(&dqm->lock);
838 return retval;
839}
840
841static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
842{
843 int retval;
844
845 BUG_ON(!dqm);
846
847 if (lock)
848 mutex_lock(&dqm->lock);
849
850 retval = destroy_queues_cpsch(dqm, false);
851 if (retval != 0) {
852 pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption");
853 goto out;
854 }
855
856 if (dqm->queue_count <= 0 || dqm->processes_count <= 0) {
857 retval = 0;
858 goto out;
859 }
860
861 if (dqm->active_runlist) {
862 retval = 0;
863 goto out;
864 }
865
866 retval = pm_send_runlist(&dqm->packets, &dqm->queues);
867 if (retval != 0) {
868 pr_err("kfd: failed to execute runlist");
869 goto out;
870 }
871 dqm->active_runlist = true;
872
873out:
874 if (lock)
875 mutex_unlock(&dqm->lock);
876 return retval;
877}
878
879static int destroy_queue_cpsch(struct device_queue_manager *dqm,
880 struct qcm_process_device *qpd,
881 struct queue *q)
882{
883 int retval;
884 struct mqd_manager *mqd;
885
886 BUG_ON(!dqm || !qpd || !q);
887
888 retval = 0;
889
890 /* remove queue from list to prevent rescheduling after preemption */
891 mutex_lock(&dqm->lock);
892
893 mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_CP);
894 if (!mqd) {
895 retval = -ENOMEM;
896 goto failed;
897 }
898
899 list_del(&q->list);
900 dqm->queue_count--;
901
902 execute_queues_cpsch(dqm, false);
903
904 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
905
906 mutex_unlock(&dqm->lock);
907
908 return 0;
909
910failed:
911 mutex_unlock(&dqm->lock);
912 return retval;
913}
914
915/*
916 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
917 * stay in user mode.
918 */
919#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
920/* APE1 limit is inclusive and 64K aligned. */
921#define APE1_LIMIT_ALIGNMENT 0xFFFF
922
923static bool set_cache_memory_policy(struct device_queue_manager *dqm,
924 struct qcm_process_device *qpd,
925 enum cache_policy default_policy,
926 enum cache_policy alternate_policy,
927 void __user *alternate_aperture_base,
928 uint64_t alternate_aperture_size)
929{
930 uint32_t default_mtype;
931 uint32_t ape1_mtype;
932
933 pr_debug("kfd: In func %s\n", __func__);
934
935 mutex_lock(&dqm->lock);
936
937 if (alternate_aperture_size == 0) {
938 /* base > limit disables APE1 */
939 qpd->sh_mem_ape1_base = 1;
940 qpd->sh_mem_ape1_limit = 0;
941 } else {
942 /*
943 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
944 * SH_MEM_APE1_BASE[31:0], 0x0000 }
945 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
946 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
947 * Verify that the base and size parameters can be
948 * represented in this format and convert them.
949 * Additionally restrict APE1 to user-mode addresses.
950 */
951
952 uint64_t base = (uintptr_t)alternate_aperture_base;
953 uint64_t limit = base + alternate_aperture_size - 1;
954
955 if (limit <= base)
956 goto out;
957
958 if ((base & APE1_FIXED_BITS_MASK) != 0)
959 goto out;
960
961 if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT)
962 goto out;
963
964 qpd->sh_mem_ape1_base = base >> 16;
965 qpd->sh_mem_ape1_limit = limit >> 16;
966 }
967
968 default_mtype = (default_policy == cache_policy_coherent) ?
969 MTYPE_NONCACHED :
970 MTYPE_CACHED;
971
972 ape1_mtype = (alternate_policy == cache_policy_coherent) ?
973 MTYPE_NONCACHED :
974 MTYPE_CACHED;
975
976 qpd->sh_mem_config = (qpd->sh_mem_config & PTR32)
977 | ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
978 | DEFAULT_MTYPE(default_mtype)
979 | APE1_MTYPE(ape1_mtype);
980
981 if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
982 program_sh_mem_settings(dqm, qpd);
983
984 pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
985 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
986 qpd->sh_mem_ape1_limit);
987
988 mutex_unlock(&dqm->lock);
989 return true;
990
991out:
992 mutex_unlock(&dqm->lock);
993 return false;
994}
995
996struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
997{
998 struct device_queue_manager *dqm;
999
1000 BUG_ON(!dev);
1001
1002 dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL);
1003 if (!dqm)
1004 return NULL;
1005
1006 dqm->dev = dev;
1007 switch (sched_policy) {
1008 case KFD_SCHED_POLICY_HWS:
1009 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1010 /* initialize dqm for cp scheduling */
1011 dqm->create_queue = create_queue_cpsch;
1012 dqm->initialize = initialize_cpsch;
1013 dqm->start = start_cpsch;
1014 dqm->stop = stop_cpsch;
1015 dqm->destroy_queue = destroy_queue_cpsch;
1016 dqm->update_queue = update_queue;
1017 dqm->get_mqd_manager = get_mqd_manager_nocpsch;
1018 dqm->register_process = register_process_nocpsch;
1019 dqm->unregister_process = unregister_process_nocpsch;
1020 dqm->uninitialize = uninitialize_nocpsch;
1021 dqm->create_kernel_queue = create_kernel_queue_cpsch;
1022 dqm->destroy_kernel_queue = destroy_kernel_queue_cpsch;
1023 dqm->set_cache_memory_policy = set_cache_memory_policy;
1024 break;
1025 case KFD_SCHED_POLICY_NO_HWS:
1026 /* initialize dqm for no cp scheduling */
1027 dqm->start = start_nocpsch;
1028 dqm->stop = stop_nocpsch;
1029 dqm->create_queue = create_queue_nocpsch;
1030 dqm->destroy_queue = destroy_queue_nocpsch;
1031 dqm->update_queue = update_queue;
1032 dqm->get_mqd_manager = get_mqd_manager_nocpsch;
1033 dqm->register_process = register_process_nocpsch;
1034 dqm->unregister_process = unregister_process_nocpsch;
1035 dqm->initialize = initialize_nocpsch;
1036 dqm->uninitialize = uninitialize_nocpsch;
1037 dqm->set_cache_memory_policy = set_cache_memory_policy;
1038 break;
1039 default:
1040 BUG();
1041 break;
1042 }
1043
1044 if (dqm->initialize(dqm) != 0) {
1045 kfree(dqm);
1046 return NULL;
1047 }
1048
1049 return dqm;
1050}
1051
1052void device_queue_manager_uninit(struct device_queue_manager *dqm)
1053{
1054 BUG_ON(!dqm);
1055
1056 dqm->uninitialize(dqm);
1057 kfree(dqm);
1058}
1059
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h+146−0
@@ -0,0 +1,146 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#ifndef KFD_DEVICE_QUEUE_MANAGER_H_
25#define KFD_DEVICE_QUEUE_MANAGER_H_
26
27#include <linux/rwsem.h>
28#include <linux/list.h>
29#include "kfd_priv.h"
30#include "kfd_mqd_manager.h"
31
32#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (500)
33#define QUEUES_PER_PIPE (8)
34#define PIPE_PER_ME_CP_SCHEDULING (3)
35#define CIK_VMID_NUM (8)
36#define KFD_VMID_START_OFFSET (8)
37#define VMID_PER_DEVICE CIK_VMID_NUM
38#define KFD_DQM_FIRST_PIPE (0)
39
40struct device_process_node {
41 struct qcm_process_device *qpd;
42 struct list_head list;
43};
44
45/**
46 * struct device_queue_manager
47 *
48 * @create_queue: Queue creation routine.
49 *
50 * @destroy_queue: Queue destruction routine.
51 *
52 * @update_queue: Queue update routine.
53 *
54 * @get_mqd_manager: Returns the mqd manager according to the mqd type.
55 *
56 * @exeute_queues: Dispatches the queues list to the H/W.
57 *
58 * @register_process: This routine associates a specific process with device.
59 *
60 * @unregister_process: destroys the associations between process to device.
61 *
62 * @initialize: Initializes the pipelines and memory module for that device.
63 *
64 * @start: Initializes the resources/modules the the device needs for queues
65 * execution. This function is called on device initialization and after the
66 * system woke up after suspension.
67 *
68 * @stop: This routine stops execution of all the active queue running on the
69 * H/W and basically this function called on system suspend.
70 *
71 * @uninitialize: Destroys all the device queue manager resources allocated in
72 * initialize routine.
73 *
74 * @create_kernel_queue: Creates kernel queue. Used for debug queue.
75 *
76 * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
77 *
78 * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
79 * memory apertures.
80 *
81 * This struct is a base class for the kfd queues scheduler in the
82 * device level. The device base class should expose the basic operations
83 * for queue creation and queue destruction. This base class hides the
84 * scheduling mode of the driver and the specific implementation of the
85 * concrete device. This class is the only class in the queues scheduler
86 * that configures the H/W.
87 */
88
89struct device_queue_manager {
90 int (*create_queue)(struct device_queue_manager *dqm,
91 struct queue *q,
92 struct qcm_process_device *qpd,
93 int *allocate_vmid);
94 int (*destroy_queue)(struct device_queue_manager *dqm,
95 struct qcm_process_device *qpd,
96 struct queue *q);
97 int (*update_queue)(struct device_queue_manager *dqm,
98 struct queue *q);
99
100 struct mqd_manager * (*get_mqd_manager)
101 (struct device_queue_manager *dqm,
102 enum KFD_MQD_TYPE type);
103
104 int (*register_process)(struct device_queue_manager *dqm,
105 struct qcm_process_device *qpd);
106 int (*unregister_process)(struct device_queue_manager *dqm,
107 struct qcm_process_device *qpd);
108 int (*initialize)(struct device_queue_manager *dqm);
109 int (*start)(struct device_queue_manager *dqm);
110 int (*stop)(struct device_queue_manager *dqm);
111 void (*uninitialize)(struct device_queue_manager *dqm);
112 int (*create_kernel_queue)(struct device_queue_manager *dqm,
113 struct kernel_queue *kq,
114 struct qcm_process_device *qpd);
115 void (*destroy_kernel_queue)(struct device_queue_manager *dqm,
116 struct kernel_queue *kq,
117 struct qcm_process_device *qpd);
118 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
119 struct qcm_process_device *qpd,
120 enum cache_policy default_policy,
121 enum cache_policy alternate_policy,
122 void __user *alternate_aperture_base,
123 uint64_t alternate_aperture_size);
124
125
126 struct mqd_manager *mqds[KFD_MQD_TYPE_MAX];
127 struct packet_manager packets;
128 struct kfd_dev *dev;
129 struct mutex lock;
130 struct list_head queues;
131 unsigned int processes_count;
132 unsigned int queue_count;
133 unsigned int next_pipe_to_allocate;
134 unsigned int *allocated_queues;
135 unsigned int vmid_bitmap;
136 uint64_t pipelines_addr;
137 struct kfd_mem_obj *pipeline_mem;
138 uint64_t fence_gpu_addr;
139 unsigned int *fence_addr;
140 struct kfd_mem_obj *fence_mem;
141 bool active_runlist;
142};
143
144
145
146#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */
drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c+255−0
@@ -0,0 +1,255 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22#include "kfd_priv.h"
23#include <linux/mm.h>
24#include <linux/mman.h>
25#include <linux/slab.h>
26
27/*
28 * This extension supports a kernel level doorbells management for
29 * the kernel queues.
30 * Basically the last doorbells page is devoted to kernel queues
31 * and that's assures that any user process won't get access to the
32 * kernel doorbells page
33 */
34static DEFINE_MUTEX(doorbell_mutex);
35static unsigned long doorbell_available_index[
36 DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)] = { 0 };
37
38#define KERNEL_DOORBELL_PASID 1
39#define KFD_SIZE_OF_DOORBELL_IN_BYTES 4
40
41/*
42 * Each device exposes a doorbell aperture, a PCI MMIO aperture that
43 * receives 32-bit writes that are passed to queues as wptr values.
44 * The doorbells are intended to be written by applications as part
45 * of queueing work on user-mode queues.
46 * We assign doorbells to applications in PAGE_SIZE-sized and aligned chunks.
47 * We map the doorbell address space into user-mode when a process creates
48 * its first queue on each device.
49 * Although the mapping is done by KFD, it is equivalent to an mmap of
50 * the /dev/kfd with the particular device encoded in the mmap offset.
51 * There will be other uses for mmap of /dev/kfd, so only a range of
52 * offsets (KFD_MMAP_DOORBELL_START-END) is used for doorbells.
53 */
54
55/* # of doorbell bytes allocated for each process. */
56static inline size_t doorbell_process_allocation(void)
57{
58 return roundup(KFD_SIZE_OF_DOORBELL_IN_BYTES *
59 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
60 PAGE_SIZE);
61}
62
63/* Doorbell calculations for device init. */
64void kfd_doorbell_init(struct kfd_dev *kfd)
65{
66 size_t doorbell_start_offset;
67 size_t doorbell_aperture_size;
68 size_t doorbell_process_limit;
69
70 /*
71 * We start with calculations in bytes because the input data might
72 * only be byte-aligned.
73 * Only after we have done the rounding can we assume any alignment.
74 */
75
76 doorbell_start_offset =
77 roundup(kfd->shared_resources.doorbell_start_offset,
78 doorbell_process_allocation());
79
80 doorbell_aperture_size =
81 rounddown(kfd->shared_resources.doorbell_aperture_size,
82 doorbell_process_allocation());
83
84 if (doorbell_aperture_size > doorbell_start_offset)
85 doorbell_process_limit =
86 (doorbell_aperture_size - doorbell_start_offset) /
87 doorbell_process_allocation();
88 else
89 doorbell_process_limit = 0;
90
91 kfd->doorbell_base = kfd->shared_resources.doorbell_physical_address +
92 doorbell_start_offset;
93
94 kfd->doorbell_id_offset = doorbell_start_offset / sizeof(u32);
95 kfd->doorbell_process_limit = doorbell_process_limit - 1;
96
97 kfd->doorbell_kernel_ptr = ioremap(kfd->doorbell_base,
98 doorbell_process_allocation());
99
100 BUG_ON(!kfd->doorbell_kernel_ptr);
101
102 pr_debug("kfd: doorbell initialization:\n");
103 pr_debug("kfd: doorbell base == 0x%08lX\n",
104 (uintptr_t)kfd->doorbell_base);
105
106 pr_debug("kfd: doorbell_id_offset == 0x%08lX\n",
107 kfd->doorbell_id_offset);
108
109 pr_debug("kfd: doorbell_process_limit == 0x%08lX\n",
110 doorbell_process_limit);
111
112 pr_debug("kfd: doorbell_kernel_offset == 0x%08lX\n",
113 (uintptr_t)kfd->doorbell_base);
114
115 pr_debug("kfd: doorbell aperture size == 0x%08lX\n",
116 kfd->shared_resources.doorbell_aperture_size);
117
118 pr_debug("kfd: doorbell kernel address == 0x%08lX\n",
119 (uintptr_t)kfd->doorbell_kernel_ptr);
120}
121
122int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma)
123{
124 phys_addr_t address;
125 struct kfd_dev *dev;
126
127 /*
128 * For simplicitly we only allow mapping of the entire doorbell
129 * allocation of a single device & process.
130 */
131 if (vma->vm_end - vma->vm_start != doorbell_process_allocation())
132 return -EINVAL;
133
134 /* Find kfd device according to gpu id */
135 dev = kfd_device_by_id(vma->vm_pgoff);
136 if (dev == NULL)
137 return -EINVAL;
138
139 /* Find if pdd exists for combination of process and gpu id */
140 if (!kfd_get_process_device_data(dev, process, 0))
141 return -EINVAL;
142
143 /* Calculate physical address of doorbell */
144 address = kfd_get_process_doorbells(dev, process);
145
146 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
147 VM_DONTDUMP | VM_PFNMAP;
148
149 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
150
151 pr_debug("kfd: mapping doorbell page in kfd_doorbell_mmap\n"
152 " target user address == 0x%08llX\n"
153 " physical address == 0x%08llX\n"
154 " vm_flags == 0x%04lX\n"
155 " size == 0x%04lX\n",
156 (unsigned long long) vma->vm_start, address, vma->vm_flags,
157 doorbell_process_allocation());
158
159
160 return io_remap_pfn_range(vma,
161 vma->vm_start,
162 address >> PAGE_SHIFT,
163 doorbell_process_allocation(),
164 vma->vm_page_prot);
165}
166
167
168/* get kernel iomem pointer for a doorbell */
169u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
170 unsigned int *doorbell_off)
171{
172 u32 inx;
173
174 BUG_ON(!kfd || !doorbell_off);
175
176 mutex_lock(&doorbell_mutex);
177 inx = find_first_zero_bit(doorbell_available_index,
178 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
179
180 __set_bit(inx, doorbell_available_index);
181 mutex_unlock(&doorbell_mutex);
182
183 if (inx >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
184 return NULL;
185
186 /*
187 * Calculating the kernel doorbell offset using "faked" kernel
188 * pasid that allocated for kernel queues only
189 */
190 *doorbell_off = KERNEL_DOORBELL_PASID * (doorbell_process_allocation() /
191 sizeof(u32)) + inx;
192
193 pr_debug("kfd: get kernel queue doorbell\n"
194 " doorbell offset == 0x%08d\n"
195 " kernel address == 0x%08lX\n",
196 *doorbell_off, (uintptr_t)(kfd->doorbell_kernel_ptr + inx));
197
198 return kfd->doorbell_kernel_ptr + inx;
199}
200
201void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr)
202{
203 unsigned int inx;
204
205 BUG_ON(!kfd || !db_addr);
206
207 inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr);
208
209 mutex_lock(&doorbell_mutex);
210 __clear_bit(inx, doorbell_available_index);
211 mutex_unlock(&doorbell_mutex);
212}
213
214inline void write_kernel_doorbell(u32 __iomem *db, u32 value)
215{
216 if (db) {
217 writel(value, db);
218 pr_debug("writing %d to doorbell address 0x%p\n", value, db);
219 }
220}
221
222/*
223 * queue_ids are in the range [0,MAX_PROCESS_QUEUES) and are mapped 1:1
224 * to doorbells with the process's doorbell page
225 */
226unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd,
227 struct kfd_process *process,
228 unsigned int queue_id)
229{
230 /*
231 * doorbell_id_offset accounts for doorbells taken by KGD.
232 * pasid * doorbell_process_allocation/sizeof(u32) adjusts
233 * to the process's doorbells
234 */
235 return kfd->doorbell_id_offset +
236 process->pasid * (doorbell_process_allocation()/sizeof(u32)) +
237 queue_id;
238}
239
240uint64_t kfd_get_number_elems(struct kfd_dev *kfd)
241{
242 uint64_t num_of_elems = (kfd->shared_resources.doorbell_aperture_size -
243 kfd->shared_resources.doorbell_start_offset) /
244 doorbell_process_allocation() + 1;
245
246 return num_of_elems;
247
248}
249
250phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
251 struct kfd_process *process)
252{
253 return dev->doorbell_base +
254 process->pasid * doorbell_process_allocation();
255}
drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c+355−0
@@ -0,0 +1,355 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/device.h>
25#include <linux/export.h>
26#include <linux/err.h>
27#include <linux/fs.h>
28#include <linux/sched.h>
29#include <linux/slab.h>
30#include <linux/uaccess.h>
31#include <linux/compat.h>
32#include <uapi/linux/kfd_ioctl.h>
33#include <linux/time.h>
34#include "kfd_priv.h"
35#include <linux/mm.h>
36#include <uapi/asm-generic/mman-common.h>
37#include <asm/processor.h>
38
39/*
40 * The primary memory I/O features being added for revisions of gfxip
41 * beyond 7.0 (Kaveri) are:
42 *
43 * Access to ATC/IOMMU mapped memory w/ associated extension of VA to 48b
44 *
45 * “Flat” shader memory access – These are new shader vector memory
46 * operations that do not reference a T#/V# so a “pointer” is what is
47 * sourced from the vector gprs for direct access to memory.
48 * This pointer space has the Shared(LDS) and Private(Scratch) memory
49 * mapped into this pointer space as apertures.
50 * The hardware then determines how to direct the memory request
51 * based on what apertures the request falls in.
52 *
53 * Unaligned support and alignment check
54 *
55 *
56 * System Unified Address - SUA
57 *
58 * The standard usage for GPU virtual addresses are that they are mapped by
59 * a set of page tables we call GPUVM and these page tables are managed by
60 * a combination of vidMM/driver software components. The current virtual
61 * address (VA) range for GPUVM is 40b.
62 *
63 * As of gfxip7.1 and beyond we’re adding the ability for compute memory
64 * clients (CP/RLC, DMA, SHADER(ifetch, scalar, and vector ops)) to access
65 * the same page tables used by host x86 processors and that are managed by
66 * the operating system. This is via a technique and hardware called ATC/IOMMU.
67 * The GPU has the capability of accessing both the GPUVM and ATC address
68 * spaces for a given VMID (process) simultaneously and we call this feature
69 * system unified address (SUA).
70 *
71 * There are three fundamental address modes of operation for a given VMID
72 * (process) on the GPU:
73 *
74 * HSA64 – 64b pointers and the default address space is ATC
75 * HSA32 – 32b pointers and the default address space is ATC
76 * GPUVM – 64b pointers and the default address space is GPUVM (driver
77 * model mode)
78 *
79 *
80 * HSA64 - ATC/IOMMU 64b
81 *
82 * A 64b pointer in the AMD64/IA64 CPU architecture is not fully utilized
83 * by the CPU so an AMD CPU can only access the high area
84 * (VA[63:47] == 0x1FFFF) and low area (VA[63:47 == 0) of the address space
85 * so the actual VA carried to translation is 48b. There is a “hole” in
86 * the middle of the 64b VA space.
87 *
88 * The GPU not only has access to all of the CPU accessible address space via
89 * ATC/IOMMU, but it also has access to the GPUVM address space. The “system
90 * unified address” feature (SUA) is the mapping of GPUVM and ATC address
91 * spaces into a unified pointer space. The method we take for 64b mode is
92 * to map the full 40b GPUVM address space into the hole of the 64b address
93 * space.
94
95 * The GPUVM_Base/GPUVM_Limit defines the aperture in the 64b space where we
96 * direct requests to be translated via GPUVM page tables instead of the
97 * IOMMU path.
98 *
99 *
100 * 64b to 49b Address conversion
101 *
102 * Note that there are still significant portions of unused regions (holes)
103 * in the 64b address space even for the GPU. There are several places in
104 * the pipeline (sw and hw), we wish to compress the 64b virtual address
105 * to a 49b address. This 49b address is constituted of an “ATC” bit
106 * plus a 48b virtual address. This 49b address is what is passed to the
107 * translation hardware. ATC==0 means the 48b address is a GPUVM address
108 * (max of 2^40 – 1) intended to be translated via GPUVM page tables.
109 * ATC==1 means the 48b address is intended to be translated via IOMMU
110 * page tables.
111 *
112 * A 64b pointer is compared to the apertures that are defined (Base/Limit), in
113 * this case the GPUVM aperture (red) is defined and if a pointer falls in this
114 * aperture, we subtract the GPUVM_Base address and set the ATC bit to zero
115 * as part of the 64b to 49b conversion.
116 *
117 * Where this 64b to 49b conversion is done is a function of the usage.
118 * Most GPU memory access is via memory objects where the driver builds
119 * a descriptor which consists of a base address and a memory access by
120 * the GPU usually consists of some kind of an offset or Cartesian coordinate
121 * that references this memory descriptor. This is the case for shader
122 * instructions that reference the T# or V# constants, or for specified
123 * locations of assets (ex. the shader program location). In these cases
124 * the driver is what handles the 64b to 49b conversion and the base
125 * address in the descriptor (ex. V# or T# or shader program location)
126 * is defined as a 48b address w/ an ATC bit. For this usage a given
127 * memory object cannot straddle multiple apertures in the 64b address
128 * space. For example a shader program cannot jump in/out between ATC
129 * and GPUVM space.
130 *
131 * In some cases we wish to pass a 64b pointer to the GPU hardware and
132 * the GPU hw does the 64b to 49b conversion before passing memory
133 * requests to the cache/memory system. This is the case for the
134 * S_LOAD and FLAT_* shader memory instructions where we have 64b pointers
135 * in scalar and vector GPRs respectively.
136 *
137 * In all cases (no matter where the 64b -> 49b conversion is done), the gfxip
138 * hardware sends a 48b address along w/ an ATC bit, to the memory controller
139 * on the memory request interfaces.
140 *
141 * <client>_MC_rdreq_atc // read request ATC bit
142 *
143 * 0 : <client>_MC_rdreq_addr is a GPUVM VA
144 *
145 * 1 : <client>_MC_rdreq_addr is a ATC VA
146 *
147 *
148 * “Spare” aperture (APE1)
149 *
150 * We use the GPUVM aperture to differentiate ATC vs. GPUVM, but we also use
151 * apertures to set the Mtype field for S_LOAD/FLAT_* ops which is input to the
152 * config tables for setting cache policies. The “spare” (APE1) aperture is
153 * motivated by getting a different Mtype from the default.
154 * The default aperture isn’t an actual base/limit aperture; it is just the
155 * address space that doesn’t hit any defined base/limit apertures.
156 * The following diagram is a complete picture of the gfxip7.x SUA apertures.
157 * The APE1 can be placed either below or above
158 * the hole (cannot be in the hole).
159 *
160 *
161 * General Aperture definitions and rules
162 *
163 * An aperture register definition consists of a Base, Limit, Mtype, and
164 * usually an ATC bit indicating which translation tables that aperture uses.
165 * In all cases (for SUA and DUA apertures discussed later), aperture base
166 * and limit definitions are 64KB aligned.
167 *
168 * <ape>_Base[63:0] = { <ape>_Base_register[63:16], 0x0000 }
169 *
170 * <ape>_Limit[63:0] = { <ape>_Limit_register[63:16], 0xFFFF }
171 *
172 * The base and limit are considered inclusive to an aperture so being
173 * inside an aperture means (address >= Base) AND (address <= Limit).
174 *
175 * In no case is a payload that straddles multiple apertures expected to work.
176 * For example a load_dword_x4 that starts in one aperture and ends in another,
177 * does not work. For the vector FLAT_* ops we have detection capability in
178 * the shader for reporting a “memory violation” back to the
179 * SQ block for use in traps.
180 * A memory violation results when an op falls into the hole,
181 * or a payload straddles multiple apertures. The S_LOAD instruction
182 * does not have this detection.
183 *
184 * Apertures cannot overlap.
185 *
186 *
187 *
188 * HSA32 - ATC/IOMMU 32b
189 *
190 * For HSA32 mode, the pointers are interpreted as 32 bits and use a single GPR
191 * instead of two for the S_LOAD and FLAT_* ops. The entire GPUVM space of 40b
192 * will not fit so there is only partial visibility to the GPUVM
193 * space (defined by the aperture) for S_LOAD and FLAT_* ops.
194 * There is no spare (APE1) aperture for HSA32 mode.
195 *
196 *
197 * GPUVM 64b mode (driver model)
198 *
199 * This mode is related to HSA64 in that the difference really is that
200 * the default aperture is GPUVM (ATC==0) and not ATC space.
201 * We have gfxip7.x hardware that has FLAT_* and S_LOAD support for
202 * SUA GPUVM mode, but does not support HSA32/HSA64.
203 *
204 *
205 * Device Unified Address - DUA
206 *
207 * Device unified address (DUA) is the name of the feature that maps the
208 * Shared(LDS) memory and Private(Scratch) memory into the overall address
209 * space for use by the new FLAT_* vector memory ops. The Shared and
210 * Private memories are mapped as apertures into the address space,
211 * and the hardware detects when a FLAT_* memory request is to be redirected
212 * to the LDS or Scratch memory when it falls into one of these apertures.
213 * Like the SUA apertures, the Shared/Private apertures are 64KB aligned and
214 * the base/limit is “in” the aperture. For both HSA64 and GPUVM SUA modes,
215 * the Shared/Private apertures are always placed in a limited selection of
216 * options in the hole of the 64b address space. For HSA32 mode, the
217 * Shared/Private apertures can be placed anywhere in the 32b space
218 * except at 0.
219 *
220 *
221 * HSA64 Apertures for FLAT_* vector ops
222 *
223 * For HSA64 SUA mode, the Shared and Private apertures are always placed
224 * in the hole w/ a limited selection of possible locations. The requests
225 * that fall in the private aperture are expanded as a function of the
226 * work-item id (tid) and redirected to the location of the
227 * “hidden private memory”. The hidden private can be placed in either GPUVM
228 * or ATC space. The addresses that fall in the shared aperture are
229 * re-directed to the on-chip LDS memory hardware.
230 *
231 *
232 * HSA32 Apertures for FLAT_* vector ops
233 *
234 * In HSA32 mode, the Private and Shared apertures can be placed anywhere
235 * in the 32b space except at 0 (Private or Shared Base at zero disables
236 * the apertures). If the base address of the apertures are non-zero
237 * (ie apertures exists), the size is always 64KB.
238 *
239 *
240 * GPUVM Apertures for FLAT_* vector ops
241 *
242 * In GPUVM mode, the Shared/Private apertures are specified identically
243 * to HSA64 mode where they are always in the hole at a limited selection
244 * of locations.
245 *
246 *
247 * Aperture Definitions for SUA and DUA
248 *
249 * The interpretation of the aperture register definitions for a given
250 * VMID is a function of the “SUA Mode” which is one of HSA64, HSA32, or
251 * GPUVM64 discussed in previous sections. The mode is first decoded, and
252 * then the remaining register decode is a function of the mode.
253 *
254 *
255 * SUA Mode Decode
256 *
257 * For the S_LOAD and FLAT_* shader operations, the SUA mode is decoded from
258 * the COMPUTE_DISPATCH_INITIATOR:DATA_ATC bit and
259 * the SH_MEM_CONFIG:PTR32 bits.
260 *
261 * COMPUTE_DISPATCH_INITIATOR:DATA_ATC SH_MEM_CONFIG:PTR32 Mode
262 *
263 * 1 0 HSA64
264 *
265 * 1 1 HSA32
266 *
267 * 0 X GPUVM64
268 *
269 * In general the hardware will ignore the PTR32 bit and treat
270 * as “0” whenever DATA_ATC = “0”, but sw should set PTR32=0
271 * when DATA_ATC=0.
272 *
273 * The DATA_ATC bit is only set for compute dispatches.
274 * All “Draw” dispatches are hardcoded to GPUVM64 mode
275 * for FLAT_* / S_LOAD operations.
276 */
277
278#define MAKE_GPUVM_APP_BASE(gpu_num) \
279 (((uint64_t)(gpu_num) << 61) + 0x1000000000000)
280
281#define MAKE_GPUVM_APP_LIMIT(base) \
282 (((uint64_t)(base) & 0xFFFFFF0000000000) | 0xFFFFFFFFFF)
283
284#define MAKE_SCRATCH_APP_BASE(gpu_num) \
285 (((uint64_t)(gpu_num) << 61) + 0x100000000)
286
287#define MAKE_SCRATCH_APP_LIMIT(base) \
288 (((uint64_t)base & 0xFFFFFFFF00000000) | 0xFFFFFFFF)
289
290#define MAKE_LDS_APP_BASE(gpu_num) \
291 (((uint64_t)(gpu_num) << 61) + 0x0)
292#define MAKE_LDS_APP_LIMIT(base) \
293 (((uint64_t)(base) & 0xFFFFFFFF00000000) | 0xFFFFFFFF)
294
295int kfd_init_apertures(struct kfd_process *process)
296{
297 uint8_t id = 0;
298 struct kfd_dev *dev;
299 struct kfd_process_device *pdd;
300
301 mutex_lock(&process->mutex);
302
303 /*Iterating over all devices*/
304 while ((dev = kfd_topology_enum_kfd_devices(id)) != NULL &&
305 id < NUM_OF_SUPPORTED_GPUS) {
306
307 pdd = kfd_get_process_device_data(dev, process, 1);
308
309 /*
310 * For 64 bit process aperture will be statically reserved in
311 * the x86_64 non canonical process address space
312 * amdkfd doesn't currently support apertures for 32 bit process
313 */
314 if (process->is_32bit_user_mode) {
315 pdd->lds_base = pdd->lds_limit = 0;
316 pdd->gpuvm_base = pdd->gpuvm_limit = 0;
317 pdd->scratch_base = pdd->scratch_limit = 0;
318 } else {
319 /*
320 * node id couldn't be 0 - the three MSB bits of
321 * aperture shoudn't be 0
322 */
323 pdd->lds_base = MAKE_LDS_APP_BASE(id + 1);
324
325 pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);
326
327 pdd->gpuvm_base = MAKE_GPUVM_APP_BASE(id + 1);
328
329 pdd->gpuvm_limit =
330 MAKE_GPUVM_APP_LIMIT(pdd->gpuvm_base);
331
332 pdd->scratch_base = MAKE_SCRATCH_APP_BASE(id + 1);
333
334 pdd->scratch_limit =
335 MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);
336 }
337
338 dev_dbg(kfd_device, "node id %u\n", id);
339 dev_dbg(kfd_device, "gpu id %u\n", pdd->dev->id);
340 dev_dbg(kfd_device, "lds_base %llX\n", pdd->lds_base);
341 dev_dbg(kfd_device, "lds_limit %llX\n", pdd->lds_limit);
342 dev_dbg(kfd_device, "gpuvm_base %llX\n", pdd->gpuvm_base);
343 dev_dbg(kfd_device, "gpuvm_limit %llX\n", pdd->gpuvm_limit);
344 dev_dbg(kfd_device, "scratch_base %llX\n", pdd->scratch_base);
345 dev_dbg(kfd_device, "scratch_limit %llX\n", pdd->scratch_limit);
346
347 id++;
348 }
349
350 mutex_unlock(&process->mutex);
351
352 return 0;
353}
354
355
drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c+176−0
@@ -0,0 +1,176 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23/*
24 * KFD Interrupts.
25 *
26 * AMD GPUs deliver interrupts by pushing an interrupt description onto the
27 * interrupt ring and then sending an interrupt. KGD receives the interrupt
28 * in ISR and sends us a pointer to each new entry on the interrupt ring.
29 *
30 * We generally can't process interrupt-signaled events from ISR, so we call
31 * out to each interrupt client module (currently only the scheduler) to ask if
32 * each interrupt is interesting. If they return true, then it requires further
33 * processing so we copy it to an internal interrupt ring and call each
34 * interrupt client again from a work-queue.
35 *
36 * There's no acknowledgment for the interrupts we use. The hardware simply
37 * queues a new interrupt each time without waiting.
38 *
39 * The fixed-size internal queue means that it's possible for us to lose
40 * interrupts because we have no back-pressure to the hardware.
41 */
42
43#include <linux/slab.h>
44#include <linux/device.h>
45#include "kfd_priv.h"
46
47#define KFD_INTERRUPT_RING_SIZE 256
48
49static void interrupt_wq(struct work_struct *);
50
51int kfd_interrupt_init(struct kfd_dev *kfd)
52{
53 void *interrupt_ring = kmalloc_array(KFD_INTERRUPT_RING_SIZE,
54 kfd->device_info->ih_ring_entry_size,
55 GFP_KERNEL);
56 if (!interrupt_ring)
57 return -ENOMEM;
58
59 kfd->interrupt_ring = interrupt_ring;
60 kfd->interrupt_ring_size =
61 KFD_INTERRUPT_RING_SIZE * kfd->device_info->ih_ring_entry_size;
62 atomic_set(&kfd->interrupt_ring_wptr, 0);
63 atomic_set(&kfd->interrupt_ring_rptr, 0);
64
65 spin_lock_init(&kfd->interrupt_lock);
66
67 INIT_WORK(&kfd->interrupt_work, interrupt_wq);
68
69 kfd->interrupts_active = true;
70
71 /*
72 * After this function returns, the interrupt will be enabled. This
73 * barrier ensures that the interrupt running on a different processor
74 * sees all the above writes.
75 */
76 smp_wmb();
77
78 return 0;
79}
80
81void kfd_interrupt_exit(struct kfd_dev *kfd)
82{
83 /*
84 * Stop the interrupt handler from writing to the ring and scheduling
85 * workqueue items. The spinlock ensures that any interrupt running
86 * after we have unlocked sees interrupts_active = false.
87 */
88 unsigned long flags;
89
90 spin_lock_irqsave(&kfd->interrupt_lock, flags);
91 kfd->interrupts_active = false;
92 spin_unlock_irqrestore(&kfd->interrupt_lock, flags);
93
94 /*
95 * Flush_scheduled_work ensures that there are no outstanding
96 * work-queue items that will access interrupt_ring. New work items
97 * can't be created because we stopped interrupt handling above.
98 */
99 flush_scheduled_work();
100
101 kfree(kfd->interrupt_ring);
102}
103
104/*
105 * This assumes that it can't be called concurrently with itself
106 * but only with dequeue_ih_ring_entry.
107 */
108bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry)
109{
110 unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr);
111 unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr);
112
113 if ((rptr - wptr) % kfd->interrupt_ring_size ==
114 kfd->device_info->ih_ring_entry_size) {
115 /* This is very bad, the system is likely to hang. */
116 dev_err_ratelimited(kfd_chardev(),
117 "Interrupt ring overflow, dropping interrupt.\n");
118 return false;
119 }
120
121 memcpy(kfd->interrupt_ring + wptr, ih_ring_entry,
122 kfd->device_info->ih_ring_entry_size);
123
124 wptr = (wptr + kfd->device_info->ih_ring_entry_size) %
125 kfd->interrupt_ring_size;
126 smp_wmb(); /* Ensure memcpy'd data is visible before wptr update. */
127 atomic_set(&kfd->interrupt_ring_wptr, wptr);
128
129 return true;
130}
131
132/*
133 * This assumes that it can't be called concurrently with itself
134 * but only with enqueue_ih_ring_entry.
135 */
136static bool dequeue_ih_ring_entry(struct kfd_dev *kfd, void *ih_ring_entry)
137{
138 /*
139 * Assume that wait queues have an implicit barrier, i.e. anything that
140 * happened in the ISR before it queued work is visible.
141 */
142
143 unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr);
144 unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr);
145
146 if (rptr == wptr)
147 return false;
148
149 memcpy(ih_ring_entry, kfd->interrupt_ring + rptr,
150 kfd->device_info->ih_ring_entry_size);
151
152 rptr = (rptr + kfd->device_info->ih_ring_entry_size) %
153 kfd->interrupt_ring_size;
154
155 /*
156 * Ensure the rptr write update is not visible until
157 * memcpy has finished reading.
158 */
159 smp_mb();
160 atomic_set(&kfd->interrupt_ring_rptr, rptr);
161
162 return true;
163}
164
165static void interrupt_wq(struct work_struct *work)
166{
167 struct kfd_dev *dev = container_of(work, struct kfd_dev,
168 interrupt_work);
169
170 uint32_t ih_ring_entry[DIV_ROUND_UP(
171 dev->device_info->ih_ring_entry_size,
172 sizeof(uint32_t))];
173
174 while (dequeue_ih_ring_entry(dev, ih_ring_entry))
175 ;
176}
drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c+347−0
@@ -0,0 +1,347 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/types.h>
25#include <linux/mutex.h>
26#include <linux/slab.h>
27#include <linux/printk.h>
28#include "kfd_kernel_queue.h"
29#include "kfd_priv.h"
30#include "kfd_device_queue_manager.h"
31#include "kfd_pm4_headers.h"
32#include "kfd_pm4_opcodes.h"
33
34#define PM4_COUNT_ZERO (((1 << 15) - 1) << 16)
35
36static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
37 enum kfd_queue_type type, unsigned int queue_size)
38{
39 struct queue_properties prop;
40 int retval;
41 union PM4_MES_TYPE_3_HEADER nop;
42
43 BUG_ON(!kq || !dev);
44 BUG_ON(type != KFD_QUEUE_TYPE_DIQ && type != KFD_QUEUE_TYPE_HIQ);
45
46 pr_debug("kfd: In func %s initializing queue type %d size %d\n",
47 __func__, KFD_QUEUE_TYPE_HIQ, queue_size);
48
49 nop.opcode = IT_NOP;
50 nop.type = PM4_TYPE_3;
51 nop.u32all |= PM4_COUNT_ZERO;
52
53 kq->dev = dev;
54 kq->nop_packet = nop.u32all;
55 switch (type) {
56 case KFD_QUEUE_TYPE_DIQ:
57 case KFD_QUEUE_TYPE_HIQ:
58 kq->mqd = dev->dqm->get_mqd_manager(dev->dqm,
59 KFD_MQD_TYPE_CIK_HIQ);
60 break;
61 default:
62 BUG();
63 break;
64 }
65
66 if (kq->mqd == NULL)
67 return false;
68
69 prop.doorbell_ptr =
70 (uint32_t *)kfd_get_kernel_doorbell(dev, &prop.doorbell_off);
71
72 if (prop.doorbell_ptr == NULL)
73 goto err_get_kernel_doorbell;
74
75 retval = kfd2kgd->allocate_mem(dev->kgd,
76 queue_size,
77 PAGE_SIZE,
78 KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
79 (struct kgd_mem **) &kq->pq);
80
81 if (retval != 0)
82 goto err_pq_allocate_vidmem;
83
84 kq->pq_kernel_addr = kq->pq->cpu_ptr;
85 kq->pq_gpu_addr = kq->pq->gpu_addr;
86
87 retval = kfd2kgd->allocate_mem(dev->kgd,
88 sizeof(*kq->rptr_kernel),
89 32,
90 KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
91 (struct kgd_mem **) &kq->rptr_mem);
92
93 if (retval != 0)
94 goto err_rptr_allocate_vidmem;
95
96 kq->rptr_kernel = kq->rptr_mem->cpu_ptr;
97 kq->rptr_gpu_addr = kq->rptr_mem->gpu_addr;
98
99 retval = kfd2kgd->allocate_mem(dev->kgd,
100 sizeof(*kq->wptr_kernel),
101 32,
102 KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
103 (struct kgd_mem **) &kq->wptr_mem);
104
105 if (retval != 0)
106 goto err_wptr_allocate_vidmem;
107
108 kq->wptr_kernel = kq->wptr_mem->cpu_ptr;
109 kq->wptr_gpu_addr = kq->wptr_mem->gpu_addr;
110
111 memset(kq->pq_kernel_addr, 0, queue_size);
112 memset(kq->rptr_kernel, 0, sizeof(*kq->rptr_kernel));
113 memset(kq->wptr_kernel, 0, sizeof(*kq->wptr_kernel));
114
115 prop.queue_size = queue_size;
116 prop.is_interop = false;
117 prop.priority = 1;
118 prop.queue_percent = 100;
119 prop.type = type;
120 prop.vmid = 0;
121 prop.queue_address = kq->pq_gpu_addr;
122 prop.read_ptr = (uint32_t *) kq->rptr_gpu_addr;
123 prop.write_ptr = (uint32_t *) kq->wptr_gpu_addr;
124
125 if (init_queue(&kq->queue, prop) != 0)
126 goto err_init_queue;
127
128 kq->queue->device = dev;
129 kq->queue->process = kfd_get_process(current);
130
131 retval = kq->mqd->init_mqd(kq->mqd, &kq->queue->mqd,
132 &kq->queue->mqd_mem_obj,
133 &kq->queue->gart_mqd_addr,
134 &kq->queue->properties);
135 if (retval != 0)
136 goto err_init_mqd;
137
138 /* assign HIQ to HQD */
139 if (type == KFD_QUEUE_TYPE_HIQ) {
140 pr_debug("assigning hiq to hqd\n");
141 kq->queue->pipe = KFD_CIK_HIQ_PIPE;
142 kq->queue->queue = KFD_CIK_HIQ_QUEUE;
143 kq->mqd->load_mqd(kq->mqd, kq->queue->mqd, kq->queue->pipe,
144 kq->queue->queue, NULL);
145 } else {
146 /* allocate fence for DIQ */
147
148 retval = kfd2kgd->allocate_mem(dev->kgd,
149 sizeof(uint32_t),
150 32,
151 KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
152 (struct kgd_mem **) &kq->fence_mem_obj);
153
154 if (retval != 0)
155 goto err_alloc_fence;
156
157 kq->fence_kernel_address = kq->fence_mem_obj->cpu_ptr;
158 kq->fence_gpu_addr = kq->fence_mem_obj->gpu_addr;
159 }
160
161 print_queue(kq->queue);
162
163 return true;
164err_alloc_fence:
165err_init_mqd:
166 uninit_queue(kq->queue);
167err_init_queue:
168 kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->wptr_mem);
169err_wptr_allocate_vidmem:
170 kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->rptr_mem);
171err_rptr_allocate_vidmem:
172 kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->pq);
173err_pq_allocate_vidmem:
174 pr_err("kfd: error init pq\n");
175 kfd_release_kernel_doorbell(dev, (u32 *)prop.doorbell_ptr);
176err_get_kernel_doorbell:
177 pr_err("kfd: error init doorbell");
178 return false;
179
180}
181
182static void uninitialize(struct kernel_queue *kq)
183{
184 BUG_ON(!kq);
185
186 if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ)
187 kq->mqd->destroy_mqd(kq->mqd,
188 NULL,
189 false,
190 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
191 kq->queue->pipe,
192 kq->queue->queue);
193
194 kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->rptr_mem);
195 kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->wptr_mem);
196 kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->pq);
197 kfd_release_kernel_doorbell(kq->dev,
198 (u32 *)kq->queue->properties.doorbell_ptr);
199 uninit_queue(kq->queue);
200}
201
202static int acquire_packet_buffer(struct kernel_queue *kq,
203 size_t packet_size_in_dwords, unsigned int **buffer_ptr)
204{
205 size_t available_size;
206 size_t queue_size_dwords;
207 uint32_t wptr, rptr;
208 unsigned int *queue_address;
209
210 BUG_ON(!kq || !buffer_ptr);
211
212 rptr = *kq->rptr_kernel;
213 wptr = *kq->wptr_kernel;
214 queue_address = (unsigned int *)kq->pq_kernel_addr;
215 queue_size_dwords = kq->queue->properties.queue_size / sizeof(uint32_t);
216
217 pr_debug("kfd: In func %s\nrptr: %d\nwptr: %d\nqueue_address 0x%p\n",
218 __func__, rptr, wptr, queue_address);
219
220 available_size = (rptr - 1 - wptr + queue_size_dwords) %
221 queue_size_dwords;
222
223 if (packet_size_in_dwords >= queue_size_dwords ||
224 packet_size_in_dwords >= available_size)
225 return -ENOMEM;
226
227 if (wptr + packet_size_in_dwords >= queue_size_dwords) {
228 while (wptr > 0) {
229 queue_address[wptr] = kq->nop_packet;
230 wptr = (wptr + 1) % queue_size_dwords;
231 }
232 }
233
234 *buffer_ptr = &queue_address[wptr];
235 kq->pending_wptr = wptr + packet_size_in_dwords;
236
237 return 0;
238}
239
240static void submit_packet(struct kernel_queue *kq)
241{
242#ifdef DEBUG
243 int i;
244#endif
245
246 BUG_ON(!kq);
247
248#ifdef DEBUG
249 for (i = *kq->wptr_kernel; i < kq->pending_wptr; i++) {
250 pr_debug("0x%2X ", kq->pq_kernel_addr[i]);
251 if (i % 15 == 0)
252 pr_debug("\n");
253 }
254 pr_debug("\n");
255#endif
256
257 *kq->wptr_kernel = kq->pending_wptr;
258 write_kernel_doorbell((u32 *)kq->queue->properties.doorbell_ptr,
259 kq->pending_wptr);
260}
261
262static int sync_with_hw(struct kernel_queue *kq, unsigned long timeout_ms)
263{
264 unsigned long org_timeout_ms;
265
266 BUG_ON(!kq);
267
268 org_timeout_ms = timeout_ms;
269 timeout_ms += jiffies * 1000 / HZ;
270 while (*kq->wptr_kernel != *kq->rptr_kernel) {
271 if (time_after(jiffies * 1000 / HZ, timeout_ms)) {
272 pr_err("kfd: kernel_queue %s timeout expired %lu\n",
273 __func__, org_timeout_ms);
274 pr_err("kfd: wptr: %d rptr: %d\n",
275 *kq->wptr_kernel, *kq->rptr_kernel);
276 return -ETIME;
277 }
278 cpu_relax();
279 }
280
281 return 0;
282}
283
284static void rollback_packet(struct kernel_queue *kq)
285{
286 BUG_ON(!kq);
287 kq->pending_wptr = *kq->queue->properties.write_ptr;
288}
289
290struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
291 enum kfd_queue_type type)
292{
293 struct kernel_queue *kq;
294
295 BUG_ON(!dev);
296
297 kq = kzalloc(sizeof(struct kernel_queue), GFP_KERNEL);
298 if (!kq)
299 return NULL;
300
301 kq->initialize = initialize;
302 kq->uninitialize = uninitialize;
303 kq->acquire_packet_buffer = acquire_packet_buffer;
304 kq->submit_packet = submit_packet;
305 kq->sync_with_hw = sync_with_hw;
306 kq->rollback_packet = rollback_packet;
307
308 if (kq->initialize(kq, dev, type, KFD_KERNEL_QUEUE_SIZE) == false) {
309 pr_err("kfd: failed to init kernel queue\n");
310 kfree(kq);
311 return NULL;
312 }
313 return kq;
314}
315
316void kernel_queue_uninit(struct kernel_queue *kq)
317{
318 BUG_ON(!kq);
319
320 kq->uninitialize(kq);
321 kfree(kq);
322}
323
324void test_kq(struct kfd_dev *dev)
325{
326 struct kernel_queue *kq;
327 uint32_t *buffer, i;
328 int retval;
329
330 BUG_ON(!dev);
331
332 pr_debug("kfd: starting kernel queue test\n");
333
334 kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_HIQ);
335 BUG_ON(!kq);
336
337 retval = kq->acquire_packet_buffer(kq, 5, &buffer);
338 BUG_ON(retval != 0);
339 for (i = 0; i < 5; i++)
340 buffer[i] = kq->nop_packet;
341 kq->submit_packet(kq);
342 kq->sync_with_hw(kq, 1000);
343
344 pr_debug("kfd: ending kernel queue test\n");
345}
346
347
drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h+69−0
@@ -0,0 +1,69 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#ifndef KFD_KERNEL_QUEUE_H_
25#define KFD_KERNEL_QUEUE_H_
26
27#include <linux/list.h>
28#include <linux/types.h>
29#include "kfd_priv.h"
30
31struct kernel_queue {
32 /* interface */
33 bool (*initialize)(struct kernel_queue *kq, struct kfd_dev *dev,
34 enum kfd_queue_type type, unsigned int queue_size);
35 void (*uninitialize)(struct kernel_queue *kq);
36 int (*acquire_packet_buffer)(struct kernel_queue *kq,
37 size_t packet_size_in_dwords,
38 unsigned int **buffer_ptr);
39
40 void (*submit_packet)(struct kernel_queue *kq);
41 int (*sync_with_hw)(struct kernel_queue *kq,
42 unsigned long timeout_ms);
43 void (*rollback_packet)(struct kernel_queue *kq);
44
45 /* data */
46 struct kfd_dev *dev;
47 struct mqd_manager *mqd;
48 struct queue *queue;
49 uint32_t pending_wptr;
50 unsigned int nop_packet;
51
52 struct kfd_mem_obj *rptr_mem;
53 uint32_t *rptr_kernel;
54 uint64_t rptr_gpu_addr;
55 struct kfd_mem_obj *wptr_mem;
56 uint32_t *wptr_kernel;
57 uint64_t wptr_gpu_addr;
58 struct kfd_mem_obj *pq;
59 uint64_t pq_gpu_addr;
60 uint32_t *pq_kernel_addr;
61
62 struct kfd_mem_obj *fence_mem_obj;
63 uint64_t fence_gpu_addr;
64 void *fence_kernel_address;
65
66 struct list_head list;
67};
68
69#endif /* KFD_KERNEL_QUEUE_H_ */
drivers/gpu/drm/amd/amdkfd/kfd_module.c+159−0
@@ -0,0 +1,159 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/module.h>
24#include <linux/sched.h>
25#include <linux/moduleparam.h>
26#include <linux/device.h>
27#include "kfd_priv.h"
28
29#define KFD_DRIVER_AUTHOR "AMD Inc. and others"
30
31#define KFD_DRIVER_DESC "Standalone HSA driver for AMD's GPUs"
32#define KFD_DRIVER_DATE "20141113"
33#define KFD_DRIVER_MAJOR 0
34#define KFD_DRIVER_MINOR 7
35#define KFD_DRIVER_PATCHLEVEL 0
36
37const struct kfd2kgd_calls *kfd2kgd;
38static const struct kgd2kfd_calls kgd2kfd = {
39 .exit = kgd2kfd_exit,
40 .probe = kgd2kfd_probe,
41 .device_init = kgd2kfd_device_init,
42 .device_exit = kgd2kfd_device_exit,
43 .interrupt = kgd2kfd_interrupt,
44 .suspend = kgd2kfd_suspend,
45 .resume = kgd2kfd_resume,
46};
47
48int sched_policy = KFD_SCHED_POLICY_HWS;
49module_param(sched_policy, int, 0444);
50MODULE_PARM_DESC(sched_policy,
51 "Kernel cmdline parameter that defines the amdkfd scheduling policy");
52
53int max_num_of_processes = KFD_MAX_NUM_OF_PROCESSES_DEFAULT;
54module_param(max_num_of_processes, int, 0444);
55MODULE_PARM_DESC(max_num_of_processes,
56 "Kernel cmdline parameter that defines the amdkfd maximum number of supported processes");
57
58int max_num_of_queues_per_process = KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT;
59module_param(max_num_of_queues_per_process, int, 0444);
60MODULE_PARM_DESC(max_num_of_queues_per_process,
61 "Kernel cmdline parameter that defines the amdkfd maximum number of supported queues per process");
62
63bool kgd2kfd_init(unsigned interface_version,
64 const struct kfd2kgd_calls *f2g,
65 const struct kgd2kfd_calls **g2f)
66{
67 /*
68 * Only one interface version is supported,
69 * no kfd/kgd version skew allowed.
70 */
71 if (interface_version != KFD_INTERFACE_VERSION)
72 return false;
73
74 /* Protection against multiple amd kgd loads */
75 if (kfd2kgd)
76 return true;
77
78 kfd2kgd = f2g;
79 *g2f = &kgd2kfd;
80
81 return true;
82}
83EXPORT_SYMBOL(kgd2kfd_init);
84
85void kgd2kfd_exit(void)
86{
87}
88
89static int __init kfd_module_init(void)
90{
91 int err;
92
93 kfd2kgd = NULL;
94
95 /* Verify module parameters */
96 if ((sched_policy < KFD_SCHED_POLICY_HWS) ||
97 (sched_policy > KFD_SCHED_POLICY_NO_HWS)) {
98 pr_err("kfd: sched_policy has invalid value\n");
99 return -1;
100 }
101
102 /* Verify module parameters */
103 if ((max_num_of_processes < 0) ||
104 (max_num_of_processes > KFD_MAX_NUM_OF_PROCESSES)) {
105 pr_err("kfd: max_num_of_processes must be between 0 to KFD_MAX_NUM_OF_PROCESSES\n");
106 return -1;
107 }
108
109 if ((max_num_of_queues_per_process < 0) ||
110 (max_num_of_queues_per_process >
111 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)) {
112 pr_err("kfd: max_num_of_queues_per_process must be between 0 to KFD_MAX_NUM_OF_QUEUES_PER_PROCESS\n");
113 return -1;
114 }
115
116 err = kfd_pasid_init();
117 if (err < 0)
118 goto err_pasid;
119
120 err = kfd_chardev_init();
121 if (err < 0)
122 goto err_ioctl;
123
124 err = kfd_topology_init();
125 if (err < 0)
126 goto err_topology;
127
128 kfd_process_create_wq();
129
130 dev_info(kfd_device, "Initialized module\n");
131
132 return 0;
133
134err_topology:
135 kfd_chardev_exit();
136err_ioctl:
137 kfd_pasid_exit();
138err_pasid:
139 return err;
140}
141
142static void __exit kfd_module_exit(void)
143{
144 kfd_process_destroy_wq();
145 kfd_topology_shutdown();
146 kfd_chardev_exit();
147 kfd_pasid_exit();
148 dev_info(kfd_device, "Removed module\n");
149}
150
151module_init(kfd_module_init);
152module_exit(kfd_module_exit);
153
154MODULE_AUTHOR(KFD_DRIVER_AUTHOR);
155MODULE_DESCRIPTION(KFD_DRIVER_DESC);
156MODULE_LICENSE("GPL and additional rights");
157MODULE_VERSION(__stringify(KFD_DRIVER_MAJOR) "."
158 __stringify(KFD_DRIVER_MINOR) "."
159 __stringify(KFD_DRIVER_PATCHLEVEL));
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c+346−0
@@ -0,0 +1,346 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/printk.h>
25#include <linux/slab.h>
26#include "kfd_priv.h"
27#include "kfd_mqd_manager.h"
28#include "cik_regs.h"
29#include "../../radeon/cik_reg.h"
30
31inline void busy_wait(unsigned long ms)
32{
33 while (time_before(jiffies, ms))
34 cpu_relax();
35}
36
37static inline struct cik_mqd *get_mqd(void *mqd)
38{
39 return (struct cik_mqd *)mqd;
40}
41
42static int init_mqd(struct mqd_manager *mm, void **mqd,
43 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
44 struct queue_properties *q)
45{
46 uint64_t addr;
47 struct cik_mqd *m;
48 int retval;
49
50 BUG_ON(!mm || !q || !mqd);
51
52 pr_debug("kfd: In func %s\n", __func__);
53
54 retval = kfd2kgd->allocate_mem(mm->dev->kgd,
55 sizeof(struct cik_mqd),
56 256,
57 KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
58 (struct kgd_mem **) mqd_mem_obj);
59
60 if (retval != 0)
61 return -ENOMEM;
62
63 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr;
64 addr = (*mqd_mem_obj)->gpu_addr;
65
66 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
67
68 m->header = 0xC0310800;
69 m->compute_pipelinestat_enable = 1;
70 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
71 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
72 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
73 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
74
75 /*
76 * Make sure to use the last queue state saved on mqd when the cp
77 * reassigns the queue, so when queue is switched on/off (e.g over
78 * subscription or quantum timeout) the context will be consistent
79 */
80 m->cp_hqd_persistent_state =
81 DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ;
82
83 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN;
84 m->cp_mqd_base_addr_lo = lower_32_bits(addr);
85 m->cp_mqd_base_addr_hi = upper_32_bits(addr);
86
87 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN;
88 /* Although WinKFD writes this, I suspect it should not be necessary */
89 m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE;
90
91 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
92 QUANTUM_DURATION(10);
93
94 /*
95 * Pipe Priority
96 * Identifies the pipe relative priority when this queue is connected
97 * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
98 * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
99 * 0 = CS_LOW (typically below GFX)
100 * 1 = CS_MEDIUM (typically between HP3D and GFX
101 * 2 = CS_HIGH (typically above HP3D)
102 */
103 m->cp_hqd_pipe_priority = 1;
104 m->cp_hqd_queue_priority = 15;
105
106 *mqd = m;
107 if (gart_addr != NULL)
108 *gart_addr = addr;
109 retval = mm->update_mqd(mm, m, q);
110
111 return retval;
112}
113
114static void uninit_mqd(struct mqd_manager *mm, void *mqd,
115 struct kfd_mem_obj *mqd_mem_obj)
116{
117 BUG_ON(!mm || !mqd);
118 kfd2kgd->free_mem(mm->dev->kgd, (struct kgd_mem *) mqd_mem_obj);
119}
120
121static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
122 uint32_t queue_id, uint32_t __user *wptr)
123{
124 return kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id, wptr);
125
126}
127
128static int update_mqd(struct mqd_manager *mm, void *mqd,
129 struct queue_properties *q)
130{
131 struct cik_mqd *m;
132
133 BUG_ON(!mm || !q || !mqd);
134
135 pr_debug("kfd: In func %s\n", __func__);
136
137 m = get_mqd(mqd);
138 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
139 DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN;
140
141 /*
142 * Calculating queue size which is log base 2 of actual queue size -1
143 * dwords and another -1 for ffs
144 */
145 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int))
146 - 1 - 1;
147 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
148 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
149 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
150 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
151 m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
152 DOORBELL_OFFSET(q->doorbell_off);
153
154 m->cp_hqd_vmid = q->vmid;
155
156 if (q->format == KFD_QUEUE_FORMAT_AQL) {
157 m->cp_hqd_iq_rptr = AQL_ENABLE;
158 m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
159 }
160
161 m->cp_hqd_active = 0;
162 q->is_active = false;
163 if (q->queue_size > 0 &&
164 q->queue_address != 0 &&
165 q->queue_percent > 0) {
166 m->cp_hqd_active = 1;
167 q->is_active = true;
168 }
169
170 return 0;
171}
172
173static int destroy_mqd(struct mqd_manager *mm, void *mqd,
174 enum kfd_preempt_type type,
175 unsigned int timeout, uint32_t pipe_id,
176 uint32_t queue_id)
177{
178 return kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout,
179 pipe_id, queue_id);
180}
181
182bool is_occupied(struct mqd_manager *mm, void *mqd,
183 uint64_t queue_address, uint32_t pipe_id,
184 uint32_t queue_id)
185{
186
187 return kfd2kgd->hqd_is_occupies(mm->dev->kgd, queue_address,
188 pipe_id, queue_id);
189
190}
191
192/*
193 * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation.
194 * The HIQ queue in Kaveri is using the same MQD structure as all the user mode
195 * queues but with different initial values.
196 */
197
198static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
199 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
200 struct queue_properties *q)
201{
202 uint64_t addr;
203 struct cik_mqd *m;
204 int retval;
205
206 BUG_ON(!mm || !q || !mqd || !mqd_mem_obj);
207
208 pr_debug("kfd: In func %s\n", __func__);
209
210 retval = kfd2kgd->allocate_mem(mm->dev->kgd,
211 sizeof(struct cik_mqd),
212 256,
213 KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
214 (struct kgd_mem **) mqd_mem_obj);
215
216 if (retval != 0)
217 return -ENOMEM;
218
219 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr;
220 addr = (*mqd_mem_obj)->gpu_addr;
221
222 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
223
224 m->header = 0xC0310800;
225 m->compute_pipelinestat_enable = 1;
226 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
227 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
228 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
229 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
230
231 m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE |
232 PRELOAD_REQ;
233 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
234 QUANTUM_DURATION(10);
235
236 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN;
237 m->cp_mqd_base_addr_lo = lower_32_bits(addr);
238 m->cp_mqd_base_addr_hi = upper_32_bits(addr);
239
240 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE;
241
242 /*
243 * Pipe Priority
244 * Identifies the pipe relative priority when this queue is connected
245 * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
246 * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
247 * 0 = CS_LOW (typically below GFX)
248 * 1 = CS_MEDIUM (typically between HP3D and GFX
249 * 2 = CS_HIGH (typically above HP3D)
250 */
251 m->cp_hqd_pipe_priority = 1;
252 m->cp_hqd_queue_priority = 15;
253
254 *mqd = m;
255 if (gart_addr)
256 *gart_addr = addr;
257 retval = mm->update_mqd(mm, m, q);
258
259 return retval;
260}
261
262static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
263 struct queue_properties *q)
264{
265 struct cik_mqd *m;
266
267 BUG_ON(!mm || !q || !mqd);
268
269 pr_debug("kfd: In func %s\n", __func__);
270
271 m = get_mqd(mqd);
272 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
273 DEFAULT_MIN_AVAIL_SIZE |
274 PRIV_STATE |
275 KMD_QUEUE;
276
277 /*
278 * Calculating queue size which is log base 2 of actual queue
279 * size -1 dwords
280 */
281 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int))
282 - 1 - 1;
283 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
284 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
285 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
286 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
287 m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
288 DOORBELL_OFFSET(q->doorbell_off);
289
290 m->cp_hqd_vmid = q->vmid;
291
292 m->cp_hqd_active = 0;
293 q->is_active = false;
294 if (q->queue_size > 0 &&
295 q->queue_address != 0 &&
296 q->queue_percent > 0) {
297 m->cp_hqd_active = 1;
298 q->is_active = true;
299 }
300
301 return 0;
302}
303
304struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
305 struct kfd_dev *dev)
306{
307 struct mqd_manager *mqd;
308
309 BUG_ON(!dev);
310 BUG_ON(type >= KFD_MQD_TYPE_MAX);
311
312 pr_debug("kfd: In func %s\n", __func__);
313
314 mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL);
315 if (!mqd)
316 return NULL;
317
318 mqd->dev = dev;
319
320 switch (type) {
321 case KFD_MQD_TYPE_CIK_CP:
322 case KFD_MQD_TYPE_CIK_COMPUTE:
323 mqd->init_mqd = init_mqd;
324 mqd->uninit_mqd = uninit_mqd;
325 mqd->load_mqd = load_mqd;
326 mqd->update_mqd = update_mqd;
327 mqd->destroy_mqd = destroy_mqd;
328 mqd->is_occupied = is_occupied;
329 break;
330 case KFD_MQD_TYPE_CIK_HIQ:
331 mqd->init_mqd = init_mqd_hiq;
332 mqd->uninit_mqd = uninit_mqd;
333 mqd->load_mqd = load_mqd;
334 mqd->update_mqd = update_mqd_hiq;
335 mqd->destroy_mqd = destroy_mqd;
336 mqd->is_occupied = is_occupied;
337 break;
338 default:
339 kfree(mqd);
340 return NULL;
341 }
342
343 return mqd;
344}
345
346/* SDMA queues should be implemented here when the cp will supports them */
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h+91−0
@@ -0,0 +1,91 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#ifndef KFD_MQD_MANAGER_H_
25#define KFD_MQD_MANAGER_H_
26
27#include "kfd_priv.h"
28
29/**
30 * struct mqd_manager
31 *
32 * @init_mqd: Allocates the mqd buffer on local gpu memory and initialize it.
33 *
34 * @load_mqd: Loads the mqd to a concrete hqd slot. Used only for no cp
35 * scheduling mode.
36 *
37 * @update_mqd: Handles a update call for the MQD
38 *
39 * @destroy_mqd: Destroys the HQD slot and by that preempt the relevant queue.
40 * Used only for no cp scheduling.
41 *
42 * @uninit_mqd: Releases the mqd buffer from local gpu memory.
43 *
44 * @is_occupied: Checks if the relevant HQD slot is occupied.
45 *
46 * @mqd_mutex: Mqd manager mutex.
47 *
48 * @dev: The kfd device structure coupled with this module.
49 *
50 * MQD stands for Memory Queue Descriptor which represents the current queue
51 * state in the memory and initiate the HQD (Hardware Queue Descriptor) state.
52 * This structure is actually a base class for the different types of MQDs
53 * structures for the variant ASICs that should be supported in the future.
54 * This base class is also contains all the MQD specific operations.
55 * Another important thing to mention is that each queue has a MQD that keeps
56 * his state (or context) after each preemption or reassignment.
57 * Basically there are a instances of the mqd manager class per MQD type per
58 * ASIC. Currently the kfd driver supports only Kaveri so there are instances
59 * per KFD_MQD_TYPE for each device.
60 *
61 */
62
63struct mqd_manager {
64 int (*init_mqd)(struct mqd_manager *mm, void **mqd,
65 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
66 struct queue_properties *q);
67
68 int (*load_mqd)(struct mqd_manager *mm, void *mqd,
69 uint32_t pipe_id, uint32_t queue_id,
70 uint32_t __user *wptr);
71
72 int (*update_mqd)(struct mqd_manager *mm, void *mqd,
73 struct queue_properties *q);
74
75 int (*destroy_mqd)(struct mqd_manager *mm, void *mqd,
76 enum kfd_preempt_type type,
77 unsigned int timeout, uint32_t pipe_id,
78 uint32_t queue_id);
79
80 void (*uninit_mqd)(struct mqd_manager *mm, void *mqd,
81 struct kfd_mem_obj *mqd_mem_obj);
82
83 bool (*is_occupied)(struct mqd_manager *mm, void *mqd,
84 uint64_t queue_address, uint32_t pipe_id,
85 uint32_t queue_id);
86
87 struct mutex mqd_mutex;
88 struct kfd_dev *dev;
89};
90
91#endif /* KFD_MQD_MANAGER_H_ */
drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c+565−0
@@ -0,0 +1,565 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/slab.h>
25#include <linux/mutex.h>
26#include "kfd_device_queue_manager.h"
27#include "kfd_kernel_queue.h"
28#include "kfd_priv.h"
29#include "kfd_pm4_headers.h"
30#include "kfd_pm4_opcodes.h"
31
32static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes,
33 unsigned int buffer_size_bytes)
34{
35 unsigned int temp = *wptr + increment_bytes / sizeof(uint32_t);
36
37 BUG_ON((temp * sizeof(uint32_t)) > buffer_size_bytes);
38 *wptr = temp;
39}
40
41static unsigned int build_pm4_header(unsigned int opcode, size_t packet_size)
42{
43 union PM4_MES_TYPE_3_HEADER header;
44
45 header.u32all = 0;
46 header.opcode = opcode;
47 header.count = packet_size/sizeof(uint32_t) - 2;
48 header.type = PM4_TYPE_3;
49
50 return header.u32all;
51}
52
53static void pm_calc_rlib_size(struct packet_manager *pm,
54 unsigned int *rlib_size,
55 bool *over_subscription)
56{
57 unsigned int process_count, queue_count;
58
59 BUG_ON(!pm || !rlib_size || !over_subscription);
60
61 process_count = pm->dqm->processes_count;
62 queue_count = pm->dqm->queue_count;
63
64 /* check if there is over subscription*/
65 *over_subscription = false;
66 if ((process_count > 1) ||
67 queue_count > PIPE_PER_ME_CP_SCHEDULING * QUEUES_PER_PIPE) {
68 *over_subscription = true;
69 pr_debug("kfd: over subscribed runlist\n");
70 }
71
72 /* calculate run list ib allocation size */
73 *rlib_size = process_count * sizeof(struct pm4_map_process) +
74 queue_count * sizeof(struct pm4_map_queues);
75
76 /*
77 * Increase the allocation size in case we need a chained run list
78 * when over subscription
79 */
80 if (*over_subscription)
81 *rlib_size += sizeof(struct pm4_runlist);
82
83 pr_debug("kfd: runlist ib size %d\n", *rlib_size);
84}
85
86static int pm_allocate_runlist_ib(struct packet_manager *pm,
87 unsigned int **rl_buffer,
88 uint64_t *rl_gpu_buffer,
89 unsigned int *rl_buffer_size,
90 bool *is_over_subscription)
91{
92 int retval;
93
94 BUG_ON(!pm);
95 BUG_ON(pm->allocated == true);
96 BUG_ON(is_over_subscription == NULL);
97
98 pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription);
99
100 retval = kfd2kgd->allocate_mem(pm->dqm->dev->kgd,
101 *rl_buffer_size,
102 PAGE_SIZE,
103 KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
104 (struct kgd_mem **) &pm->ib_buffer_obj);
105
106 if (retval != 0) {
107 pr_err("kfd: failed to allocate runlist IB\n");
108 return retval;
109 }
110
111 *(void **)rl_buffer = pm->ib_buffer_obj->cpu_ptr;
112 *rl_gpu_buffer = pm->ib_buffer_obj->gpu_addr;
113
114 memset(*rl_buffer, 0, *rl_buffer_size);
115 pm->allocated = true;
116 return retval;
117}
118
119static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer,
120 uint64_t ib, size_t ib_size_in_dwords, bool chain)
121{
122 struct pm4_runlist *packet;
123
124 BUG_ON(!pm || !buffer || !ib);
125
126 packet = (struct pm4_runlist *)buffer;
127
128 memset(buffer, 0, sizeof(struct pm4_runlist));
129 packet->header.u32all = build_pm4_header(IT_RUN_LIST,
130 sizeof(struct pm4_runlist));
131
132 packet->bitfields4.ib_size = ib_size_in_dwords;
133 packet->bitfields4.chain = chain ? 1 : 0;
134 packet->bitfields4.offload_polling = 0;
135 packet->bitfields4.valid = 1;
136 packet->ordinal2 = lower_32_bits(ib);
137 packet->bitfields3.ib_base_hi = upper_32_bits(ib);
138
139 return 0;
140}
141
142static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer,
143 struct qcm_process_device *qpd)
144{
145 struct pm4_map_process *packet;
146 struct queue *cur;
147 uint32_t num_queues;
148
149 BUG_ON(!pm || !buffer || !qpd);
150
151 packet = (struct pm4_map_process *)buffer;
152
153 pr_debug("kfd: In func %s\n", __func__);
154
155 memset(buffer, 0, sizeof(struct pm4_map_process));
156
157 packet->header.u32all = build_pm4_header(IT_MAP_PROCESS,
158 sizeof(struct pm4_map_process));
159 packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
160 packet->bitfields2.process_quantum = 1;
161 packet->bitfields2.pasid = qpd->pqm->process->pasid;
162 packet->bitfields3.page_table_base = qpd->page_table_base;
163 packet->bitfields10.gds_size = qpd->gds_size;
164 packet->bitfields10.num_gws = qpd->num_gws;
165 packet->bitfields10.num_oac = qpd->num_oac;
166 num_queues = 0;
167 list_for_each_entry(cur, &qpd->queues_list, list)
168 num_queues++;
169 packet->bitfields10.num_queues = num_queues;
170
171 packet->sh_mem_config = qpd->sh_mem_config;
172 packet->sh_mem_bases = qpd->sh_mem_bases;
173 packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base;
174 packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit;
175
176 packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
177 packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
178
179 return 0;
180}
181
182static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer,
183 struct queue *q)
184{
185 struct pm4_map_queues *packet;
186
187 BUG_ON(!pm || !buffer || !q);
188
189 pr_debug("kfd: In func %s\n", __func__);
190
191 packet = (struct pm4_map_queues *)buffer;
192 memset(buffer, 0, sizeof(struct pm4_map_queues));
193
194 packet->header.u32all = build_pm4_header(IT_MAP_QUEUES,
195 sizeof(struct pm4_map_queues));
196 packet->bitfields2.alloc_format =
197 alloc_format__mes_map_queues__one_per_pipe;
198 packet->bitfields2.num_queues = 1;
199 packet->bitfields2.queue_sel =
200 queue_sel__mes_map_queues__map_to_hws_determined_queue_slots;
201
202 packet->bitfields2.vidmem = (q->properties.is_interop) ?
203 vidmem__mes_map_queues__uses_video_memory :
204 vidmem__mes_map_queues__uses_no_video_memory;
205
206 switch (q->properties.type) {
207 case KFD_QUEUE_TYPE_COMPUTE:
208 case KFD_QUEUE_TYPE_DIQ:
209 packet->bitfields2.engine_sel =
210 engine_sel__mes_map_queues__compute;
211 break;
212 case KFD_QUEUE_TYPE_SDMA:
213 packet->bitfields2.engine_sel =
214 engine_sel__mes_map_queues__sdma0;
215 break;
216 default:
217 BUG();
218 break;
219 }
220
221 packet->mes_map_queues_ordinals[0].bitfields3.doorbell_offset =
222 q->properties.doorbell_off;
223
224 packet->mes_map_queues_ordinals[0].mqd_addr_lo =
225 lower_32_bits(q->gart_mqd_addr);
226
227 packet->mes_map_queues_ordinals[0].mqd_addr_hi =
228 upper_32_bits(q->gart_mqd_addr);
229
230 packet->mes_map_queues_ordinals[0].wptr_addr_lo =
231 lower_32_bits((uint64_t)q->properties.write_ptr);
232
233 packet->mes_map_queues_ordinals[0].wptr_addr_hi =
234 upper_32_bits((uint64_t)q->properties.write_ptr);
235
236 return 0;
237}
238
239static int pm_create_runlist_ib(struct packet_manager *pm,
240 struct list_head *queues,
241 uint64_t *rl_gpu_addr,
242 size_t *rl_size_bytes)
243{
244 unsigned int alloc_size_bytes;
245 unsigned int *rl_buffer, rl_wptr, i;
246 int retval, proccesses_mapped;
247 struct device_process_node *cur;
248 struct qcm_process_device *qpd;
249 struct queue *q;
250 struct kernel_queue *kq;
251 bool is_over_subscription;
252
253 BUG_ON(!pm || !queues || !rl_size_bytes || !rl_gpu_addr);
254
255 rl_wptr = retval = proccesses_mapped = 0;
256
257 retval = pm_allocate_runlist_ib(pm, &rl_buffer, rl_gpu_addr,
258 &alloc_size_bytes, &is_over_subscription);
259 if (retval != 0)
260 return retval;
261
262 *rl_size_bytes = alloc_size_bytes;
263
264 pr_debug("kfd: In func %s\n", __func__);
265 pr_debug("kfd: building runlist ib process count: %d queues count %d\n",
266 pm->dqm->processes_count, pm->dqm->queue_count);
267
268 /* build the run list ib packet */
269 list_for_each_entry(cur, queues, list) {
270 qpd = cur->qpd;
271 /* build map process packet */
272 if (proccesses_mapped >= pm->dqm->processes_count) {
273 pr_debug("kfd: not enough space left in runlist IB\n");
274 pm_release_ib(pm);
275 return -ENOMEM;
276 }
277 retval = pm_create_map_process(pm, &rl_buffer[rl_wptr], qpd);
278 if (retval != 0)
279 return retval;
280 proccesses_mapped++;
281 inc_wptr(&rl_wptr, sizeof(struct pm4_map_process),
282 alloc_size_bytes);
283
284 list_for_each_entry(kq, &qpd->priv_queue_list, list) {
285 if (kq->queue->properties.is_active != true)
286 continue;
287 retval = pm_create_map_queue(pm, &rl_buffer[rl_wptr],
288 kq->queue);
289 if (retval != 0)
290 return retval;
291 inc_wptr(&rl_wptr, sizeof(struct pm4_map_queues),
292 alloc_size_bytes);
293 }
294
295 list_for_each_entry(q, &qpd->queues_list, list) {
296 if (q->properties.is_active != true)
297 continue;
298 retval = pm_create_map_queue(pm,
299 &rl_buffer[rl_wptr], q);
300 if (retval != 0)
301 return retval;
302 inc_wptr(&rl_wptr, sizeof(struct pm4_map_queues),
303 alloc_size_bytes);
304 }
305 }
306
307 pr_debug("kfd: finished map process and queues to runlist\n");
308
309 if (is_over_subscription)
310 pm_create_runlist(pm, &rl_buffer[rl_wptr], *rl_gpu_addr,
311 alloc_size_bytes / sizeof(uint32_t), true);
312
313 for (i = 0; i < alloc_size_bytes / sizeof(uint32_t); i++)
314 pr_debug("0x%2X ", rl_buffer[i]);
315 pr_debug("\n");
316
317 return 0;
318}
319
320int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)
321{
322 BUG_ON(!dqm);
323
324 pm->dqm = dqm;
325 mutex_init(&pm->lock);
326 pm->priv_queue = kernel_queue_init(dqm->dev, KFD_QUEUE_TYPE_HIQ);
327 if (pm->priv_queue == NULL) {
328 mutex_destroy(&pm->lock);
329 return -ENOMEM;
330 }
331 pm->allocated = false;
332
333 return 0;
334}
335
336void pm_uninit(struct packet_manager *pm)
337{
338 BUG_ON(!pm);
339
340 mutex_destroy(&pm->lock);
341 kernel_queue_uninit(pm->priv_queue);
342}
343
344int pm_send_set_resources(struct packet_manager *pm,
345 struct scheduling_resources *res)
346{
347 struct pm4_set_resources *packet;
348
349 BUG_ON(!pm || !res);
350
351 pr_debug("kfd: In func %s\n", __func__);
352
353 mutex_lock(&pm->lock);
354 pm->priv_queue->acquire_packet_buffer(pm->priv_queue,
355 sizeof(*packet) / sizeof(uint32_t),
356 (unsigned int **)&packet);
357 if (packet == NULL) {
358 mutex_unlock(&pm->lock);
359 pr_err("kfd: failed to allocate buffer on kernel queue\n");
360 return -ENOMEM;
361 }
362
363 memset(packet, 0, sizeof(struct pm4_set_resources));
364 packet->header.u32all = build_pm4_header(IT_SET_RESOURCES,
365 sizeof(struct pm4_set_resources));
366
367 packet->bitfields2.queue_type =
368 queue_type__mes_set_resources__hsa_interface_queue_hiq;
369 packet->bitfields2.vmid_mask = res->vmid_mask;
370 packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY;
371 packet->bitfields7.oac_mask = res->oac_mask;
372 packet->bitfields8.gds_heap_base = res->gds_heap_base;
373 packet->bitfields8.gds_heap_size = res->gds_heap_size;
374
375 packet->gws_mask_lo = lower_32_bits(res->gws_mask);
376 packet->gws_mask_hi = upper_32_bits(res->gws_mask);
377
378 packet->queue_mask_lo = lower_32_bits(res->queue_mask);
379 packet->queue_mask_hi = upper_32_bits(res->queue_mask);
380
381 pm->priv_queue->submit_packet(pm->priv_queue);
382 pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT);
383
384 mutex_unlock(&pm->lock);
385
386 return 0;
387}
388
389int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues)
390{
391 uint64_t rl_gpu_ib_addr;
392 uint32_t *rl_buffer;
393 size_t rl_ib_size, packet_size_dwords;
394 int retval;
395
396 BUG_ON(!pm || !dqm_queues);
397
398 retval = pm_create_runlist_ib(pm, dqm_queues, &rl_gpu_ib_addr,
399 &rl_ib_size);
400 if (retval != 0)
401 goto fail_create_runlist_ib;
402
403 pr_debug("kfd: runlist IB address: 0x%llX\n", rl_gpu_ib_addr);
404
405 packet_size_dwords = sizeof(struct pm4_runlist) / sizeof(uint32_t);
406 mutex_lock(&pm->lock);
407
408 retval = pm->priv_queue->acquire_packet_buffer(pm->priv_queue,
409 packet_size_dwords, &rl_buffer);
410 if (retval != 0)
411 goto fail_acquire_packet_buffer;
412
413 retval = pm_create_runlist(pm, rl_buffer, rl_gpu_ib_addr,
414 rl_ib_size / sizeof(uint32_t), false);
415 if (retval != 0)
416 goto fail_create_runlist;
417
418 pm->priv_queue->submit_packet(pm->priv_queue);
419 pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT);
420
421 mutex_unlock(&pm->lock);
422
423 return retval;
424
425fail_create_runlist:
426 pm->priv_queue->rollback_packet(pm->priv_queue);
427fail_acquire_packet_buffer:
428 mutex_unlock(&pm->lock);
429fail_create_runlist_ib:
430 if (pm->allocated == true)
431 pm_release_ib(pm);
432 return retval;
433}
434
435int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
436 uint32_t fence_value)
437{
438 int retval;
439 struct pm4_query_status *packet;
440
441 BUG_ON(!pm || !fence_address);
442
443 mutex_lock(&pm->lock);
444 retval = pm->priv_queue->acquire_packet_buffer(
445 pm->priv_queue,
446 sizeof(struct pm4_query_status) / sizeof(uint32_t),
447 (unsigned int **)&packet);
448 if (retval != 0)
449 goto fail_acquire_packet_buffer;
450
451 packet->header.u32all = build_pm4_header(IT_QUERY_STATUS,
452 sizeof(struct pm4_query_status));
453
454 packet->bitfields2.context_id = 0;
455 packet->bitfields2.interrupt_sel =
456 interrupt_sel__mes_query_status__completion_status;
457 packet->bitfields2.command =
458 command__mes_query_status__fence_only_after_write_ack;
459
460 packet->addr_hi = upper_32_bits((uint64_t)fence_address);
461 packet->addr_lo = lower_32_bits((uint64_t)fence_address);
462 packet->data_hi = upper_32_bits((uint64_t)fence_value);
463 packet->data_lo = lower_32_bits((uint64_t)fence_value);
464
465 pm->priv_queue->submit_packet(pm->priv_queue);
466 pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT);
467 mutex_unlock(&pm->lock);
468
469 return 0;
470
471fail_acquire_packet_buffer:
472 mutex_unlock(&pm->lock);
473 return retval;
474}
475
476int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
477 enum kfd_preempt_type_filter mode,
478 uint32_t filter_param, bool reset,
479 unsigned int sdma_engine)
480{
481 int retval;
482 uint32_t *buffer;
483 struct pm4_unmap_queues *packet;
484
485 BUG_ON(!pm);
486
487 mutex_lock(&pm->lock);
488 retval = pm->priv_queue->acquire_packet_buffer(
489 pm->priv_queue,
490 sizeof(struct pm4_unmap_queues) / sizeof(uint32_t),
491 &buffer);
492 if (retval != 0)
493 goto err_acquire_packet_buffer;
494
495 packet = (struct pm4_unmap_queues *)buffer;
496 memset(buffer, 0, sizeof(struct pm4_unmap_queues));
497
498 packet->header.u32all = build_pm4_header(IT_UNMAP_QUEUES,
499 sizeof(struct pm4_unmap_queues));
500 switch (type) {
501 case KFD_QUEUE_TYPE_COMPUTE:
502 case KFD_QUEUE_TYPE_DIQ:
503 packet->bitfields2.engine_sel =
504 engine_sel__mes_unmap_queues__compute;
505 break;
506 case KFD_QUEUE_TYPE_SDMA:
507 packet->bitfields2.engine_sel =
508 engine_sel__mes_unmap_queues__sdma0 + sdma_engine;
509 break;
510 default:
511 BUG();
512 break;
513 }
514
515 if (reset)
516 packet->bitfields2.action =
517 action__mes_unmap_queues__reset_queues;
518 else
519 packet->bitfields2.action =
520 action__mes_unmap_queues__preempt_queues;
521
522 switch (mode) {
523 case KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE:
524 packet->bitfields2.queue_sel =
525 queue_sel__mes_unmap_queues__perform_request_on_specified_queues;
526 packet->bitfields2.num_queues = 1;
527 packet->bitfields3b.doorbell_offset0 = filter_param;
528 break;
529 case KFD_PREEMPT_TYPE_FILTER_BY_PASID:
530 packet->bitfields2.queue_sel =
531 queue_sel__mes_unmap_queues__perform_request_on_pasid_queues;
532 packet->bitfields3a.pasid = filter_param;
533 break;
534 case KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES:
535 packet->bitfields2.queue_sel =
536 queue_sel__mes_unmap_queues__perform_request_on_all_active_queues;
537 break;
538 default:
539 BUG();
540 break;
541 };
542
543 pm->priv_queue->submit_packet(pm->priv_queue);
544 pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT);
545
546 mutex_unlock(&pm->lock);
547 return 0;
548
549err_acquire_packet_buffer:
550 mutex_unlock(&pm->lock);
551 return retval;
552}
553
554void pm_release_ib(struct packet_manager *pm)
555{
556 BUG_ON(!pm);
557
558 mutex_lock(&pm->lock);
559 if (pm->allocated) {
560 kfd2kgd->free_mem(pm->dqm->dev->kgd,
561 (struct kgd_mem *) pm->ib_buffer_obj);
562 pm->allocated = false;
563 }
564 mutex_unlock(&pm->lock);
565}
drivers/gpu/drm/amd/amdkfd/kfd_pasid.c+97−0
@@ -0,0 +1,97 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/slab.h>
24#include <linux/types.h>
25#include "kfd_priv.h"
26
27static unsigned long *pasid_bitmap;
28static unsigned int pasid_limit;
29static DEFINE_MUTEX(pasid_mutex);
30
31int kfd_pasid_init(void)
32{
33 pasid_limit = max_num_of_processes;
34
35 pasid_bitmap = kzalloc(DIV_ROUND_UP(pasid_limit, BITS_PER_BYTE),
36 GFP_KERNEL);
37 if (!pasid_bitmap)
38 return -ENOMEM;
39
40 set_bit(0, pasid_bitmap); /* PASID 0 is reserved. */
41
42 return 0;
43}
44
45void kfd_pasid_exit(void)
46{
47 kfree(pasid_bitmap);
48}
49
50bool kfd_set_pasid_limit(unsigned int new_limit)
51{
52 if (new_limit < pasid_limit) {
53 bool ok;
54
55 mutex_lock(&pasid_mutex);
56
57 /* ensure that no pasids >= new_limit are in-use */
58 ok = (find_next_bit(pasid_bitmap, pasid_limit, new_limit) ==
59 pasid_limit);
60 if (ok)
61 pasid_limit = new_limit;
62
63 mutex_unlock(&pasid_mutex);
64
65 return ok;
66 }
67
68 return true;
69}
70
71inline unsigned int kfd_get_pasid_limit(void)
72{
73 return pasid_limit;
74}
75
76unsigned int kfd_pasid_alloc(void)
77{
78 unsigned int found;
79
80 mutex_lock(&pasid_mutex);
81
82 found = find_first_zero_bit(pasid_bitmap, pasid_limit);
83 if (found == pasid_limit)
84 found = 0;
85 else
86 set_bit(found, pasid_bitmap);
87
88 mutex_unlock(&pasid_mutex);
89
90 return found;
91}
92
93void kfd_pasid_free(unsigned int pasid)
94{
95 BUG_ON(pasid == 0 || pasid >= pasid_limit);
96 clear_bit(pasid, pasid_bitmap);
97}
drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h+405−0
@@ -0,0 +1,405 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#ifndef KFD_PM4_HEADERS_H_
25#define KFD_PM4_HEADERS_H_
26
27#ifndef PM4_MES_HEADER_DEFINED
28#define PM4_MES_HEADER_DEFINED
29union PM4_MES_TYPE_3_HEADER {
30 struct {
31 uint32_t reserved1:8; /* < reserved */
32 uint32_t opcode:8; /* < IT opcode */
33 uint32_t count:14; /* < number of DWORDs - 1
34 * in the information body.
35 */
36 uint32_t type:2; /* < packet identifier.
37 * It should be 3 for type 3 packets
38 */
39 };
40 uint32_t u32all;
41};
42#endif /* PM4_MES_HEADER_DEFINED */
43
44/* --------------------MES_SET_RESOURCES-------------------- */
45
46#ifndef PM4_MES_SET_RESOURCES_DEFINED
47#define PM4_MES_SET_RESOURCES_DEFINED
48enum set_resources_queue_type_enum {
49 queue_type__mes_set_resources__kernel_interface_queue_kiq = 0,
50 queue_type__mes_set_resources__hsa_interface_queue_hiq = 1,
51 queue_type__mes_set_resources__hsa_debug_interface_queue = 4
52};
53
54struct pm4_set_resources {
55 union {
56 union PM4_MES_TYPE_3_HEADER header; /* header */
57 uint32_t ordinal1;
58 };
59
60 union {
61 struct {
62 uint32_t vmid_mask:16;
63 uint32_t unmap_latency:8;
64 uint32_t reserved1:5;
65 enum set_resources_queue_type_enum queue_type:3;
66 } bitfields2;
67 uint32_t ordinal2;
68 };
69
70 uint32_t queue_mask_lo;
71 uint32_t queue_mask_hi;
72 uint32_t gws_mask_lo;
73 uint32_t gws_mask_hi;
74
75 union {
76 struct {
77 uint32_t oac_mask:16;
78 uint32_t reserved2:16;
79 } bitfields7;
80 uint32_t ordinal7;
81 };
82
83 union {
84 struct {
85 uint32_t gds_heap_base:6;
86 uint32_t reserved3:5;
87 uint32_t gds_heap_size:6;
88 uint32_t reserved4:15;
89 } bitfields8;
90 uint32_t ordinal8;
91 };
92
93};
94#endif
95
96/*--------------------MES_RUN_LIST-------------------- */
97
98#ifndef PM4_MES_RUN_LIST_DEFINED
99#define PM4_MES_RUN_LIST_DEFINED
100
101struct pm4_runlist {
102 union {
103 union PM4_MES_TYPE_3_HEADER header; /* header */
104 uint32_t ordinal1;
105 };
106
107 union {
108 struct {
109 uint32_t reserved1:2;
110 uint32_t ib_base_lo:30;
111 } bitfields2;
112 uint32_t ordinal2;
113 };
114
115 union {
116 struct {
117 uint32_t ib_base_hi:16;
118 uint32_t reserved2:16;
119 } bitfields3;
120 uint32_t ordinal3;
121 };
122
123 union {
124 struct {
125 uint32_t ib_size:20;
126 uint32_t chain:1;
127 uint32_t offload_polling:1;
128 uint32_t reserved3:1;
129 uint32_t valid:1;
130 uint32_t reserved4:8;
131 } bitfields4;
132 uint32_t ordinal4;
133 };
134
135};
136#endif
137
138/*--------------------MES_MAP_PROCESS-------------------- */
139
140#ifndef PM4_MES_MAP_PROCESS_DEFINED
141#define PM4_MES_MAP_PROCESS_DEFINED
142
143struct pm4_map_process {
144 union {
145 union PM4_MES_TYPE_3_HEADER header; /* header */
146 uint32_t ordinal1;
147 };
148
149 union {
150 struct {
151 uint32_t pasid:16;
152 uint32_t reserved1:8;
153 uint32_t diq_enable:1;
154 uint32_t process_quantum:7;
155 } bitfields2;
156 uint32_t ordinal2;
157 };
158
159 union {
160 struct {
161 uint32_t page_table_base:28;
162 uint32_t reserved3:4;
163 } bitfields3;
164 uint32_t ordinal3;
165 };
166
167 uint32_t sh_mem_bases;
168 uint32_t sh_mem_ape1_base;
169 uint32_t sh_mem_ape1_limit;
170 uint32_t sh_mem_config;
171 uint32_t gds_addr_lo;
172 uint32_t gds_addr_hi;
173
174 union {
175 struct {
176 uint32_t num_gws:6;
177 uint32_t reserved4:2;
178 uint32_t num_oac:4;
179 uint32_t reserved5:4;
180 uint32_t gds_size:6;
181 uint32_t num_queues:10;
182 } bitfields10;
183 uint32_t ordinal10;
184 };
185
186};
187#endif
188
189/*--------------------MES_MAP_QUEUES--------------------*/
190
191#ifndef PM4_MES_MAP_QUEUES_DEFINED
192#define PM4_MES_MAP_QUEUES_DEFINED
193enum map_queues_queue_sel_enum {
194 queue_sel__mes_map_queues__map_to_specified_queue_slots = 0,
195 queue_sel__mes_map_queues__map_to_hws_determined_queue_slots = 1,
196 queue_sel__mes_map_queues__enable_process_queues = 2
197};
198
199enum map_queues_vidmem_enum {
200 vidmem__mes_map_queues__uses_no_video_memory = 0,
201 vidmem__mes_map_queues__uses_video_memory = 1
202};
203
204enum map_queues_alloc_format_enum {
205 alloc_format__mes_map_queues__one_per_pipe = 0,
206 alloc_format__mes_map_queues__all_on_one_pipe = 1
207};
208
209enum map_queues_engine_sel_enum {
210 engine_sel__mes_map_queues__compute = 0,
211 engine_sel__mes_map_queues__sdma0 = 2,
212 engine_sel__mes_map_queues__sdma1 = 3
213};
214
215struct pm4_map_queues {
216 union {
217 union PM4_MES_TYPE_3_HEADER header; /* header */
218 uint32_t ordinal1;
219 };
220
221 union {
222 struct {
223 uint32_t reserved1:4;
224 enum map_queues_queue_sel_enum queue_sel:2;
225 uint32_t reserved2:2;
226 uint32_t vmid:4;
227 uint32_t reserved3:4;
228 enum map_queues_vidmem_enum vidmem:2;
229 uint32_t reserved4:6;
230 enum map_queues_alloc_format_enum alloc_format:2;
231 enum map_queues_engine_sel_enum engine_sel:3;
232 uint32_t num_queues:3;
233 } bitfields2;
234 uint32_t ordinal2;
235 };
236
237 struct {
238 union {
239 struct {
240 uint32_t reserved5:2;
241 uint32_t doorbell_offset:21;
242 uint32_t reserved6:3;
243 uint32_t queue:6;
244 } bitfields3;
245 uint32_t ordinal3;
246 };
247
248 uint32_t mqd_addr_lo;
249 uint32_t mqd_addr_hi;
250 uint32_t wptr_addr_lo;
251 uint32_t wptr_addr_hi;
252
253 } mes_map_queues_ordinals[1]; /* 1..N of these ordinal groups */
254
255};
256#endif
257
258/*--------------------MES_QUERY_STATUS--------------------*/
259
260#ifndef PM4_MES_QUERY_STATUS_DEFINED
261#define PM4_MES_QUERY_STATUS_DEFINED
262enum query_status_interrupt_sel_enum {
263 interrupt_sel__mes_query_status__completion_status = 0,
264 interrupt_sel__mes_query_status__process_status = 1,
265 interrupt_sel__mes_query_status__queue_status = 2
266};
267
268enum query_status_command_enum {
269 command__mes_query_status__interrupt_only = 0,
270 command__mes_query_status__fence_only_immediate = 1,
271 command__mes_query_status__fence_only_after_write_ack = 2,
272 command__mes_query_status__fence_wait_for_write_ack_send_interrupt = 3
273};
274
275enum query_status_engine_sel_enum {
276 engine_sel__mes_query_status__compute = 0,
277 engine_sel__mes_query_status__sdma0_queue = 2,
278 engine_sel__mes_query_status__sdma1_queue = 3
279};
280
281struct pm4_query_status {
282 union {
283 union PM4_MES_TYPE_3_HEADER header; /* header */
284 uint32_t ordinal1;
285 };
286
287 union {
288 struct {
289 uint32_t context_id:28;
290 enum query_status_interrupt_sel_enum interrupt_sel:2;
291 enum query_status_command_enum command:2;
292 } bitfields2;
293 uint32_t ordinal2;
294 };
295
296 union {
297 struct {
298 uint32_t pasid:16;
299 uint32_t reserved1:16;
300 } bitfields3a;
301 struct {
302 uint32_t reserved2:2;
303 uint32_t doorbell_offset:21;
304 uint32_t reserved3:3;
305 enum query_status_engine_sel_enum engine_sel:3;
306 uint32_t reserved4:3;
307 } bitfields3b;
308 uint32_t ordinal3;
309 };
310
311 uint32_t addr_lo;
312 uint32_t addr_hi;
313 uint32_t data_lo;
314 uint32_t data_hi;
315};
316#endif
317
318/*--------------------MES_UNMAP_QUEUES--------------------*/
319
320#ifndef PM4_MES_UNMAP_QUEUES_DEFINED
321#define PM4_MES_UNMAP_QUEUES_DEFINED
322enum unmap_queues_action_enum {
323 action__mes_unmap_queues__preempt_queues = 0,
324 action__mes_unmap_queues__reset_queues = 1,
325 action__mes_unmap_queues__disable_process_queues = 2
326};
327
328enum unmap_queues_queue_sel_enum {
329 queue_sel__mes_unmap_queues__perform_request_on_specified_queues = 0,
330 queue_sel__mes_unmap_queues__perform_request_on_pasid_queues = 1,
331 queue_sel__mes_unmap_queues__perform_request_on_all_active_queues = 2
332};
333
334enum unmap_queues_engine_sel_enum {
335 engine_sel__mes_unmap_queues__compute = 0,
336 engine_sel__mes_unmap_queues__sdma0 = 2,
337 engine_sel__mes_unmap_queues__sdma1 = 3
338};
339
340struct pm4_unmap_queues {
341 union {
342 union PM4_MES_TYPE_3_HEADER header; /* header */
343 uint32_t ordinal1;
344 };
345
346 union {
347 struct {
348 enum unmap_queues_action_enum action:2;
349 uint32_t reserved1:2;
350 enum unmap_queues_queue_sel_enum queue_sel:2;
351 uint32_t reserved2:20;
352 enum unmap_queues_engine_sel_enum engine_sel:3;
353 uint32_t num_queues:3;
354 } bitfields2;
355 uint32_t ordinal2;
356 };
357
358 union {
359 struct {
360 uint32_t pasid:16;
361 uint32_t reserved3:16;
362 } bitfields3a;
363 struct {
364 uint32_t reserved4:2;
365 uint32_t doorbell_offset0:21;
366 uint32_t reserved5:9;
367 } bitfields3b;
368 uint32_t ordinal3;
369 };
370
371 union {
372 struct {
373 uint32_t reserved6:2;
374 uint32_t doorbell_offset1:21;
375 uint32_t reserved7:9;
376 } bitfields4;
377 uint32_t ordinal4;
378 };
379
380 union {
381 struct {
382 uint32_t reserved8:2;
383 uint32_t doorbell_offset2:21;
384 uint32_t reserved9:9;
385 } bitfields5;
386 uint32_t ordinal5;
387 };
388
389 union {
390 struct {
391 uint32_t reserved10:2;
392 uint32_t doorbell_offset3:21;
393 uint32_t reserved11:9;
394 } bitfields6;
395 uint32_t ordinal6;
396 };
397
398};
399#endif
400
401enum {
402 CACHE_FLUSH_AND_INV_TS_EVENT = 0x00000014
403};
404
405#endif /* KFD_PM4_HEADERS_H_ */
drivers/gpu/drm/amd/amdkfd/kfd_pm4_opcodes.h+107−0
@@ -0,0 +1,107 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24
25#ifndef KFD_PM4_OPCODES_H
26#define KFD_PM4_OPCODES_H
27
28enum it_opcode_type {
29 IT_NOP = 0x10,
30 IT_SET_BASE = 0x11,
31 IT_CLEAR_STATE = 0x12,
32 IT_INDEX_BUFFER_SIZE = 0x13,
33 IT_DISPATCH_DIRECT = 0x15,
34 IT_DISPATCH_INDIRECT = 0x16,
35 IT_ATOMIC_GDS = 0x1D,
36 IT_OCCLUSION_QUERY = 0x1F,
37 IT_SET_PREDICATION = 0x20,
38 IT_REG_RMW = 0x21,
39 IT_COND_EXEC = 0x22,
40 IT_PRED_EXEC = 0x23,
41 IT_DRAW_INDIRECT = 0x24,
42 IT_DRAW_INDEX_INDIRECT = 0x25,
43 IT_INDEX_BASE = 0x26,
44 IT_DRAW_INDEX_2 = 0x27,
45 IT_CONTEXT_CONTROL = 0x28,
46 IT_INDEX_TYPE = 0x2A,
47 IT_DRAW_INDIRECT_MULTI = 0x2C,
48 IT_DRAW_INDEX_AUTO = 0x2D,
49 IT_NUM_INSTANCES = 0x2F,
50 IT_DRAW_INDEX_MULTI_AUTO = 0x30,
51 IT_INDIRECT_BUFFER_CNST = 0x33,
52 IT_STRMOUT_BUFFER_UPDATE = 0x34,
53 IT_DRAW_INDEX_OFFSET_2 = 0x35,
54 IT_DRAW_PREAMBLE = 0x36,
55 IT_WRITE_DATA = 0x37,
56 IT_DRAW_INDEX_INDIRECT_MULTI = 0x38,
57 IT_MEM_SEMAPHORE = 0x39,
58 IT_COPY_DW = 0x3B,
59 IT_WAIT_REG_MEM = 0x3C,
60 IT_INDIRECT_BUFFER = 0x3F,
61 IT_COPY_DATA = 0x40,
62 IT_PFP_SYNC_ME = 0x42,
63 IT_SURFACE_SYNC = 0x43,
64 IT_COND_WRITE = 0x45,
65 IT_EVENT_WRITE = 0x46,
66 IT_EVENT_WRITE_EOP = 0x47,
67 IT_EVENT_WRITE_EOS = 0x48,
68 IT_RELEASE_MEM = 0x49,
69 IT_PREAMBLE_CNTL = 0x4A,
70 IT_DMA_DATA = 0x50,
71 IT_ACQUIRE_MEM = 0x58,
72 IT_REWIND = 0x59,
73 IT_LOAD_UCONFIG_REG = 0x5E,
74 IT_LOAD_SH_REG = 0x5F,
75 IT_LOAD_CONFIG_REG = 0x60,
76 IT_LOAD_CONTEXT_REG = 0x61,
77 IT_SET_CONFIG_REG = 0x68,
78 IT_SET_CONTEXT_REG = 0x69,
79 IT_SET_CONTEXT_REG_INDIRECT = 0x73,
80 IT_SET_SH_REG = 0x76,
81 IT_SET_SH_REG_OFFSET = 0x77,
82 IT_SET_QUEUE_REG = 0x78,
83 IT_SET_UCONFIG_REG = 0x79,
84 IT_SCRATCH_RAM_WRITE = 0x7D,
85 IT_SCRATCH_RAM_READ = 0x7E,
86 IT_LOAD_CONST_RAM = 0x80,
87 IT_WRITE_CONST_RAM = 0x81,
88 IT_DUMP_CONST_RAM = 0x83,
89 IT_INCREMENT_CE_COUNTER = 0x84,
90 IT_INCREMENT_DE_COUNTER = 0x85,
91 IT_WAIT_ON_CE_COUNTER = 0x86,
92 IT_WAIT_ON_DE_COUNTER_DIFF = 0x88,
93 IT_SWITCH_BUFFER = 0x8B,
94 IT_SET_RESOURCES = 0xA0,
95 IT_MAP_PROCESS = 0xA1,
96 IT_MAP_QUEUES = 0xA2,
97 IT_UNMAP_QUEUES = 0xA3,
98 IT_QUERY_STATUS = 0xA4,
99 IT_RUN_LIST = 0xA5,
100};
101
102#define PM4_TYPE_0 0
103#define PM4_TYPE_2 2
104#define PM4_TYPE_3 3
105
106#endif /* KFD_PM4_OPCODES_H */
107
drivers/gpu/drm/amd/amdkfd/kfd_priv.h+598−0
@@ -0,0 +1,598 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef KFD_PRIV_H_INCLUDED
24#define KFD_PRIV_H_INCLUDED
25
26#include <linux/hashtable.h>
27#include <linux/mmu_notifier.h>
28#include <linux/mutex.h>
29#include <linux/types.h>
30#include <linux/atomic.h>
31#include <linux/workqueue.h>
32#include <linux/spinlock.h>
33#include <linux/kfd_ioctl.h>
34#include <kgd_kfd_interface.h>
35
36#define KFD_SYSFS_FILE_MODE 0444
37
38/*
39 * When working with cp scheduler we should assign the HIQ manually or via
40 * the radeon driver to a fixed hqd slot, here are the fixed HIQ hqd slot
41 * definitions for Kaveri. In Kaveri only the first ME queues participates
42 * in the cp scheduling taking that in mind we set the HIQ slot in the
43 * second ME.
44 */
45#define KFD_CIK_HIQ_PIPE 4
46#define KFD_CIK_HIQ_QUEUE 0
47
48/* GPU ID hash width in bits */
49#define KFD_GPU_ID_HASH_WIDTH 16
50
51/* Macro for allocating structures */
52#define kfd_alloc_struct(ptr_to_struct) \
53 ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))
54
55/* Kernel module parameter to specify maximum number of supported processes */
56extern int max_num_of_processes;
57
58#define KFD_MAX_NUM_OF_PROCESSES_DEFAULT 32
59#define KFD_MAX_NUM_OF_PROCESSES 512
60
61/*
62 * Kernel module parameter to specify maximum number of supported queues
63 * per process
64 */
65extern int max_num_of_queues_per_process;
66
67#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT 128
68#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
69
70#define KFD_KERNEL_QUEUE_SIZE 2048
71
72/* Kernel module parameter to specify the scheduling policy */
73extern int sched_policy;
74
75/**
76 * enum kfd_sched_policy
77 *
78 * @KFD_SCHED_POLICY_HWS: H/W scheduling policy known as command processor (cp)
79 * scheduling. In this scheduling mode we're using the firmware code to
80 * schedule the user mode queues and kernel queues such as HIQ and DIQ.
81 * the HIQ queue is used as a special queue that dispatches the configuration
82 * to the cp and the user mode queues list that are currently running.
83 * the DIQ queue is a debugging queue that dispatches debugging commands to the
84 * firmware.
85 * in this scheduling mode user mode queues over subscription feature is
86 * enabled.
87 *
88 * @KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: The same as above but the over
89 * subscription feature disabled.
90 *
91 * @KFD_SCHED_POLICY_NO_HWS: no H/W scheduling policy is a mode which directly
92 * set the command processor registers and sets the queues "manually". This
93 * mode is used *ONLY* for debugging proposes.
94 *
95 */
96enum kfd_sched_policy {
97 KFD_SCHED_POLICY_HWS = 0,
98 KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION,
99 KFD_SCHED_POLICY_NO_HWS
100};
101
102enum cache_policy {
103 cache_policy_coherent,
104 cache_policy_noncoherent
105};
106
107struct kfd_device_info {
108 unsigned int max_pasid_bits;
109 size_t ih_ring_entry_size;
110 uint16_t mqd_size_aligned;
111};
112
113struct kfd_dev {
114 struct kgd_dev *kgd;
115
116 const struct kfd_device_info *device_info;
117 struct pci_dev *pdev;
118
119 unsigned int id; /* topology stub index */
120
121 phys_addr_t doorbell_base; /* Start of actual doorbells used by
122 * KFD. It is aligned for mapping
123 * into user mode
124 */
125 size_t doorbell_id_offset; /* Doorbell offset (from KFD doorbell
126 * to HW doorbell, GFX reserved some
127 * at the start)
128 */
129 size_t doorbell_process_limit; /* Number of processes we have doorbell
130 * space for.
131 */
132 u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells
133 * page used by kernel queue
134 */
135
136 struct kgd2kfd_shared_resources shared_resources;
137
138 void *interrupt_ring;
139 size_t interrupt_ring_size;
140 atomic_t interrupt_ring_rptr;
141 atomic_t interrupt_ring_wptr;
142 struct work_struct interrupt_work;
143 spinlock_t interrupt_lock;
144
145 /* QCM Device instance */
146 struct device_queue_manager *dqm;
147
148 bool init_complete;
149 /*
150 * Interrupts of interest to KFD are copied
151 * from the HW ring into a SW ring.
152 */
153 bool interrupts_active;
154};
155
156/* KGD2KFD callbacks */
157void kgd2kfd_exit(void);
158struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev);
159bool kgd2kfd_device_init(struct kfd_dev *kfd,
160 const struct kgd2kfd_shared_resources *gpu_resources);
161void kgd2kfd_device_exit(struct kfd_dev *kfd);
162
163extern const struct kfd2kgd_calls *kfd2kgd;
164
165struct kfd_mem_obj {
166 void *bo;
167 uint64_t gpu_addr;
168 uint32_t *cpu_ptr;
169};
170
171enum kfd_mempool {
172 KFD_MEMPOOL_SYSTEM_CACHEABLE = 1,
173 KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2,
174 KFD_MEMPOOL_FRAMEBUFFER = 3,
175};
176
177/* Character device interface */
178int kfd_chardev_init(void);
179void kfd_chardev_exit(void);
180struct device *kfd_chardev(void);
181
182/**
183 * enum kfd_preempt_type_filter
184 *
185 * @KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE: Preempts single queue.
186 *
187 * @KFD_PRERMPT_TYPE_FILTER_ALL_QUEUES: Preempts all queues in the
188 * running queues list.
189 *
190 * @KFD_PRERMPT_TYPE_FILTER_BY_PASID: Preempts queues that belongs to
191 * specific process.
192 *
193 */
194enum kfd_preempt_type_filter {
195 KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE,
196 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES,
197 KFD_PREEMPT_TYPE_FILTER_BY_PASID
198};
199
200enum kfd_preempt_type {
201 KFD_PREEMPT_TYPE_WAVEFRONT,
202 KFD_PREEMPT_TYPE_WAVEFRONT_RESET
203};
204
205/**
206 * enum kfd_queue_type
207 *
208 * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type.
209 *
210 * @KFD_QUEUE_TYPE_SDMA: Sdma user mode queue type.
211 *
212 * @KFD_QUEUE_TYPE_HIQ: HIQ queue type.
213 *
214 * @KFD_QUEUE_TYPE_DIQ: DIQ queue type.
215 */
216enum kfd_queue_type {
217 KFD_QUEUE_TYPE_COMPUTE,
218 KFD_QUEUE_TYPE_SDMA,
219 KFD_QUEUE_TYPE_HIQ,
220 KFD_QUEUE_TYPE_DIQ
221};
222
223enum kfd_queue_format {
224 KFD_QUEUE_FORMAT_PM4,
225 KFD_QUEUE_FORMAT_AQL
226};
227
228/**
229 * struct queue_properties
230 *
231 * @type: The queue type.
232 *
233 * @queue_id: Queue identifier.
234 *
235 * @queue_address: Queue ring buffer address.
236 *
237 * @queue_size: Queue ring buffer size.
238 *
239 * @priority: Defines the queue priority relative to other queues in the
240 * process.
241 * This is just an indication and HW scheduling may override the priority as
242 * necessary while keeping the relative prioritization.
243 * the priority granularity is from 0 to f which f is the highest priority.
244 * currently all queues are initialized with the highest priority.
245 *
246 * @queue_percent: This field is partially implemented and currently a zero in
247 * this field defines that the queue is non active.
248 *
249 * @read_ptr: User space address which points to the number of dwords the
250 * cp read from the ring buffer. This field updates automatically by the H/W.
251 *
252 * @write_ptr: Defines the number of dwords written to the ring buffer.
253 *
254 * @doorbell_ptr: This field aim is to notify the H/W of new packet written to
255 * the queue ring buffer. This field should be similar to write_ptr and the user
256 * should update this field after he updated the write_ptr.
257 *
258 * @doorbell_off: The doorbell offset in the doorbell pci-bar.
259 *
260 * @is_interop: Defines if this is a interop queue. Interop queue means that the
261 * queue can access both graphics and compute resources.
262 *
263 * @is_active: Defines if the queue is active or not.
264 *
265 * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid
266 * of the queue.
267 *
268 * This structure represents the queue properties for each queue no matter if
269 * it's user mode or kernel mode queue.
270 *
271 */
272struct queue_properties {
273 enum kfd_queue_type type;
274 enum kfd_queue_format format;
275 unsigned int queue_id;
276 uint64_t queue_address;
277 uint64_t queue_size;
278 uint32_t priority;
279 uint32_t queue_percent;
280 uint32_t *read_ptr;
281 uint32_t *write_ptr;
282 uint32_t *doorbell_ptr;
283 uint32_t doorbell_off;
284 bool is_interop;
285 bool is_active;
286 /* Not relevant for user mode queues in cp scheduling */
287 unsigned int vmid;
288};
289
290/**
291 * struct queue
292 *
293 * @list: Queue linked list.
294 *
295 * @mqd: The queue MQD.
296 *
297 * @mqd_mem_obj: The MQD local gpu memory object.
298 *
299 * @gart_mqd_addr: The MQD gart mc address.
300 *
301 * @properties: The queue properties.
302 *
303 * @mec: Used only in no cp scheduling mode and identifies to micro engine id
304 * that the queue should be execute on.
305 *
306 * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe id.
307 *
308 * @queue: Used only in no cp scheduliong mode and identifies the queue's slot.
309 *
310 * @process: The kfd process that created this queue.
311 *
312 * @device: The kfd device that created this queue.
313 *
314 * This structure represents user mode compute queues.
315 * It contains all the necessary data to handle such queues.
316 *
317 */
318
319struct queue {
320 struct list_head list;
321 void *mqd;
322 struct kfd_mem_obj *mqd_mem_obj;
323 uint64_t gart_mqd_addr;
324 struct queue_properties properties;
325
326 uint32_t mec;
327 uint32_t pipe;
328 uint32_t queue;
329
330 struct kfd_process *process;
331 struct kfd_dev *device;
332};
333
334/*
335 * Please read the kfd_mqd_manager.h description.
336 */
337enum KFD_MQD_TYPE {
338 KFD_MQD_TYPE_CIK_COMPUTE = 0, /* for no cp scheduling */
339 KFD_MQD_TYPE_CIK_HIQ, /* for hiq */
340 KFD_MQD_TYPE_CIK_CP, /* for cp queues and diq */
341 KFD_MQD_TYPE_CIK_SDMA, /* for sdma queues */
342 KFD_MQD_TYPE_MAX
343};
344
345struct scheduling_resources {
346 unsigned int vmid_mask;
347 enum kfd_queue_type type;
348 uint64_t queue_mask;
349 uint64_t gws_mask;
350 uint32_t oac_mask;
351 uint32_t gds_heap_base;
352 uint32_t gds_heap_size;
353};
354
355struct process_queue_manager {
356 /* data */
357 struct kfd_process *process;
358 unsigned int num_concurrent_processes;
359 struct list_head queues;
360 unsigned long *queue_slot_bitmap;
361};
362
363struct qcm_process_device {
364 /* The Device Queue Manager that owns this data */
365 struct device_queue_manager *dqm;
366 struct process_queue_manager *pqm;
367 /* Device Queue Manager lock */
368 struct mutex *lock;
369 /* Queues list */
370 struct list_head queues_list;
371 struct list_head priv_queue_list;
372
373 unsigned int queue_count;
374 unsigned int vmid;
375 bool is_debug;
376 /*
377 * All the memory management data should be here too
378 */
379 uint64_t gds_context_area;
380 uint32_t sh_mem_config;
381 uint32_t sh_mem_bases;
382 uint32_t sh_mem_ape1_base;
383 uint32_t sh_mem_ape1_limit;
384 uint32_t page_table_base;
385 uint32_t gds_size;
386 uint32_t num_gws;
387 uint32_t num_oac;
388};
389
390/* Data that is per-process-per device. */
391struct kfd_process_device {
392 /*
393 * List of all per-device data for a process.
394 * Starts from kfd_process.per_device_data.
395 */
396 struct list_head per_device_list;
397
398 /* The device that owns this data. */
399 struct kfd_dev *dev;
400
401
402 /* per-process-per device QCM data structure */
403 struct qcm_process_device qpd;
404
405 /*Apertures*/
406 uint64_t lds_base;
407 uint64_t lds_limit;
408 uint64_t gpuvm_base;
409 uint64_t gpuvm_limit;
410 uint64_t scratch_base;
411 uint64_t scratch_limit;
412
413 /* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */
414 bool bound;
415};
416
417/* Process data */
418struct kfd_process {
419 /*
420 * kfd_process are stored in an mm_struct*->kfd_process*
421 * hash table (kfd_processes in kfd_process.c)
422 */
423 struct hlist_node kfd_processes;
424
425 struct mm_struct *mm;
426
427 struct mutex mutex;
428
429 /*
430 * In any process, the thread that started main() is the lead
431 * thread and outlives the rest.
432 * It is here because amd_iommu_bind_pasid wants a task_struct.
433 */
434 struct task_struct *lead_thread;
435
436 /* We want to receive a notification when the mm_struct is destroyed */
437 struct mmu_notifier mmu_notifier;
438
439 /* Use for delayed freeing of kfd_process structure */
440 struct rcu_head rcu;
441
442 unsigned int pasid;
443
444 /*
445 * List of kfd_process_device structures,
446 * one for each device the process is using.
447 */
448 struct list_head per_device_data;
449
450 struct process_queue_manager pqm;
451
452 /* The process's queues. */
453 size_t queue_array_size;
454
455 /* Size is queue_array_size, up to MAX_PROCESS_QUEUES. */
456 struct kfd_queue **queues;
457
458 unsigned long allocated_queue_bitmap[DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)];
459
460 /*Is the user space process 32 bit?*/
461 bool is_32bit_user_mode;
462};
463
464void kfd_process_create_wq(void);
465void kfd_process_destroy_wq(void);
466struct kfd_process *kfd_create_process(const struct task_struct *);
467struct kfd_process *kfd_get_process(const struct task_struct *);
468
469struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
470 struct kfd_process *p);
471void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid);
472struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
473 struct kfd_process *p,
474 int create_pdd);
475
476/* Process device data iterator */
477struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p);
478struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
479 struct kfd_process_device *pdd);
480bool kfd_has_process_device_data(struct kfd_process *p);
481
482/* PASIDs */
483int kfd_pasid_init(void);
484void kfd_pasid_exit(void);
485bool kfd_set_pasid_limit(unsigned int new_limit);
486unsigned int kfd_get_pasid_limit(void);
487unsigned int kfd_pasid_alloc(void);
488void kfd_pasid_free(unsigned int pasid);
489
490/* Doorbells */
491void kfd_doorbell_init(struct kfd_dev *kfd);
492int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma);
493u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
494 unsigned int *doorbell_off);
495void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
496u32 read_kernel_doorbell(u32 __iomem *db);
497void write_kernel_doorbell(u32 __iomem *db, u32 value);
498unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd,
499 struct kfd_process *process,
500 unsigned int queue_id);
501
502extern struct device *kfd_device;
503
504/* Topology */
505int kfd_topology_init(void);
506void kfd_topology_shutdown(void);
507int kfd_topology_add_device(struct kfd_dev *gpu);
508int kfd_topology_remove_device(struct kfd_dev *gpu);
509struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
510struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
511struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx);
512
513/* Interrupts */
514int kfd_interrupt_init(struct kfd_dev *dev);
515void kfd_interrupt_exit(struct kfd_dev *dev);
516void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);
517bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry);
518
519/* Power Management */
520void kgd2kfd_suspend(struct kfd_dev *kfd);
521int kgd2kfd_resume(struct kfd_dev *kfd);
522
523/* amdkfd Apertures */
524int kfd_init_apertures(struct kfd_process *process);
525
526/* Queue Context Management */
527inline uint32_t lower_32(uint64_t x);
528inline uint32_t upper_32(uint64_t x);
529
530int init_queue(struct queue **q, struct queue_properties properties);
531void uninit_queue(struct queue *q);
532void print_queue_properties(struct queue_properties *q);
533void print_queue(struct queue *q);
534
535struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
536 struct kfd_dev *dev);
537struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
538void device_queue_manager_uninit(struct device_queue_manager *dqm);
539struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
540 enum kfd_queue_type type);
541void kernel_queue_uninit(struct kernel_queue *kq);
542
543/* Process Queue Manager */
544struct process_queue_node {
545 struct queue *q;
546 struct kernel_queue *kq;
547 struct list_head process_queue_list;
548};
549
550int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
551void pqm_uninit(struct process_queue_manager *pqm);
552int pqm_create_queue(struct process_queue_manager *pqm,
553 struct kfd_dev *dev,
554 struct file *f,
555 struct queue_properties *properties,
556 unsigned int flags,
557 enum kfd_queue_type type,
558 unsigned int *qid);
559int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
560int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
561 struct queue_properties *p);
562
563/* Packet Manager */
564
565#define KFD_HIQ_TIMEOUT (500)
566
567#define KFD_FENCE_COMPLETED (100)
568#define KFD_FENCE_INIT (10)
569#define KFD_UNMAP_LATENCY (150)
570
571struct packet_manager {
572 struct device_queue_manager *dqm;
573 struct kernel_queue *priv_queue;
574 struct mutex lock;
575 bool allocated;
576 struct kfd_mem_obj *ib_buffer_obj;
577};
578
579int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
580void pm_uninit(struct packet_manager *pm);
581int pm_send_set_resources(struct packet_manager *pm,
582 struct scheduling_resources *res);
583int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues);
584int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
585 uint32_t fence_value);
586
587int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
588 enum kfd_preempt_type_filter mode,
589 uint32_t filter_param, bool reset,
590 unsigned int sdma_engine);
591
592void pm_release_ib(struct packet_manager *pm);
593
594uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
595phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
596 struct kfd_process *process);
597
598#endif
drivers/gpu/drm/amd/amdkfd/kfd_process.c+415−0
@@ -0,0 +1,415 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/mutex.h>
24#include <linux/log2.h>
25#include <linux/sched.h>
26#include <linux/slab.h>
27#include <linux/amd-iommu.h>
28#include <linux/notifier.h>
29struct mm_struct;
30
31#include "kfd_priv.h"
32
33/*
34 * Initial size for the array of queues.
35 * The allocated size is doubled each time
36 * it is exceeded up to MAX_PROCESS_QUEUES.
37 */
38#define INITIAL_QUEUE_ARRAY_SIZE 16
39
40/*
41 * List of struct kfd_process (field kfd_process).
42 * Unique/indexed by mm_struct*
43 */
44#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
45static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
46static DEFINE_MUTEX(kfd_processes_mutex);
47
48DEFINE_STATIC_SRCU(kfd_processes_srcu);
49
50static struct workqueue_struct *kfd_process_wq;
51
52struct kfd_process_release_work {
53 struct work_struct kfd_work;
54 struct kfd_process *p;
55};
56
57static struct kfd_process *find_process(const struct task_struct *thread);
58static struct kfd_process *create_process(const struct task_struct *thread);
59
60void kfd_process_create_wq(void)
61{
62 if (!kfd_process_wq)
63 kfd_process_wq = create_workqueue("kfd_process_wq");
64}
65
66void kfd_process_destroy_wq(void)
67{
68 if (kfd_process_wq) {
69 flush_workqueue(kfd_process_wq);
70 destroy_workqueue(kfd_process_wq);
71 kfd_process_wq = NULL;
72 }
73}
74
75struct kfd_process *kfd_create_process(const struct task_struct *thread)
76{
77 struct kfd_process *process;
78
79 BUG_ON(!kfd_process_wq);
80
81 if (thread->mm == NULL)
82 return ERR_PTR(-EINVAL);
83
84 /* Only the pthreads threading model is supported. */
85 if (thread->group_leader->mm != thread->mm)
86 return ERR_PTR(-EINVAL);
87
88 /* Take mmap_sem because we call __mmu_notifier_register inside */
89 down_write(&thread->mm->mmap_sem);
90
91 /*
92 * take kfd processes mutex before starting of process creation
93 * so there won't be a case where two threads of the same process
94 * create two kfd_process structures
95 */
96 mutex_lock(&kfd_processes_mutex);
97
98 /* A prior open of /dev/kfd could have already created the process. */
99 process = find_process(thread);
100 if (process)
101 pr_debug("kfd: process already found\n");
102
103 if (!process)
104 process = create_process(thread);
105
106 mutex_unlock(&kfd_processes_mutex);
107
108 up_write(&thread->mm->mmap_sem);
109
110 return process;
111}
112
113struct kfd_process *kfd_get_process(const struct task_struct *thread)
114{
115 struct kfd_process *process;
116
117 if (thread->mm == NULL)
118 return ERR_PTR(-EINVAL);
119
120 /* Only the pthreads threading model is supported. */
121 if (thread->group_leader->mm != thread->mm)
122 return ERR_PTR(-EINVAL);
123
124 process = find_process(thread);
125
126 return process;
127}
128
129static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
130{
131 struct kfd_process *process;
132
133 hash_for_each_possible_rcu(kfd_processes_table, process,
134 kfd_processes, (uintptr_t)mm)
135 if (process->mm == mm)
136 return process;
137
138 return NULL;
139}
140
141static struct kfd_process *find_process(const struct task_struct *thread)
142{
143 struct kfd_process *p;
144 int idx;
145
146 idx = srcu_read_lock(&kfd_processes_srcu);
147 p = find_process_by_mm(thread->mm);
148 srcu_read_unlock(&kfd_processes_srcu, idx);
149
150 return p;
151}
152
153static void kfd_process_wq_release(struct work_struct *work)
154{
155 struct kfd_process_release_work *my_work;
156 struct kfd_process_device *pdd, *temp;
157 struct kfd_process *p;
158
159 my_work = (struct kfd_process_release_work *) work;
160
161 p = my_work->p;
162
163 mutex_lock(&p->mutex);
164
165 list_for_each_entry_safe(pdd, temp, &p->per_device_data,
166 per_device_list) {
167 amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
168 list_del(&pdd->per_device_list);
169
170 kfree(pdd);
171 }
172
173 kfd_pasid_free(p->pasid);
174
175 mutex_unlock(&p->mutex);
176
177 mutex_destroy(&p->mutex);
178
179 kfree(p->queues);
180
181 kfree(p);
182
183 kfree((void *)work);
184}
185
186static void kfd_process_destroy_delayed(struct rcu_head *rcu)
187{
188 struct kfd_process_release_work *work;
189 struct kfd_process *p;
190
191 BUG_ON(!kfd_process_wq);
192
193 p = container_of(rcu, struct kfd_process, rcu);
194 BUG_ON(atomic_read(&p->mm->mm_count) <= 0);
195
196 mmdrop(p->mm);
197
198 work = (struct kfd_process_release_work *)
199 kmalloc(sizeof(struct kfd_process_release_work), GFP_KERNEL);
200
201 if (work) {
202 INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
203 work->p = p;
204 queue_work(kfd_process_wq, (struct work_struct *) work);
205 }
206}
207
208static void kfd_process_notifier_release(struct mmu_notifier *mn,
209 struct mm_struct *mm)
210{
211 struct kfd_process *p;
212
213 /*
214 * The kfd_process structure can not be free because the
215 * mmu_notifier srcu is read locked
216 */
217 p = container_of(mn, struct kfd_process, mmu_notifier);
218 BUG_ON(p->mm != mm);
219
220 mutex_lock(&kfd_processes_mutex);
221 hash_del_rcu(&p->kfd_processes);
222 mutex_unlock(&kfd_processes_mutex);
223 synchronize_srcu(&kfd_processes_srcu);
224
225 mutex_lock(&p->mutex);
226
227 /* In case our notifier is called before IOMMU notifier */
228 pqm_uninit(&p->pqm);
229
230 mutex_unlock(&p->mutex);
231
232 /*
233 * Because we drop mm_count inside kfd_process_destroy_delayed
234 * and because the mmu_notifier_unregister function also drop
235 * mm_count we need to take an extra count here.
236 */
237 atomic_inc(&p->mm->mm_count);
238 mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
239 mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
240}
241
242static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
243 .release = kfd_process_notifier_release,
244};
245
246static struct kfd_process *create_process(const struct task_struct *thread)
247{
248 struct kfd_process *process;
249 int err = -ENOMEM;
250
251 process = kzalloc(sizeof(*process), GFP_KERNEL);
252
253 if (!process)
254 goto err_alloc_process;
255
256 process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
257 sizeof(process->queues[0]), GFP_KERNEL);
258 if (!process->queues)
259 goto err_alloc_queues;
260
261 process->pasid = kfd_pasid_alloc();
262 if (process->pasid == 0)
263 goto err_alloc_pasid;
264
265 mutex_init(&process->mutex);
266
267 process->mm = thread->mm;
268
269 /* register notifier */
270 process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
271 err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
272 if (err)
273 goto err_mmu_notifier;
274
275 hash_add_rcu(kfd_processes_table, &process->kfd_processes,
276 (uintptr_t)process->mm);
277
278 process->lead_thread = thread->group_leader;
279
280 process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
281
282 INIT_LIST_HEAD(&process->per_device_data);
283
284 err = pqm_init(&process->pqm, process);
285 if (err != 0)
286 goto err_process_pqm_init;
287
288 return process;
289
290err_process_pqm_init:
291 hash_del_rcu(&process->kfd_processes);
292 synchronize_rcu();
293 mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
294err_mmu_notifier:
295 kfd_pasid_free(process->pasid);
296err_alloc_pasid:
297 kfree(process->queues);
298err_alloc_queues:
299 kfree(process);
300err_alloc_process:
301 return ERR_PTR(err);
302}
303
304struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
305 struct kfd_process *p,
306 int create_pdd)
307{
308 struct kfd_process_device *pdd = NULL;
309
310 list_for_each_entry(pdd, &p->per_device_data, per_device_list)
311 if (pdd->dev == dev)
312 return pdd;
313
314 if (create_pdd) {
315 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
316 if (pdd != NULL) {
317 pdd->dev = dev;
318 INIT_LIST_HEAD(&pdd->qpd.queues_list);
319 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
320 pdd->qpd.dqm = dev->dqm;
321 list_add(&pdd->per_device_list, &p->per_device_data);
322 }
323 }
324
325 return pdd;
326}
327
328/*
329 * Direct the IOMMU to bind the process (specifically the pasid->mm)
330 * to the device.
331 * Unbinding occurs when the process dies or the device is removed.
332 *
333 * Assumes that the process lock is held.
334 */
335struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
336 struct kfd_process *p)
337{
338 struct kfd_process_device *pdd = kfd_get_process_device_data(dev, p, 1);
339 int err;
340
341 if (pdd == NULL)
342 return ERR_PTR(-ENOMEM);
343
344 if (pdd->bound)
345 return pdd;
346
347 err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
348 if (err < 0)
349 return ERR_PTR(err);
350
351 if (err < 0) {
352 amd_iommu_unbind_pasid(dev->pdev, p->pasid);
353 return ERR_PTR(err);
354 }
355
356 pdd->bound = true;
357
358 return pdd;
359}
360
361void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
362{
363 struct kfd_process *p;
364 struct kfd_process_device *pdd;
365 int idx, i;
366
367 BUG_ON(dev == NULL);
368
369 idx = srcu_read_lock(&kfd_processes_srcu);
370
371 hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes)
372 if (p->pasid == pasid)
373 break;
374
375 srcu_read_unlock(&kfd_processes_srcu, idx);
376
377 BUG_ON(p->pasid != pasid);
378
379 mutex_lock(&p->mutex);
380
381 pqm_uninit(&p->pqm);
382
383 pdd = kfd_get_process_device_data(dev, p, 0);
384
385 /*
386 * Just mark pdd as unbound, because we still need it to call
387 * amd_iommu_unbind_pasid() in when the process exits.
388 * We don't call amd_iommu_unbind_pasid() here
389 * because the IOMMU called us.
390 */
391 if (pdd)
392 pdd->bound = false;
393
394 mutex_unlock(&p->mutex);
395}
396
397struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
398{
399 return list_first_entry(&p->per_device_data,
400 struct kfd_process_device,
401 per_device_list);
402}
403
404struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
405 struct kfd_process_device *pdd)
406{
407 if (list_is_last(&pdd->per_device_list, &p->per_device_data))
408 return NULL;
409 return list_next_entry(pdd, per_device_list);
410}
411
412bool kfd_has_process_device_data(struct kfd_process *p)
413{
414 return !(list_empty(&p->per_device_data));
415}
drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c+342−0
@@ -0,0 +1,342 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/slab.h>
25#include <linux/list.h>
26#include "kfd_device_queue_manager.h"
27#include "kfd_priv.h"
28#include "kfd_kernel_queue.h"
29
30static inline struct process_queue_node *get_queue_by_qid(
31 struct process_queue_manager *pqm, unsigned int qid)
32{
33 struct process_queue_node *pqn;
34
35 BUG_ON(!pqm);
36
37 list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
38 if (pqn->q && pqn->q->properties.queue_id == qid)
39 return pqn;
40 if (pqn->kq && pqn->kq->queue->properties.queue_id == qid)
41 return pqn;
42 }
43
44 return NULL;
45}
46
47static int find_available_queue_slot(struct process_queue_manager *pqm,
48 unsigned int *qid)
49{
50 unsigned long found;
51
52 BUG_ON(!pqm || !qid);
53
54 pr_debug("kfd: in %s\n", __func__);
55
56 found = find_first_zero_bit(pqm->queue_slot_bitmap,
57 max_num_of_queues_per_process);
58
59 pr_debug("kfd: the new slot id %lu\n", found);
60
61 if (found >= max_num_of_queues_per_process) {
62 pr_info("amdkfd: Can not open more queues for process with pasid %d\n",
63 pqm->process->pasid);
64 return -ENOMEM;
65 }
66
67 set_bit(found, pqm->queue_slot_bitmap);
68 *qid = found;
69
70 return 0;
71}
72
73int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
74{
75 BUG_ON(!pqm);
76
77 INIT_LIST_HEAD(&pqm->queues);
78 pqm->queue_slot_bitmap =
79 kzalloc(DIV_ROUND_UP(max_num_of_queues_per_process,
80 BITS_PER_BYTE), GFP_KERNEL);
81 if (pqm->queue_slot_bitmap == NULL)
82 return -ENOMEM;
83 pqm->process = p;
84
85 return 0;
86}
87
88void pqm_uninit(struct process_queue_manager *pqm)
89{
90 int retval;
91 struct process_queue_node *pqn, *next;
92
93 BUG_ON(!pqm);
94
95 pr_debug("In func %s\n", __func__);
96
97 list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
98 retval = pqm_destroy_queue(
99 pqm,
100 (pqn->q != NULL) ?
101 pqn->q->properties.queue_id :
102 pqn->kq->queue->properties.queue_id);
103
104 if (retval != 0) {
105 pr_err("kfd: failed to destroy queue\n");
106 return;
107 }
108 }
109 kfree(pqm->queue_slot_bitmap);
110 pqm->queue_slot_bitmap = NULL;
111}
112
113static int create_cp_queue(struct process_queue_manager *pqm,
114 struct kfd_dev *dev, struct queue **q,
115 struct queue_properties *q_properties,
116 struct file *f, unsigned int qid)
117{
118 int retval;
119
120 retval = 0;
121
122 /* Doorbell initialized in user space*/
123 q_properties->doorbell_ptr = NULL;
124
125 q_properties->doorbell_off =
126 kfd_queue_id_to_doorbell(dev, pqm->process, qid);
127
128 /* let DQM handle it*/
129 q_properties->vmid = 0;
130 q_properties->queue_id = qid;
131 q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
132
133 retval = init_queue(q, *q_properties);
134 if (retval != 0)
135 goto err_init_queue;
136
137 (*q)->device = dev;
138 (*q)->process = pqm->process;
139
140 pr_debug("kfd: PQM After init queue");
141
142 return retval;
143
144err_init_queue:
145 return retval;
146}
147
148int pqm_create_queue(struct process_queue_manager *pqm,
149 struct kfd_dev *dev,
150 struct file *f,
151 struct queue_properties *properties,
152 unsigned int flags,
153 enum kfd_queue_type type,
154 unsigned int *qid)
155{
156 int retval;
157 struct kfd_process_device *pdd;
158 struct queue_properties q_properties;
159 struct queue *q;
160 struct process_queue_node *pqn;
161 struct kernel_queue *kq;
162
163 BUG_ON(!pqm || !dev || !properties || !qid);
164
165 memset(&q_properties, 0, sizeof(struct queue_properties));
166 memcpy(&q_properties, properties, sizeof(struct queue_properties));
167 q = NULL;
168 kq = NULL;
169
170 pdd = kfd_get_process_device_data(dev, pqm->process, 1);
171 BUG_ON(!pdd);
172
173 retval = find_available_queue_slot(pqm, qid);
174 if (retval != 0)
175 return retval;
176
177 if (list_empty(&pqm->queues)) {
178 pdd->qpd.pqm = pqm;
179 dev->dqm->register_process(dev->dqm, &pdd->qpd);
180 }
181
182 pqn = kzalloc(sizeof(struct process_queue_node), GFP_KERNEL);
183 if (!pqn) {
184 retval = -ENOMEM;
185 goto err_allocate_pqn;
186 }
187
188 switch (type) {
189 case KFD_QUEUE_TYPE_COMPUTE:
190 /* check if there is over subscription */
191 if ((sched_policy == KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
192 ((dev->dqm->processes_count >= VMID_PER_DEVICE) ||
193 (dev->dqm->queue_count >= PIPE_PER_ME_CP_SCHEDULING * QUEUES_PER_PIPE))) {
194 pr_err("kfd: over-subscription is not allowed in radeon_kfd.sched_policy == 1\n");
195 retval = -EPERM;
196 goto err_create_queue;
197 }
198
199 retval = create_cp_queue(pqm, dev, &q, &q_properties, f, *qid);
200 if (retval != 0)
201 goto err_create_queue;
202 pqn->q = q;
203 pqn->kq = NULL;
204 retval = dev->dqm->create_queue(dev->dqm, q, &pdd->qpd,
205 &q->properties.vmid);
206 print_queue(q);
207 break;
208 case KFD_QUEUE_TYPE_DIQ:
209 kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ);
210 if (kq == NULL) {
211 kernel_queue_uninit(kq);
212 goto err_create_queue;
213 }
214 kq->queue->properties.queue_id = *qid;
215 pqn->kq = kq;
216 pqn->q = NULL;
217 retval = dev->dqm->create_kernel_queue(dev->dqm, kq, &pdd->qpd);
218 break;
219 default:
220 BUG();
221 break;
222 }
223
224 if (retval != 0) {
225 pr_err("kfd: error dqm create queue\n");
226 goto err_create_queue;
227 }
228
229 pr_debug("kfd: PQM After DQM create queue\n");
230
231 list_add(&pqn->process_queue_list, &pqm->queues);
232
233 if (q) {
234 *properties = q->properties;
235 pr_debug("kfd: PQM done creating queue\n");
236 print_queue_properties(properties);
237 }
238
239 return retval;
240
241err_create_queue:
242 kfree(pqn);
243err_allocate_pqn:
244 clear_bit(*qid, pqm->queue_slot_bitmap);
245 return retval;
246}
247
248int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
249{
250 struct process_queue_node *pqn;
251 struct kfd_process_device *pdd;
252 struct device_queue_manager *dqm;
253 struct kfd_dev *dev;
254 int retval;
255
256 dqm = NULL;
257
258 BUG_ON(!pqm);
259 retval = 0;
260
261 pr_debug("kfd: In Func %s\n", __func__);
262
263 pqn = get_queue_by_qid(pqm, qid);
264 if (pqn == NULL) {
265 pr_err("kfd: queue id does not match any known queue\n");
266 return -EINVAL;
267 }
268
269 dev = NULL;
270 if (pqn->kq)
271 dev = pqn->kq->dev;
272 if (pqn->q)
273 dev = pqn->q->device;
274 BUG_ON(!dev);
275
276 pdd = kfd_get_process_device_data(dev, pqm->process, 1);
277 BUG_ON(!pdd);
278
279 if (pqn->kq) {
280 /* destroy kernel queue (DIQ) */
281 dqm = pqn->kq->dev->dqm;
282 dqm->destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);
283 kernel_queue_uninit(pqn->kq);
284 }
285
286 if (pqn->q) {
287 dqm = pqn->q->device->dqm;
288 retval = dqm->destroy_queue(dqm, &pdd->qpd, pqn->q);
289 if (retval != 0)
290 return retval;
291
292 uninit_queue(pqn->q);
293 }
294
295 list_del(&pqn->process_queue_list);
296 kfree(pqn);
297 clear_bit(qid, pqm->queue_slot_bitmap);
298
299 if (list_empty(&pqm->queues))
300 dqm->unregister_process(dqm, &pdd->qpd);
301
302 return retval;
303}
304
305int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
306 struct queue_properties *p)
307{
308 int retval;
309 struct process_queue_node *pqn;
310
311 BUG_ON(!pqm);
312
313 pqn = get_queue_by_qid(pqm, qid);
314 BUG_ON(!pqn);
315
316 pqn->q->properties.queue_address = p->queue_address;
317 pqn->q->properties.queue_size = p->queue_size;
318 pqn->q->properties.queue_percent = p->queue_percent;
319 pqn->q->properties.priority = p->priority;
320
321 retval = pqn->q->device->dqm->update_queue(pqn->q->device->dqm, pqn->q);
322 if (retval != 0)
323 return retval;
324
325 return 0;
326}
327
328struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
329 unsigned int qid)
330{
331 struct process_queue_node *pqn;
332
333 BUG_ON(!pqm);
334
335 pqn = get_queue_by_qid(pqm, qid);
336 if (pqn && pqn->kq)
337 return pqn->kq;
338
339 return NULL;
340}
341
342
drivers/gpu/drm/amd/amdkfd/kfd_queue.c+85−0
@@ -0,0 +1,85 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/slab.h>
25#include "kfd_priv.h"
26
27void print_queue_properties(struct queue_properties *q)
28{
29 if (!q)
30 return;
31
32 pr_debug("Printing queue properties:\n");
33 pr_debug("Queue Type: %u\n", q->type);
34 pr_debug("Queue Size: %llu\n", q->queue_size);
35 pr_debug("Queue percent: %u\n", q->queue_percent);
36 pr_debug("Queue Address: 0x%llX\n", q->queue_address);
37 pr_debug("Queue Id: %u\n", q->queue_id);
38 pr_debug("Queue Process Vmid: %u\n", q->vmid);
39 pr_debug("Queue Read Pointer: 0x%p\n", q->read_ptr);
40 pr_debug("Queue Write Pointer: 0x%p\n", q->write_ptr);
41 pr_debug("Queue Doorbell Pointer: 0x%p\n", q->doorbell_ptr);
42 pr_debug("Queue Doorbell Offset: %u\n", q->doorbell_off);
43}
44
45void print_queue(struct queue *q)
46{
47 if (!q)
48 return;
49 pr_debug("Printing queue:\n");
50 pr_debug("Queue Type: %u\n", q->properties.type);
51 pr_debug("Queue Size: %llu\n", q->properties.queue_size);
52 pr_debug("Queue percent: %u\n", q->properties.queue_percent);
53 pr_debug("Queue Address: 0x%llX\n", q->properties.queue_address);
54 pr_debug("Queue Id: %u\n", q->properties.queue_id);
55 pr_debug("Queue Process Vmid: %u\n", q->properties.vmid);
56 pr_debug("Queue Read Pointer: 0x%p\n", q->properties.read_ptr);
57 pr_debug("Queue Write Pointer: 0x%p\n", q->properties.write_ptr);
58 pr_debug("Queue Doorbell Pointer: 0x%p\n", q->properties.doorbell_ptr);
59 pr_debug("Queue Doorbell Offset: %u\n", q->properties.doorbell_off);
60 pr_debug("Queue MQD Address: 0x%p\n", q->mqd);
61 pr_debug("Queue MQD Gart: 0x%llX\n", q->gart_mqd_addr);
62 pr_debug("Queue Process Address: 0x%p\n", q->process);
63 pr_debug("Queue Device Address: 0x%p\n", q->device);
64}
65
66int init_queue(struct queue **q, struct queue_properties properties)
67{
68 struct queue *tmp;
69
70 BUG_ON(!q);
71
72 tmp = kzalloc(sizeof(struct queue), GFP_KERNEL);
73 if (!tmp)
74 return -ENOMEM;
75
76 memcpy(&tmp->properties, &properties, sizeof(struct queue_properties));
77
78 *q = tmp;
79 return 0;
80}
81
82void uninit_queue(struct queue *q)
83{
84 kfree(q);
85}
drivers/gpu/drm/amd/amdkfd/kfd_topology.c+1235−0
@@ -0,0 +1,1235 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/types.h>
24#include <linux/kernel.h>
25#include <linux/pci.h>
26#include <linux/errno.h>
27#include <linux/acpi.h>
28#include <linux/hash.h>
29#include <linux/cpufreq.h>
30
31#include "kfd_priv.h"
32#include "kfd_crat.h"
33#include "kfd_topology.h"
34
35static struct list_head topology_device_list;
36static int topology_crat_parsed;
37static struct kfd_system_properties sys_props;
38
39static DECLARE_RWSEM(topology_lock);
40
41struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
42{
43 struct kfd_topology_device *top_dev;
44 struct kfd_dev *device = NULL;
45
46 down_read(&topology_lock);
47
48 list_for_each_entry(top_dev, &topology_device_list, list)
49 if (top_dev->gpu_id == gpu_id) {
50 device = top_dev->gpu;
51 break;
52 }
53
54 up_read(&topology_lock);
55
56 return device;
57}
58
59struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
60{
61 struct kfd_topology_device *top_dev;
62 struct kfd_dev *device = NULL;
63
64 down_read(&topology_lock);
65
66 list_for_each_entry(top_dev, &topology_device_list, list)
67 if (top_dev->gpu->pdev == pdev) {
68 device = top_dev->gpu;
69 break;
70 }
71
72 up_read(&topology_lock);
73
74 return device;
75}
76
77static int kfd_topology_get_crat_acpi(void *crat_image, size_t *size)
78{
79 struct acpi_table_header *crat_table;
80 acpi_status status;
81
82 if (!size)
83 return -EINVAL;
84
85 /*
86 * Fetch the CRAT table from ACPI
87 */
88 status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
89 if (status == AE_NOT_FOUND) {
90 pr_warn("CRAT table not found\n");
91 return -ENODATA;
92 } else if (ACPI_FAILURE(status)) {
93 const char *err = acpi_format_exception(status);
94
95 pr_err("CRAT table error: %s\n", err);
96 return -EINVAL;
97 }
98
99 if (*size >= crat_table->length && crat_image != 0)
100 memcpy(crat_image, crat_table, crat_table->length);
101
102 *size = crat_table->length;
103
104 return 0;
105}
106
107static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
108 struct crat_subtype_computeunit *cu)
109{
110 BUG_ON(!dev);
111 BUG_ON(!cu);
112
113 dev->node_props.cpu_cores_count = cu->num_cpu_cores;
114 dev->node_props.cpu_core_id_base = cu->processor_id_low;
115 if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
116 dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
117
118 pr_info("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores,
119 cu->processor_id_low);
120}
121
122static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
123 struct crat_subtype_computeunit *cu)
124{
125 BUG_ON(!dev);
126 BUG_ON(!cu);
127
128 dev->node_props.simd_id_base = cu->processor_id_low;
129 dev->node_props.simd_count = cu->num_simd_cores;
130 dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
131 dev->node_props.max_waves_per_simd = cu->max_waves_simd;
132 dev->node_props.wave_front_size = cu->wave_front_size;
133 dev->node_props.mem_banks_count = cu->num_banks;
134 dev->node_props.array_count = cu->num_arrays;
135 dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
136 dev->node_props.simd_per_cu = cu->num_simd_per_cu;
137 dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
138 if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE)
139 dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE;
140 pr_info("CU GPU: simds=%d id_base=%d\n", cu->num_simd_cores,
141 cu->processor_id_low);
142}
143
144/* kfd_parse_subtype_cu is called when the topology mutex is already acquired */
145static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu)
146{
147 struct kfd_topology_device *dev;
148 int i = 0;
149
150 BUG_ON(!cu);
151
152 pr_info("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
153 cu->proximity_domain, cu->hsa_capability);
154 list_for_each_entry(dev, &topology_device_list, list) {
155 if (cu->proximity_domain == i) {
156 if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT)
157 kfd_populated_cu_info_cpu(dev, cu);
158
159 if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT)
160 kfd_populated_cu_info_gpu(dev, cu);
161 break;
162 }
163 i++;
164 }
165
166 return 0;
167}
168
169/*
170 * kfd_parse_subtype_mem is called when the topology mutex is
171 * already acquired
172 */
173static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem)
174{
175 struct kfd_mem_properties *props;
176 struct kfd_topology_device *dev;
177 int i = 0;
178
179 BUG_ON(!mem);
180
181 pr_info("Found memory entry in CRAT table with proximity_domain=%d\n",
182 mem->promixity_domain);
183 list_for_each_entry(dev, &topology_device_list, list) {
184 if (mem->promixity_domain == i) {
185 props = kfd_alloc_struct(props);
186 if (props == 0)
187 return -ENOMEM;
188
189 if (dev->node_props.cpu_cores_count == 0)
190 props->heap_type = HSA_MEM_HEAP_TYPE_FB_PRIVATE;
191 else
192 props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
193
194 if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
195 props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
196 if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
197 props->flags |= HSA_MEM_FLAGS_NON_VOLATILE;
198
199 props->size_in_bytes =
200 ((uint64_t)mem->length_high << 32) +
201 mem->length_low;
202 props->width = mem->width;
203
204 dev->mem_bank_count++;
205 list_add_tail(&props->list, &dev->mem_props);
206
207 break;
208 }
209 i++;
210 }
211
212 return 0;
213}
214
215/*
216 * kfd_parse_subtype_cache is called when the topology mutex
217 * is already acquired
218 */
219static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache)
220{
221 struct kfd_cache_properties *props;
222 struct kfd_topology_device *dev;
223 uint32_t id;
224
225 BUG_ON(!cache);
226
227 id = cache->processor_id_low;
228
229 pr_info("Found cache entry in CRAT table with processor_id=%d\n", id);
230 list_for_each_entry(dev, &topology_device_list, list)
231 if (id == dev->node_props.cpu_core_id_base ||
232 id == dev->node_props.simd_id_base) {
233 props = kfd_alloc_struct(props);
234 if (props == 0)
235 return -ENOMEM;
236
237 props->processor_id_low = id;
238 props->cache_level = cache->cache_level;
239 props->cache_size = cache->cache_size;
240 props->cacheline_size = cache->cache_line_size;
241 props->cachelines_per_tag = cache->lines_per_tag;
242 props->cache_assoc = cache->associativity;
243 props->cache_latency = cache->cache_latency;
244
245 if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
246 props->cache_type |= HSA_CACHE_TYPE_DATA;
247 if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE)
248 props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
249 if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE)
250 props->cache_type |= HSA_CACHE_TYPE_CPU;
251 if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
252 props->cache_type |= HSA_CACHE_TYPE_HSACU;
253
254 dev->cache_count++;
255 dev->node_props.caches_count++;
256 list_add_tail(&props->list, &dev->cache_props);
257
258 break;
259 }
260
261 return 0;
262}
263
264/*
265 * kfd_parse_subtype_iolink is called when the topology mutex
266 * is already acquired
267 */
268static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink)
269{
270 struct kfd_iolink_properties *props;
271 struct kfd_topology_device *dev;
272 uint32_t i = 0;
273 uint32_t id_from;
274 uint32_t id_to;
275
276 BUG_ON(!iolink);
277
278 id_from = iolink->proximity_domain_from;
279 id_to = iolink->proximity_domain_to;
280
281 pr_info("Found IO link entry in CRAT table with id_from=%d\n", id_from);
282 list_for_each_entry(dev, &topology_device_list, list) {
283 if (id_from == i) {
284 props = kfd_alloc_struct(props);
285 if (props == 0)
286 return -ENOMEM;
287
288 props->node_from = id_from;
289 props->node_to = id_to;
290 props->ver_maj = iolink->version_major;
291 props->ver_min = iolink->version_minor;
292
293 /*
294 * weight factor (derived from CDIR), currently always 1
295 */
296 props->weight = 1;
297
298 props->min_latency = iolink->minimum_latency;
299 props->max_latency = iolink->maximum_latency;
300 props->min_bandwidth = iolink->minimum_bandwidth_mbs;
301 props->max_bandwidth = iolink->maximum_bandwidth_mbs;
302 props->rec_transfer_size =
303 iolink->recommended_transfer_size;
304
305 dev->io_link_count++;
306 dev->node_props.io_links_count++;
307 list_add_tail(&props->list, &dev->io_link_props);
308
309 break;
310 }
311 i++;
312 }
313
314 return 0;
315}
316
317static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr)
318{
319 struct crat_subtype_computeunit *cu;
320 struct crat_subtype_memory *mem;
321 struct crat_subtype_cache *cache;
322 struct crat_subtype_iolink *iolink;
323 int ret = 0;
324
325 BUG_ON(!sub_type_hdr);
326
327 switch (sub_type_hdr->type) {
328 case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
329 cu = (struct crat_subtype_computeunit *)sub_type_hdr;
330 ret = kfd_parse_subtype_cu(cu);
331 break;
332 case CRAT_SUBTYPE_MEMORY_AFFINITY:
333 mem = (struct crat_subtype_memory *)sub_type_hdr;
334 ret = kfd_parse_subtype_mem(mem);
335 break;
336 case CRAT_SUBTYPE_CACHE_AFFINITY:
337 cache = (struct crat_subtype_cache *)sub_type_hdr;
338 ret = kfd_parse_subtype_cache(cache);
339 break;
340 case CRAT_SUBTYPE_TLB_AFFINITY:
341 /*
342 * For now, nothing to do here
343 */
344 pr_info("Found TLB entry in CRAT table (not processing)\n");
345 break;
346 case CRAT_SUBTYPE_CCOMPUTE_AFFINITY:
347 /*
348 * For now, nothing to do here
349 */
350 pr_info("Found CCOMPUTE entry in CRAT table (not processing)\n");
351 break;
352 case CRAT_SUBTYPE_IOLINK_AFFINITY:
353 iolink = (struct crat_subtype_iolink *)sub_type_hdr;
354 ret = kfd_parse_subtype_iolink(iolink);
355 break;
356 default:
357 pr_warn("Unknown subtype (%d) in CRAT\n",
358 sub_type_hdr->type);
359 }
360
361 return ret;
362}
363
364static void kfd_release_topology_device(struct kfd_topology_device *dev)
365{
366 struct kfd_mem_properties *mem;
367 struct kfd_cache_properties *cache;
368 struct kfd_iolink_properties *iolink;
369
370 BUG_ON(!dev);
371
372 list_del(&dev->list);
373
374 while (dev->mem_props.next != &dev->mem_props) {
375 mem = container_of(dev->mem_props.next,
376 struct kfd_mem_properties, list);
377 list_del(&mem->list);
378 kfree(mem);
379 }
380
381 while (dev->cache_props.next != &dev->cache_props) {
382 cache = container_of(dev->cache_props.next,
383 struct kfd_cache_properties, list);
384 list_del(&cache->list);
385 kfree(cache);
386 }
387
388 while (dev->io_link_props.next != &dev->io_link_props) {
389 iolink = container_of(dev->io_link_props.next,
390 struct kfd_iolink_properties, list);
391 list_del(&iolink->list);
392 kfree(iolink);
393 }
394
395 kfree(dev);
396
397 sys_props.num_devices--;
398}
399
400static void kfd_release_live_view(void)
401{
402 struct kfd_topology_device *dev;
403
404 while (topology_device_list.next != &topology_device_list) {
405 dev = container_of(topology_device_list.next,
406 struct kfd_topology_device, list);
407 kfd_release_topology_device(dev);
408}
409
410 memset(&sys_props, 0, sizeof(sys_props));
411}
412
413static struct kfd_topology_device *kfd_create_topology_device(void)
414{
415 struct kfd_topology_device *dev;
416
417 dev = kfd_alloc_struct(dev);
418 if (dev == 0) {
419 pr_err("No memory to allocate a topology device");
420 return 0;
421 }
422
423 INIT_LIST_HEAD(&dev->mem_props);
424 INIT_LIST_HEAD(&dev->cache_props);
425 INIT_LIST_HEAD(&dev->io_link_props);
426
427 list_add_tail(&dev->list, &topology_device_list);
428 sys_props.num_devices++;
429
430 return dev;
431 }
432
433static int kfd_parse_crat_table(void *crat_image)
434{
435 struct kfd_topology_device *top_dev;
436 struct crat_subtype_generic *sub_type_hdr;
437 uint16_t node_id;
438 int ret;
439 struct crat_header *crat_table = (struct crat_header *)crat_image;
440 uint16_t num_nodes;
441 uint32_t image_len;
442
443 if (!crat_image)
444 return -EINVAL;
445
446 num_nodes = crat_table->num_domains;
447 image_len = crat_table->length;
448
449 pr_info("Parsing CRAT table with %d nodes\n", num_nodes);
450
451 for (node_id = 0; node_id < num_nodes; node_id++) {
452 top_dev = kfd_create_topology_device();
453 if (!top_dev) {
454 kfd_release_live_view();
455 return -ENOMEM;
456 }
457 }
458
459 sys_props.platform_id =
460 (*((uint64_t *)crat_table->oem_id)) & CRAT_OEMID_64BIT_MASK;
461 sys_props.platform_oem = *((uint64_t *)crat_table->oem_table_id);
462 sys_props.platform_rev = crat_table->revision;
463
464 sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
465 while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) <
466 ((char *)crat_image) + image_len) {
467 if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
468 ret = kfd_parse_subtype(sub_type_hdr);
469 if (ret != 0) {
470 kfd_release_live_view();
471 return ret;
472 }
473 }
474
475 sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
476 sub_type_hdr->length);
477 }
478
479 sys_props.generation_count++;
480 topology_crat_parsed = 1;
481
482 return 0;
483}
484
485
486#define sysfs_show_gen_prop(buffer, fmt, ...) \
487 snprintf(buffer, PAGE_SIZE, "%s"fmt, buffer, __VA_ARGS__)
488#define sysfs_show_32bit_prop(buffer, name, value) \
489 sysfs_show_gen_prop(buffer, "%s %u\n", name, value)
490#define sysfs_show_64bit_prop(buffer, name, value) \
491 sysfs_show_gen_prop(buffer, "%s %llu\n", name, value)
492#define sysfs_show_32bit_val(buffer, value) \
493 sysfs_show_gen_prop(buffer, "%u\n", value)
494#define sysfs_show_str_val(buffer, value) \
495 sysfs_show_gen_prop(buffer, "%s\n", value)
496
497static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr,
498 char *buffer)
499{
500 ssize_t ret;
501
502 /* Making sure that the buffer is an empty string */
503 buffer[0] = 0;
504
505 if (attr == &sys_props.attr_genid) {
506 ret = sysfs_show_32bit_val(buffer, sys_props.generation_count);
507 } else if (attr == &sys_props.attr_props) {
508 sysfs_show_64bit_prop(buffer, "platform_oem",
509 sys_props.platform_oem);
510 sysfs_show_64bit_prop(buffer, "platform_id",
511 sys_props.platform_id);
512 ret = sysfs_show_64bit_prop(buffer, "platform_rev",
513 sys_props.platform_rev);
514 } else {
515 ret = -EINVAL;
516 }
517
518 return ret;
519}
520
521static const struct sysfs_ops sysprops_ops = {
522 .show = sysprops_show,
523};
524
525static struct kobj_type sysprops_type = {
526 .sysfs_ops = &sysprops_ops,
527};
528
529static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr,
530 char *buffer)
531{
532 ssize_t ret;
533 struct kfd_iolink_properties *iolink;
534
535 /* Making sure that the buffer is an empty string */
536 buffer[0] = 0;
537
538 iolink = container_of(attr, struct kfd_iolink_properties, attr);
539 sysfs_show_32bit_prop(buffer, "type", iolink->iolink_type);
540 sysfs_show_32bit_prop(buffer, "version_major", iolink->ver_maj);
541 sysfs_show_32bit_prop(buffer, "version_minor", iolink->ver_min);
542 sysfs_show_32bit_prop(buffer, "node_from", iolink->node_from);
543 sysfs_show_32bit_prop(buffer, "node_to", iolink->node_to);
544 sysfs_show_32bit_prop(buffer, "weight", iolink->weight);
545 sysfs_show_32bit_prop(buffer, "min_latency", iolink->min_latency);
546 sysfs_show_32bit_prop(buffer, "max_latency", iolink->max_latency);
547 sysfs_show_32bit_prop(buffer, "min_bandwidth", iolink->min_bandwidth);
548 sysfs_show_32bit_prop(buffer, "max_bandwidth", iolink->max_bandwidth);
549 sysfs_show_32bit_prop(buffer, "recommended_transfer_size",
550 iolink->rec_transfer_size);
551 ret = sysfs_show_32bit_prop(buffer, "flags", iolink->flags);
552
553 return ret;
554}
555
556static const struct sysfs_ops iolink_ops = {
557 .show = iolink_show,
558};
559
560static struct kobj_type iolink_type = {
561 .sysfs_ops = &iolink_ops,
562};
563
564static ssize_t mem_show(struct kobject *kobj, struct attribute *attr,
565 char *buffer)
566{
567 ssize_t ret;
568 struct kfd_mem_properties *mem;
569
570 /* Making sure that the buffer is an empty string */
571 buffer[0] = 0;
572
573 mem = container_of(attr, struct kfd_mem_properties, attr);
574 sysfs_show_32bit_prop(buffer, "heap_type", mem->heap_type);
575 sysfs_show_64bit_prop(buffer, "size_in_bytes", mem->size_in_bytes);
576 sysfs_show_32bit_prop(buffer, "flags", mem->flags);
577 sysfs_show_32bit_prop(buffer, "width", mem->width);
578 ret = sysfs_show_32bit_prop(buffer, "mem_clk_max", mem->mem_clk_max);
579
580 return ret;
581}
582
583static const struct sysfs_ops mem_ops = {
584 .show = mem_show,
585};
586
587static struct kobj_type mem_type = {
588 .sysfs_ops = &mem_ops,
589};
590
591static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr,
592 char *buffer)
593{
594 ssize_t ret;
595 uint32_t i;
596 struct kfd_cache_properties *cache;
597
598 /* Making sure that the buffer is an empty string */
599 buffer[0] = 0;
600
601 cache = container_of(attr, struct kfd_cache_properties, attr);
602 sysfs_show_32bit_prop(buffer, "processor_id_low",
603 cache->processor_id_low);
604 sysfs_show_32bit_prop(buffer, "level", cache->cache_level);
605 sysfs_show_32bit_prop(buffer, "size", cache->cache_size);
606 sysfs_show_32bit_prop(buffer, "cache_line_size", cache->cacheline_size);
607 sysfs_show_32bit_prop(buffer, "cache_lines_per_tag",
608 cache->cachelines_per_tag);
609 sysfs_show_32bit_prop(buffer, "association", cache->cache_assoc);
610 sysfs_show_32bit_prop(buffer, "latency", cache->cache_latency);
611 sysfs_show_32bit_prop(buffer, "type", cache->cache_type);
612 snprintf(buffer, PAGE_SIZE, "%ssibling_map ", buffer);
613 for (i = 0; i < KFD_TOPOLOGY_CPU_SIBLINGS; i++)
614 ret = snprintf(buffer, PAGE_SIZE, "%s%d%s",
615 buffer, cache->sibling_map[i],
616 (i == KFD_TOPOLOGY_CPU_SIBLINGS-1) ?
617 "\n" : ",");
618
619 return ret;
620}
621
622static const struct sysfs_ops cache_ops = {
623 .show = kfd_cache_show,
624};
625
626static struct kobj_type cache_type = {
627 .sysfs_ops = &cache_ops,
628};
629
630static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
631 char *buffer)
632{
633 ssize_t ret;
634 struct kfd_topology_device *dev;
635 char public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE];
636 uint32_t i;
637
638 /* Making sure that the buffer is an empty string */
639 buffer[0] = 0;
640
641 if (strcmp(attr->name, "gpu_id") == 0) {
642 dev = container_of(attr, struct kfd_topology_device,
643 attr_gpuid);
644 ret = sysfs_show_32bit_val(buffer, dev->gpu_id);
645 } else if (strcmp(attr->name, "name") == 0) {
646 dev = container_of(attr, struct kfd_topology_device,
647 attr_name);
648 for (i = 0; i < KFD_TOPOLOGY_PUBLIC_NAME_SIZE; i++) {
649 public_name[i] =
650 (char)dev->node_props.marketing_name[i];
651 if (dev->node_props.marketing_name[i] == 0)
652 break;
653 }
654 public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE-1] = 0x0;
655 ret = sysfs_show_str_val(buffer, public_name);
656 } else {
657 dev = container_of(attr, struct kfd_topology_device,
658 attr_props);
659 sysfs_show_32bit_prop(buffer, "cpu_cores_count",
660 dev->node_props.cpu_cores_count);
661 sysfs_show_32bit_prop(buffer, "simd_count",
662 dev->node_props.simd_count);
663
664 if (dev->mem_bank_count < dev->node_props.mem_banks_count) {
665 pr_warn("kfd: mem_banks_count truncated from %d to %d\n",
666 dev->node_props.mem_banks_count,
667 dev->mem_bank_count);
668 sysfs_show_32bit_prop(buffer, "mem_banks_count",
669 dev->mem_bank_count);
670 } else {
671 sysfs_show_32bit_prop(buffer, "mem_banks_count",
672 dev->node_props.mem_banks_count);
673 }
674
675 sysfs_show_32bit_prop(buffer, "caches_count",
676 dev->node_props.caches_count);
677 sysfs_show_32bit_prop(buffer, "io_links_count",
678 dev->node_props.io_links_count);
679 sysfs_show_32bit_prop(buffer, "cpu_core_id_base",
680 dev->node_props.cpu_core_id_base);
681 sysfs_show_32bit_prop(buffer, "simd_id_base",
682 dev->node_props.simd_id_base);
683 sysfs_show_32bit_prop(buffer, "capability",
684 dev->node_props.capability);
685 sysfs_show_32bit_prop(buffer, "max_waves_per_simd",
686 dev->node_props.max_waves_per_simd);
687 sysfs_show_32bit_prop(buffer, "lds_size_in_kb",
688 dev->node_props.lds_size_in_kb);
689 sysfs_show_32bit_prop(buffer, "gds_size_in_kb",
690 dev->node_props.gds_size_in_kb);
691 sysfs_show_32bit_prop(buffer, "wave_front_size",
692 dev->node_props.wave_front_size);
693 sysfs_show_32bit_prop(buffer, "array_count",
694 dev->node_props.array_count);
695 sysfs_show_32bit_prop(buffer, "simd_arrays_per_engine",
696 dev->node_props.simd_arrays_per_engine);
697 sysfs_show_32bit_prop(buffer, "cu_per_simd_array",
698 dev->node_props.cu_per_simd_array);
699 sysfs_show_32bit_prop(buffer, "simd_per_cu",
700 dev->node_props.simd_per_cu);
701 sysfs_show_32bit_prop(buffer, "max_slots_scratch_cu",
702 dev->node_props.max_slots_scratch_cu);
703 sysfs_show_32bit_prop(buffer, "engine_id",
704 dev->node_props.engine_id);
705 sysfs_show_32bit_prop(buffer, "vendor_id",
706 dev->node_props.vendor_id);
707 sysfs_show_32bit_prop(buffer, "device_id",
708 dev->node_props.device_id);
709 sysfs_show_32bit_prop(buffer, "location_id",
710 dev->node_props.location_id);
711
712 if (dev->gpu) {
713 sysfs_show_32bit_prop(buffer, "max_engine_clk_fcompute",
714 kfd2kgd->get_max_engine_clock_in_mhz(
715 dev->gpu->kgd));
716 sysfs_show_64bit_prop(buffer, "local_mem_size",
717 kfd2kgd->get_vmem_size(dev->gpu->kgd));
718 }
719
720 ret = sysfs_show_32bit_prop(buffer, "max_engine_clk_ccompute",
721 cpufreq_quick_get_max(0)/1000);
722 }
723
724 return ret;
725}
726
727static const struct sysfs_ops node_ops = {
728 .show = node_show,
729};
730
731static struct kobj_type node_type = {
732 .sysfs_ops = &node_ops,
733};
734
735static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr)
736{
737 sysfs_remove_file(kobj, attr);
738 kobject_del(kobj);
739 kobject_put(kobj);
740}
741
742static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
743{
744 struct kfd_iolink_properties *iolink;
745 struct kfd_cache_properties *cache;
746 struct kfd_mem_properties *mem;
747
748 BUG_ON(!dev);
749
750 if (dev->kobj_iolink) {
751 list_for_each_entry(iolink, &dev->io_link_props, list)
752 if (iolink->kobj) {
753 kfd_remove_sysfs_file(iolink->kobj,
754 &iolink->attr);
755 iolink->kobj = 0;
756 }
757 kobject_del(dev->kobj_iolink);
758 kobject_put(dev->kobj_iolink);
759 dev->kobj_iolink = 0;
760 }
761
762 if (dev->kobj_cache) {
763 list_for_each_entry(cache, &dev->cache_props, list)
764 if (cache->kobj) {
765 kfd_remove_sysfs_file(cache->kobj,
766 &cache->attr);
767 cache->kobj = 0;
768 }
769 kobject_del(dev->kobj_cache);
770 kobject_put(dev->kobj_cache);
771 dev->kobj_cache = 0;
772 }
773
774 if (dev->kobj_mem) {
775 list_for_each_entry(mem, &dev->mem_props, list)
776 if (mem->kobj) {
777 kfd_remove_sysfs_file(mem->kobj, &mem->attr);
778 mem->kobj = 0;
779 }
780 kobject_del(dev->kobj_mem);
781 kobject_put(dev->kobj_mem);
782 dev->kobj_mem = 0;
783 }
784
785 if (dev->kobj_node) {
786 sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid);
787 sysfs_remove_file(dev->kobj_node, &dev->attr_name);
788 sysfs_remove_file(dev->kobj_node, &dev->attr_props);
789 kobject_del(dev->kobj_node);
790 kobject_put(dev->kobj_node);
791 dev->kobj_node = 0;
792 }
793}
794
795static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
796 uint32_t id)
797{
798 struct kfd_iolink_properties *iolink;
799 struct kfd_cache_properties *cache;
800 struct kfd_mem_properties *mem;
801 int ret;
802 uint32_t i;
803
804 BUG_ON(!dev);
805
806 /*
807 * Creating the sysfs folders
808 */
809 BUG_ON(dev->kobj_node);
810 dev->kobj_node = kfd_alloc_struct(dev->kobj_node);
811 if (!dev->kobj_node)
812 return -ENOMEM;
813
814 ret = kobject_init_and_add(dev->kobj_node, &node_type,
815 sys_props.kobj_nodes, "%d", id);
816 if (ret < 0)
817 return ret;
818
819 dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node);
820 if (!dev->kobj_mem)
821 return -ENOMEM;
822
823 dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node);
824 if (!dev->kobj_cache)
825 return -ENOMEM;
826
827 dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node);
828 if (!dev->kobj_iolink)
829 return -ENOMEM;
830
831 /*
832 * Creating sysfs files for node properties
833 */
834 dev->attr_gpuid.name = "gpu_id";
835 dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE;
836 sysfs_attr_init(&dev->attr_gpuid);
837 dev->attr_name.name = "name";
838 dev->attr_name.mode = KFD_SYSFS_FILE_MODE;
839 sysfs_attr_init(&dev->attr_name);
840 dev->attr_props.name = "properties";
841 dev->attr_props.mode = KFD_SYSFS_FILE_MODE;
842 sysfs_attr_init(&dev->attr_props);
843 ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid);
844 if (ret < 0)
845 return ret;
846 ret = sysfs_create_file(dev->kobj_node, &dev->attr_name);
847 if (ret < 0)
848 return ret;
849 ret = sysfs_create_file(dev->kobj_node, &dev->attr_props);
850 if (ret < 0)
851 return ret;
852
853 i = 0;
854 list_for_each_entry(mem, &dev->mem_props, list) {
855 mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
856 if (!mem->kobj)
857 return -ENOMEM;
858 ret = kobject_init_and_add(mem->kobj, &mem_type,
859 dev->kobj_mem, "%d", i);
860 if (ret < 0)
861 return ret;
862
863 mem->attr.name = "properties";
864 mem->attr.mode = KFD_SYSFS_FILE_MODE;
865 sysfs_attr_init(&mem->attr);
866 ret = sysfs_create_file(mem->kobj, &mem->attr);
867 if (ret < 0)
868 return ret;
869 i++;
870 }
871
872 i = 0;
873 list_for_each_entry(cache, &dev->cache_props, list) {
874 cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
875 if (!cache->kobj)
876 return -ENOMEM;
877 ret = kobject_init_and_add(cache->kobj, &cache_type,
878 dev->kobj_cache, "%d", i);
879 if (ret < 0)
880 return ret;
881
882 cache->attr.name = "properties";
883 cache->attr.mode = KFD_SYSFS_FILE_MODE;
884 sysfs_attr_init(&cache->attr);
885 ret = sysfs_create_file(cache->kobj, &cache->attr);
886 if (ret < 0)
887 return ret;
888 i++;
889 }
890
891 i = 0;
892 list_for_each_entry(iolink, &dev->io_link_props, list) {
893 iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
894 if (!iolink->kobj)
895 return -ENOMEM;
896 ret = kobject_init_and_add(iolink->kobj, &iolink_type,
897 dev->kobj_iolink, "%d", i);
898 if (ret < 0)
899 return ret;
900
901 iolink->attr.name = "properties";
902 iolink->attr.mode = KFD_SYSFS_FILE_MODE;
903 sysfs_attr_init(&iolink->attr);
904 ret = sysfs_create_file(iolink->kobj, &iolink->attr);
905 if (ret < 0)
906 return ret;
907 i++;
908}
909
910 return 0;
911}
912
913static int kfd_build_sysfs_node_tree(void)
914{
915 struct kfd_topology_device *dev;
916 int ret;
917 uint32_t i = 0;
918
919 list_for_each_entry(dev, &topology_device_list, list) {
920 ret = kfd_build_sysfs_node_entry(dev, 0);
921 if (ret < 0)
922 return ret;
923 i++;
924 }
925
926 return 0;
927}
928
929static void kfd_remove_sysfs_node_tree(void)
930{
931 struct kfd_topology_device *dev;
932
933 list_for_each_entry(dev, &topology_device_list, list)
934 kfd_remove_sysfs_node_entry(dev);
935}
936
937static int kfd_topology_update_sysfs(void)
938{
939 int ret;
940
941 pr_info("Creating topology SYSFS entries\n");
942 if (sys_props.kobj_topology == 0) {
943 sys_props.kobj_topology =
944 kfd_alloc_struct(sys_props.kobj_topology);
945 if (!sys_props.kobj_topology)
946 return -ENOMEM;
947
948 ret = kobject_init_and_add(sys_props.kobj_topology,
949 &sysprops_type, &kfd_device->kobj,
950 "topology");
951 if (ret < 0)
952 return ret;
953
954 sys_props.kobj_nodes = kobject_create_and_add("nodes",
955 sys_props.kobj_topology);
956 if (!sys_props.kobj_nodes)
957 return -ENOMEM;
958
959 sys_props.attr_genid.name = "generation_id";
960 sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE;
961 sysfs_attr_init(&sys_props.attr_genid);
962 ret = sysfs_create_file(sys_props.kobj_topology,
963 &sys_props.attr_genid);
964 if (ret < 0)
965 return ret;
966
967 sys_props.attr_props.name = "system_properties";
968 sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE;
969 sysfs_attr_init(&sys_props.attr_props);
970 ret = sysfs_create_file(sys_props.kobj_topology,
971 &sys_props.attr_props);
972 if (ret < 0)
973 return ret;
974 }
975
976 kfd_remove_sysfs_node_tree();
977
978 return kfd_build_sysfs_node_tree();
979}
980
981static void kfd_topology_release_sysfs(void)
982{
983 kfd_remove_sysfs_node_tree();
984 if (sys_props.kobj_topology) {
985 sysfs_remove_file(sys_props.kobj_topology,
986 &sys_props.attr_genid);
987 sysfs_remove_file(sys_props.kobj_topology,
988 &sys_props.attr_props);
989 if (sys_props.kobj_nodes) {
990 kobject_del(sys_props.kobj_nodes);
991 kobject_put(sys_props.kobj_nodes);
992 sys_props.kobj_nodes = 0;
993 }
994 kobject_del(sys_props.kobj_topology);
995 kobject_put(sys_props.kobj_topology);
996 sys_props.kobj_topology = 0;
997 }
998}
999
1000int kfd_topology_init(void)
1001{
1002 void *crat_image = 0;
1003 size_t image_size = 0;
1004 int ret;
1005
1006 /*
1007 * Initialize the head for the topology device list
1008 */
1009 INIT_LIST_HEAD(&topology_device_list);
1010 init_rwsem(&topology_lock);
1011 topology_crat_parsed = 0;
1012
1013 memset(&sys_props, 0, sizeof(sys_props));
1014
1015 /*
1016 * Get the CRAT image from the ACPI
1017 */
1018 ret = kfd_topology_get_crat_acpi(crat_image, &image_size);
1019 if (ret == 0 && image_size > 0) {
1020 pr_info("Found CRAT image with size=%zd\n", image_size);
1021 crat_image = kmalloc(image_size, GFP_KERNEL);
1022 if (!crat_image) {
1023 ret = -ENOMEM;
1024 pr_err("No memory for allocating CRAT image\n");
1025 goto err;
1026 }
1027 ret = kfd_topology_get_crat_acpi(crat_image, &image_size);
1028
1029 if (ret == 0) {
1030 down_write(&topology_lock);
1031 ret = kfd_parse_crat_table(crat_image);
1032 if (ret == 0)
1033 ret = kfd_topology_update_sysfs();
1034 up_write(&topology_lock);
1035 } else {
1036 pr_err("Couldn't get CRAT table size from ACPI\n");
1037 }
1038 kfree(crat_image);
1039 } else if (ret == -ENODATA) {
1040 ret = 0;
1041 } else {
1042 pr_err("Couldn't get CRAT table size from ACPI\n");
1043 }
1044
1045err:
1046 pr_info("Finished initializing topology ret=%d\n", ret);
1047 return ret;
1048}
1049
1050void kfd_topology_shutdown(void)
1051{
1052 kfd_topology_release_sysfs();
1053 kfd_release_live_view();
1054}
1055
1056static void kfd_debug_print_topology(void)
1057{
1058 struct kfd_topology_device *dev;
1059 uint32_t i = 0;
1060
1061 pr_info("DEBUG PRINT OF TOPOLOGY:");
1062 list_for_each_entry(dev, &topology_device_list, list) {
1063 pr_info("Node: %d\n", i);
1064 pr_info("\tGPU assigned: %s\n", (dev->gpu ? "yes" : "no"));
1065 pr_info("\tCPU count: %d\n", dev->node_props.cpu_cores_count);
1066 pr_info("\tSIMD count: %d", dev->node_props.simd_count);
1067 i++;
1068 }
1069}
1070
1071static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
1072{
1073 uint32_t hashout;
1074 uint32_t buf[7];
1075 int i;
1076
1077 if (!gpu)
1078 return 0;
1079
1080 buf[0] = gpu->pdev->devfn;
1081 buf[1] = gpu->pdev->subsystem_vendor;
1082 buf[2] = gpu->pdev->subsystem_device;
1083 buf[3] = gpu->pdev->device;
1084 buf[4] = gpu->pdev->bus->number;
1085 buf[5] = (uint32_t)(kfd2kgd->get_vmem_size(gpu->kgd) & 0xffffffff);
1086 buf[6] = (uint32_t)(kfd2kgd->get_vmem_size(gpu->kgd) >> 32);
1087
1088 for (i = 0, hashout = 0; i < 7; i++)
1089 hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH);
1090
1091 return hashout;
1092}
1093
1094static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu)
1095{
1096 struct kfd_topology_device *dev;
1097 struct kfd_topology_device *out_dev = 0;
1098
1099 BUG_ON(!gpu);
1100
1101 list_for_each_entry(dev, &topology_device_list, list)
1102 if (dev->gpu == 0 && dev->node_props.simd_count > 0) {
1103 dev->gpu = gpu;
1104 out_dev = dev;
1105 break;
1106 }
1107
1108 return out_dev;
1109}
1110
1111static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival)
1112{
1113 /*
1114 * TODO: Generate an event for thunk about the arrival/removal
1115 * of the GPU
1116 */
1117}
1118
1119int kfd_topology_add_device(struct kfd_dev *gpu)
1120{
1121 uint32_t gpu_id;
1122 struct kfd_topology_device *dev;
1123 int res;
1124
1125 BUG_ON(!gpu);
1126
1127 gpu_id = kfd_generate_gpu_id(gpu);
1128
1129 pr_debug("kfd: Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
1130
1131 down_write(&topology_lock);
1132 /*
1133 * Try to assign the GPU to existing topology device (generated from
1134 * CRAT table
1135 */
1136 dev = kfd_assign_gpu(gpu);
1137 if (!dev) {
1138 pr_info("GPU was not found in the current topology. Extending.\n");
1139 kfd_debug_print_topology();
1140 dev = kfd_create_topology_device();
1141 if (!dev) {
1142 res = -ENOMEM;
1143 goto err;
1144 }
1145 dev->gpu = gpu;
1146
1147 /*
1148 * TODO: Make a call to retrieve topology information from the
1149 * GPU vBIOS
1150 */
1151
1152 /*
1153 * Update the SYSFS tree, since we added another topology device
1154 */
1155 if (kfd_topology_update_sysfs() < 0)
1156 kfd_topology_release_sysfs();
1157
1158 }
1159
1160 dev->gpu_id = gpu_id;
1161 gpu->id = gpu_id;
1162 dev->node_props.vendor_id = gpu->pdev->vendor;
1163 dev->node_props.device_id = gpu->pdev->device;
1164 dev->node_props.location_id = (gpu->pdev->bus->number << 24) +
1165 (gpu->pdev->devfn & 0xffffff);
1166 /*
1167 * TODO: Retrieve max engine clock values from KGD
1168 */
1169
1170 res = 0;
1171
1172err:
1173 up_write(&topology_lock);
1174
1175 if (res == 0)
1176 kfd_notify_gpu_change(gpu_id, 1);
1177
1178 return res;
1179}
1180
1181int kfd_topology_remove_device(struct kfd_dev *gpu)
1182{
1183 struct kfd_topology_device *dev;
1184 uint32_t gpu_id;
1185 int res = -ENODEV;
1186
1187 BUG_ON(!gpu);
1188
1189 down_write(&topology_lock);
1190
1191 list_for_each_entry(dev, &topology_device_list, list)
1192 if (dev->gpu == gpu) {
1193 gpu_id = dev->gpu_id;
1194 kfd_remove_sysfs_node_entry(dev);
1195 kfd_release_topology_device(dev);
1196 res = 0;
1197 if (kfd_topology_update_sysfs() < 0)
1198 kfd_topology_release_sysfs();
1199 break;
1200 }
1201
1202 up_write(&topology_lock);
1203
1204 if (res == 0)
1205 kfd_notify_gpu_change(gpu_id, 0);
1206
1207 return res;
1208}
1209
1210/*
1211 * When idx is out of bounds, the function will return NULL
1212 */
1213struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx)
1214{
1215
1216 struct kfd_topology_device *top_dev;
1217 struct kfd_dev *device = NULL;
1218 uint8_t device_idx = 0;
1219
1220 down_read(&topology_lock);
1221
1222 list_for_each_entry(top_dev, &topology_device_list, list) {
1223 if (device_idx == idx) {
1224 device = top_dev->gpu;
1225 break;
1226 }
1227
1228 device_idx++;
1229 }
1230
1231 up_read(&topology_lock);
1232
1233 return device;
1234
1235}
drivers/gpu/drm/amd/amdkfd/kfd_topology.h+168−0
@@ -0,0 +1,168 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef __KFD_TOPOLOGY_H__
24#define __KFD_TOPOLOGY_H__
25
26#include <linux/types.h>
27#include <linux/list.h>
28#include "kfd_priv.h"
29
30#define KFD_TOPOLOGY_PUBLIC_NAME_SIZE 128
31
32#define HSA_CAP_HOT_PLUGGABLE 0x00000001
33#define HSA_CAP_ATS_PRESENT 0x00000002
34#define HSA_CAP_SHARED_WITH_GRAPHICS 0x00000004
35#define HSA_CAP_QUEUE_SIZE_POW2 0x00000008
36#define HSA_CAP_QUEUE_SIZE_32BIT 0x00000010
37#define HSA_CAP_QUEUE_IDLE_EVENT 0x00000020
38#define HSA_CAP_VA_LIMIT 0x00000040
39#define HSA_CAP_WATCH_POINTS_SUPPORTED 0x00000080
40#define HSA_CAP_WATCH_POINTS_TOTALBITS_MASK 0x00000f00
41#define HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT 8
42#define HSA_CAP_RESERVED 0xfffff000
43
44struct kfd_node_properties {
45 uint32_t cpu_cores_count;
46 uint32_t simd_count;
47 uint32_t mem_banks_count;
48 uint32_t caches_count;
49 uint32_t io_links_count;
50 uint32_t cpu_core_id_base;
51 uint32_t simd_id_base;
52 uint32_t capability;
53 uint32_t max_waves_per_simd;
54 uint32_t lds_size_in_kb;
55 uint32_t gds_size_in_kb;
56 uint32_t wave_front_size;
57 uint32_t array_count;
58 uint32_t simd_arrays_per_engine;
59 uint32_t cu_per_simd_array;
60 uint32_t simd_per_cu;
61 uint32_t max_slots_scratch_cu;
62 uint32_t engine_id;
63 uint32_t vendor_id;
64 uint32_t device_id;
65 uint32_t location_id;
66 uint32_t max_engine_clk_fcompute;
67 uint32_t max_engine_clk_ccompute;
68 uint16_t marketing_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE];
69};
70
71#define HSA_MEM_HEAP_TYPE_SYSTEM 0
72#define HSA_MEM_HEAP_TYPE_FB_PUBLIC 1
73#define HSA_MEM_HEAP_TYPE_FB_PRIVATE 2
74#define HSA_MEM_HEAP_TYPE_GPU_GDS 3
75#define HSA_MEM_HEAP_TYPE_GPU_LDS 4
76#define HSA_MEM_HEAP_TYPE_GPU_SCRATCH 5
77
78#define HSA_MEM_FLAGS_HOT_PLUGGABLE 0x00000001
79#define HSA_MEM_FLAGS_NON_VOLATILE 0x00000002
80#define HSA_MEM_FLAGS_RESERVED 0xfffffffc
81
82struct kfd_mem_properties {
83 struct list_head list;
84 uint32_t heap_type;
85 uint64_t size_in_bytes;
86 uint32_t flags;
87 uint32_t width;
88 uint32_t mem_clk_max;
89 struct kobject *kobj;
90 struct attribute attr;
91};
92
93#define KFD_TOPOLOGY_CPU_SIBLINGS 256
94
95#define HSA_CACHE_TYPE_DATA 0x00000001
96#define HSA_CACHE_TYPE_INSTRUCTION 0x00000002
97#define HSA_CACHE_TYPE_CPU 0x00000004
98#define HSA_CACHE_TYPE_HSACU 0x00000008
99#define HSA_CACHE_TYPE_RESERVED 0xfffffff0
100
101struct kfd_cache_properties {
102 struct list_head list;
103 uint32_t processor_id_low;
104 uint32_t cache_level;
105 uint32_t cache_size;
106 uint32_t cacheline_size;
107 uint32_t cachelines_per_tag;
108 uint32_t cache_assoc;
109 uint32_t cache_latency;
110 uint32_t cache_type;
111 uint8_t sibling_map[KFD_TOPOLOGY_CPU_SIBLINGS];
112 struct kobject *kobj;
113 struct attribute attr;
114};
115
116struct kfd_iolink_properties {
117 struct list_head list;
118 uint32_t iolink_type;
119 uint32_t ver_maj;
120 uint32_t ver_min;
121 uint32_t node_from;
122 uint32_t node_to;
123 uint32_t weight;
124 uint32_t min_latency;
125 uint32_t max_latency;
126 uint32_t min_bandwidth;
127 uint32_t max_bandwidth;
128 uint32_t rec_transfer_size;
129 uint32_t flags;
130 struct kobject *kobj;
131 struct attribute attr;
132};
133
134struct kfd_topology_device {
135 struct list_head list;
136 uint32_t gpu_id;
137 struct kfd_node_properties node_props;
138 uint32_t mem_bank_count;
139 struct list_head mem_props;
140 uint32_t cache_count;
141 struct list_head cache_props;
142 uint32_t io_link_count;
143 struct list_head io_link_props;
144 struct kfd_dev *gpu;
145 struct kobject *kobj_node;
146 struct kobject *kobj_mem;
147 struct kobject *kobj_cache;
148 struct kobject *kobj_iolink;
149 struct attribute attr_gpuid;
150 struct attribute attr_name;
151 struct attribute attr_props;
152};
153
154struct kfd_system_properties {
155 uint32_t num_devices; /* Number of H-NUMA nodes */
156 uint32_t generation_count;
157 uint64_t platform_oem;
158 uint64_t platform_id;
159 uint64_t platform_rev;
160 struct kobject *kobj_topology;
161 struct kobject *kobj_nodes;
162 struct attribute attr_genid;
163 struct attribute attr_props;
164};
165
166
167
168#endif /* __KFD_TOPOLOGY_H__ */
drivers/gpu/drm/amd/include/kgd_kfd_interface.h+185−0
@@ -0,0 +1,185 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23/*
24 * This file defines the private interface between the
25 * AMD kernel graphics drivers and the AMD KFD.
26 */
27
28#ifndef KGD_KFD_INTERFACE_H_INCLUDED
29#define KGD_KFD_INTERFACE_H_INCLUDED
30
31#include <linux/types.h>
32
33struct pci_dev;
34
35#define KFD_INTERFACE_VERSION 1
36
37struct kfd_dev;
38struct kgd_dev;
39
40struct kgd_mem;
41
42enum kgd_memory_pool {
43 KGD_POOL_SYSTEM_CACHEABLE = 1,
44 KGD_POOL_SYSTEM_WRITECOMBINE = 2,
45 KGD_POOL_FRAMEBUFFER = 3,
46};
47
48struct kgd2kfd_shared_resources {
49 /* Bit n == 1 means VMID n is available for KFD. */
50 unsigned int compute_vmid_bitmap;
51
52 /* Compute pipes are counted starting from MEC0/pipe0 as 0. */
53 unsigned int first_compute_pipe;
54
55 /* Number of MEC pipes available for KFD. */
56 unsigned int compute_pipe_count;
57
58 /* Base address of doorbell aperture. */
59 phys_addr_t doorbell_physical_address;
60
61 /* Size in bytes of doorbell aperture. */
62 size_t doorbell_aperture_size;
63
64 /* Number of bytes at start of aperture reserved for KGD. */
65 size_t doorbell_start_offset;
66};
67
68/**
69 * struct kgd2kfd_calls
70 *
71 * @exit: Notifies amdkfd that kgd module is unloaded
72 *
73 * @probe: Notifies amdkfd about a probe done on a device in the kgd driver.
74 *
75 * @device_init: Initialize the newly probed device (if it is a device that
76 * amdkfd supports)
77 *
78 * @device_exit: Notifies amdkfd about a removal of a kgd device
79 *
80 * @suspend: Notifies amdkfd about a suspend action done to a kgd device
81 *
82 * @resume: Notifies amdkfd about a resume action done to a kgd device
83 *
84 * This structure contains function callback pointers so the kgd driver
85 * will notify to the amdkfd about certain status changes.
86 *
87 */
88struct kgd2kfd_calls {
89 void (*exit)(void);
90 struct kfd_dev* (*probe)(struct kgd_dev *kgd, struct pci_dev *pdev);
91 bool (*device_init)(struct kfd_dev *kfd,
92 const struct kgd2kfd_shared_resources *gpu_resources);
93 void (*device_exit)(struct kfd_dev *kfd);
94 void (*interrupt)(struct kfd_dev *kfd, const void *ih_ring_entry);
95 void (*suspend)(struct kfd_dev *kfd);
96 int (*resume)(struct kfd_dev *kfd);
97};
98
99/**
100 * struct kfd2kgd_calls
101 *
102 * @init_sa_manager: Initialize an instance of the sa manager, used by
103 * amdkfd for all system memory allocations that are mapped to the GART
104 * address space
105 *
106 * @fini_sa_manager: Releases all memory allocations for amdkfd that are
107 * handled by kgd sa manager
108 *
109 * @allocate_mem: Allocate a buffer from amdkfd's sa manager. The buffer can
110 * be used for mqds, hpds, kernel queue, fence and runlists
111 *
112 * @free_mem: Frees a buffer that was allocated by amdkfd's sa manager
113 *
114 * @get_vmem_size: Retrieves (physical) size of VRAM
115 *
116 * @get_gpu_clock_counter: Retrieves GPU clock counter
117 *
118 * @get_max_engine_clock_in_mhz: Retrieves maximum GPU clock in MHz
119 *
120 * @program_sh_mem_settings: A function that should initiate the memory
121 * properties such as main aperture memory type (cache / non cached) and
122 * secondary aperture base address, size and memory type.
123 * This function is used only for no cp scheduling mode.
124 *
125 * @set_pasid_vmid_mapping: Exposes pasid/vmid pair to the H/W for no cp
126 * scheduling mode. Only used for no cp scheduling mode.
127 *
128 * @init_memory: Initializes memory apertures to fixed base/limit address
129 * and non cached memory types.
130 *
131 * @init_pipeline: Initialized the compute pipelines.
132 *
133 * @hqd_load: Loads the mqd structure to a H/W hqd slot. used only for no cp
134 * sceduling mode.
135 *
136 * @hqd_is_occupies: Checks if a hqd slot is occupied.
137 *
138 * @hqd_destroy: Destructs and preempts the queue assigned to that hqd slot.
139 *
140 * This structure contains function pointers to services that the kgd driver
141 * provides to amdkfd driver.
142 *
143 */
144struct kfd2kgd_calls {
145 /* Memory management. */
146 int (*init_sa_manager)(struct kgd_dev *kgd, unsigned int size);
147 void (*fini_sa_manager)(struct kgd_dev *kgd);
148 int (*allocate_mem)(struct kgd_dev *kgd, size_t size, size_t alignment,
149 enum kgd_memory_pool pool, struct kgd_mem **mem);
150
151 void (*free_mem)(struct kgd_dev *kgd, struct kgd_mem *mem);
152
153 uint64_t (*get_vmem_size)(struct kgd_dev *kgd);
154 uint64_t (*get_gpu_clock_counter)(struct kgd_dev *kgd);
155
156 uint32_t (*get_max_engine_clock_in_mhz)(struct kgd_dev *kgd);
157
158 /* Register access functions */
159 void (*program_sh_mem_settings)(struct kgd_dev *kgd, uint32_t vmid,
160 uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
161 uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
162
163 int (*set_pasid_vmid_mapping)(struct kgd_dev *kgd, unsigned int pasid,
164 unsigned int vmid);
165
166 int (*init_memory)(struct kgd_dev *kgd);
167 int (*init_pipeline)(struct kgd_dev *kgd, uint32_t pipe_id,
168 uint32_t hpd_size, uint64_t hpd_gpu_addr);
169
170 int (*hqd_load)(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
171 uint32_t queue_id, uint32_t __user *wptr);
172
173 bool (*hqd_is_occupies)(struct kgd_dev *kgd, uint64_t queue_address,
174 uint32_t pipe_id, uint32_t queue_id);
175
176 int (*hqd_destroy)(struct kgd_dev *kgd, uint32_t reset_type,
177 unsigned int timeout, uint32_t pipe_id,
178 uint32_t queue_id);
179};
180
181bool kgd2kfd_init(unsigned interface_version,
182 const struct kfd2kgd_calls *f2g,
183 const struct kgd2kfd_calls **g2f);
184
185#endif /* KGD_KFD_INTERFACE_H_INCLUDED */
drivers/gpu/drm/radeon/Makefile+1−0
@@ -104,6 +104,7 @@ radeon-y += \ · 103 unmodified lines
104 radeon_vce.o \104 radeon_vce.o \
105 vce_v1_0.o \105 vce_v1_0.o \
106 vce_v2_0.o \106 vce_v2_0.o \
107 radeon_kfd.o
107 108
108radeon-$(CONFIG_COMPAT) += radeon_ioc32.o109radeon-$(CONFIG_COMPAT) += radeon_ioc32.o
109radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o110radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o
drivers/gpu/drm/radeon/cik.c+61−94
@@ -32,6 +32,7 @@ · 31 unmodified lines
32#include "cik_blit_shaders.h"32#include "cik_blit_shaders.h"
33#include "radeon_ucode.h"33#include "radeon_ucode.h"
34#include "clearstate_ci.h"34#include "clearstate_ci.h"
35#include "radeon_kfd.h"
35 36
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");37MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");38MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
@@ -1563,6 +1564,8 @@ static const u32 godavari_golden_registers[] = · 1525 unmodified lines
1563 1564
1564static void cik_init_golden_registers(struct radeon_device *rdev)1565static void cik_init_golden_registers(struct radeon_device *rdev)
1565{1566{
1567 /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1568 mutex_lock(&rdev->grbm_idx_mutex);
1566 switch (rdev->family) {1569 switch (rdev->family) {
1567 case CHIP_BONAIRE:1570 case CHIP_BONAIRE:
1568 radeon_program_register_sequence(rdev,1571 radeon_program_register_sequence(rdev,
@@ -1637,6 +1640,7 @@ static void cik_init_golden_registers(struct radeon_device *rdev) · 68 unmodified lines
1637 default:1640 default:
1638 break;1641 break;
1639 }1642 }
1643 mutex_unlock(&rdev->grbm_idx_mutex);
1640}1644}
1641 1645
1642/**1646/**
@@ -3428,6 +3432,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 1785 unmodified lines
3428 u32 disabled_rbs = 0;3432 u32 disabled_rbs = 0;
3429 u32 enabled_rbs = 0;3433 u32 enabled_rbs = 0;
3430 3434
3435 mutex_lock(&rdev->grbm_idx_mutex);
3431 for (i = 0; i < se_num; i++) {3436 for (i = 0; i < se_num; i++) {
3432 for (j = 0; j < sh_per_se; j++) {3437 for (j = 0; j < sh_per_se; j++) {
3433 cik_select_se_sh(rdev, i, j);3438 cik_select_se_sh(rdev, i, j);
@@ -3439,6 +3444,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 5 unmodified lines
3439 }3444 }
3440 }3445 }
3441 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);3446 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3447 mutex_unlock(&rdev->grbm_idx_mutex);
3442 3448
3443 mask = 1;3449 mask = 1;
3444 for (i = 0; i < max_rb_num_per_se * se_num; i++) {3450 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
@@ -3449,6 +3455,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 4 unmodified lines
3449 3455
3450 rdev->config.cik.backend_enable_mask = enabled_rbs;3456 rdev->config.cik.backend_enable_mask = enabled_rbs;
3451 3457
3458 mutex_lock(&rdev->grbm_idx_mutex);
3452 for (i = 0; i < se_num; i++) {3459 for (i = 0; i < se_num; i++) {
3453 cik_select_se_sh(rdev, i, 0xffffffff);3460 cik_select_se_sh(rdev, i, 0xffffffff);
3454 data = 0;3461 data = 0;
@@ -3476,6 +3483,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 21 unmodified lines
3476 WREG32(PA_SC_RASTER_CONFIG, data);3483 WREG32(PA_SC_RASTER_CONFIG, data);
3477 }3484 }
3478 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);3485 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3486 mutex_unlock(&rdev->grbm_idx_mutex);
3479}3487}
3480 3488
3481/**3489/**
@@ -3693,6 +3701,12 @@ static void cik_gpu_init(struct radeon_device *rdev) · 211 unmodified lines
3693 /* set HW defaults for 3D engine */3701 /* set HW defaults for 3D engine */
3694 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));3702 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3695 3703
3704 mutex_lock(&rdev->grbm_idx_mutex);
3705 /*
3706 * making sure that the following register writes will be broadcasted
3707 * to all the shaders
3708 */
3709 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3696 WREG32(SX_DEBUG_1, 0x20);3710 WREG32(SX_DEBUG_1, 0x20);
3697 3711
3698 WREG32(TA_CNTL_AUX, 0x00010000);3712 WREG32(TA_CNTL_AUX, 0x00010000);
@@ -3748,6 +3762,7 @@ static void cik_gpu_init(struct radeon_device *rdev) · 49 unmodified lines
3748 3762
3749 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));3763 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3750 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);3764 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3765 mutex_unlock(&rdev->grbm_idx_mutex);
3751 3766
3752 udelay(50);3767 udelay(50);
3753}3768}
@@ -4684,12 +4699,11 @@ static int cik_mec_init(struct radeon_device *rdev) · 930 unmodified lines
4684 /*4699 /*
4685 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total4700 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4686 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total4701 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4702 * Nonetheless, we assign only 1 pipe because all other pipes will
4703 * be handled by KFD
4687 */4704 */
4688 if (rdev->family == CHIP_KAVERI)4705 rdev->mec.num_mec = 1;
4689 rdev->mec.num_mec = 2;4706 rdev->mec.num_pipe = 1;
4690 else
4691 rdev->mec.num_mec = 1;
4692 rdev->mec.num_pipe = 4;
4693 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;4707 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4694 4708
4695 if (rdev->mec.hpd_eop_obj == NULL) {4709 if (rdev->mec.hpd_eop_obj == NULL) {
@@ -4831,28 +4845,24 @@ static int cik_cp_compute_resume(struct radeon_device *rdev) · 135 unmodified lines
4831 4845
4832 /* init the pipes */4846 /* init the pipes */
4833 mutex_lock(&rdev->srbm_mutex);4847 mutex_lock(&rdev->srbm_mutex);
4834 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4835 int me = (i < 4) ? 1 : 2;
4836 int pipe = (i < 4) ? i : (i - 4);
4837 4848
4838 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);4849 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;
4839 4850
4840 cik_srbm_select(rdev, me, pipe, 0, 0);4851 cik_srbm_select(rdev, 0, 0, 0, 0);
4841 4852
4842 /* write the EOP addr */4853 /* write the EOP addr */
4843 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);4854 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4844 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);4855 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4845 4856
4846 /* set the VMID assigned */4857 /* set the VMID assigned */
4847 WREG32(CP_HPD_EOP_VMID, 0);4858 WREG32(CP_HPD_EOP_VMID, 0);
4859
4860 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4861 tmp = RREG32(CP_HPD_EOP_CONTROL);
4862 tmp &= ~EOP_SIZE_MASK;
4863 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4864 WREG32(CP_HPD_EOP_CONTROL, tmp);
4848 4865
4849 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4850 tmp = RREG32(CP_HPD_EOP_CONTROL);
4851 tmp &= ~EOP_SIZE_MASK;
4852 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4853 WREG32(CP_HPD_EOP_CONTROL, tmp);
4854 }
4855 cik_srbm_select(rdev, 0, 0, 0, 0);
4856 mutex_unlock(&rdev->srbm_mutex);4866 mutex_unlock(&rdev->srbm_mutex);
4857 4867
4858 /* init the queues. Just two for now. */4868 /* init the queues. Just two for now. */
@@ -5906,8 +5916,13 @@ int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) · 1047 unmodified lines
5906 */5916 */
5907int cik_vm_init(struct radeon_device *rdev)5917int cik_vm_init(struct radeon_device *rdev)
5908{5918{
5909 /* number of VMs */5919 /*
5910 rdev->vm_manager.nvm = 16;5920 * number of VMs
5921 * VMID 0 is reserved for System
5922 * radeon graphics/compute will use VMIDs 1-7
5923 * amdkfd will use VMIDs 8-15
5924 */
5925 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;
5911 /* base offset of vram pages */5926 /* base offset of vram pages */
5912 if (rdev->flags & RADEON_IS_IGP) {5927 if (rdev->flags & RADEON_IS_IGP) {
5913 u64 tmp = RREG32(MC_VM_FB_OFFSET);5928 u64 tmp = RREG32(MC_VM_FB_OFFSET);
@@ -6068,6 +6083,7 @@ static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) · 154 unmodified lines
6068 u32 i, j, k;6083 u32 i, j, k;
6069 u32 mask;6084 u32 mask;
6070 6085
6086 mutex_lock(&rdev->grbm_idx_mutex);
6071 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {6087 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6072 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {6088 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6073 cik_select_se_sh(rdev, i, j);6089 cik_select_se_sh(rdev, i, j);
@@ -6079,6 +6095,7 @@ static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) · 5 unmodified lines
6079 }6095 }
6080 }6096 }
6081 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);6097 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6098 mutex_unlock(&rdev->grbm_idx_mutex);
6082 6099
6083 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;6100 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6084 for (k = 0; k < rdev->usec_timeout; k++) {6101 for (k = 0; k < rdev->usec_timeout; k++) {
@@ -6213,10 +6230,12 @@ static int cik_rlc_resume(struct radeon_device *rdev) · 128 unmodified lines
6213 WREG32(RLC_LB_CNTR_INIT, 0);6230 WREG32(RLC_LB_CNTR_INIT, 0);
6214 WREG32(RLC_LB_CNTR_MAX, 0x00008000);6231 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
6215 6232
6233 mutex_lock(&rdev->grbm_idx_mutex);
6216 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);6234 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6217 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);6235 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6218 WREG32(RLC_LB_PARAMS, 0x00600408);6236 WREG32(RLC_LB_PARAMS, 0x00600408);
6219 WREG32(RLC_LB_CNTL, 0x80000004);6237 WREG32(RLC_LB_CNTL, 0x80000004);
6238 mutex_unlock(&rdev->grbm_idx_mutex);
6220 6239
6221 WREG32(RLC_MC_CNTL, 0);6240 WREG32(RLC_MC_CNTL, 0);
6222 WREG32(RLC_UCODE_CNTL, 0);6241 WREG32(RLC_UCODE_CNTL, 0);
@@ -6283,11 +6302,13 @@ static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) · 60 unmodified lines
6283 6302
6284 tmp = cik_halt_rlc(rdev);6303 tmp = cik_halt_rlc(rdev);
6285 6304
6305 mutex_lock(&rdev->grbm_idx_mutex);
6286 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);6306 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6287 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);6307 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6288 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);6308 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6289 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;6309 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6290 WREG32(RLC_SERDES_WR_CTRL, tmp2);6310 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6311 mutex_unlock(&rdev->grbm_idx_mutex);
6291 6312
6292 cik_update_rlc(rdev, tmp);6313 cik_update_rlc(rdev, tmp);
6293 6314
@@ -6329,11 +6350,13 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) · 35 unmodified lines
6329 6350
6330 tmp = cik_halt_rlc(rdev);6351 tmp = cik_halt_rlc(rdev);
6331 6352
6353 mutex_lock(&rdev->grbm_idx_mutex);
6332 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);6354 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6333 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);6355 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6334 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);6356 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6335 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;6357 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6336 WREG32(RLC_SERDES_WR_CTRL, data);6358 WREG32(RLC_SERDES_WR_CTRL, data);
6359 mutex_unlock(&rdev->grbm_idx_mutex);
6337 6360
6338 cik_update_rlc(rdev, tmp);6361 cik_update_rlc(rdev, tmp);
6339 6362
@@ -6377,11 +6400,13 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) · 37 unmodified lines
6377 6400
6378 tmp = cik_halt_rlc(rdev);6401 tmp = cik_halt_rlc(rdev);
6379 6402
6403 mutex_lock(&rdev->grbm_idx_mutex);
6380 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);6404 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6381 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);6405 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6382 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);6406 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6383 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;6407 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6384 WREG32(RLC_SERDES_WR_CTRL, data);6408 WREG32(RLC_SERDES_WR_CTRL, data);
6409 mutex_unlock(&rdev->grbm_idx_mutex);
6385 6410
6386 cik_update_rlc(rdev, tmp);6411 cik_update_rlc(rdev, tmp);
6387 }6412 }
@@ -6810,10 +6835,12 @@ static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) · 422 unmodified lines
6810 u32 mask = 0, tmp, tmp1;6835 u32 mask = 0, tmp, tmp1;
6811 int i;6836 int i;
6812 6837
6838 mutex_lock(&rdev->grbm_idx_mutex);
6813 cik_select_se_sh(rdev, se, sh);6839 cik_select_se_sh(rdev, se, sh);
6814 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);6840 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6815 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);6841 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6816 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);6842 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6843 mutex_unlock(&rdev->grbm_idx_mutex);
6817 6844
6818 tmp &= 0xffff0000;6845 tmp &= 0xffff0000;
6819 6846
@@ -7297,8 +7324,7 @@ static int cik_irq_init(struct radeon_device *rdev) · 477 unmodified lines
7297int cik_irq_set(struct radeon_device *rdev)7324int cik_irq_set(struct radeon_device *rdev)
7298{7325{
7299 u32 cp_int_cntl;7326 u32 cp_int_cntl;
7300 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;7327 u32 cp_m1p0;
7301 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
7302 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;7328 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7303 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;7329 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7304 u32 grbm_int_cntl = 0;7330 u32 grbm_int_cntl = 0;
@@ -7332,13 +7358,6 @@ int cik_irq_set(struct radeon_device *rdev) · 27 unmodified lines
7332 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;7358 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7333 7359
7334 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;7360 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7335 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7336 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7337 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7338 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7339 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7340 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7341 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7342 7361
7343 if (rdev->flags & RADEON_IS_IGP)7362 if (rdev->flags & RADEON_IS_IGP)
7344 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &7363 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
@@ -7360,33 +7379,6 @@ int cik_irq_set(struct radeon_device *rdev) · 15 unmodified lines
7360 case 0:7379 case 0:
7361 cp_m1p0 |= TIME_STAMP_INT_ENABLE;7380 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7362 break;7381 break;
7363 case 1:
7364 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7365 break;
7366 case 2:
7367 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7368 break;
7369 case 3:
7370 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7371 break;
7372 default:
7373 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7374 break;
7375 }
7376 } else if (ring->me == 2) {
7377 switch (ring->pipe) {
7378 case 0:
7379 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7380 break;
7381 case 1:
7382 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7383 break;
7384 case 2:
7385 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7386 break;
7387 case 3:
7388 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7389 break;
7390 default:7382 default:
7391 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);7383 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7392 break;7384 break;
@@ -7403,33 +7395,6 @@ int cik_irq_set(struct radeon_device *rdev) · 10 unmodified lines
7403 case 0:7395 case 0:
7404 cp_m1p0 |= TIME_STAMP_INT_ENABLE;7396 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7405 break;7397 break;
7406 case 1:
7407 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7408 break;
7409 case 2:
7410 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7411 break;
7412 case 3:
7413 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7414 break;
7415 default:
7416 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7417 break;
7418 }
7419 } else if (ring->me == 2) {
7420 switch (ring->pipe) {
7421 case 0:
7422 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7423 break;
7424 case 1:
7425 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7426 break;
7427 case 2:
7428 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7429 break;
7430 case 3:
7431 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7432 break;
7433 default:7398 default:
7434 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);7399 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7435 break;7400 break;
@@ -7518,13 +7483,6 @@ int cik_irq_set(struct radeon_device *rdev) · 82 unmodified lines
7518 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);7483 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7519 7484
7520 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);7485 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7521 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7522 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7523 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7524 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7525 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7526 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7527 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7528 7486
7529 WREG32(GRBM_INT_CNTL, grbm_int_cntl);7487 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7530 7488
@@ -7841,6 +7799,10 @@ restart_ih: · 310 unmodified lines
7841 while (rptr != wptr) {7799 while (rptr != wptr) {
7842 /* wptr/rptr are in bytes! */7800 /* wptr/rptr are in bytes! */
7843 ring_index = rptr / 4;7801 ring_index = rptr / 4;
7802
7803 radeon_kfd_interrupt(rdev,
7804 (const void *) &rdev->ih.ring[ring_index]);
7805
7844 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;7806 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7845 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;7807 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7846 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;7808 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
@@ -8530,6 +8492,10 @@ static int cik_startup(struct radeon_device *rdev) · 683 unmodified lines
8530 if (r)8492 if (r)
8531 return r;8493 return r;
8532 8494
8495 r = radeon_kfd_resume(rdev);
8496 if (r)
8497 return r;
8498
8533 return 0;8499 return 0;
8534}8500}
8535 8501
@@ -8578,6 +8544,7 @@ int cik_resume(struct radeon_device *rdev) · 42 unmodified lines
8578 */8544 */
8579int cik_suspend(struct radeon_device *rdev)8545int cik_suspend(struct radeon_device *rdev)
8580{8546{
8547 radeon_kfd_suspend(rdev);
8581 radeon_pm_suspend(rdev);8548 radeon_pm_suspend(rdev);
8582 dce6_audio_fini(rdev);8549 dce6_audio_fini(rdev);
8583 radeon_vm_manager_fini(rdev);8550 radeon_vm_manager_fini(rdev);
drivers/gpu/drm/radeon/cik_reg.h+136−0
@@ -147,4 +147,140 @@ · 146 unmodified lines
147 147
148#define CIK_LB_DESKTOP_HEIGHT 0x6b0c148#define CIK_LB_DESKTOP_HEIGHT 0x6b0c
149 149
150#define CP_HQD_IQ_RPTR 0xC970u
151#define AQL_ENABLE (1U << 0)
152
153#define IDLE (1 << 2)
154
155struct cik_mqd {
156 uint32_t header;
157 uint32_t compute_dispatch_initiator;
158 uint32_t compute_dim_x;
159 uint32_t compute_dim_y;
160 uint32_t compute_dim_z;
161 uint32_t compute_start_x;
162 uint32_t compute_start_y;
163 uint32_t compute_start_z;
164 uint32_t compute_num_thread_x;
165 uint32_t compute_num_thread_y;
166 uint32_t compute_num_thread_z;
167 uint32_t compute_pipelinestat_enable;
168 uint32_t compute_perfcount_enable;
169 uint32_t compute_pgm_lo;
170 uint32_t compute_pgm_hi;
171 uint32_t compute_tba_lo;
172 uint32_t compute_tba_hi;
173 uint32_t compute_tma_lo;
174 uint32_t compute_tma_hi;
175 uint32_t compute_pgm_rsrc1;
176 uint32_t compute_pgm_rsrc2;
177 uint32_t compute_vmid;
178 uint32_t compute_resource_limits;
179 uint32_t compute_static_thread_mgmt_se0;
180 uint32_t compute_static_thread_mgmt_se1;
181 uint32_t compute_tmpring_size;
182 uint32_t compute_static_thread_mgmt_se2;
183 uint32_t compute_static_thread_mgmt_se3;
184 uint32_t compute_restart_x;
185 uint32_t compute_restart_y;
186 uint32_t compute_restart_z;
187 uint32_t compute_thread_trace_enable;
188 uint32_t compute_misc_reserved;
189 uint32_t compute_user_data_0;
190 uint32_t compute_user_data_1;
191 uint32_t compute_user_data_2;
192 uint32_t compute_user_data_3;
193 uint32_t compute_user_data_4;
194 uint32_t compute_user_data_5;
195 uint32_t compute_user_data_6;
196 uint32_t compute_user_data_7;
197 uint32_t compute_user_data_8;
198 uint32_t compute_user_data_9;
199 uint32_t compute_user_data_10;
200 uint32_t compute_user_data_11;
201 uint32_t compute_user_data_12;
202 uint32_t compute_user_data_13;
203 uint32_t compute_user_data_14;
204 uint32_t compute_user_data_15;
205 uint32_t cp_compute_csinvoc_count_lo;
206 uint32_t cp_compute_csinvoc_count_hi;
207 uint32_t cp_mqd_base_addr_lo;
208 uint32_t cp_mqd_base_addr_hi;
209 uint32_t cp_hqd_active;
210 uint32_t cp_hqd_vmid;
211 uint32_t cp_hqd_persistent_state;
212 uint32_t cp_hqd_pipe_priority;
213 uint32_t cp_hqd_queue_priority;
214 uint32_t cp_hqd_quantum;
215 uint32_t cp_hqd_pq_base_lo;
216 uint32_t cp_hqd_pq_base_hi;
217 uint32_t cp_hqd_pq_rptr;
218 uint32_t cp_hqd_pq_rptr_report_addr_lo;
219 uint32_t cp_hqd_pq_rptr_report_addr_hi;
220 uint32_t cp_hqd_pq_wptr_poll_addr_lo;
221 uint32_t cp_hqd_pq_wptr_poll_addr_hi;
222 uint32_t cp_hqd_pq_doorbell_control;
223 uint32_t cp_hqd_pq_wptr;
224 uint32_t cp_hqd_pq_control;
225 uint32_t cp_hqd_ib_base_addr_lo;
226 uint32_t cp_hqd_ib_base_addr_hi;
227 uint32_t cp_hqd_ib_rptr;
228 uint32_t cp_hqd_ib_control;
229 uint32_t cp_hqd_iq_timer;
230 uint32_t cp_hqd_iq_rptr;
231 uint32_t cp_hqd_dequeue_request;
232 uint32_t cp_hqd_dma_offload;
233 uint32_t cp_hqd_sema_cmd;
234 uint32_t cp_hqd_msg_type;
235 uint32_t cp_hqd_atomic0_preop_lo;
236 uint32_t cp_hqd_atomic0_preop_hi;
237 uint32_t cp_hqd_atomic1_preop_lo;
238 uint32_t cp_hqd_atomic1_preop_hi;
239 uint32_t cp_hqd_hq_status0;
240 uint32_t cp_hqd_hq_control0;
241 uint32_t cp_mqd_control;
242 uint32_t cp_mqd_query_time_lo;
243 uint32_t cp_mqd_query_time_hi;
244 uint32_t cp_mqd_connect_start_time_lo;
245 uint32_t cp_mqd_connect_start_time_hi;
246 uint32_t cp_mqd_connect_end_time_lo;
247 uint32_t cp_mqd_connect_end_time_hi;
248 uint32_t cp_mqd_connect_end_wf_count;
249 uint32_t cp_mqd_connect_end_pq_rptr;
250 uint32_t cp_mqd_connect_end_pq_wptr;
251 uint32_t cp_mqd_connect_end_ib_rptr;
252 uint32_t reserved_96;
253 uint32_t reserved_97;
254 uint32_t reserved_98;
255 uint32_t reserved_99;
256 uint32_t iqtimer_pkt_header;
257 uint32_t iqtimer_pkt_dw0;
258 uint32_t iqtimer_pkt_dw1;
259 uint32_t iqtimer_pkt_dw2;
260 uint32_t iqtimer_pkt_dw3;
261 uint32_t iqtimer_pkt_dw4;
262 uint32_t iqtimer_pkt_dw5;
263 uint32_t iqtimer_pkt_dw6;
264 uint32_t reserved_108;
265 uint32_t reserved_109;
266 uint32_t reserved_110;
267 uint32_t reserved_111;
268 uint32_t queue_doorbell_id0;
269 uint32_t queue_doorbell_id1;
270 uint32_t queue_doorbell_id2;
271 uint32_t queue_doorbell_id3;
272 uint32_t queue_doorbell_id4;
273 uint32_t queue_doorbell_id5;
274 uint32_t queue_doorbell_id6;
275 uint32_t queue_doorbell_id7;
276 uint32_t queue_doorbell_id8;
277 uint32_t queue_doorbell_id9;
278 uint32_t queue_doorbell_id10;
279 uint32_t queue_doorbell_id11;
280 uint32_t queue_doorbell_id12;
281 uint32_t queue_doorbell_id13;
282 uint32_t queue_doorbell_id14;
283 uint32_t queue_doorbell_id15;
284};
285
150#endif286#endif
drivers/gpu/drm/radeon/cikd.h+52−1
@@ -30,6 +30,8 @@ · 29 unmodified lines
30#define CIK_RB_BITMAP_WIDTH_PER_SH 230#define CIK_RB_BITMAP_WIDTH_PER_SH 2
31#define HAWAII_RB_BITMAP_WIDTH_PER_SH 431#define HAWAII_RB_BITMAP_WIDTH_PER_SH 4
32 32
33#define RADEON_NUM_OF_VMIDS 8
34
33/* DIDT IND registers */35/* DIDT IND registers */
34#define DIDT_SQ_CTRL0 0x036#define DIDT_SQ_CTRL0 0x0
35# define DIDT_CTRL_EN (1 << 0)37# define DIDT_CTRL_EN (1 << 0)
@@ -1137,6 +1139,9 @@ · 1101 unmodified lines
1137#define SH_MEM_ALIGNMENT_MODE_UNALIGNED 31139#define SH_MEM_ALIGNMENT_MODE_UNALIGNED 3
1138#define DEFAULT_MTYPE(x) ((x) << 4)1140#define DEFAULT_MTYPE(x) ((x) << 4)
1139#define APE1_MTYPE(x) ((x) << 7)1141#define APE1_MTYPE(x) ((x) << 7)
1142/* valid for both DEFAULT_MTYPE and APE1_MTYPE */
1143#define MTYPE_CACHED 0
1144#define MTYPE_NONCACHED 3
1140 1145
1141#define SX_DEBUG_1 0x90601146#define SX_DEBUG_1 0x9060
1142 1147
@@ -1447,6 +1452,16 @@ · 304 unmodified lines
1447#define CP_HQD_ACTIVE 0xC91C1452#define CP_HQD_ACTIVE 0xC91C
1448#define CP_HQD_VMID 0xC9201453#define CP_HQD_VMID 0xC920
1449 1454
1455#define CP_HQD_PERSISTENT_STATE 0xC924u
1456#define DEFAULT_CP_HQD_PERSISTENT_STATE (0x33U << 8)
1457
1458#define CP_HQD_PIPE_PRIORITY 0xC928u
1459#define CP_HQD_QUEUE_PRIORITY 0xC92Cu
1460#define CP_HQD_QUANTUM 0xC930u
1461#define QUANTUM_EN 1U
1462#define QUANTUM_SCALE_1MS (1U << 4)
1463#define QUANTUM_DURATION(x) ((x) << 8)
1464
1450#define CP_HQD_PQ_BASE 0xC9341465#define CP_HQD_PQ_BASE 0xC934
1451#define CP_HQD_PQ_BASE_HI 0xC9381466#define CP_HQD_PQ_BASE_HI 0xC938
1452#define CP_HQD_PQ_RPTR 0xC93C1467#define CP_HQD_PQ_RPTR 0xC93C
@@ -1474,12 +1489,32 @@ · 21 unmodified lines
1474#define PRIV_STATE (1 << 30)1489#define PRIV_STATE (1 << 30)
1475#define KMD_QUEUE (1 << 31)1490#define KMD_QUEUE (1 << 31)
1476 1491
1477#define CP_HQD_DEQUEUE_REQUEST 0xC9741492#define CP_HQD_IB_BASE_ADDR 0xC95Cu
1493#define CP_HQD_IB_BASE_ADDR_HI 0xC960u
1494#define CP_HQD_IB_RPTR 0xC964u
1495#define CP_HQD_IB_CONTROL 0xC968u
1496#define IB_ATC_EN (1U << 23)
1497#define DEFAULT_MIN_IB_AVAIL_SIZE (3U << 20)
1498
1499#define CP_HQD_DEQUEUE_REQUEST 0xC974
1500#define DEQUEUE_REQUEST_DRAIN 1
1501#define DEQUEUE_REQUEST_RESET 2
1478 1502
1479#define CP_MQD_CONTROL 0xC99C1503#define CP_MQD_CONTROL 0xC99C
1480#define MQD_VMID(x) ((x) << 0)1504#define MQD_VMID(x) ((x) << 0)
1481#define MQD_VMID_MASK (0xf << 0)1505#define MQD_VMID_MASK (0xf << 0)
1482 1506
1507#define CP_HQD_SEMA_CMD 0xC97Cu
1508#define CP_HQD_MSG_TYPE 0xC980u
1509#define CP_HQD_ATOMIC0_PREOP_LO 0xC984u
1510#define CP_HQD_ATOMIC0_PREOP_HI 0xC988u
1511#define CP_HQD_ATOMIC1_PREOP_LO 0xC98Cu
1512#define CP_HQD_ATOMIC1_PREOP_HI 0xC990u
1513#define CP_HQD_HQ_SCHEDULER0 0xC994u
1514#define CP_HQD_HQ_SCHEDULER1 0xC998u
1515
1516#define SH_STATIC_MEM_CONFIG 0x9604u
1517
1483#define DB_RENDER_CONTROL 0x280001518#define DB_RENDER_CONTROL 0x28000
1484 1519
1485#define PA_SC_RASTER_CONFIG 0x283501520#define PA_SC_RASTER_CONFIG 0x28350
@@ -2069,4 +2104,20 @@ · 583 unmodified lines
2069#define VCE_CMD_IB_AUTO 0x000000052104#define VCE_CMD_IB_AUTO 0x00000005
2070#define VCE_CMD_SEMAPHORE 0x000000062105#define VCE_CMD_SEMAPHORE 0x00000006
2071 2106
2107#define ATC_VMID0_PASID_MAPPING 0x339Cu
2108#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS 0x3398u
2109#define ATC_VMID_PASID_MAPPING_VALID (1U << 31)
2110
2111#define ATC_VM_APERTURE0_CNTL 0x3310u
2112#define ATS_ACCESS_MODE_NEVER 0
2113#define ATS_ACCESS_MODE_ALWAYS 1
2114
2115#define ATC_VM_APERTURE0_CNTL2 0x3318u
2116#define ATC_VM_APERTURE0_HIGH_ADDR 0x3308u
2117#define ATC_VM_APERTURE0_LOW_ADDR 0x3300u
2118#define ATC_VM_APERTURE1_CNTL 0x3314u
2119#define ATC_VM_APERTURE1_CNTL2 0x331Cu
2120#define ATC_VM_APERTURE1_HIGH_ADDR 0x330Cu
2121#define ATC_VM_APERTURE1_LOW_ADDR 0x3304u
2122
2072#endif2123#endif
drivers/gpu/drm/radeon/radeon.h+10−0
@@ -701,6 +701,10 @@ struct radeon_doorbell { · 700 unmodified lines
701 701
702int radeon_doorbell_get(struct radeon_device *rdev, u32 *page);702int radeon_doorbell_get(struct radeon_device *rdev, u32 *page);
703void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell);703void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell);
704void radeon_doorbell_get_kfd_info(struct radeon_device *rdev,
705 phys_addr_t *aperture_base,
706 size_t *aperture_size,
707 size_t *start_offset);
704 708
705/*709/*
706 * IRQS.710 * IRQS.
@@ -2393,6 +2397,8 @@ struct radeon_device { · 1686 unmodified lines
2393 struct radeon_atcs atcs;2397 struct radeon_atcs atcs;
2394 /* srbm instance registers */2398 /* srbm instance registers */
2395 struct mutex srbm_mutex;2399 struct mutex srbm_mutex;
2400 /* GRBM index mutex. Protects concurrents access to GRBM index */
2401 struct mutex grbm_idx_mutex;
2396 /* clock, powergating flags */2402 /* clock, powergating flags */
2397 u32 cg_flags;2403 u32 cg_flags;
2398 u32 pg_flags;2404 u32 pg_flags;
@@ -2405,6 +2411,10 @@ struct radeon_device { · 6 unmodified lines
2405 u64 vram_pin_size;2411 u64 vram_pin_size;
2406 u64 gart_pin_size;2412 u64 gart_pin_size;
2407 2413
2414 /* amdkfd interface */
2415 struct kfd_dev *kfd;
2416 struct radeon_sa_manager kfd_bo;
2417
2408 struct mutex mn_lock;2418 struct mutex mn_lock;
2409 DECLARE_HASHTABLE(mn_hash, 7);2419 DECLARE_HASHTABLE(mn_hash, 7);
2410};2420};
drivers/gpu/drm/radeon/radeon_device.c+32−0
@@ -377,6 +377,37 @@ void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell) · 376 unmodified lines
377 __clear_bit(doorbell, rdev->doorbell.used);377 __clear_bit(doorbell, rdev->doorbell.used);
378}378}
379 379
380/**
381 * radeon_doorbell_get_kfd_info - Report doorbell configuration required to
382 * setup KFD
383 *
384 * @rdev: radeon_device pointer
385 * @aperture_base: output returning doorbell aperture base physical address
386 * @aperture_size: output returning doorbell aperture size in bytes
387 * @start_offset: output returning # of doorbell bytes reserved for radeon.
388 *
389 * Radeon and the KFD share the doorbell aperture. Radeon sets it up,
390 * takes doorbells required for its own rings and reports the setup to KFD.
391 * Radeon reserved doorbells are at the start of the doorbell aperture.
392 */
393void radeon_doorbell_get_kfd_info(struct radeon_device *rdev,
394 phys_addr_t *aperture_base,
395 size_t *aperture_size,
396 size_t *start_offset)
397{
398 /* The first num_doorbells are used by radeon.
399 * KFD takes whatever's left in the aperture. */
400 if (rdev->doorbell.size > rdev->doorbell.num_doorbells * sizeof(u32)) {
401 *aperture_base = rdev->doorbell.base;
402 *aperture_size = rdev->doorbell.size;
403 *start_offset = rdev->doorbell.num_doorbells * sizeof(u32);
404 } else {
405 *aperture_base = 0;
406 *aperture_size = 0;
407 *start_offset = 0;
408 }
409}
410
380/*411/*
381 * radeon_wb_*()412 * radeon_wb_*()
382 * Writeback is the the method by which the the GPU updates special pages413 * Writeback is the the method by which the the GPU updates special pages
@@ -1272,6 +1303,7 @@ int radeon_device_init(struct radeon_device *rdev, · 889 unmodified lines
1272 mutex_init(&rdev->pm.mutex);1303 mutex_init(&rdev->pm.mutex);
1273 mutex_init(&rdev->gpu_clock_mutex);1304 mutex_init(&rdev->gpu_clock_mutex);
1274 mutex_init(&rdev->srbm_mutex);1305 mutex_init(&rdev->srbm_mutex);
1306 mutex_init(&rdev->grbm_idx_mutex);
1275 init_rwsem(&rdev->pm.mclk_lock);1307 init_rwsem(&rdev->pm.mclk_lock);
1276 init_rwsem(&rdev->exclusive_lock);1308 init_rwsem(&rdev->exclusive_lock);
1277 init_waitqueue_head(&rdev->irq.vblank_queue);1309 init_waitqueue_head(&rdev->irq.vblank_queue);
drivers/gpu/drm/radeon/radeon_drv.c+5−0
@@ -41,6 +41,8 @@ · 40 unmodified lines
41#include <drm/drm_gem.h>41#include <drm/drm_gem.h>
42 42
43#include "drm_crtc_helper.h"43#include "drm_crtc_helper.h"
44#include "radeon_kfd.h"
45
44/*46/*
45 * KMS wrapper.47 * KMS wrapper.
46 * - 2.0.0 - initial interface48 * - 2.0.0 - initial interface
@@ -654,12 +656,15 @@ static int __init radeon_init(void) · 607 unmodified lines
654#endif656#endif
655 }657 }
656 658
659 radeon_kfd_init();
660
657 /* let modprobe override vga console setting */661 /* let modprobe override vga console setting */
658 return drm_pci_init(driver, pdriver);662 return drm_pci_init(driver, pdriver);
659}663}
660 664
661static void __exit radeon_exit(void)665static void __exit radeon_exit(void)
662{666{
667 radeon_kfd_fini();
663 drm_pci_exit(driver, pdriver);668 drm_pci_exit(driver, pdriver);
664 radeon_unregister_atpx_handler();669 radeon_unregister_atpx_handler();
665}670}
drivers/gpu/drm/radeon/radeon_kfd.c+563−0
@@ -0,0 +1,563 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/module.h>
24#include <linux/fdtable.h>
25#include <linux/uaccess.h>
26#include <drm/drmP.h>
27#include "radeon.h"
28#include "cikd.h"
29#include "cik_reg.h"
30#include "radeon_kfd.h"
31
32#define CIK_PIPE_PER_MEC (4)
33
34struct kgd_mem {
35 struct radeon_sa_bo *sa_bo;
36 uint64_t gpu_addr;
37 void *ptr;
38};
39
40static int init_sa_manager(struct kgd_dev *kgd, unsigned int size);
41static void fini_sa_manager(struct kgd_dev *kgd);
42
43static int allocate_mem(struct kgd_dev *kgd, size_t size, size_t alignment,
44 enum kgd_memory_pool pool, struct kgd_mem **mem);
45
46static void free_mem(struct kgd_dev *kgd, struct kgd_mem *mem);
47
48static uint64_t get_vmem_size(struct kgd_dev *kgd);
49static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
50
51static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
52
53/*
54 * Register access functions
55 */
56
57static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
58 uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
59 uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
60
61static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
62 unsigned int vmid);
63
64static int kgd_init_memory(struct kgd_dev *kgd);
65
66static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
67 uint32_t hpd_size, uint64_t hpd_gpu_addr);
68
69static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
70 uint32_t queue_id, uint32_t __user *wptr);
71
72static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address,
73 uint32_t pipe_id, uint32_t queue_id);
74
75static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
76 unsigned int timeout, uint32_t pipe_id,
77 uint32_t queue_id);
78
79static const struct kfd2kgd_calls kfd2kgd = {
80 .init_sa_manager = init_sa_manager,
81 .fini_sa_manager = fini_sa_manager,
82 .allocate_mem = allocate_mem,
83 .free_mem = free_mem,
84 .get_vmem_size = get_vmem_size,
85 .get_gpu_clock_counter = get_gpu_clock_counter,
86 .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
87 .program_sh_mem_settings = kgd_program_sh_mem_settings,
88 .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
89 .init_memory = kgd_init_memory,
90 .init_pipeline = kgd_init_pipeline,
91 .hqd_load = kgd_hqd_load,
92 .hqd_is_occupies = kgd_hqd_is_occupies,
93 .hqd_destroy = kgd_hqd_destroy,
94};
95
96static const struct kgd2kfd_calls *kgd2kfd;
97
98bool radeon_kfd_init(void)
99{
100 bool (*kgd2kfd_init_p)(unsigned, const struct kfd2kgd_calls*,
101 const struct kgd2kfd_calls**);
102
103 kgd2kfd_init_p = symbol_request(kgd2kfd_init);
104
105 if (kgd2kfd_init_p == NULL)
106 return false;
107
108 if (!kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kfd2kgd, &kgd2kfd)) {
109 symbol_put(kgd2kfd_init);
110 kgd2kfd = NULL;
111
112 return false;
113 }
114
115 return true;
116}
117
118void radeon_kfd_fini(void)
119{
120 if (kgd2kfd) {
121 kgd2kfd->exit();
122 symbol_put(kgd2kfd_init);
123 }
124}
125
126void radeon_kfd_device_probe(struct radeon_device *rdev)
127{
128 if (kgd2kfd)
129 rdev->kfd = kgd2kfd->probe((struct kgd_dev *)rdev, rdev->pdev);
130}
131
132void radeon_kfd_device_init(struct radeon_device *rdev)
133{
134 if (rdev->kfd) {
135 struct kgd2kfd_shared_resources gpu_resources = {
136 .compute_vmid_bitmap = 0xFF00,
137
138 .first_compute_pipe = 1,
139 .compute_pipe_count = 8 - 1,
140 };
141
142 radeon_doorbell_get_kfd_info(rdev,
143 &gpu_resources.doorbell_physical_address,
144 &gpu_resources.doorbell_aperture_size,
145 &gpu_resources.doorbell_start_offset);
146
147 kgd2kfd->device_init(rdev->kfd, &gpu_resources);
148 }
149}
150
151void radeon_kfd_device_fini(struct radeon_device *rdev)
152{
153 if (rdev->kfd) {
154 kgd2kfd->device_exit(rdev->kfd);
155 rdev->kfd = NULL;
156 }
157}
158
159void radeon_kfd_interrupt(struct radeon_device *rdev, const void *ih_ring_entry)
160{
161 if (rdev->kfd)
162 kgd2kfd->interrupt(rdev->kfd, ih_ring_entry);
163}
164
165void radeon_kfd_suspend(struct radeon_device *rdev)
166{
167 if (rdev->kfd)
168 kgd2kfd->suspend(rdev->kfd);
169}
170
171int radeon_kfd_resume(struct radeon_device *rdev)
172{
173 int r = 0;
174
175 if (rdev->kfd)
176 r = kgd2kfd->resume(rdev->kfd);
177
178 return r;
179}
180
181static u32 pool_to_domain(enum kgd_memory_pool p)
182{
183 switch (p) {
184 case KGD_POOL_FRAMEBUFFER: return RADEON_GEM_DOMAIN_VRAM;
185 default: return RADEON_GEM_DOMAIN_GTT;
186 }
187}
188
189static int init_sa_manager(struct kgd_dev *kgd, unsigned int size)
190{
191 struct radeon_device *rdev = (struct radeon_device *)kgd;
192 int r;
193
194 BUG_ON(kgd == NULL);
195
196 r = radeon_sa_bo_manager_init(rdev, &rdev->kfd_bo,
197 size,
198 RADEON_GPU_PAGE_SIZE,
199 RADEON_GEM_DOMAIN_GTT,
200 RADEON_GEM_GTT_WC);
201
202 if (r)
203 return r;
204
205 r = radeon_sa_bo_manager_start(rdev, &rdev->kfd_bo);
206 if (r)
207 radeon_sa_bo_manager_fini(rdev, &rdev->kfd_bo);
208
209 return r;
210}
211
212static void fini_sa_manager(struct kgd_dev *kgd)
213{
214 struct radeon_device *rdev = (struct radeon_device *)kgd;
215
216 BUG_ON(kgd == NULL);
217
218 radeon_sa_bo_manager_suspend(rdev, &rdev->kfd_bo);
219 radeon_sa_bo_manager_fini(rdev, &rdev->kfd_bo);
220}
221
222static int allocate_mem(struct kgd_dev *kgd, size_t size, size_t alignment,
223 enum kgd_memory_pool pool, struct kgd_mem **mem)
224{
225 struct radeon_device *rdev = (struct radeon_device *)kgd;
226 u32 domain;
227 int r;
228
229 BUG_ON(kgd == NULL);
230
231 domain = pool_to_domain(pool);
232 if (domain != RADEON_GEM_DOMAIN_GTT) {
233 dev_err(rdev->dev,
234 "Only allowed to allocate gart memory for kfd\n");
235 return -EINVAL;
236 }
237
238 *mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL);
239 if ((*mem) == NULL)
240 return -ENOMEM;
241
242 r = radeon_sa_bo_new(rdev, &rdev->kfd_bo, &(*mem)->sa_bo, size,
243 alignment);
244 if (r) {
245 dev_err(rdev->dev, "failed to get memory for kfd (%d)\n", r);
246 return r;
247 }
248
249 (*mem)->ptr = radeon_sa_bo_cpu_addr((*mem)->sa_bo);
250 (*mem)->gpu_addr = radeon_sa_bo_gpu_addr((*mem)->sa_bo);
251
252 return 0;
253}
254
255static void free_mem(struct kgd_dev *kgd, struct kgd_mem *mem)
256{
257 struct radeon_device *rdev = (struct radeon_device *)kgd;
258
259 BUG_ON(kgd == NULL);
260
261 radeon_sa_bo_free(rdev, &mem->sa_bo, NULL);
262 kfree(mem);
263}
264
265static uint64_t get_vmem_size(struct kgd_dev *kgd)
266{
267 struct radeon_device *rdev = (struct radeon_device *)kgd;
268
269 BUG_ON(kgd == NULL);
270
271 return rdev->mc.real_vram_size;
272}
273
274static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
275{
276 struct radeon_device *rdev = (struct radeon_device *)kgd;
277
278 return rdev->asic->get_gpu_clock_counter(rdev);
279}
280
281static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
282{
283 struct radeon_device *rdev = (struct radeon_device *)kgd;
284
285 /* The sclk is in quantas of 10kHz */
286 return rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100;
287}
288
289static inline struct radeon_device *get_radeon_device(struct kgd_dev *kgd)
290{
291 return (struct radeon_device *)kgd;
292}
293
294static void write_register(struct kgd_dev *kgd, uint32_t offset, uint32_t value)
295{
296 struct radeon_device *rdev = get_radeon_device(kgd);
297
298 writel(value, (void __iomem *)(rdev->rmmio + offset));
299}
300
301static uint32_t read_register(struct kgd_dev *kgd, uint32_t offset)
302{
303 struct radeon_device *rdev = get_radeon_device(kgd);
304
305 return readl((void __iomem *)(rdev->rmmio + offset));
306}
307
308static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
309 uint32_t queue, uint32_t vmid)
310{
311 struct radeon_device *rdev = get_radeon_device(kgd);
312 uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue);
313
314 mutex_lock(&rdev->srbm_mutex);
315 write_register(kgd, SRBM_GFX_CNTL, value);
316}
317
318static void unlock_srbm(struct kgd_dev *kgd)
319{
320 struct radeon_device *rdev = get_radeon_device(kgd);
321
322 write_register(kgd, SRBM_GFX_CNTL, 0);
323 mutex_unlock(&rdev->srbm_mutex);
324}
325
326static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
327 uint32_t queue_id)
328{
329 uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1;
330 uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
331
332 lock_srbm(kgd, mec, pipe, queue_id, 0);
333}
334
335static void release_queue(struct kgd_dev *kgd)
336{
337 unlock_srbm(kgd);
338}
339
340static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
341 uint32_t sh_mem_config,
342 uint32_t sh_mem_ape1_base,
343 uint32_t sh_mem_ape1_limit,
344 uint32_t sh_mem_bases)
345{
346 lock_srbm(kgd, 0, 0, 0, vmid);
347
348 write_register(kgd, SH_MEM_CONFIG, sh_mem_config);
349 write_register(kgd, SH_MEM_APE1_BASE, sh_mem_ape1_base);
350 write_register(kgd, SH_MEM_APE1_LIMIT, sh_mem_ape1_limit);
351 write_register(kgd, SH_MEM_BASES, sh_mem_bases);
352
353 unlock_srbm(kgd);
354}
355
356static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
357 unsigned int vmid)
358{
359 /*
360 * We have to assume that there is no outstanding mapping.
361 * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0
362 * because a mapping is in progress or because a mapping finished and
363 * the SW cleared it.
364 * So the protocol is to always wait & clear.
365 */
366 uint32_t pasid_mapping = (pasid == 0) ? 0 :
367 (uint32_t)pasid | ATC_VMID_PASID_MAPPING_VALID;
368
369 write_register(kgd, ATC_VMID0_PASID_MAPPING + vmid*sizeof(uint32_t),
370 pasid_mapping);
371
372 while (!(read_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS) &
373 (1U << vmid)))
374 cpu_relax();
375 write_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid);
376
377 return 0;
378}
379
380static int kgd_init_memory(struct kgd_dev *kgd)
381{
382 /*
383 * Configure apertures:
384 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
385 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
386 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
387 */
388 int i;
389 uint32_t sh_mem_bases = PRIVATE_BASE(0x6000) | SHARED_BASE(0x6000);
390
391 for (i = 8; i < 16; i++) {
392 uint32_t sh_mem_config;
393
394 lock_srbm(kgd, 0, 0, 0, i);
395
396 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
397 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
398
399 write_register(kgd, SH_MEM_CONFIG, sh_mem_config);
400
401 write_register(kgd, SH_MEM_BASES, sh_mem_bases);
402
403 /* Scratch aperture is not supported for now. */
404 write_register(kgd, SH_STATIC_MEM_CONFIG, 0);
405
406 /* APE1 disabled for now. */
407 write_register(kgd, SH_MEM_APE1_BASE, 1);
408 write_register(kgd, SH_MEM_APE1_LIMIT, 0);
409
410 unlock_srbm(kgd);
411 }
412
413 return 0;
414}
415
416static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
417 uint32_t hpd_size, uint64_t hpd_gpu_addr)
418{
419 uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1;
420 uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
421
422 lock_srbm(kgd, mec, pipe, 0, 0);
423 write_register(kgd, CP_HPD_EOP_BASE_ADDR,
424 lower_32_bits(hpd_gpu_addr >> 8));
425 write_register(kgd, CP_HPD_EOP_BASE_ADDR_HI,
426 upper_32_bits(hpd_gpu_addr >> 8));
427 write_register(kgd, CP_HPD_EOP_VMID, 0);
428 write_register(kgd, CP_HPD_EOP_CONTROL, hpd_size);
429 unlock_srbm(kgd);
430
431 return 0;
432}
433
434static inline struct cik_mqd *get_mqd(void *mqd)
435{
436 return (struct cik_mqd *)mqd;
437}
438
439static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
440 uint32_t queue_id, uint32_t __user *wptr)
441{
442 uint32_t wptr_shadow, is_wptr_shadow_valid;
443 struct cik_mqd *m;
444
445 m = get_mqd(mqd);
446
447 is_wptr_shadow_valid = !get_user(wptr_shadow, wptr);
448
449 acquire_queue(kgd, pipe_id, queue_id);
450 write_register(kgd, CP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo);
451 write_register(kgd, CP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi);
452 write_register(kgd, CP_MQD_CONTROL, m->cp_mqd_control);
453
454 write_register(kgd, CP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo);
455 write_register(kgd, CP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi);
456 write_register(kgd, CP_HQD_PQ_CONTROL, m->cp_hqd_pq_control);
457
458 write_register(kgd, CP_HQD_IB_CONTROL, m->cp_hqd_ib_control);
459 write_register(kgd, CP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo);
460 write_register(kgd, CP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi);
461
462 write_register(kgd, CP_HQD_IB_RPTR, m->cp_hqd_ib_rptr);
463
464 write_register(kgd, CP_HQD_PERSISTENT_STATE,
465 m->cp_hqd_persistent_state);
466 write_register(kgd, CP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd);
467 write_register(kgd, CP_HQD_MSG_TYPE, m->cp_hqd_msg_type);
468
469 write_register(kgd, CP_HQD_ATOMIC0_PREOP_LO,
470 m->cp_hqd_atomic0_preop_lo);
471
472 write_register(kgd, CP_HQD_ATOMIC0_PREOP_HI,
473 m->cp_hqd_atomic0_preop_hi);
474
475 write_register(kgd, CP_HQD_ATOMIC1_PREOP_LO,
476 m->cp_hqd_atomic1_preop_lo);
477
478 write_register(kgd, CP_HQD_ATOMIC1_PREOP_HI,
479 m->cp_hqd_atomic1_preop_hi);
480
481 write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR,
482 m->cp_hqd_pq_rptr_report_addr_lo);
483
484 write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
485 m->cp_hqd_pq_rptr_report_addr_hi);
486
487 write_register(kgd, CP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr);
488
489 write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR,
490 m->cp_hqd_pq_wptr_poll_addr_lo);
491
492 write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR_HI,
493 m->cp_hqd_pq_wptr_poll_addr_hi);
494
495 write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL,
496 m->cp_hqd_pq_doorbell_control);
497
498 write_register(kgd, CP_HQD_VMID, m->cp_hqd_vmid);
499
500 write_register(kgd, CP_HQD_QUANTUM, m->cp_hqd_quantum);
501
502 write_register(kgd, CP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority);
503 write_register(kgd, CP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority);
504
505 write_register(kgd, CP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr);
506
507 if (is_wptr_shadow_valid)
508 write_register(kgd, CP_HQD_PQ_WPTR, wptr_shadow);
509
510 write_register(kgd, CP_HQD_ACTIVE, m->cp_hqd_active);
511 release_queue(kgd);
512
513 return 0;
514}
515
516static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address,
517 uint32_t pipe_id, uint32_t queue_id)
518{
519 uint32_t act;
520 bool retval = false;
521 uint32_t low, high;
522
523 acquire_queue(kgd, pipe_id, queue_id);
524 act = read_register(kgd, CP_HQD_ACTIVE);
525 if (act) {
526 low = lower_32_bits(queue_address >> 8);
527 high = upper_32_bits(queue_address >> 8);
528
529 if (low == read_register(kgd, CP_HQD_PQ_BASE) &&
530 high == read_register(kgd, CP_HQD_PQ_BASE_HI))
531 retval = true;
532 }
533 release_queue(kgd);
534 return retval;
535}
536
537static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
538 unsigned int timeout, uint32_t pipe_id,
539 uint32_t queue_id)
540{
541 uint32_t temp;
542
543 acquire_queue(kgd, pipe_id, queue_id);
544 write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, 0);
545
546 write_register(kgd, CP_HQD_DEQUEUE_REQUEST, reset_type);
547
548 while (true) {
549 temp = read_register(kgd, CP_HQD_ACTIVE);
550 if (temp & 0x1)
551 break;
552 if (timeout == 0) {
553 pr_err("kfd: cp queue preemption time out (%dms)\n",
554 temp);
555 return -ETIME;
556 }
557 msleep(20);
558 timeout -= 20;
559 }
560
561 release_queue(kgd);
562 return 0;
563}
drivers/gpu/drm/radeon/radeon_kfd.h+47−0
@@ -0,0 +1,47 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23/*
24 * radeon_kfd.h defines the private interface between the
25 * AMD kernel graphics drivers and the AMD KFD.
26 */
27
28#ifndef RADEON_KFD_H_INCLUDED
29#define RADEON_KFD_H_INCLUDED
30
31#include <linux/types.h>
32#include "../amd/include/kgd_kfd_interface.h"
33
34struct radeon_device;
35
36bool radeon_kfd_init(void);
37void radeon_kfd_fini(void);
38
39void radeon_kfd_suspend(struct radeon_device *rdev);
40int radeon_kfd_resume(struct radeon_device *rdev);
41void radeon_kfd_interrupt(struct radeon_device *rdev,
42 const void *ih_ring_entry);
43void radeon_kfd_device_probe(struct radeon_device *rdev);
44void radeon_kfd_device_init(struct radeon_device *rdev);
45void radeon_kfd_device_fini(struct radeon_device *rdev);
46
47#endif /* RADEON_KFD_H_INCLUDED */
drivers/gpu/drm/radeon/radeon_kms.c+7−0
@@ -34,6 +34,8 @@ · 33 unmodified lines
34#include <linux/slab.h>34#include <linux/slab.h>
35#include <linux/pm_runtime.h>35#include <linux/pm_runtime.h>
36 36
37#include "radeon_kfd.h"
38
37#if defined(CONFIG_VGA_SWITCHEROO)39#if defined(CONFIG_VGA_SWITCHEROO)
38bool radeon_has_atpx(void);40bool radeon_has_atpx(void);
39#else41#else
@@ -63,6 +65,8 @@ int radeon_driver_unload_kms(struct drm_device *dev) · 23 unmodified lines
63 65
64 pm_runtime_get_sync(dev->dev);66 pm_runtime_get_sync(dev->dev);
65 67
68 radeon_kfd_device_fini(rdev);
69
66 radeon_acpi_fini(rdev);70 radeon_acpi_fini(rdev);
67 71
68 radeon_modeset_fini(rdev);72 radeon_modeset_fini(rdev);
@@ -142,6 +146,9 @@ int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags) · 73 unmodified lines
142 "Error during ACPI methods call\n");146 "Error during ACPI methods call\n");
143 }147 }
144 148
149 radeon_kfd_device_probe(rdev);
150 radeon_kfd_device_init(rdev);
151
145 if (radeon_is_px(dev)) {152 if (radeon_is_px(dev)) {
146 pm_runtime_use_autosuspend(dev->dev);153 pm_runtime_use_autosuspend(dev->dev);
147 pm_runtime_set_autosuspend_delay(dev->dev, 5000);154 pm_runtime_set_autosuspend_delay(dev->dev, 5000);
drivers/iommu/amd_iommu_v2.c+11−58
@@ -92,13 +92,6 @@ static spinlock_t state_lock; · 91 unmodified lines
92 92
93static struct workqueue_struct *iommu_wq;93static struct workqueue_struct *iommu_wq;
94 94
95/*
96 * Empty page table - Used between
97 * mmu_notifier_invalidate_range_start and
98 * mmu_notifier_invalidate_range_end
99 */
100static u64 *empty_page_table;
101
102static void free_pasid_states(struct device_state *dev_state);95static void free_pasid_states(struct device_state *dev_state);
103 96
104static u16 device_id(struct pci_dev *pdev)97static u16 device_id(struct pci_dev *pdev)
@@ -279,10 +272,8 @@ static void free_pasid_state(struct pasid_state *pasid_state) · 174 unmodified lines
279 272
280static void put_pasid_state(struct pasid_state *pasid_state)273static void put_pasid_state(struct pasid_state *pasid_state)
281{274{
282 if (atomic_dec_and_test(&pasid_state->count)) {275 if (atomic_dec_and_test(&pasid_state->count))
283 put_device_state(pasid_state->device_state);
284 wake_up(&pasid_state->wq);276 wake_up(&pasid_state->wq);
285 }
286}277}
287 278
288static void put_pasid_state_wait(struct pasid_state *pasid_state)279static void put_pasid_state_wait(struct pasid_state *pasid_state)
@@ -291,9 +282,7 @@ static void put_pasid_state_wait(struct pasid_state *pasid_state) · 2 unmodified lines
291 282
292 prepare_to_wait(&pasid_state->wq, &wait, TASK_UNINTERRUPTIBLE);283 prepare_to_wait(&pasid_state->wq, &wait, TASK_UNINTERRUPTIBLE);
293 284
294 if (atomic_dec_and_test(&pasid_state->count))285 if (!atomic_dec_and_test(&pasid_state->count))
295 put_device_state(pasid_state->device_state);
296 else
297 schedule();286 schedule();
298 287
299 finish_wait(&pasid_state->wq, &wait);288 finish_wait(&pasid_state->wq, &wait);
@@ -418,46 +407,21 @@ static void mn_invalidate_page(struct mmu_notifier *mn, · 118 unmodified lines
418 __mn_flush_page(mn, address);407 __mn_flush_page(mn, address);
419}408}
420 409
421static void mn_invalidate_range_start(struct mmu_notifier *mn,410static void mn_invalidate_range(struct mmu_notifier *mn,
422 struct mm_struct *mm,411 struct mm_struct *mm,
423 unsigned long start, unsigned long end)412 unsigned long start, unsigned long end)
424{
425 struct pasid_state *pasid_state;
426 struct device_state *dev_state;
427 unsigned long flags;
428
429 pasid_state = mn_to_state(mn);
430 dev_state = pasid_state->device_state;
431
432 spin_lock_irqsave(&pasid_state->lock, flags);
433 if (pasid_state->mmu_notifier_count == 0) {
434 amd_iommu_domain_set_gcr3(dev_state->domain,
435 pasid_state->pasid,
436 __pa(empty_page_table));
437 }
438 pasid_state->mmu_notifier_count += 1;
439 spin_unlock_irqrestore(&pasid_state->lock, flags);
440}
441
442static void mn_invalidate_range_end(struct mmu_notifier *mn,
443 struct mm_struct *mm,
444 unsigned long start, unsigned long end)
445{413{
446 struct pasid_state *pasid_state;414 struct pasid_state *pasid_state;
447 struct device_state *dev_state;415 struct device_state *dev_state;
448 unsigned long flags;
449 416
450 pasid_state = mn_to_state(mn);417 pasid_state = mn_to_state(mn);
451 dev_state = pasid_state->device_state;418 dev_state = pasid_state->device_state;
452 419
453 spin_lock_irqsave(&pasid_state->lock, flags);420 if ((start ^ (end - 1)) < PAGE_SIZE)
454 pasid_state->mmu_notifier_count -= 1;421 amd_iommu_flush_page(dev_state->domain, pasid_state->pasid,
455 if (pasid_state->mmu_notifier_count == 0) {422 start);
456 amd_iommu_domain_set_gcr3(dev_state->domain,423 else
457 pasid_state->pasid,424 amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid);
458 __pa(pasid_state->mm->pgd));
459 }
460 spin_unlock_irqrestore(&pasid_state->lock, flags);
461}425}
462 426
463static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)427static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
@@ -482,8 +446,7 @@ static struct mmu_notifier_ops iommu_mn = { · 18 unmodified lines
482 .release = mn_release,446 .release = mn_release,
483 .clear_flush_young = mn_clear_flush_young,447 .clear_flush_young = mn_clear_flush_young,
484 .invalidate_page = mn_invalidate_page,448 .invalidate_page = mn_invalidate_page,
485 .invalidate_range_start = mn_invalidate_range_start,449 .invalidate_range = mn_invalidate_range,
486 .invalidate_range_end = mn_invalidate_range_end,
487};450};
488 451
489static void set_pri_tag_status(struct pasid_state *pasid_state,452static void set_pri_tag_status(struct pasid_state *pasid_state,
@@ -954,18 +917,10 @@ static int __init amd_iommu_v2_init(void) · 464 unmodified lines
954 if (iommu_wq == NULL)917 if (iommu_wq == NULL)
955 goto out;918 goto out;
956 919
957 ret = -ENOMEM;
958 empty_page_table = (u64 *)get_zeroed_page(GFP_KERNEL);
959 if (empty_page_table == NULL)
960 goto out_destroy_wq;
961
962 amd_iommu_register_ppr_notifier(&ppr_nb);920 amd_iommu_register_ppr_notifier(&ppr_nb);
963 921
964 return 0;922 return 0;
965 923
966out_destroy_wq:
967 destroy_workqueue(iommu_wq);
968
969out:924out:
970 return ret;925 return ret;
971}926}
@@ -999,8 +954,6 @@ static void __exit amd_iommu_v2_exit(void) · 27 unmodified lines
999 }954 }
1000 955
1001 destroy_workqueue(iommu_wq);956 destroy_workqueue(iommu_wq);
1002
1003 free_page((unsigned long)empty_page_table);
1004}957}
1005 958
1006module_init(amd_iommu_v2_init);959module_init(amd_iommu_v2_init);
include/linux/mmu_notifier.h+83−5
@@ -98,11 +98,11 @@ struct mmu_notifier_ops { · 97 unmodified lines
98 /*98 /*
99 * invalidate_range_start() and invalidate_range_end() must be99 * invalidate_range_start() and invalidate_range_end() must be
100 * paired and are called only when the mmap_sem and/or the100 * paired and are called only when the mmap_sem and/or the
101 * locks protecting the reverse maps are held. The subsystem101 * locks protecting the reverse maps are held. If the subsystem
102 * must guarantee that no additional references are taken to102 * can't guarantee that no additional references are taken to
103 * the pages in the range established between the call to103 * the pages in the range, it has to implement the
104 * invalidate_range_start() and the matching call to104 * invalidate_range() notifier to remove any references taken
105 * invalidate_range_end().105 * after invalidate_range_start().
106 *106 *
107 * Invalidation of multiple concurrent ranges may be107 * Invalidation of multiple concurrent ranges may be
108 * optionally permitted by the driver. Either way the108 * optionally permitted by the driver. Either way the
@@ -144,6 +144,29 @@ struct mmu_notifier_ops { · 35 unmodified lines
144 void (*invalidate_range_end)(struct mmu_notifier *mn,144 void (*invalidate_range_end)(struct mmu_notifier *mn,
145 struct mm_struct *mm,145 struct mm_struct *mm,
146 unsigned long start, unsigned long end);146 unsigned long start, unsigned long end);
147
148 /*
149 * invalidate_range() is either called between
150 * invalidate_range_start() and invalidate_range_end() when the
151 * VM has to free pages that where unmapped, but before the
152 * pages are actually freed, or outside of _start()/_end() when
153 * a (remote) TLB is necessary.
154 *
155 * If invalidate_range() is used to manage a non-CPU TLB with
156 * shared page-tables, it not necessary to implement the
157 * invalidate_range_start()/end() notifiers, as
158 * invalidate_range() alread catches the points in time when an
159 * external TLB range needs to be flushed.
160 *
161 * The invalidate_range() function is called under the ptl
162 * spin-lock and not allowed to sleep.
163 *
164 * Note that this function might be called with just a sub-range
165 * of what was passed to invalidate_range_start()/end(), if
166 * called between those functions.
167 */
168 void (*invalidate_range)(struct mmu_notifier *mn, struct mm_struct *mm,
169 unsigned long start, unsigned long end);
147};170};
148 171
149/*172/*
@@ -190,6 +213,8 @@ extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm, · 40 unmodified lines
190 unsigned long start, unsigned long end);213 unsigned long start, unsigned long end);
191extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,214extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
192 unsigned long start, unsigned long end);215 unsigned long start, unsigned long end);
216extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,
217 unsigned long start, unsigned long end);
193 218
194static inline void mmu_notifier_release(struct mm_struct *mm)219static inline void mmu_notifier_release(struct mm_struct *mm)
195{220{
@@ -242,6 +267,13 @@ static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 46 unmodified lines
242 __mmu_notifier_invalidate_range_end(mm, start, end);267 __mmu_notifier_invalidate_range_end(mm, start, end);
243}268}
244 269
270static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
271 unsigned long start, unsigned long end)
272{
273 if (mm_has_notifiers(mm))
274 __mmu_notifier_invalidate_range(mm, start, end);
275}
276
245static inline void mmu_notifier_mm_init(struct mm_struct *mm)277static inline void mmu_notifier_mm_init(struct mm_struct *mm)
246{278{
247 mm->mmu_notifier_mm = NULL;279 mm->mmu_notifier_mm = NULL;
@@ -279,6 +311,44 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) · 31 unmodified lines
279 __young; \311 __young; \
280})312})
281 313
314#define ptep_clear_flush_notify(__vma, __address, __ptep) \
315({ \
316 unsigned long ___addr = __address & PAGE_MASK; \
317 struct mm_struct *___mm = (__vma)->vm_mm; \
318 pte_t ___pte; \
319 \
320 ___pte = ptep_clear_flush(__vma, __address, __ptep); \
321 mmu_notifier_invalidate_range(___mm, ___addr, \
322 ___addr + PAGE_SIZE); \
323 \
324 ___pte; \
325})
326
327#define pmdp_clear_flush_notify(__vma, __haddr, __pmd) \
328({ \
329 unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \
330 struct mm_struct *___mm = (__vma)->vm_mm; \
331 pmd_t ___pmd; \
332 \
333 ___pmd = pmdp_clear_flush(__vma, __haddr, __pmd); \
334 mmu_notifier_invalidate_range(___mm, ___haddr, \
335 ___haddr + HPAGE_PMD_SIZE); \
336 \
337 ___pmd; \
338})
339
340#define pmdp_get_and_clear_notify(__mm, __haddr, __pmd) \
341({ \
342 unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \
343 pmd_t ___pmd; \
344 \
345 ___pmd = pmdp_get_and_clear(__mm, __haddr, __pmd); \
346 mmu_notifier_invalidate_range(__mm, ___haddr, \
347 ___haddr + HPAGE_PMD_SIZE); \
348 \
349 ___pmd; \
350})
351
282/*352/*
283 * set_pte_at_notify() sets the pte _after_ running the notifier.353 * set_pte_at_notify() sets the pte _after_ running the notifier.
284 * This is safe to start by updating the secondary MMUs, because the primary MMU354 * This is safe to start by updating the secondary MMUs, because the primary MMU
@@ -342,6 +412,11 @@ static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 57 unmodified lines
342{412{
343}413}
344 414
415static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
416 unsigned long start, unsigned long end)
417{
418}
419
345static inline void mmu_notifier_mm_init(struct mm_struct *mm)420static inline void mmu_notifier_mm_init(struct mm_struct *mm)
346{421{
347}422}
@@ -352,6 +427,9 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) · 4 unmodified lines
352 427
353#define ptep_clear_flush_young_notify ptep_clear_flush_young428#define ptep_clear_flush_young_notify ptep_clear_flush_young
354#define pmdp_clear_flush_young_notify pmdp_clear_flush_young429#define pmdp_clear_flush_young_notify pmdp_clear_flush_young
430#define ptep_clear_flush_notify ptep_clear_flush
431#define pmdp_clear_flush_notify pmdp_clear_flush
432#define pmdp_get_and_clear_notify pmdp_get_and_clear
355#define set_pte_at_notify set_pte_at433#define set_pte_at_notify set_pte_at
356 434
357#endif /* CONFIG_MMU_NOTIFIER */435#endif /* CONFIG_MMU_NOTIFIER */
include/uapi/linux/kfd_ioctl.h+154−0
@@ -0,0 +1,154 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef KFD_IOCTL_H_INCLUDED
24#define KFD_IOCTL_H_INCLUDED
25
26#include <linux/types.h>
27#include <linux/ioctl.h>
28
29#define KFD_IOCTL_MAJOR_VERSION 1
30#define KFD_IOCTL_MINOR_VERSION 0
31
32struct kfd_ioctl_get_version_args {
33 uint32_t major_version; /* from KFD */
34 uint32_t minor_version; /* from KFD */
35};
36
37/* For kfd_ioctl_create_queue_args.queue_type. */
38#define KFD_IOC_QUEUE_TYPE_COMPUTE 0
39#define KFD_IOC_QUEUE_TYPE_SDMA 1
40#define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 2
41
42#define KFD_MAX_QUEUE_PERCENTAGE 100
43#define KFD_MAX_QUEUE_PRIORITY 15
44
45struct kfd_ioctl_create_queue_args {
46 uint64_t ring_base_address; /* to KFD */
47 uint64_t write_pointer_address; /* from KFD */
48 uint64_t read_pointer_address; /* from KFD */
49 uint64_t doorbell_offset; /* from KFD */
50
51 uint32_t ring_size; /* to KFD */
52 uint32_t gpu_id; /* to KFD */
53 uint32_t queue_type; /* to KFD */
54 uint32_t queue_percentage; /* to KFD */
55 uint32_t queue_priority; /* to KFD */
56 uint32_t queue_id; /* from KFD */
57
58 uint64_t eop_buffer_address; /* to KFD */
59 uint64_t eop_buffer_size; /* to KFD */
60 uint64_t ctx_save_restore_address; /* to KFD */
61 uint64_t ctx_save_restore_size; /* to KFD */
62};
63
64struct kfd_ioctl_destroy_queue_args {
65 uint32_t queue_id; /* to KFD */
66 uint32_t pad;
67};
68
69struct kfd_ioctl_update_queue_args {
70 uint64_t ring_base_address; /* to KFD */
71
72 uint32_t queue_id; /* to KFD */
73 uint32_t ring_size; /* to KFD */
74 uint32_t queue_percentage; /* to KFD */
75 uint32_t queue_priority; /* to KFD */
76};
77
78/* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */
79#define KFD_IOC_CACHE_POLICY_COHERENT 0
80#define KFD_IOC_CACHE_POLICY_NONCOHERENT 1
81
82struct kfd_ioctl_set_memory_policy_args {
83 uint64_t alternate_aperture_base; /* to KFD */
84 uint64_t alternate_aperture_size; /* to KFD */
85
86 uint32_t gpu_id; /* to KFD */
87 uint32_t default_policy; /* to KFD */
88 uint32_t alternate_policy; /* to KFD */
89 uint32_t pad;
90};
91
92/*
93 * All counters are monotonic. They are used for profiling of compute jobs.
94 * The profiling is done by userspace.
95 *
96 * In case of GPU reset, the counter should not be affected.
97 */
98
99struct kfd_ioctl_get_clock_counters_args {
100 uint64_t gpu_clock_counter; /* from KFD */
101 uint64_t cpu_clock_counter; /* from KFD */
102 uint64_t system_clock_counter; /* from KFD */
103 uint64_t system_clock_freq; /* from KFD */
104
105 uint32_t gpu_id; /* to KFD */
106 uint32_t pad;
107};
108
109#define NUM_OF_SUPPORTED_GPUS 7
110
111struct kfd_process_device_apertures {
112 uint64_t lds_base; /* from KFD */
113 uint64_t lds_limit; /* from KFD */
114 uint64_t scratch_base; /* from KFD */
115 uint64_t scratch_limit; /* from KFD */
116 uint64_t gpuvm_base; /* from KFD */
117 uint64_t gpuvm_limit; /* from KFD */
118 uint32_t gpu_id; /* from KFD */
119 uint32_t pad;
120};
121
122struct kfd_ioctl_get_process_apertures_args {
123 struct kfd_process_device_apertures
124 process_apertures[NUM_OF_SUPPORTED_GPUS];/* from KFD */
125
126 /* from KFD, should be in the range [1 - NUM_OF_SUPPORTED_GPUS] */
127 uint32_t num_of_nodes;
128 uint32_t pad;
129};
130
131#define KFD_IOC_MAGIC 'K'
132
133#define KFD_IOC_GET_VERSION \
134 _IOR(KFD_IOC_MAGIC, 1, struct kfd_ioctl_get_version_args)
135
136#define KFD_IOC_CREATE_QUEUE \
137 _IOWR(KFD_IOC_MAGIC, 2, struct kfd_ioctl_create_queue_args)
138
139#define KFD_IOC_DESTROY_QUEUE \
140 _IOWR(KFD_IOC_MAGIC, 3, struct kfd_ioctl_destroy_queue_args)
141
142#define KFD_IOC_SET_MEMORY_POLICY \
143 _IOW(KFD_IOC_MAGIC, 4, struct kfd_ioctl_set_memory_policy_args)
144
145#define KFD_IOC_GET_CLOCK_COUNTERS \
146 _IOWR(KFD_IOC_MAGIC, 5, struct kfd_ioctl_get_clock_counters_args)
147
148#define KFD_IOC_GET_PROCESS_APERTURES \
149 _IOR(KFD_IOC_MAGIC, 6, struct kfd_ioctl_get_process_apertures_args)
150
151#define KFD_IOC_UPDATE_QUEUE \
152 _IOW(KFD_IOC_MAGIC, 7, struct kfd_ioctl_update_queue_args)
153
154#endif
kernel/events/uprobes.c+1−1
@@ -193,7 +193,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, · 192 unmodified lines
193 }193 }
194 194
195 flush_cache_page(vma, addr, pte_pfn(*ptep));195 flush_cache_page(vma, addr, pte_pfn(*ptep));
196 ptep_clear_flush(vma, addr, ptep);196 ptep_clear_flush_notify(vma, addr, ptep);
197 set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));197 set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
198 198
199 page_remove_rmap(page);199 page_remove_rmap(page);
mm/fremap.c+1−1
@@ -37,7 +37,7 @@ static void zap_pte(struct mm_struct *mm, struct vm_area_struct *vma, · 36 unmodified lines
37 37
38 if (pte_present(pte)) {38 if (pte_present(pte)) {
39 flush_cache_page(vma, addr, pte_pfn(pte));39 flush_cache_page(vma, addr, pte_pfn(pte));
40 pte = ptep_clear_flush(vma, addr, ptep);40 pte = ptep_clear_flush_notify(vma, addr, ptep);
41 page = vm_normal_page(vma, addr, pte);41 page = vm_normal_page(vma, addr, pte);
42 if (page) {42 if (page) {
43 if (pte_dirty(pte))43 if (pte_dirty(pte))
mm/huge_memory.c+5−4
@@ -1036,7 +1036,7 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, · 1035 unmodified lines
1036 goto out_free_pages;1036 goto out_free_pages;
1037 VM_BUG_ON_PAGE(!PageHead(page), page);1037 VM_BUG_ON_PAGE(!PageHead(page), page);
1038 1038
1039 pmdp_clear_flush(vma, haddr, pmd);1039 pmdp_clear_flush_notify(vma, haddr, pmd);
1040 /* leave pmd empty until pte is filled */1040 /* leave pmd empty until pte is filled */
1041 1041
1042 pgtable = pgtable_trans_huge_withdraw(mm, pmd);1042 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
@@ -1179,7 +1179,7 @@ alloc: · 136 unmodified lines
1179 pmd_t entry;1179 pmd_t entry;
1180 entry = mk_huge_pmd(new_page, vma->vm_page_prot);1180 entry = mk_huge_pmd(new_page, vma->vm_page_prot);
1181 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);1181 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1182 pmdp_clear_flush(vma, haddr, pmd);1182 pmdp_clear_flush_notify(vma, haddr, pmd);
1183 page_add_new_anon_rmap(new_page, vma, haddr);1183 page_add_new_anon_rmap(new_page, vma, haddr);
1184 mem_cgroup_commit_charge(new_page, memcg, false);1184 mem_cgroup_commit_charge(new_page, memcg, false);
1185 lru_cache_add_active_or_unevictable(new_page, vma);1185 lru_cache_add_active_or_unevictable(new_page, vma);
@@ -1512,7 +1512,7 @@ int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, · 326 unmodified lines
1512 pmd_t entry;1512 pmd_t entry;
1513 ret = 1;1513 ret = 1;
1514 if (!prot_numa) {1514 if (!prot_numa) {
1515 entry = pmdp_get_and_clear(mm, addr, pmd);1515 entry = pmdp_get_and_clear_notify(mm, addr, pmd);
1516 if (pmd_numa(entry))1516 if (pmd_numa(entry))
1517 entry = pmd_mknonnuma(entry);1517 entry = pmd_mknonnuma(entry);
1518 entry = pmd_modify(entry, newprot);1518 entry = pmd_modify(entry, newprot);
@@ -1644,6 +1644,7 @@ static int __split_huge_page_splitting(struct page *page, · 125 unmodified lines
1644 * serialize against split_huge_page*.1644 * serialize against split_huge_page*.
1645 */1645 */
1646 pmdp_splitting_flush(vma, address, pmd);1646 pmdp_splitting_flush(vma, address, pmd);
1647
1647 ret = 1;1648 ret = 1;
1648 spin_unlock(ptl);1649 spin_unlock(ptl);
1649 }1650 }
@@ -2834,7 +2835,7 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, · 1184 unmodified lines
2834 pmd_t _pmd;2835 pmd_t _pmd;
2835 int i;2836 int i;
2836 2837
2837 pmdp_clear_flush(vma, haddr, pmd);2838 pmdp_clear_flush_notify(vma, haddr, pmd);
2838 /* leave pmd empty until pte is filled */2839 /* leave pmd empty until pte is filled */
2839 2840
2840 pgtable = pgtable_trans_huge_withdraw(mm, pmd);2841 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
mm/hugetlb.c+6−1
@@ -2598,8 +2598,11 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, · 2597 unmodified lines
2598 }2598 }
2599 set_huge_pte_at(dst, addr, dst_pte, entry);2599 set_huge_pte_at(dst, addr, dst_pte, entry);
2600 } else {2600 } else {
2601 if (cow)2601 if (cow) {
2602 huge_ptep_set_wrprotect(src, addr, src_pte);2602 huge_ptep_set_wrprotect(src, addr, src_pte);
2603 mmu_notifier_invalidate_range(src, mmun_start,
2604 mmun_end);
2605 }
2603 entry = huge_ptep_get(src_pte);2606 entry = huge_ptep_get(src_pte);
2604 ptepage = pte_page(entry);2607 ptepage = pte_page(entry);
2605 get_page(ptepage);2608 get_page(ptepage);
@@ -2899,6 +2902,7 @@ retry_avoidcopy: · 293 unmodified lines
2899 2902
2900 /* Break COW */2903 /* Break COW */
2901 huge_ptep_clear_flush(vma, address, ptep);2904 huge_ptep_clear_flush(vma, address, ptep);
2905 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
2902 set_huge_pte_at(mm, address, ptep,2906 set_huge_pte_at(mm, address, ptep,
2903 make_huge_pte(vma, new_page, 1));2907 make_huge_pte(vma, new_page, 1));
2904 page_remove_rmap(old_page);2908 page_remove_rmap(old_page);
@@ -3374,6 +3378,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, · 469 unmodified lines
3374 * and that page table be reused and filled with junk.3378 * and that page table be reused and filled with junk.
3375 */3379 */
3376 flush_tlb_range(vma, start, end);3380 flush_tlb_range(vma, start, end);
3381 mmu_notifier_invalidate_range(mm, start, end);
3377 mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);3382 mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
3378 mmu_notifier_invalidate_range_end(mm, start, end);3383 mmu_notifier_invalidate_range_end(mm, start, end);
3379 3384
mm/ksm.c+2−2
@@ -892,7 +892,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page, · 891 unmodified lines
892 * this assure us that no O_DIRECT can happen after the check892 * this assure us that no O_DIRECT can happen after the check
893 * or in the middle of the check.893 * or in the middle of the check.
894 */894 */
895 entry = ptep_clear_flush(vma, addr, ptep);895 entry = ptep_clear_flush_notify(vma, addr, ptep);
896 /*896 /*
897 * Check that no O_DIRECT or similar I/O is in progress on the897 * Check that no O_DIRECT or similar I/O is in progress on the
898 * page898 * page
@@ -960,7 +960,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, · 61 unmodified lines
960 page_add_anon_rmap(kpage, vma, addr);960 page_add_anon_rmap(kpage, vma, addr);
961 961
962 flush_cache_page(vma, addr, pte_pfn(*ptep));962 flush_cache_page(vma, addr, pte_pfn(*ptep));
963 ptep_clear_flush(vma, addr, ptep);963 ptep_clear_flush_notify(vma, addr, ptep);
964 set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));964 set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
965 965
966 page_remove_rmap(page);966 page_remove_rmap(page);
mm/memory.c+2−1
@@ -238,6 +238,7 @@ static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) · 237 unmodified lines
238{238{
239 tlb->need_flush = 0;239 tlb->need_flush = 0;
240 tlb_flush(tlb);240 tlb_flush(tlb);
241 mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end);
241#ifdef CONFIG_HAVE_RCU_TABLE_FREE242#ifdef CONFIG_HAVE_RCU_TABLE_FREE
242 tlb_table_flush(tlb);243 tlb_table_flush(tlb);
243#endif244#endif
@@ -2234,7 +2235,7 @@ gotten: · 1990 unmodified lines
2234 * seen in the presence of one thread doing SMC and another2235 * seen in the presence of one thread doing SMC and another
2235 * thread doing COW.2236 * thread doing COW.
2236 */2237 */
2237 ptep_clear_flush(vma, address, page_table);2238 ptep_clear_flush_notify(vma, address, page_table);
2238 page_add_new_anon_rmap(new_page, vma, address);2239 page_add_new_anon_rmap(new_page, vma, address);
2239 mem_cgroup_commit_charge(new_page, memcg, false);2240 mem_cgroup_commit_charge(new_page, memcg, false);
2240 lru_cache_add_active_or_unevictable(new_page, vma);2241 lru_cache_add_active_or_unevictable(new_page, vma);
mm/migrate.c+2−1
@@ -1854,7 +1854,7 @@ fail_putback: · 1853 unmodified lines
1854 */1854 */
1855 flush_cache_range(vma, mmun_start, mmun_end);1855 flush_cache_range(vma, mmun_start, mmun_end);
1856 page_add_anon_rmap(new_page, vma, mmun_start);1856 page_add_anon_rmap(new_page, vma, mmun_start);
1857 pmdp_clear_flush(vma, mmun_start, pmd);1857 pmdp_clear_flush_notify(vma, mmun_start, pmd);
1858 set_pmd_at(mm, mmun_start, pmd, entry);1858 set_pmd_at(mm, mmun_start, pmd, entry);
1859 flush_tlb_range(vma, mmun_start, mmun_end);1859 flush_tlb_range(vma, mmun_start, mmun_end);
1860 update_mmu_cache_pmd(vma, address, &entry);1860 update_mmu_cache_pmd(vma, address, &entry);
@@ -1862,6 +1862,7 @@ fail_putback: · 1 unmodified line
1862 if (page_count(page) != 2) {1862 if (page_count(page) != 2) {
1863 set_pmd_at(mm, mmun_start, pmd, orig_entry);1863 set_pmd_at(mm, mmun_start, pmd, orig_entry);
1864 flush_tlb_range(vma, mmun_start, mmun_end);1864 flush_tlb_range(vma, mmun_start, mmun_end);
1865 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
1865 update_mmu_cache_pmd(vma, address, &entry);1866 update_mmu_cache_pmd(vma, address, &entry);
1866 page_remove_rmap(new_page);1867 page_remove_rmap(new_page);
1867 goto fail_putback;1868 goto fail_putback;
mm/mmu_notifier.c+25−0
@@ -193,6 +193,16 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 192 unmodified lines
193 193
194 id = srcu_read_lock(&srcu);194 id = srcu_read_lock(&srcu);
195 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {195 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
196 /*
197 * Call invalidate_range here too to avoid the need for the
198 * subsystem of having to register an invalidate_range_end
199 * call-back when there is invalidate_range already. Usually a
200 * subsystem registers either invalidate_range_start()/end() or
201 * invalidate_range(), so this will be no additional overhead
202 * (besides the pointer check).
203 */
204 if (mn->ops->invalidate_range)
205 mn->ops->invalidate_range(mn, mm, start, end);
196 if (mn->ops->invalidate_range_end)206 if (mn->ops->invalidate_range_end)
197 mn->ops->invalidate_range_end(mn, mm, start, end);207 mn->ops->invalidate_range_end(mn, mm, start, end);
198 }208 }
@@ -200,6 +210,21 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 1 unmodified line
200}210}
201EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_end);211EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_end);
202 212
213void __mmu_notifier_invalidate_range(struct mm_struct *mm,
214 unsigned long start, unsigned long end)
215{
216 struct mmu_notifier *mn;
217 int id;
218
219 id = srcu_read_lock(&srcu);
220 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
221 if (mn->ops->invalidate_range)
222 mn->ops->invalidate_range(mn, mm, start, end);
223 }
224 srcu_read_unlock(&srcu, id);
225}
226EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range);
227
203static int do_mmu_notifier_register(struct mmu_notifier *mn,228static int do_mmu_notifier_register(struct mmu_notifier *mn,
204 struct mm_struct *mm,229 struct mm_struct *mm,
205 int take_mmap_sem)230 int take_mmap_sem)
mm/rmap.c+1−1
@@ -1378,7 +1378,7 @@ static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount, · 1377 unmodified lines
1378 1378
1379 /* Nuke the page table entry. */1379 /* Nuke the page table entry. */
1380 flush_cache_page(vma, address, pte_pfn(*pte));1380 flush_cache_page(vma, address, pte_pfn(*pte));
1381 pteval = ptep_clear_flush(vma, address, pte);1381 pteval = ptep_clear_flush_notify(vma, address, pte);
1382 1382
1383 /* If nonlinear, store the file page offset in the pte. */1383 /* If nonlinear, store the file page offset in the pte. */
1384 if (page->index != linear_page_index(vma, address)) {1384 if (page->index != linear_page_index(vma, address)) {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t