Merge AMDKFD it seems clean enough. * 'amdkfd-v6' of git://people.freedesktop.org/~gabbayo/linux: (29 commits) amdkfd: Implement the Get Version IOCTL amdkfd: Implement the Get Process Aperture IOCTL amdkfd: Implement the Get Clock Counters IOCTL amdkfd: Implement the Set Memory Policy IOCTL amdkfd: Implement the create/destroy/update queue IOCTLs amdkfd: Add interrupt handling module amdkfd: Add device queue manager module amdkfd: Add process queue manager module amdkfd: Add packet manager module amdkfd: Add module parameter of scheduling policy amdkfd: Add kernel queue module amdkfd: Add mqd_manager module amdkfd: Add queue module amdkfd: Add binding/unbinding calls to amd_iommu driver amdkfd: Add basic modules to amdkfd amdkfd: Add topology module to amdkfd amdkfd: Add amdkfd skeleton driver amdkfd: Add IOCTL set definitions of amdkfd Update MAINTAINERS and CREDITS files with amdkfd info drm/radeon: Add radeon <--> amdkfd interface ...
Unified · Split
| @@ -1197,6 +1197,13 @@ S: R. Tocantins, 89 - Cristo Rei · 1196 unmodified lines | |||
| 1197 | S: 80050-430 - Curitiba - Paraná | 1197 | S: 80050-430 - Curitiba - Paraná |
| 1198 | S: Brazil | 1198 | S: Brazil |
| 1199 | 1199 | ||
| 1200 | N: Oded Gabbay | ||
| 1201 | E: oded.gabbay@gmail.com | ||
| 1202 | D: AMD KFD maintainer | ||
| 1203 | S: 12 Shraga Raphaeli | ||
| 1204 | S: Petah-Tikva, 4906418 | ||
| 1205 | S: Israel | ||
| 1206 | |||
| 1200 | N: Kumar Gala | 1207 | N: Kumar Gala |
| 1201 | E: galak@kernel.crashing.org | 1208 | E: galak@kernel.crashing.org |
| 1202 | D: Embedded PowerPC 6xx/7xx/74xx/82xx/83xx/85xx support | 1209 | D: Embedded PowerPC 6xx/7xx/74xx/82xx/83xx/85xx support |
| @@ -618,6 +618,16 @@ S: Maintained · 617 unmodified lines | |||
| 618 | F: drivers/iommu/amd_iommu*.[ch] | 618 | F: drivers/iommu/amd_iommu*.[ch] |
| 619 | F: include/linux/amd-iommu.h | 619 | F: include/linux/amd-iommu.h |
| 620 | 620 | ||
| 621 | AMD KFD | ||
| 622 | M: Oded Gabbay <oded.gabbay@amd.com> | ||
| 623 | L: dri-devel@lists.freedesktop.org | ||
| 624 | T: git git://people.freedesktop.org/~gabbayo/linux.git | ||
| 625 | S: Supported | ||
| 626 | F: drivers/gpu/drm/amd/amdkfd/ | ||
| 627 | F: drivers/gpu/drm/radeon/radeon_kfd.c | ||
| 628 | F: drivers/gpu/drm/radeon/radeon_kfd.h | ||
| 629 | F: include/uapi/linux/kfd_ioctl.h | ||
| 630 | |||
| 621 | AMD MICROCODE UPDATE SUPPORT | 631 | AMD MICROCODE UPDATE SUPPORT |
| 622 | M: Andreas Herrmann <herrmann.der.user@googlemail.com> | 632 | M: Andreas Herrmann <herrmann.der.user@googlemail.com> |
| 623 | L: amd64-microcode@amd64.org | 633 | L: amd64-microcode@amd64.org |
| @@ -200,3 +200,5 @@ source "drivers/gpu/drm/tegra/Kconfig" · 199 unmodified lines | |||
| 200 | source "drivers/gpu/drm/panel/Kconfig" | 200 | source "drivers/gpu/drm/panel/Kconfig" |
| 201 | 201 | ||
| 202 | source "drivers/gpu/drm/sti/Kconfig" | 202 | source "drivers/gpu/drm/sti/Kconfig" |
| 203 | |||
| 204 | source "drivers/gpu/drm/amd/amdkfd/Kconfig" | ||
| @@ -65,3 +65,4 @@ obj-$(CONFIG_DRM_STI) += sti/ · 64 unmodified lines | |||
| 65 | obj-y += i2c/ | 65 | obj-y += i2c/ |
| 66 | obj-y += panel/ | 66 | obj-y += panel/ |
| 67 | obj-y += bridge/ | 67 | obj-y += bridge/ |
| 68 | obj-$(CONFIG_HSA_AMD) += amd/amdkfd/ | ||
| @@ -0,0 +1,9 @@ | |||
| 1 | # | ||
| 2 | # Heterogenous system architecture configuration | ||
| 3 | # | ||
| 4 | |||
| 5 | config HSA_AMD | ||
| 6 | tristate "HSA kernel driver for AMD GPU devices" | ||
| 7 | depends on (DRM_RADEON || DRM_AMDGPU) && AMD_IOMMU_V2 && X86_64 | ||
| 8 | help | ||
| 9 | Enable this if you want to use HSA features on AMD GPU devices. | ||
| @@ -0,0 +1,14 @@ | |||
| 1 | # | ||
| 2 | # Makefile for Heterogenous System Architecture support for AMD GPU devices | ||
| 3 | # | ||
| 4 | |||
| 5 | ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/amd/include/ | ||
| 6 | |||
| 7 | amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \ | ||
| 8 | kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \ | ||
| 9 | kfd_process.o kfd_queue.o kfd_mqd_manager.o \ | ||
| 10 | kfd_kernel_queue.o kfd_packet_manager.o \ | ||
| 11 | kfd_process_queue_manager.o kfd_device_queue_manager.o \ | ||
| 12 | kfd_interrupt.o | ||
| 13 | |||
| 14 | obj-$(CONFIG_HSA_AMD) += amdkfd.o | ||
| @@ -0,0 +1,221 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | #ifndef CIK_REGS_H | ||
| 24 | #define CIK_REGS_H | ||
| 25 | |||
| 26 | #define IH_VMID_0_LUT 0x3D40u | ||
| 27 | |||
| 28 | #define BIF_DOORBELL_CNTL 0x530Cu | ||
| 29 | |||
| 30 | #define SRBM_GFX_CNTL 0xE44 | ||
| 31 | #define PIPEID(x) ((x) << 0) | ||
| 32 | #define MEID(x) ((x) << 2) | ||
| 33 | #define VMID(x) ((x) << 4) | ||
| 34 | #define QUEUEID(x) ((x) << 8) | ||
| 35 | |||
| 36 | #define SQ_CONFIG 0x8C00 | ||
| 37 | |||
| 38 | #define SH_MEM_BASES 0x8C28 | ||
| 39 | /* if PTR32, these are the bases for scratch and lds */ | ||
| 40 | #define PRIVATE_BASE(x) ((x) << 0) /* scratch */ | ||
| 41 | #define SHARED_BASE(x) ((x) << 16) /* LDS */ | ||
| 42 | #define SH_MEM_APE1_BASE 0x8C2C | ||
| 43 | /* if PTR32, this is the base location of GPUVM */ | ||
| 44 | #define SH_MEM_APE1_LIMIT 0x8C30 | ||
| 45 | /* if PTR32, this is the upper limit of GPUVM */ | ||
| 46 | #define SH_MEM_CONFIG 0x8C34 | ||
| 47 | #define PTR32 (1 << 0) | ||
| 48 | #define PRIVATE_ATC (1 << 1) | ||
| 49 | #define ALIGNMENT_MODE(x) ((x) << 2) | ||
| 50 | #define SH_MEM_ALIGNMENT_MODE_DWORD 0 | ||
| 51 | #define SH_MEM_ALIGNMENT_MODE_DWORD_STRICT 1 | ||
| 52 | #define SH_MEM_ALIGNMENT_MODE_STRICT 2 | ||
| 53 | #define SH_MEM_ALIGNMENT_MODE_UNALIGNED 3 | ||
| 54 | #define DEFAULT_MTYPE(x) ((x) << 4) | ||
| 55 | #define APE1_MTYPE(x) ((x) << 7) | ||
| 56 | |||
| 57 | /* valid for both DEFAULT_MTYPE and APE1_MTYPE */ | ||
| 58 | #define MTYPE_CACHED 0 | ||
| 59 | #define MTYPE_NONCACHED 3 | ||
| 60 | |||
| 61 | |||
| 62 | #define SH_STATIC_MEM_CONFIG 0x9604u | ||
| 63 | |||
| 64 | #define TC_CFG_L1_LOAD_POLICY0 0xAC68 | ||
| 65 | #define TC_CFG_L1_LOAD_POLICY1 0xAC6C | ||
| 66 | #define TC_CFG_L1_STORE_POLICY 0xAC70 | ||
| 67 | #define TC_CFG_L2_LOAD_POLICY0 0xAC74 | ||
| 68 | #define TC_CFG_L2_LOAD_POLICY1 0xAC78 | ||
| 69 | #define TC_CFG_L2_STORE_POLICY0 0xAC7C | ||
| 70 | #define TC_CFG_L2_STORE_POLICY1 0xAC80 | ||
| 71 | #define TC_CFG_L2_ATOMIC_POLICY 0xAC84 | ||
| 72 | #define TC_CFG_L1_VOLATILE 0xAC88 | ||
| 73 | #define TC_CFG_L2_VOLATILE 0xAC8C | ||
| 74 | |||
| 75 | #define CP_PQ_WPTR_POLL_CNTL 0xC20C | ||
| 76 | #define WPTR_POLL_EN (1 << 31) | ||
| 77 | |||
| 78 | #define CPC_INT_CNTL 0xC2D0 | ||
| 79 | #define CP_ME1_PIPE0_INT_CNTL 0xC214 | ||
| 80 | #define CP_ME1_PIPE1_INT_CNTL 0xC218 | ||
| 81 | #define CP_ME1_PIPE2_INT_CNTL 0xC21C | ||
| 82 | #define CP_ME1_PIPE3_INT_CNTL 0xC220 | ||
| 83 | #define CP_ME2_PIPE0_INT_CNTL 0xC224 | ||
| 84 | #define CP_ME2_PIPE1_INT_CNTL 0xC228 | ||
| 85 | #define CP_ME2_PIPE2_INT_CNTL 0xC22C | ||
| 86 | #define CP_ME2_PIPE3_INT_CNTL 0xC230 | ||
| 87 | #define DEQUEUE_REQUEST_INT_ENABLE (1 << 13) | ||
| 88 | #define WRM_POLL_TIMEOUT_INT_ENABLE (1 << 17) | ||
| 89 | #define PRIV_REG_INT_ENABLE (1 << 23) | ||
| 90 | #define TIME_STAMP_INT_ENABLE (1 << 26) | ||
| 91 | #define GENERIC2_INT_ENABLE (1 << 29) | ||
| 92 | #define GENERIC1_INT_ENABLE (1 << 30) | ||
| 93 | #define GENERIC0_INT_ENABLE (1 << 31) | ||
| 94 | #define CP_ME1_PIPE0_INT_STATUS 0xC214 | ||
| 95 | #define CP_ME1_PIPE1_INT_STATUS 0xC218 | ||
| 96 | #define CP_ME1_PIPE2_INT_STATUS 0xC21C | ||
| 97 | #define CP_ME1_PIPE3_INT_STATUS 0xC220 | ||
| 98 | #define CP_ME2_PIPE0_INT_STATUS 0xC224 | ||
| 99 | #define CP_ME2_PIPE1_INT_STATUS 0xC228 | ||
| 100 | #define CP_ME2_PIPE2_INT_STATUS 0xC22C | ||
| 101 | #define CP_ME2_PIPE3_INT_STATUS 0xC230 | ||
| 102 | #define DEQUEUE_REQUEST_INT_STATUS (1 << 13) | ||
| 103 | #define WRM_POLL_TIMEOUT_INT_STATUS (1 << 17) | ||
| 104 | #define PRIV_REG_INT_STATUS (1 << 23) | ||
| 105 | #define TIME_STAMP_INT_STATUS (1 << 26) | ||
| 106 | #define GENERIC2_INT_STATUS (1 << 29) | ||
| 107 | #define GENERIC1_INT_STATUS (1 << 30) | ||
| 108 | #define GENERIC0_INT_STATUS (1 << 31) | ||
| 109 | |||
| 110 | #define CP_HPD_EOP_BASE_ADDR 0xC904 | ||
| 111 | #define CP_HPD_EOP_BASE_ADDR_HI 0xC908 | ||
| 112 | #define CP_HPD_EOP_VMID 0xC90C | ||
| 113 | #define CP_HPD_EOP_CONTROL 0xC910 | ||
| 114 | #define EOP_SIZE(x) ((x) << 0) | ||
| 115 | #define EOP_SIZE_MASK (0x3f << 0) | ||
| 116 | #define CP_MQD_BASE_ADDR 0xC914 | ||
| 117 | #define CP_MQD_BASE_ADDR_HI 0xC918 | ||
| 118 | #define CP_HQD_ACTIVE 0xC91C | ||
| 119 | #define CP_HQD_VMID 0xC920 | ||
| 120 | |||
| 121 | #define CP_HQD_PERSISTENT_STATE 0xC924u | ||
| 122 | #define DEFAULT_CP_HQD_PERSISTENT_STATE (0x33U << 8) | ||
| 123 | #define PRELOAD_REQ (1 << 0) | ||
| 124 | |||
| 125 | #define CP_HQD_PIPE_PRIORITY 0xC928u | ||
| 126 | #define CP_HQD_QUEUE_PRIORITY 0xC92Cu | ||
| 127 | #define CP_HQD_QUANTUM 0xC930u | ||
| 128 | #define QUANTUM_EN 1U | ||
| 129 | #define QUANTUM_SCALE_1MS (1U << 4) | ||
| 130 | #define QUANTUM_DURATION(x) ((x) << 8) | ||
| 131 | |||
| 132 | #define CP_HQD_PQ_BASE 0xC934 | ||
| 133 | #define CP_HQD_PQ_BASE_HI 0xC938 | ||
| 134 | #define CP_HQD_PQ_RPTR 0xC93C | ||
| 135 | #define CP_HQD_PQ_RPTR_REPORT_ADDR 0xC940 | ||
| 136 | #define CP_HQD_PQ_RPTR_REPORT_ADDR_HI 0xC944 | ||
| 137 | #define CP_HQD_PQ_WPTR_POLL_ADDR 0xC948 | ||
| 138 | #define CP_HQD_PQ_WPTR_POLL_ADDR_HI 0xC94C | ||
| 139 | #define CP_HQD_PQ_DOORBELL_CONTROL 0xC950 | ||
| 140 | #define DOORBELL_OFFSET(x) ((x) << 2) | ||
| 141 | #define DOORBELL_OFFSET_MASK (0x1fffff << 2) | ||
| 142 | #define DOORBELL_SOURCE (1 << 28) | ||
| 143 | #define DOORBELL_SCHD_HIT (1 << 29) | ||
| 144 | #define DOORBELL_EN (1 << 30) | ||
| 145 | #define DOORBELL_HIT (1 << 31) | ||
| 146 | #define CP_HQD_PQ_WPTR 0xC954 | ||
| 147 | #define CP_HQD_PQ_CONTROL 0xC958 | ||
| 148 | #define QUEUE_SIZE(x) ((x) << 0) | ||
| 149 | #define QUEUE_SIZE_MASK (0x3f << 0) | ||
| 150 | #define RPTR_BLOCK_SIZE(x) ((x) << 8) | ||
| 151 | #define RPTR_BLOCK_SIZE_MASK (0x3f << 8) | ||
| 152 | #define MIN_AVAIL_SIZE(x) ((x) << 20) | ||
| 153 | #define PQ_ATC_EN (1 << 23) | ||
| 154 | #define PQ_VOLATILE (1 << 26) | ||
| 155 | #define NO_UPDATE_RPTR (1 << 27) | ||
| 156 | #define UNORD_DISPATCH (1 << 28) | ||
| 157 | #define ROQ_PQ_IB_FLIP (1 << 29) | ||
| 158 | #define PRIV_STATE (1 << 30) | ||
| 159 | #define KMD_QUEUE (1 << 31) | ||
| 160 | |||
| 161 | #define DEFAULT_RPTR_BLOCK_SIZE RPTR_BLOCK_SIZE(5) | ||
| 162 | #define DEFAULT_MIN_AVAIL_SIZE MIN_AVAIL_SIZE(3) | ||
| 163 | |||
| 164 | #define CP_HQD_IB_BASE_ADDR 0xC95Cu | ||
| 165 | #define CP_HQD_IB_BASE_ADDR_HI 0xC960u | ||
| 166 | #define CP_HQD_IB_RPTR 0xC964u | ||
| 167 | #define CP_HQD_IB_CONTROL 0xC968u | ||
| 168 | #define IB_ATC_EN (1U << 23) | ||
| 169 | #define DEFAULT_MIN_IB_AVAIL_SIZE (3U << 20) | ||
| 170 | |||
| 171 | #define CP_HQD_DEQUEUE_REQUEST 0xC974 | ||
| 172 | #define DEQUEUE_REQUEST_DRAIN 1 | ||
| 173 | #define DEQUEUE_REQUEST_RESET 2 | ||
| 174 | #define DEQUEUE_INT (1U << 8) | ||
| 175 | |||
| 176 | #define CP_HQD_SEMA_CMD 0xC97Cu | ||
| 177 | #define CP_HQD_MSG_TYPE 0xC980u | ||
| 178 | #define CP_HQD_ATOMIC0_PREOP_LO 0xC984u | ||
| 179 | #define CP_HQD_ATOMIC0_PREOP_HI 0xC988u | ||
| 180 | #define CP_HQD_ATOMIC1_PREOP_LO 0xC98Cu | ||
| 181 | #define CP_HQD_ATOMIC1_PREOP_HI 0xC990u | ||
| 182 | #define CP_HQD_HQ_SCHEDULER0 0xC994u | ||
| 183 | #define CP_HQD_HQ_SCHEDULER1 0xC998u | ||
| 184 | |||
| 185 | |||
| 186 | #define CP_MQD_CONTROL 0xC99C | ||
| 187 | #define MQD_VMID(x) ((x) << 0) | ||
| 188 | #define MQD_VMID_MASK (0xf << 0) | ||
| 189 | #define MQD_CONTROL_PRIV_STATE_EN (1U << 8) | ||
| 190 | |||
| 191 | #define GRBM_GFX_INDEX 0x30800 | ||
| 192 | #define INSTANCE_INDEX(x) ((x) << 0) | ||
| 193 | #define SH_INDEX(x) ((x) << 8) | ||
| 194 | #define SE_INDEX(x) ((x) << 16) | ||
| 195 | #define SH_BROADCAST_WRITES (1 << 29) | ||
| 196 | #define INSTANCE_BROADCAST_WRITES (1 << 30) | ||
| 197 | #define SE_BROADCAST_WRITES (1 << 31) | ||
| 198 | |||
| 199 | #define SQC_CACHES 0x30d20 | ||
| 200 | #define SQC_POLICY 0x8C38u | ||
| 201 | #define SQC_VOLATILE 0x8C3Cu | ||
| 202 | |||
| 203 | #define CP_PERFMON_CNTL 0x36020 | ||
| 204 | |||
| 205 | #define ATC_VMID0_PASID_MAPPING 0x339Cu | ||
| 206 | #define ATC_VMID_PASID_MAPPING_UPDATE_STATUS 0x3398u | ||
| 207 | #define ATC_VMID_PASID_MAPPING_VALID (1U << 31) | ||
| 208 | |||
| 209 | #define ATC_VM_APERTURE0_CNTL 0x3310u | ||
| 210 | #define ATS_ACCESS_MODE_NEVER 0 | ||
| 211 | #define ATS_ACCESS_MODE_ALWAYS 1 | ||
| 212 | |||
| 213 | #define ATC_VM_APERTURE0_CNTL2 0x3318u | ||
| 214 | #define ATC_VM_APERTURE0_HIGH_ADDR 0x3308u | ||
| 215 | #define ATC_VM_APERTURE0_LOW_ADDR 0x3300u | ||
| 216 | #define ATC_VM_APERTURE1_CNTL 0x3314u | ||
| 217 | #define ATC_VM_APERTURE1_CNTL2 0x331Cu | ||
| 218 | #define ATC_VM_APERTURE1_HIGH_ADDR 0x330Cu | ||
| 219 | #define ATC_VM_APERTURE1_LOW_ADDR 0x3304u | ||
| 220 | |||
| 221 | #endif | ||
| @@ -0,0 +1,576 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | #include <linux/device.h> | ||
| 24 | #include <linux/export.h> | ||
| 25 | #include <linux/err.h> | ||
| 26 | #include <linux/fs.h> | ||
| 27 | #include <linux/sched.h> | ||
| 28 | #include <linux/slab.h> | ||
| 29 | #include <linux/uaccess.h> | ||
| 30 | #include <linux/compat.h> | ||
| 31 | #include <uapi/linux/kfd_ioctl.h> | ||
| 32 | #include <linux/time.h> | ||
| 33 | #include <linux/mm.h> | ||
| 34 | #include <linux/uaccess.h> | ||
| 35 | #include <uapi/asm-generic/mman-common.h> | ||
| 36 | #include <asm/processor.h> | ||
| 37 | #include "kfd_priv.h" | ||
| 38 | #include "kfd_device_queue_manager.h" | ||
| 39 | |||
| 40 | static long kfd_ioctl(struct file *, unsigned int, unsigned long); | ||
| 41 | static int kfd_open(struct inode *, struct file *); | ||
| 42 | static int kfd_mmap(struct file *, struct vm_area_struct *); | ||
| 43 | |||
| 44 | static const char kfd_dev_name[] = "kfd"; | ||
| 45 | |||
| 46 | static const struct file_operations kfd_fops = { | ||
| 47 | .owner = THIS_MODULE, | ||
| 48 | .unlocked_ioctl = kfd_ioctl, | ||
| 49 | .compat_ioctl = kfd_ioctl, | ||
| 50 | .open = kfd_open, | ||
| 51 | .mmap = kfd_mmap, | ||
| 52 | }; | ||
| 53 | |||
| 54 | static int kfd_char_dev_major = -1; | ||
| 55 | static struct class *kfd_class; | ||
| 56 | struct device *kfd_device; | ||
| 57 | |||
| 58 | int kfd_chardev_init(void) | ||
| 59 | { | ||
| 60 | int err = 0; | ||
| 61 | |||
| 62 | kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops); | ||
| 63 | err = kfd_char_dev_major; | ||
| 64 | if (err < 0) | ||
| 65 | goto err_register_chrdev; | ||
| 66 | |||
| 67 | kfd_class = class_create(THIS_MODULE, kfd_dev_name); | ||
| 68 | err = PTR_ERR(kfd_class); | ||
| 69 | if (IS_ERR(kfd_class)) | ||
| 70 | goto err_class_create; | ||
| 71 | |||
| 72 | kfd_device = device_create(kfd_class, NULL, | ||
| 73 | MKDEV(kfd_char_dev_major, 0), | ||
| 74 | NULL, kfd_dev_name); | ||
| 75 | err = PTR_ERR(kfd_device); | ||
| 76 | if (IS_ERR(kfd_device)) | ||
| 77 | goto err_device_create; | ||
| 78 | |||
| 79 | return 0; | ||
| 80 | |||
| 81 | err_device_create: | ||
| 82 | class_destroy(kfd_class); | ||
| 83 | err_class_create: | ||
| 84 | unregister_chrdev(kfd_char_dev_major, kfd_dev_name); | ||
| 85 | err_register_chrdev: | ||
| 86 | return err; | ||
| 87 | } | ||
| 88 | |||
| 89 | void kfd_chardev_exit(void) | ||
| 90 | { | ||
| 91 | device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0)); | ||
| 92 | class_destroy(kfd_class); | ||
| 93 | unregister_chrdev(kfd_char_dev_major, kfd_dev_name); | ||
| 94 | } | ||
| 95 | |||
| 96 | struct device *kfd_chardev(void) | ||
| 97 | { | ||
| 98 | return kfd_device; | ||
| 99 | } | ||
| 100 | |||
| 101 | |||
| 102 | static int kfd_open(struct inode *inode, struct file *filep) | ||
| 103 | { | ||
| 104 | struct kfd_process *process; | ||
| 105 | |||
| 106 | if (iminor(inode) != 0) | ||
| 107 | return -ENODEV; | ||
| 108 | |||
| 109 | process = kfd_create_process(current); | ||
| 110 | if (IS_ERR(process)) | ||
| 111 | return PTR_ERR(process); | ||
| 112 | |||
| 113 | process->is_32bit_user_mode = is_compat_task(); | ||
| 114 | |||
| 115 | dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n", | ||
| 116 | process->pasid, process->is_32bit_user_mode); | ||
| 117 | |||
| 118 | kfd_init_apertures(process); | ||
| 119 | |||
| 120 | return 0; | ||
| 121 | } | ||
| 122 | |||
| 123 | static long kfd_ioctl_get_version(struct file *filep, struct kfd_process *p, | ||
| 124 | void __user *arg) | ||
| 125 | { | ||
| 126 | struct kfd_ioctl_get_version_args args; | ||
| 127 | int err = 0; | ||
| 128 | |||
| 129 | args.major_version = KFD_IOCTL_MAJOR_VERSION; | ||
| 130 | args.minor_version = KFD_IOCTL_MINOR_VERSION; | ||
| 131 | |||
| 132 | if (copy_to_user(arg, &args, sizeof(args))) | ||
| 133 | err = -EFAULT; | ||
| 134 | |||
| 135 | return err; | ||
| 136 | } | ||
| 137 | |||
| 138 | static int set_queue_properties_from_user(struct queue_properties *q_properties, | ||
| 139 | struct kfd_ioctl_create_queue_args *args) | ||
| 140 | { | ||
| 141 | if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { | ||
| 142 | pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); | ||
| 143 | return -EINVAL; | ||
| 144 | } | ||
| 145 | |||
| 146 | if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { | ||
| 147 | pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); | ||
| 148 | return -EINVAL; | ||
| 149 | } | ||
| 150 | |||
| 151 | if ((args->ring_base_address) && | ||
| 152 | (!access_ok(VERIFY_WRITE, args->ring_base_address, sizeof(uint64_t)))) { | ||
| 153 | pr_err("kfd: can't access ring base address\n"); | ||
| 154 | return -EFAULT; | ||
| 155 | } | ||
| 156 | |||
| 157 | if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { | ||
| 158 | pr_err("kfd: ring size must be a power of 2 or 0\n"); | ||
| 159 | return -EINVAL; | ||
| 160 | } | ||
| 161 | |||
| 162 | if (!access_ok(VERIFY_WRITE, args->read_pointer_address, sizeof(uint32_t))) { | ||
| 163 | pr_err("kfd: can't access read pointer\n"); | ||
| 164 | return -EFAULT; | ||
| 165 | } | ||
| 166 | |||
| 167 | if (!access_ok(VERIFY_WRITE, args->write_pointer_address, sizeof(uint32_t))) { | ||
| 168 | pr_err("kfd: can't access write pointer\n"); | ||
| 169 | return -EFAULT; | ||
| 170 | } | ||
| 171 | |||
| 172 | q_properties->is_interop = false; | ||
| 173 | q_properties->queue_percent = args->queue_percentage; | ||
| 174 | q_properties->priority = args->queue_priority; | ||
| 175 | q_properties->queue_address = args->ring_base_address; | ||
| 176 | q_properties->queue_size = args->ring_size; | ||
| 177 | q_properties->read_ptr = (uint32_t *) args->read_pointer_address; | ||
| 178 | q_properties->write_ptr = (uint32_t *) args->write_pointer_address; | ||
| 179 | if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE || | ||
| 180 | args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) | ||
| 181 | q_properties->type = KFD_QUEUE_TYPE_COMPUTE; | ||
| 182 | else | ||
| 183 | return -ENOTSUPP; | ||
| 184 | |||
| 185 | if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) | ||
| 186 | q_properties->format = KFD_QUEUE_FORMAT_AQL; | ||
| 187 | else | ||
| 188 | q_properties->format = KFD_QUEUE_FORMAT_PM4; | ||
| 189 | |||
| 190 | pr_debug("Queue Percentage (%d, %d)\n", | ||
| 191 | q_properties->queue_percent, args->queue_percentage); | ||
| 192 | |||
| 193 | pr_debug("Queue Priority (%d, %d)\n", | ||
| 194 | q_properties->priority, args->queue_priority); | ||
| 195 | |||
| 196 | pr_debug("Queue Address (0x%llX, 0x%llX)\n", | ||
| 197 | q_properties->queue_address, args->ring_base_address); | ||
| 198 | |||
| 199 | pr_debug("Queue Size (0x%llX, %u)\n", | ||
| 200 | q_properties->queue_size, args->ring_size); | ||
| 201 | |||
| 202 | pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n", | ||
| 203 | (uint64_t) q_properties->read_ptr, | ||
| 204 | (uint64_t) q_properties->write_ptr); | ||
| 205 | |||
| 206 | pr_debug("Queue Format (%d)\n", q_properties->format); | ||
| 207 | |||
| 208 | return 0; | ||
| 209 | } | ||
| 210 | |||
| 211 | static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, | ||
| 212 | void __user *arg) | ||
| 213 | { | ||
| 214 | struct kfd_ioctl_create_queue_args args; | ||
| 215 | struct kfd_dev *dev; | ||
| 216 | int err = 0; | ||
| 217 | unsigned int queue_id; | ||
| 218 | struct kfd_process_device *pdd; | ||
| 219 | struct queue_properties q_properties; | ||
| 220 | |||
| 221 | memset(&q_properties, 0, sizeof(struct queue_properties)); | ||
| 222 | |||
| 223 | if (copy_from_user(&args, arg, sizeof(args))) | ||
| 224 | return -EFAULT; | ||
| 225 | |||
| 226 | pr_debug("kfd: creating queue ioctl\n"); | ||
| 227 | |||
| 228 | err = set_queue_properties_from_user(&q_properties, &args); | ||
| 229 | if (err) | ||
| 230 | return err; | ||
| 231 | |||
| 232 | dev = kfd_device_by_id(args.gpu_id); | ||
| 233 | if (dev == NULL) | ||
| 234 | return -EINVAL; | ||
| 235 | |||
| 236 | mutex_lock(&p->mutex); | ||
| 237 | |||
| 238 | pdd = kfd_bind_process_to_device(dev, p); | ||
| 239 | if (IS_ERR(pdd) < 0) { | ||
| 240 | err = PTR_ERR(pdd); | ||
| 241 | goto err_bind_process; | ||
| 242 | } | ||
| 243 | |||
| 244 | pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n", | ||
| 245 | p->pasid, | ||
| 246 | dev->id); | ||
| 247 | |||
| 248 | err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, 0, | ||
| 249 | KFD_QUEUE_TYPE_COMPUTE, &queue_id); | ||
| 250 | if (err != 0) | ||
| 251 | goto err_create_queue; | ||
| 252 | |||
| 253 | args.queue_id = queue_id; | ||
| 254 | |||
| 255 | /* Return gpu_id as doorbell offset for mmap usage */ | ||
| 256 | args.doorbell_offset = args.gpu_id << PAGE_SHIFT; | ||
| 257 | |||
| 258 | if (copy_to_user(arg, &args, sizeof(args))) { | ||
| 259 | err = -EFAULT; | ||
| 260 | goto err_copy_args_out; | ||
| 261 | } | ||
| 262 | |||
| 263 | mutex_unlock(&p->mutex); | ||
| 264 | |||
| 265 | pr_debug("kfd: queue id %d was created successfully\n", args.queue_id); | ||
| 266 | |||
| 267 | pr_debug("ring buffer address == 0x%016llX\n", | ||
| 268 | args.ring_base_address); | ||
| 269 | |||
| 270 | pr_debug("read ptr address == 0x%016llX\n", | ||
| 271 | args.read_pointer_address); | ||
| 272 | |||
| 273 | pr_debug("write ptr address == 0x%016llX\n", | ||
| 274 | args.write_pointer_address); | ||
| 275 | |||
| 276 | return 0; | ||
| 277 | |||
| 278 | err_copy_args_out: | ||
| 279 | pqm_destroy_queue(&p->pqm, queue_id); | ||
| 280 | err_create_queue: | ||
| 281 | err_bind_process: | ||
| 282 | mutex_unlock(&p->mutex); | ||
| 283 | return err; | ||
| 284 | } | ||
| 285 | |||
| 286 | static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, | ||
| 287 | void __user *arg) | ||
| 288 | { | ||
| 289 | int retval; | ||
| 290 | struct kfd_ioctl_destroy_queue_args args; | ||
| 291 | |||
| 292 | if (copy_from_user(&args, arg, sizeof(args))) | ||
| 293 | return -EFAULT; | ||
| 294 | |||
| 295 | pr_debug("kfd: destroying queue id %d for PASID %d\n", | ||
| 296 | args.queue_id, | ||
| 297 | p->pasid); | ||
| 298 | |||
| 299 | mutex_lock(&p->mutex); | ||
| 300 | |||
| 301 | retval = pqm_destroy_queue(&p->pqm, args.queue_id); | ||
| 302 | |||
| 303 | mutex_unlock(&p->mutex); | ||
| 304 | return retval; | ||
| 305 | } | ||
| 306 | |||
| 307 | static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, | ||
| 308 | void __user *arg) | ||
| 309 | { | ||
| 310 | int retval; | ||
| 311 | struct kfd_ioctl_update_queue_args args; | ||
| 312 | struct queue_properties properties; | ||
| 313 | |||
| 314 | if (copy_from_user(&args, arg, sizeof(args))) | ||
| 315 | return -EFAULT; | ||
| 316 | |||
| 317 | if (args.queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { | ||
| 318 | pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); | ||
| 319 | return -EINVAL; | ||
| 320 | } | ||
| 321 | |||
| 322 | if (args.queue_priority > KFD_MAX_QUEUE_PRIORITY) { | ||
| 323 | pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); | ||
| 324 | return -EINVAL; | ||
| 325 | } | ||
| 326 | |||
| 327 | if ((args.ring_base_address) && | ||
| 328 | (!access_ok(VERIFY_WRITE, args.ring_base_address, sizeof(uint64_t)))) { | ||
| 329 | pr_err("kfd: can't access ring base address\n"); | ||
| 330 | return -EFAULT; | ||
| 331 | } | ||
| 332 | |||
| 333 | if (!is_power_of_2(args.ring_size) && (args.ring_size != 0)) { | ||
| 334 | pr_err("kfd: ring size must be a power of 2 or 0\n"); | ||
| 335 | return -EINVAL; | ||
| 336 | } | ||
| 337 | |||
| 338 | properties.queue_address = args.ring_base_address; | ||
| 339 | properties.queue_size = args.ring_size; | ||
| 340 | properties.queue_percent = args.queue_percentage; | ||
| 341 | properties.priority = args.queue_priority; | ||
| 342 | |||
| 343 | pr_debug("kfd: updating queue id %d for PASID %d\n", | ||
| 344 | args.queue_id, p->pasid); | ||
| 345 | |||
| 346 | mutex_lock(&p->mutex); | ||
| 347 | |||
| 348 | retval = pqm_update_queue(&p->pqm, args.queue_id, &properties); | ||
| 349 | |||
| 350 | mutex_unlock(&p->mutex); | ||
| 351 | |||
| 352 | return retval; | ||
| 353 | } | ||
| 354 | |||
| 355 | static long kfd_ioctl_set_memory_policy(struct file *filep, | ||
| 356 | struct kfd_process *p, void __user *arg) | ||
| 357 | { | ||
| 358 | struct kfd_ioctl_set_memory_policy_args args; | ||
| 359 | struct kfd_dev *dev; | ||
| 360 | int err = 0; | ||
| 361 | struct kfd_process_device *pdd; | ||
| 362 | enum cache_policy default_policy, alternate_policy; | ||
| 363 | |||
| 364 | if (copy_from_user(&args, arg, sizeof(args))) | ||
| 365 | return -EFAULT; | ||
| 366 | |||
| 367 | if (args.default_policy != KFD_IOC_CACHE_POLICY_COHERENT | ||
| 368 | && args.default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { | ||
| 369 | return -EINVAL; | ||
| 370 | } | ||
| 371 | |||
| 372 | if (args.alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT | ||
| 373 | && args.alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { | ||
| 374 | return -EINVAL; | ||
| 375 | } | ||
| 376 | |||
| 377 | dev = kfd_device_by_id(args.gpu_id); | ||
| 378 | if (dev == NULL) | ||
| 379 | return -EINVAL; | ||
| 380 | |||
| 381 | mutex_lock(&p->mutex); | ||
| 382 | |||
| 383 | pdd = kfd_bind_process_to_device(dev, p); | ||
| 384 | if (IS_ERR(pdd) < 0) { | ||
| 385 | err = PTR_ERR(pdd); | ||
| 386 | goto out; | ||
| 387 | } | ||
| 388 | |||
| 389 | default_policy = (args.default_policy == KFD_IOC_CACHE_POLICY_COHERENT) | ||
| 390 | ? cache_policy_coherent : cache_policy_noncoherent; | ||
| 391 | |||
| 392 | alternate_policy = | ||
| 393 | (args.alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT) | ||
| 394 | ? cache_policy_coherent : cache_policy_noncoherent; | ||
| 395 | |||
| 396 | if (!dev->dqm->set_cache_memory_policy(dev->dqm, | ||
| 397 | &pdd->qpd, | ||
| 398 | default_policy, | ||
| 399 | alternate_policy, | ||
| 400 | (void __user *)args.alternate_aperture_base, | ||
| 401 | args.alternate_aperture_size)) | ||
| 402 | err = -EINVAL; | ||
| 403 | |||
| 404 | out: | ||
| 405 | mutex_unlock(&p->mutex); | ||
| 406 | |||
| 407 | return err; | ||
| 408 | } | ||
| 409 | |||
| 410 | static long kfd_ioctl_get_clock_counters(struct file *filep, | ||
| 411 | struct kfd_process *p, void __user *arg) | ||
| 412 | { | ||
| 413 | struct kfd_ioctl_get_clock_counters_args args; | ||
| 414 | struct kfd_dev *dev; | ||
| 415 | struct timespec time; | ||
| 416 | |||
| 417 | if (copy_from_user(&args, arg, sizeof(args))) | ||
| 418 | return -EFAULT; | ||
| 419 | |||
| 420 | dev = kfd_device_by_id(args.gpu_id); | ||
| 421 | if (dev == NULL) | ||
| 422 | return -EINVAL; | ||
| 423 | |||
| 424 | /* Reading GPU clock counter from KGD */ | ||
| 425 | args.gpu_clock_counter = kfd2kgd->get_gpu_clock_counter(dev->kgd); | ||
| 426 | |||
| 427 | /* No access to rdtsc. Using raw monotonic time */ | ||
| 428 | getrawmonotonic(&time); | ||
| 429 | args.cpu_clock_counter = (uint64_t)timespec_to_ns(&time); | ||
| 430 | |||
| 431 | get_monotonic_boottime(&time); | ||
| 432 | args.system_clock_counter = (uint64_t)timespec_to_ns(&time); | ||
| 433 | |||
| 434 | /* Since the counter is in nano-seconds we use 1GHz frequency */ | ||
| 435 | args.system_clock_freq = 1000000000; | ||
| 436 | |||
| 437 | if (copy_to_user(arg, &args, sizeof(args))) | ||
| 438 | return -EFAULT; | ||
| 439 | |||
| 440 | return 0; | ||
| 441 | } | ||
| 442 | |||
| 443 | |||
| 444 | static int kfd_ioctl_get_process_apertures(struct file *filp, | ||
| 445 | struct kfd_process *p, void __user *arg) | ||
| 446 | { | ||
| 447 | struct kfd_ioctl_get_process_apertures_args args; | ||
| 448 | struct kfd_process_device_apertures *pAperture; | ||
| 449 | struct kfd_process_device *pdd; | ||
| 450 | |||
| 451 | dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid); | ||
| 452 | |||
| 453 | if (copy_from_user(&args, arg, sizeof(args))) | ||
| 454 | return -EFAULT; | ||
| 455 | |||
| 456 | args.num_of_nodes = 0; | ||
| 457 | |||
| 458 | mutex_lock(&p->mutex); | ||
| 459 | |||
| 460 | /*if the process-device list isn't empty*/ | ||
| 461 | if (kfd_has_process_device_data(p)) { | ||
| 462 | /* Run over all pdd of the process */ | ||
| 463 | pdd = kfd_get_first_process_device_data(p); | ||
| 464 | do { | ||
| 465 | pAperture = &args.process_apertures[args.num_of_nodes]; | ||
| 466 | pAperture->gpu_id = pdd->dev->id; | ||
| 467 | pAperture->lds_base = pdd->lds_base; | ||
| 468 | pAperture->lds_limit = pdd->lds_limit; | ||
| 469 | pAperture->gpuvm_base = pdd->gpuvm_base; | ||
| 470 | pAperture->gpuvm_limit = pdd->gpuvm_limit; | ||
| 471 | pAperture->scratch_base = pdd->scratch_base; | ||
| 472 | pAperture->scratch_limit = pdd->scratch_limit; | ||
| 473 | |||
| 474 | dev_dbg(kfd_device, | ||
| 475 | "node id %u\n", args.num_of_nodes); | ||
| 476 | dev_dbg(kfd_device, | ||
| 477 | "gpu id %u\n", pdd->dev->id); | ||
| 478 | dev_dbg(kfd_device, | ||
| 479 | "lds_base %llX\n", pdd->lds_base); | ||
| 480 | dev_dbg(kfd_device, | ||
| 481 | "lds_limit %llX\n", pdd->lds_limit); | ||
| 482 | dev_dbg(kfd_device, | ||
| 483 | "gpuvm_base %llX\n", pdd->gpuvm_base); | ||
| 484 | dev_dbg(kfd_device, | ||
| 485 | "gpuvm_limit %llX\n", pdd->gpuvm_limit); | ||
| 486 | dev_dbg(kfd_device, | ||
| 487 | "scratch_base %llX\n", pdd->scratch_base); | ||
| 488 | dev_dbg(kfd_device, | ||
| 489 | "scratch_limit %llX\n", pdd->scratch_limit); | ||
| 490 | |||
| 491 | args.num_of_nodes++; | ||
| 492 | } while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL && | ||
| 493 | (args.num_of_nodes < NUM_OF_SUPPORTED_GPUS)); | ||
| 494 | } | ||
| 495 | |||
| 496 | mutex_unlock(&p->mutex); | ||
| 497 | |||
| 498 | if (copy_to_user(arg, &args, sizeof(args))) | ||
| 499 | return -EFAULT; | ||
| 500 | |||
| 501 | return 0; | ||
| 502 | } | ||
| 503 | |||
| 504 | static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) | ||
| 505 | { | ||
| 506 | struct kfd_process *process; | ||
| 507 | long err = -EINVAL; | ||
| 508 | |||
| 509 | dev_dbg(kfd_device, | ||
| 510 | "ioctl cmd 0x%x (#%d), arg 0x%lx\n", | ||
| 511 | cmd, _IOC_NR(cmd), arg); | ||
| 512 | |||
| 513 | process = kfd_get_process(current); | ||
| 514 | if (IS_ERR(process)) | ||
| 515 | return PTR_ERR(process); | ||
| 516 | |||
| 517 | switch (cmd) { | ||
| 518 | case KFD_IOC_GET_VERSION: | ||
| 519 | err = kfd_ioctl_get_version(filep, process, (void __user *)arg); | ||
| 520 | break; | ||
| 521 | case KFD_IOC_CREATE_QUEUE: | ||
| 522 | err = kfd_ioctl_create_queue(filep, process, | ||
| 523 | (void __user *)arg); | ||
| 524 | break; | ||
| 525 | |||
| 526 | case KFD_IOC_DESTROY_QUEUE: | ||
| 527 | err = kfd_ioctl_destroy_queue(filep, process, | ||
| 528 | (void __user *)arg); | ||
| 529 | break; | ||
| 530 | |||
| 531 | case KFD_IOC_SET_MEMORY_POLICY: | ||
| 532 | err = kfd_ioctl_set_memory_policy(filep, process, | ||
| 533 | (void __user *)arg); | ||
| 534 | break; | ||
| 535 | |||
| 536 | case KFD_IOC_GET_CLOCK_COUNTERS: | ||
| 537 | err = kfd_ioctl_get_clock_counters(filep, process, | ||
| 538 | (void __user *)arg); | ||
| 539 | break; | ||
| 540 | |||
| 541 | case KFD_IOC_GET_PROCESS_APERTURES: | ||
| 542 | err = kfd_ioctl_get_process_apertures(filep, process, | ||
| 543 | (void __user *)arg); | ||
| 544 | break; | ||
| 545 | |||
| 546 | case KFD_IOC_UPDATE_QUEUE: | ||
| 547 | err = kfd_ioctl_update_queue(filep, process, | ||
| 548 | (void __user *)arg); | ||
| 549 | break; | ||
| 550 | |||
| 551 | default: | ||
| 552 | dev_err(kfd_device, | ||
| 553 | "unknown ioctl cmd 0x%x, arg 0x%lx)\n", | ||
| 554 | cmd, arg); | ||
| 555 | err = -EINVAL; | ||
| 556 | break; | ||
| 557 | } | ||
| 558 | |||
| 559 | if (err < 0) | ||
| 560 | dev_err(kfd_device, | ||
| 561 | "ioctl error %ld for ioctl cmd 0x%x (#%d)\n", | ||
| 562 | err, cmd, _IOC_NR(cmd)); | ||
| 563 | |||
| 564 | return err; | ||
| 565 | } | ||
| 566 | |||
| 567 | static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) | ||
| 568 | { | ||
| 569 | struct kfd_process *process; | ||
| 570 | |||
| 571 | process = kfd_get_process(current); | ||
| 572 | if (IS_ERR(process)) | ||
| 573 | return PTR_ERR(process); | ||
| 574 | |||
| 575 | return kfd_doorbell_mmap(process, vma); | ||
| 576 | } | ||
| @@ -0,0 +1,294 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | #ifndef KFD_CRAT_H_INCLUDED | ||
| 24 | #define KFD_CRAT_H_INCLUDED | ||
| 25 | |||
| 26 | #include <linux/types.h> | ||
| 27 | |||
| 28 | #pragma pack(1) | ||
| 29 | |||
| 30 | /* | ||
| 31 | * 4CC signature values for the CRAT and CDIT ACPI tables | ||
| 32 | */ | ||
| 33 | |||
| 34 | #define CRAT_SIGNATURE "CRAT" | ||
| 35 | #define CDIT_SIGNATURE "CDIT" | ||
| 36 | |||
| 37 | /* | ||
| 38 | * Component Resource Association Table (CRAT) | ||
| 39 | */ | ||
| 40 | |||
| 41 | #define CRAT_OEMID_LENGTH 6 | ||
| 42 | #define CRAT_OEMTABLEID_LENGTH 8 | ||
| 43 | #define CRAT_RESERVED_LENGTH 6 | ||
| 44 | |||
| 45 | #define CRAT_OEMID_64BIT_MASK ((1ULL << (CRAT_OEMID_LENGTH * 8)) - 1) | ||
| 46 | |||
| 47 | struct crat_header { | ||
| 48 | uint32_t signature; | ||
| 49 | uint32_t length; | ||
| 50 | uint8_t revision; | ||
| 51 | uint8_t checksum; | ||
| 52 | uint8_t oem_id[CRAT_OEMID_LENGTH]; | ||
| 53 | uint8_t oem_table_id[CRAT_OEMTABLEID_LENGTH]; | ||
| 54 | uint32_t oem_revision; | ||
| 55 | uint32_t creator_id; | ||
| 56 | uint32_t creator_revision; | ||
| 57 | uint32_t total_entries; | ||
| 58 | uint16_t num_domains; | ||
| 59 | uint8_t reserved[CRAT_RESERVED_LENGTH]; | ||
| 60 | }; | ||
| 61 | |||
| 62 | /* | ||
| 63 | * The header structure is immediately followed by total_entries of the | ||
| 64 | * data definitions | ||
| 65 | */ | ||
| 66 | |||
| 67 | /* | ||
| 68 | * The currently defined subtype entries in the CRAT | ||
| 69 | */ | ||
| 70 | #define CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY 0 | ||
| 71 | #define CRAT_SUBTYPE_MEMORY_AFFINITY 1 | ||
| 72 | #define CRAT_SUBTYPE_CACHE_AFFINITY 2 | ||
| 73 | #define CRAT_SUBTYPE_TLB_AFFINITY 3 | ||
| 74 | #define CRAT_SUBTYPE_CCOMPUTE_AFFINITY 4 | ||
| 75 | #define CRAT_SUBTYPE_IOLINK_AFFINITY 5 | ||
| 76 | #define CRAT_SUBTYPE_MAX 6 | ||
| 77 | |||
| 78 | #define CRAT_SIBLINGMAP_SIZE 32 | ||
| 79 | |||
| 80 | /* | ||
| 81 | * ComputeUnit Affinity structure and definitions | ||
| 82 | */ | ||
| 83 | #define CRAT_CU_FLAGS_ENABLED 0x00000001 | ||
| 84 | #define CRAT_CU_FLAGS_HOT_PLUGGABLE 0x00000002 | ||
| 85 | #define CRAT_CU_FLAGS_CPU_PRESENT 0x00000004 | ||
| 86 | #define CRAT_CU_FLAGS_GPU_PRESENT 0x00000008 | ||
| 87 | #define CRAT_CU_FLAGS_IOMMU_PRESENT 0x00000010 | ||
| 88 | #define CRAT_CU_FLAGS_RESERVED 0xffffffe0 | ||
| 89 | |||
| 90 | #define CRAT_COMPUTEUNIT_RESERVED_LENGTH 4 | ||
| 91 | |||
| 92 | struct crat_subtype_computeunit { | ||
| 93 | uint8_t type; | ||
| 94 | uint8_t length; | ||
| 95 | uint16_t reserved; | ||
| 96 | uint32_t flags; | ||
| 97 | uint32_t proximity_domain; | ||
| 98 | uint32_t processor_id_low; | ||
| 99 | uint16_t num_cpu_cores; | ||
| 100 | uint16_t num_simd_cores; | ||
| 101 | uint16_t max_waves_simd; | ||
| 102 | uint16_t io_count; | ||
| 103 | uint16_t hsa_capability; | ||
| 104 | uint16_t lds_size_in_kb; | ||
| 105 | uint8_t wave_front_size; | ||
| 106 | uint8_t num_banks; | ||
| 107 | uint16_t micro_engine_id; | ||
| 108 | uint8_t num_arrays; | ||
| 109 | uint8_t num_cu_per_array; | ||
| 110 | uint8_t num_simd_per_cu; | ||
| 111 | uint8_t max_slots_scatch_cu; | ||
| 112 | uint8_t reserved2[CRAT_COMPUTEUNIT_RESERVED_LENGTH]; | ||
| 113 | }; | ||
| 114 | |||
| 115 | /* | ||
| 116 | * HSA Memory Affinity structure and definitions | ||
| 117 | */ | ||
| 118 | #define CRAT_MEM_FLAGS_ENABLED 0x00000001 | ||
| 119 | #define CRAT_MEM_FLAGS_HOT_PLUGGABLE 0x00000002 | ||
| 120 | #define CRAT_MEM_FLAGS_NON_VOLATILE 0x00000004 | ||
| 121 | #define CRAT_MEM_FLAGS_RESERVED 0xfffffff8 | ||
| 122 | |||
| 123 | #define CRAT_MEMORY_RESERVED_LENGTH 8 | ||
| 124 | |||
| 125 | struct crat_subtype_memory { | ||
| 126 | uint8_t type; | ||
| 127 | uint8_t length; | ||
| 128 | uint16_t reserved; | ||
| 129 | uint32_t flags; | ||
| 130 | uint32_t promixity_domain; | ||
| 131 | uint32_t base_addr_low; | ||
| 132 | uint32_t base_addr_high; | ||
| 133 | uint32_t length_low; | ||
| 134 | uint32_t length_high; | ||
| 135 | uint32_t width; | ||
| 136 | uint8_t reserved2[CRAT_MEMORY_RESERVED_LENGTH]; | ||
| 137 | }; | ||
| 138 | |||
| 139 | /* | ||
| 140 | * HSA Cache Affinity structure and definitions | ||
| 141 | */ | ||
| 142 | #define CRAT_CACHE_FLAGS_ENABLED 0x00000001 | ||
| 143 | #define CRAT_CACHE_FLAGS_DATA_CACHE 0x00000002 | ||
| 144 | #define CRAT_CACHE_FLAGS_INST_CACHE 0x00000004 | ||
| 145 | #define CRAT_CACHE_FLAGS_CPU_CACHE 0x00000008 | ||
| 146 | #define CRAT_CACHE_FLAGS_SIMD_CACHE 0x00000010 | ||
| 147 | #define CRAT_CACHE_FLAGS_RESERVED 0xffffffe0 | ||
| 148 | |||
| 149 | #define CRAT_CACHE_RESERVED_LENGTH 8 | ||
| 150 | |||
| 151 | struct crat_subtype_cache { | ||
| 152 | uint8_t type; | ||
| 153 | uint8_t length; | ||
| 154 | uint16_t reserved; | ||
| 155 | uint32_t flags; | ||
| 156 | uint32_t processor_id_low; | ||
| 157 | uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE]; | ||
| 158 | uint32_t cache_size; | ||
| 159 | uint8_t cache_level; | ||
| 160 | uint8_t lines_per_tag; | ||
| 161 | uint16_t cache_line_size; | ||
| 162 | uint8_t associativity; | ||
| 163 | uint8_t cache_properties; | ||
| 164 | uint16_t cache_latency; | ||
| 165 | uint8_t reserved2[CRAT_CACHE_RESERVED_LENGTH]; | ||
| 166 | }; | ||
| 167 | |||
| 168 | /* | ||
| 169 | * HSA TLB Affinity structure and definitions | ||
| 170 | */ | ||
| 171 | #define CRAT_TLB_FLAGS_ENABLED 0x00000001 | ||
| 172 | #define CRAT_TLB_FLAGS_DATA_TLB 0x00000002 | ||
| 173 | #define CRAT_TLB_FLAGS_INST_TLB 0x00000004 | ||
| 174 | #define CRAT_TLB_FLAGS_CPU_TLB 0x00000008 | ||
| 175 | #define CRAT_TLB_FLAGS_SIMD_TLB 0x00000010 | ||
| 176 | #define CRAT_TLB_FLAGS_RESERVED 0xffffffe0 | ||
| 177 | |||
| 178 | #define CRAT_TLB_RESERVED_LENGTH 4 | ||
| 179 | |||
| 180 | struct crat_subtype_tlb { | ||
| 181 | uint8_t type; | ||
| 182 | uint8_t length; | ||
| 183 | uint16_t reserved; | ||
| 184 | uint32_t flags; | ||
| 185 | uint32_t processor_id_low; | ||
| 186 | uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE]; | ||
| 187 | uint32_t tlb_level; | ||
| 188 | uint8_t data_tlb_associativity_2mb; | ||
| 189 | uint8_t data_tlb_size_2mb; | ||
| 190 | uint8_t instruction_tlb_associativity_2mb; | ||
| 191 | uint8_t instruction_tlb_size_2mb; | ||
| 192 | uint8_t data_tlb_associativity_4k; | ||
| 193 | uint8_t data_tlb_size_4k; | ||
| 194 | uint8_t instruction_tlb_associativity_4k; | ||
| 195 | uint8_t instruction_tlb_size_4k; | ||
| 196 | uint8_t data_tlb_associativity_1gb; | ||
| 197 | uint8_t data_tlb_size_1gb; | ||
| 198 | uint8_t instruction_tlb_associativity_1gb; | ||
| 199 | uint8_t instruction_tlb_size_1gb; | ||
| 200 | uint8_t reserved2[CRAT_TLB_RESERVED_LENGTH]; | ||
| 201 | }; | ||
| 202 | |||
| 203 | /* | ||
| 204 | * HSA CCompute/APU Affinity structure and definitions | ||
| 205 | */ | ||
| 206 | #define CRAT_CCOMPUTE_FLAGS_ENABLED 0x00000001 | ||
| 207 | #define CRAT_CCOMPUTE_FLAGS_RESERVED 0xfffffffe | ||
| 208 | |||
| 209 | #define CRAT_CCOMPUTE_RESERVED_LENGTH 16 | ||
| 210 | |||
| 211 | struct crat_subtype_ccompute { | ||
| 212 | uint8_t type; | ||
| 213 | uint8_t length; | ||
| 214 | uint16_t reserved; | ||
| 215 | uint32_t flags; | ||
| 216 | uint32_t processor_id_low; | ||
| 217 | uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE]; | ||
| 218 | uint32_t apu_size; | ||
| 219 | uint8_t reserved2[CRAT_CCOMPUTE_RESERVED_LENGTH]; | ||
| 220 | }; | ||
| 221 | |||
| 222 | /* | ||
| 223 | * HSA IO Link Affinity structure and definitions | ||
| 224 | */ | ||
| 225 | #define CRAT_IOLINK_FLAGS_ENABLED 0x00000001 | ||
| 226 | #define CRAT_IOLINK_FLAGS_COHERENCY 0x00000002 | ||
| 227 | #define CRAT_IOLINK_FLAGS_RESERVED 0xfffffffc | ||
| 228 | |||
| 229 | /* | ||
| 230 | * IO interface types | ||
| 231 | */ | ||
| 232 | #define CRAT_IOLINK_TYPE_UNDEFINED 0 | ||
| 233 | #define CRAT_IOLINK_TYPE_HYPERTRANSPORT 1 | ||
| 234 | #define CRAT_IOLINK_TYPE_PCIEXPRESS 2 | ||
| 235 | #define CRAT_IOLINK_TYPE_OTHER 3 | ||
| 236 | #define CRAT_IOLINK_TYPE_MAX 255 | ||
| 237 | |||
| 238 | #define CRAT_IOLINK_RESERVED_LENGTH 24 | ||
| 239 | |||
| 240 | struct crat_subtype_iolink { | ||
| 241 | uint8_t type; | ||
| 242 | uint8_t length; | ||
| 243 | uint16_t reserved; | ||
| 244 | uint32_t flags; | ||
| 245 | uint32_t proximity_domain_from; | ||
| 246 | uint32_t proximity_domain_to; | ||
| 247 | uint8_t io_interface_type; | ||
| 248 | uint8_t version_major; | ||
| 249 | uint16_t version_minor; | ||
| 250 | uint32_t minimum_latency; | ||
| 251 | uint32_t maximum_latency; | ||
| 252 | uint32_t minimum_bandwidth_mbs; | ||
| 253 | uint32_t maximum_bandwidth_mbs; | ||
| 254 | uint32_t recommended_transfer_size; | ||
| 255 | uint8_t reserved2[CRAT_IOLINK_RESERVED_LENGTH]; | ||
| 256 | }; | ||
| 257 | |||
| 258 | /* | ||
| 259 | * HSA generic sub-type header | ||
| 260 | */ | ||
| 261 | |||
| 262 | #define CRAT_SUBTYPE_FLAGS_ENABLED 0x00000001 | ||
| 263 | |||
| 264 | struct crat_subtype_generic { | ||
| 265 | uint8_t type; | ||
| 266 | uint8_t length; | ||
| 267 | uint16_t reserved; | ||
| 268 | uint32_t flags; | ||
| 269 | }; | ||
| 270 | |||
| 271 | /* | ||
| 272 | * Component Locality Distance Information Table (CDIT) | ||
| 273 | */ | ||
| 274 | #define CDIT_OEMID_LENGTH 6 | ||
| 275 | #define CDIT_OEMTABLEID_LENGTH 8 | ||
| 276 | |||
| 277 | struct cdit_header { | ||
| 278 | uint32_t signature; | ||
| 279 | uint32_t length; | ||
| 280 | uint8_t revision; | ||
| 281 | uint8_t checksum; | ||
| 282 | uint8_t oem_id[CDIT_OEMID_LENGTH]; | ||
| 283 | uint8_t oem_table_id[CDIT_OEMTABLEID_LENGTH]; | ||
| 284 | uint32_t oem_revision; | ||
| 285 | uint32_t creator_id; | ||
| 286 | uint32_t creator_revision; | ||
| 287 | uint32_t total_entries; | ||
| 288 | uint16_t num_domains; | ||
| 289 | uint8_t entry[1]; | ||
| 290 | }; | ||
| 291 | |||
| 292 | #pragma pack() | ||
| 293 | |||
| 294 | #endif /* KFD_CRAT_H_INCLUDED */ | ||
| @@ -0,0 +1,307 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | #include <linux/amd-iommu.h> | ||
| 24 | #include <linux/bsearch.h> | ||
| 25 | #include <linux/pci.h> | ||
| 26 | #include <linux/slab.h> | ||
| 27 | #include "kfd_priv.h" | ||
| 28 | #include "kfd_device_queue_manager.h" | ||
| 29 | |||
| 30 | #define MQD_SIZE_ALIGNED 768 | ||
| 31 | |||
| 32 | static const struct kfd_device_info kaveri_device_info = { | ||
| 33 | .max_pasid_bits = 16, | ||
| 34 | .ih_ring_entry_size = 4 * sizeof(uint32_t), | ||
| 35 | .mqd_size_aligned = MQD_SIZE_ALIGNED | ||
| 36 | }; | ||
| 37 | |||
| 38 | struct kfd_deviceid { | ||
| 39 | unsigned short did; | ||
| 40 | const struct kfd_device_info *device_info; | ||
| 41 | }; | ||
| 42 | |||
| 43 | /* Please keep this sorted by increasing device id. */ | ||
| 44 | static const struct kfd_deviceid supported_devices[] = { | ||
| 45 | { 0x1304, &kaveri_device_info }, /* Kaveri */ | ||
| 46 | { 0x1305, &kaveri_device_info }, /* Kaveri */ | ||
| 47 | { 0x1306, &kaveri_device_info }, /* Kaveri */ | ||
| 48 | { 0x1307, &kaveri_device_info }, /* Kaveri */ | ||
| 49 | { 0x1309, &kaveri_device_info }, /* Kaveri */ | ||
| 50 | { 0x130A, &kaveri_device_info }, /* Kaveri */ | ||
| 51 | { 0x130B, &kaveri_device_info }, /* Kaveri */ | ||
| 52 | { 0x130C, &kaveri_device_info }, /* Kaveri */ | ||
| 53 | { 0x130D, &kaveri_device_info }, /* Kaveri */ | ||
| 54 | { 0x130E, &kaveri_device_info }, /* Kaveri */ | ||
| 55 | { 0x130F, &kaveri_device_info }, /* Kaveri */ | ||
| 56 | { 0x1310, &kaveri_device_info }, /* Kaveri */ | ||
| 57 | { 0x1311, &kaveri_device_info }, /* Kaveri */ | ||
| 58 | { 0x1312, &kaveri_device_info }, /* Kaveri */ | ||
| 59 | { 0x1313, &kaveri_device_info }, /* Kaveri */ | ||
| 60 | { 0x1315, &kaveri_device_info }, /* Kaveri */ | ||
| 61 | { 0x1316, &kaveri_device_info }, /* Kaveri */ | ||
| 62 | { 0x1317, &kaveri_device_info }, /* Kaveri */ | ||
| 63 | { 0x1318, &kaveri_device_info }, /* Kaveri */ | ||
| 64 | { 0x131B, &kaveri_device_info }, /* Kaveri */ | ||
| 65 | { 0x131C, &kaveri_device_info }, /* Kaveri */ | ||
| 66 | { 0x131D, &kaveri_device_info }, /* Kaveri */ | ||
| 67 | }; | ||
| 68 | |||
| 69 | static const struct kfd_device_info *lookup_device_info(unsigned short did) | ||
| 70 | { | ||
| 71 | size_t i; | ||
| 72 | |||
| 73 | for (i = 0; i < ARRAY_SIZE(supported_devices); i++) { | ||
| 74 | if (supported_devices[i].did == did) { | ||
| 75 | BUG_ON(supported_devices[i].device_info == NULL); | ||
| 76 | return supported_devices[i].device_info; | ||
| 77 | } | ||
| 78 | } | ||
| 79 | |||
| 80 | return NULL; | ||
| 81 | } | ||
| 82 | |||
| 83 | struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev) | ||
| 84 | { | ||
| 85 | struct kfd_dev *kfd; | ||
| 86 | |||
| 87 | const struct kfd_device_info *device_info = | ||
| 88 | lookup_device_info(pdev->device); | ||
| 89 | |||
| 90 | if (!device_info) | ||
| 91 | return NULL; | ||
| 92 | |||
| 93 | kfd = kzalloc(sizeof(*kfd), GFP_KERNEL); | ||
| 94 | if (!kfd) | ||
| 95 | return NULL; | ||
| 96 | |||
| 97 | kfd->kgd = kgd; | ||
| 98 | kfd->device_info = device_info; | ||
| 99 | kfd->pdev = pdev; | ||
| 100 | kfd->init_complete = false; | ||
| 101 | |||
| 102 | return kfd; | ||
| 103 | } | ||
| 104 | |||
| 105 | static bool device_iommu_pasid_init(struct kfd_dev *kfd) | ||
| 106 | { | ||
| 107 | const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP | | ||
| 108 | AMD_IOMMU_DEVICE_FLAG_PRI_SUP | | ||
| 109 | AMD_IOMMU_DEVICE_FLAG_PASID_SUP; | ||
| 110 | |||
| 111 | struct amd_iommu_device_info iommu_info; | ||
| 112 | unsigned int pasid_limit; | ||
| 113 | int err; | ||
| 114 | |||
| 115 | err = amd_iommu_device_info(kfd->pdev, &iommu_info); | ||
| 116 | if (err < 0) { | ||
| 117 | dev_err(kfd_device, | ||
| 118 | "error getting iommu info. is the iommu enabled?\n"); | ||
| 119 | return false; | ||
| 120 | } | ||
| 121 | |||
| 122 | if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) { | ||
| 123 | dev_err(kfd_device, "error required iommu flags ats(%i), pri(%i), pasid(%i)\n", | ||
| 124 | (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0, | ||
| 125 | (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0, | ||
| 126 | (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) != 0); | ||
| 127 | return false; | ||
| 128 | } | ||
| 129 | |||
| 130 | pasid_limit = min_t(unsigned int, | ||
| 131 | (unsigned int)1 << kfd->device_info->max_pasid_bits, | ||
| 132 | iommu_info.max_pasids); | ||
| 133 | /* | ||
| 134 | * last pasid is used for kernel queues doorbells | ||
| 135 | * in the future the last pasid might be used for a kernel thread. | ||
| 136 | */ | ||
| 137 | pasid_limit = min_t(unsigned int, | ||
| 138 | pasid_limit, | ||
| 139 | kfd->doorbell_process_limit - 1); | ||
| 140 | |||
| 141 | err = amd_iommu_init_device(kfd->pdev, pasid_limit); | ||
| 142 | if (err < 0) { | ||
| 143 | dev_err(kfd_device, "error initializing iommu device\n"); | ||
| 144 | return false; | ||
| 145 | } | ||
| 146 | |||
| 147 | if (!kfd_set_pasid_limit(pasid_limit)) { | ||
| 148 | dev_err(kfd_device, "error setting pasid limit\n"); | ||
| 149 | amd_iommu_free_device(kfd->pdev); | ||
| 150 | return false; | ||
| 151 | } | ||
| 152 | |||
| 153 | return true; | ||
| 154 | } | ||
| 155 | |||
| 156 | static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int pasid) | ||
| 157 | { | ||
| 158 | struct kfd_dev *dev = kfd_device_by_pci_dev(pdev); | ||
| 159 | |||
| 160 | if (dev) | ||
| 161 | kfd_unbind_process_from_device(dev, pasid); | ||
| 162 | } | ||
| 163 | |||
| 164 | bool kgd2kfd_device_init(struct kfd_dev *kfd, | ||
| 165 | const struct kgd2kfd_shared_resources *gpu_resources) | ||
| 166 | { | ||
| 167 | unsigned int size; | ||
| 168 | |||
| 169 | kfd->shared_resources = *gpu_resources; | ||
| 170 | |||
| 171 | /* calculate max size of mqds needed for queues */ | ||
| 172 | size = max_num_of_processes * | ||
| 173 | max_num_of_queues_per_process * | ||
| 174 | kfd->device_info->mqd_size_aligned; | ||
| 175 | |||
| 176 | /* add another 512KB for all other allocations on gart */ | ||
| 177 | size += 512 * 1024; | ||
| 178 | |||
| 179 | if (kfd2kgd->init_sa_manager(kfd->kgd, size)) { | ||
| 180 | dev_err(kfd_device, | ||
| 181 | "Error initializing sa manager for device (%x:%x)\n", | ||
| 182 | kfd->pdev->vendor, kfd->pdev->device); | ||
| 183 | goto out; | ||
| 184 | } | ||
| 185 | |||
| 186 | kfd_doorbell_init(kfd); | ||
| 187 | |||
| 188 | if (kfd_topology_add_device(kfd) != 0) { | ||
| 189 | dev_err(kfd_device, | ||
| 190 | "Error adding device (%x:%x) to topology\n", | ||
| 191 | kfd->pdev->vendor, kfd->pdev->device); | ||
| 192 | goto kfd_topology_add_device_error; | ||
| 193 | } | ||
| 194 | |||
| 195 | if (kfd_interrupt_init(kfd)) { | ||
| 196 | dev_err(kfd_device, | ||
| 197 | "Error initializing interrupts for device (%x:%x)\n", | ||
| 198 | kfd->pdev->vendor, kfd->pdev->device); | ||
| 199 | goto kfd_interrupt_error; | ||
| 200 | } | ||
| 201 | |||
| 202 | if (!device_iommu_pasid_init(kfd)) { | ||
| 203 | dev_err(kfd_device, | ||
| 204 | "Error initializing iommuv2 for device (%x:%x)\n", | ||
| 205 | kfd->pdev->vendor, kfd->pdev->device); | ||
| 206 | goto device_iommu_pasid_error; | ||
| 207 | } | ||
| 208 | amd_iommu_set_invalidate_ctx_cb(kfd->pdev, | ||
| 209 | iommu_pasid_shutdown_callback); | ||
| 210 | |||
| 211 | kfd->dqm = device_queue_manager_init(kfd); | ||
| 212 | if (!kfd->dqm) { | ||
| 213 | dev_err(kfd_device, | ||
| 214 | "Error initializing queue manager for device (%x:%x)\n", | ||
| 215 | kfd->pdev->vendor, kfd->pdev->device); | ||
| 216 | goto device_queue_manager_error; | ||
| 217 | } | ||
| 218 | |||
| 219 | if (kfd->dqm->start(kfd->dqm) != 0) { | ||
| 220 | dev_err(kfd_device, | ||
| 221 | "Error starting queuen manager for device (%x:%x)\n", | ||
| 222 | kfd->pdev->vendor, kfd->pdev->device); | ||
| 223 | goto dqm_start_error; | ||
| 224 | } | ||
| 225 | |||
| 226 | kfd->init_complete = true; | ||
| 227 | dev_info(kfd_device, "added device (%x:%x)\n", kfd->pdev->vendor, | ||
| 228 | kfd->pdev->device); | ||
| 229 | |||
| 230 | pr_debug("kfd: Starting kfd with the following scheduling policy %d\n", | ||
| 231 | sched_policy); | ||
| 232 | |||
| 233 | goto out; | ||
| 234 | |||
| 235 | dqm_start_error: | ||
| 236 | device_queue_manager_uninit(kfd->dqm); | ||
| 237 | device_queue_manager_error: | ||
| 238 | amd_iommu_free_device(kfd->pdev); | ||
| 239 | device_iommu_pasid_error: | ||
| 240 | kfd_interrupt_exit(kfd); | ||
| 241 | kfd_interrupt_error: | ||
| 242 | kfd_topology_remove_device(kfd); | ||
| 243 | kfd_topology_add_device_error: | ||
| 244 | kfd2kgd->fini_sa_manager(kfd->kgd); | ||
| 245 | dev_err(kfd_device, | ||
| 246 | "device (%x:%x) NOT added due to errors\n", | ||
| 247 | kfd->pdev->vendor, kfd->pdev->device); | ||
| 248 | out: | ||
| 249 | return kfd->init_complete; | ||
| 250 | } | ||
| 251 | |||
| 252 | void kgd2kfd_device_exit(struct kfd_dev *kfd) | ||
| 253 | { | ||
| 254 | if (kfd->init_complete) { | ||
| 255 | device_queue_manager_uninit(kfd->dqm); | ||
| 256 | amd_iommu_free_device(kfd->pdev); | ||
| 257 | kfd_interrupt_exit(kfd); | ||
| 258 | kfd_topology_remove_device(kfd); | ||
| 259 | } | ||
| 260 | |||
| 261 | kfree(kfd); | ||
| 262 | } | ||
| 263 | |||
| 264 | void kgd2kfd_suspend(struct kfd_dev *kfd) | ||
| 265 | { | ||
| 266 | BUG_ON(kfd == NULL); | ||
| 267 | |||
| 268 | if (kfd->init_complete) { | ||
| 269 | kfd->dqm->stop(kfd->dqm); | ||
| 270 | amd_iommu_free_device(kfd->pdev); | ||
| 271 | } | ||
| 272 | } | ||
| 273 | |||
| 274 | int kgd2kfd_resume(struct kfd_dev *kfd) | ||
| 275 | { | ||
| 276 | unsigned int pasid_limit; | ||
| 277 | int err; | ||
| 278 | |||
| 279 | BUG_ON(kfd == NULL); | ||
| 280 | |||
| 281 | pasid_limit = kfd_get_pasid_limit(); | ||
| 282 | |||
| 283 | if (kfd->init_complete) { | ||
| 284 | err = amd_iommu_init_device(kfd->pdev, pasid_limit); | ||
| 285 | if (err < 0) | ||
| 286 | return -ENXIO; | ||
| 287 | amd_iommu_set_invalidate_ctx_cb(kfd->pdev, | ||
| 288 | iommu_pasid_shutdown_callback); | ||
| 289 | kfd->dqm->start(kfd->dqm); | ||
| 290 | } | ||
| 291 | |||
| 292 | return 0; | ||
| 293 | } | ||
| 294 | |||
| 295 | /* This is called directly from KGD at ISR. */ | ||
| 296 | void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) | ||
| 297 | { | ||
| 298 | if (kfd->init_complete) { | ||
| 299 | spin_lock(&kfd->interrupt_lock); | ||
| 300 | |||
| 301 | if (kfd->interrupts_active | ||
| 302 | && enqueue_ih_ring_entry(kfd, ih_ring_entry)) | ||
| 303 | schedule_work(&kfd->interrupt_work); | ||
| 304 | |||
| 305 | spin_unlock(&kfd->interrupt_lock); | ||
| 306 | } | ||
| 307 | } | ||
| @@ -0,0 +1,1059 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | #include <linux/slab.h> | ||
| 25 | #include <linux/list.h> | ||
| 26 | #include <linux/types.h> | ||
| 27 | #include <linux/printk.h> | ||
| 28 | #include <linux/bitops.h> | ||
| 29 | #include "kfd_priv.h" | ||
| 30 | #include "kfd_device_queue_manager.h" | ||
| 31 | #include "kfd_mqd_manager.h" | ||
| 32 | #include "cik_regs.h" | ||
| 33 | #include "kfd_kernel_queue.h" | ||
| 34 | #include "../../radeon/cik_reg.h" | ||
| 35 | |||
| 36 | /* Size of the per-pipe EOP queue */ | ||
| 37 | #define CIK_HPD_EOP_BYTES_LOG2 11 | ||
| 38 | #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) | ||
| 39 | |||
| 40 | static bool is_mem_initialized; | ||
| 41 | |||
| 42 | static int init_memory(struct device_queue_manager *dqm); | ||
| 43 | static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, | ||
| 44 | unsigned int pasid, unsigned int vmid); | ||
| 45 | |||
| 46 | static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, | ||
| 47 | struct queue *q, | ||
| 48 | struct qcm_process_device *qpd); | ||
| 49 | static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock); | ||
| 50 | static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock); | ||
| 51 | |||
| 52 | |||
| 53 | static inline unsigned int get_pipes_num(struct device_queue_manager *dqm) | ||
| 54 | { | ||
| 55 | BUG_ON(!dqm || !dqm->dev); | ||
| 56 | return dqm->dev->shared_resources.compute_pipe_count; | ||
| 57 | } | ||
| 58 | |||
| 59 | static inline unsigned int get_first_pipe(struct device_queue_manager *dqm) | ||
| 60 | { | ||
| 61 | BUG_ON(!dqm); | ||
| 62 | return dqm->dev->shared_resources.first_compute_pipe; | ||
| 63 | } | ||
| 64 | |||
| 65 | static inline unsigned int get_pipes_num_cpsch(void) | ||
| 66 | { | ||
| 67 | return PIPE_PER_ME_CP_SCHEDULING; | ||
| 68 | } | ||
| 69 | |||
| 70 | static unsigned int get_sh_mem_bases_nybble_64(struct kfd_process *process, | ||
| 71 | struct kfd_dev *dev) | ||
| 72 | { | ||
| 73 | struct kfd_process_device *pdd; | ||
| 74 | uint32_t nybble; | ||
| 75 | |||
| 76 | pdd = kfd_get_process_device_data(dev, process, 1); | ||
| 77 | nybble = (pdd->lds_base >> 60) & 0x0E; | ||
| 78 | |||
| 79 | return nybble; | ||
| 80 | |||
| 81 | } | ||
| 82 | |||
| 83 | static unsigned int get_sh_mem_bases_32(struct kfd_process *process, | ||
| 84 | struct kfd_dev *dev) | ||
| 85 | { | ||
| 86 | struct kfd_process_device *pdd; | ||
| 87 | unsigned int shared_base; | ||
| 88 | |||
| 89 | pdd = kfd_get_process_device_data(dev, process, 1); | ||
| 90 | shared_base = (pdd->lds_base >> 16) & 0xFF; | ||
| 91 | |||
| 92 | return shared_base; | ||
| 93 | } | ||
| 94 | |||
| 95 | static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble); | ||
| 96 | static void init_process_memory(struct device_queue_manager *dqm, | ||
| 97 | struct qcm_process_device *qpd) | ||
| 98 | { | ||
| 99 | unsigned int temp; | ||
| 100 | |||
| 101 | BUG_ON(!dqm || !qpd); | ||
| 102 | |||
| 103 | /* check if sh_mem_config register already configured */ | ||
| 104 | if (qpd->sh_mem_config == 0) { | ||
| 105 | qpd->sh_mem_config = | ||
| 106 | ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) | | ||
| 107 | DEFAULT_MTYPE(MTYPE_NONCACHED) | | ||
| 108 | APE1_MTYPE(MTYPE_NONCACHED); | ||
| 109 | qpd->sh_mem_ape1_limit = 0; | ||
| 110 | qpd->sh_mem_ape1_base = 0; | ||
| 111 | } | ||
| 112 | |||
| 113 | if (qpd->pqm->process->is_32bit_user_mode) { | ||
| 114 | temp = get_sh_mem_bases_32(qpd->pqm->process, dqm->dev); | ||
| 115 | qpd->sh_mem_bases = SHARED_BASE(temp); | ||
| 116 | qpd->sh_mem_config |= PTR32; | ||
| 117 | } else { | ||
| 118 | temp = get_sh_mem_bases_nybble_64(qpd->pqm->process, dqm->dev); | ||
| 119 | qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp); | ||
| 120 | } | ||
| 121 | |||
| 122 | pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n", | ||
| 123 | qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases); | ||
| 124 | } | ||
| 125 | |||
| 126 | static void program_sh_mem_settings(struct device_queue_manager *dqm, | ||
| 127 | struct qcm_process_device *qpd) | ||
| 128 | { | ||
| 129 | return kfd2kgd->program_sh_mem_settings(dqm->dev->kgd, qpd->vmid, | ||
| 130 | qpd->sh_mem_config, | ||
| 131 | qpd->sh_mem_ape1_base, | ||
| 132 | qpd->sh_mem_ape1_limit, | ||
| 133 | qpd->sh_mem_bases); | ||
| 134 | } | ||
| 135 | |||
| 136 | static int allocate_vmid(struct device_queue_manager *dqm, | ||
| 137 | struct qcm_process_device *qpd, | ||
| 138 | struct queue *q) | ||
| 139 | { | ||
| 140 | int bit, allocated_vmid; | ||
| 141 | |||
| 142 | if (dqm->vmid_bitmap == 0) | ||
| 143 | return -ENOMEM; | ||
| 144 | |||
| 145 | bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM); | ||
| 146 | clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap); | ||
| 147 | |||
| 148 | /* Kaveri kfd vmid's starts from vmid 8 */ | ||
| 149 | allocated_vmid = bit + KFD_VMID_START_OFFSET; | ||
| 150 | pr_debug("kfd: vmid allocation %d\n", allocated_vmid); | ||
| 151 | qpd->vmid = allocated_vmid; | ||
| 152 | q->properties.vmid = allocated_vmid; | ||
| 153 | |||
| 154 | set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid); | ||
| 155 | program_sh_mem_settings(dqm, qpd); | ||
| 156 | |||
| 157 | return 0; | ||
| 158 | } | ||
| 159 | |||
| 160 | static void deallocate_vmid(struct device_queue_manager *dqm, | ||
| 161 | struct qcm_process_device *qpd, | ||
| 162 | struct queue *q) | ||
| 163 | { | ||
| 164 | int bit = qpd->vmid - KFD_VMID_START_OFFSET; | ||
| 165 | |||
| 166 | set_bit(bit, (unsigned long *)&dqm->vmid_bitmap); | ||
| 167 | qpd->vmid = 0; | ||
| 168 | q->properties.vmid = 0; | ||
| 169 | } | ||
| 170 | |||
| 171 | static int create_queue_nocpsch(struct device_queue_manager *dqm, | ||
| 172 | struct queue *q, | ||
| 173 | struct qcm_process_device *qpd, | ||
| 174 | int *allocated_vmid) | ||
| 175 | { | ||
| 176 | int retval; | ||
| 177 | |||
| 178 | BUG_ON(!dqm || !q || !qpd || !allocated_vmid); | ||
| 179 | |||
| 180 | pr_debug("kfd: In func %s\n", __func__); | ||
| 181 | print_queue(q); | ||
| 182 | |||
| 183 | mutex_lock(&dqm->lock); | ||
| 184 | |||
| 185 | if (list_empty(&qpd->queues_list)) { | ||
| 186 | retval = allocate_vmid(dqm, qpd, q); | ||
| 187 | if (retval != 0) { | ||
| 188 | mutex_unlock(&dqm->lock); | ||
| 189 | return retval; | ||
| 190 | } | ||
| 191 | } | ||
| 192 | *allocated_vmid = qpd->vmid; | ||
| 193 | q->properties.vmid = qpd->vmid; | ||
| 194 | |||
| 195 | retval = create_compute_queue_nocpsch(dqm, q, qpd); | ||
| 196 | |||
| 197 | if (retval != 0) { | ||
| 198 | if (list_empty(&qpd->queues_list)) { | ||
| 199 | deallocate_vmid(dqm, qpd, q); | ||
| 200 | *allocated_vmid = 0; | ||
| 201 | } | ||
| 202 | mutex_unlock(&dqm->lock); | ||
| 203 | return retval; | ||
| 204 | } | ||
| 205 | |||
| 206 | list_add(&q->list, &qpd->queues_list); | ||
| 207 | dqm->queue_count++; | ||
| 208 | |||
| 209 | mutex_unlock(&dqm->lock); | ||
| 210 | return 0; | ||
| 211 | } | ||
| 212 | |||
| 213 | static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) | ||
| 214 | { | ||
| 215 | bool set; | ||
| 216 | int pipe, bit; | ||
| 217 | |||
| 218 | set = false; | ||
| 219 | |||
| 220 | for (pipe = dqm->next_pipe_to_allocate; pipe < get_pipes_num(dqm); | ||
| 221 | pipe = (pipe + 1) % get_pipes_num(dqm)) { | ||
| 222 | if (dqm->allocated_queues[pipe] != 0) { | ||
| 223 | bit = find_first_bit( | ||
| 224 | (unsigned long *)&dqm->allocated_queues[pipe], | ||
| 225 | QUEUES_PER_PIPE); | ||
| 226 | |||
| 227 | clear_bit(bit, | ||
| 228 | (unsigned long *)&dqm->allocated_queues[pipe]); | ||
| 229 | q->pipe = pipe; | ||
| 230 | q->queue = bit; | ||
| 231 | set = true; | ||
| 232 | break; | ||
| 233 | } | ||
| 234 | } | ||
| 235 | |||
| 236 | if (set == false) | ||
| 237 | return -EBUSY; | ||
| 238 | |||
| 239 | pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n", | ||
| 240 | __func__, q->pipe, q->queue); | ||
| 241 | /* horizontal hqd allocation */ | ||
| 242 | dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm); | ||
| 243 | |||
| 244 | return 0; | ||
| 245 | } | ||
| 246 | |||
| 247 | static inline void deallocate_hqd(struct device_queue_manager *dqm, | ||
| 248 | struct queue *q) | ||
| 249 | { | ||
| 250 | set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]); | ||
| 251 | } | ||
| 252 | |||
| 253 | static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, | ||
| 254 | struct queue *q, | ||
| 255 | struct qcm_process_device *qpd) | ||
| 256 | { | ||
| 257 | int retval; | ||
| 258 | struct mqd_manager *mqd; | ||
| 259 | |||
| 260 | BUG_ON(!dqm || !q || !qpd); | ||
| 261 | |||
| 262 | mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); | ||
| 263 | if (mqd == NULL) | ||
| 264 | return -ENOMEM; | ||
| 265 | |||
| 266 | retval = allocate_hqd(dqm, q); | ||
| 267 | if (retval != 0) | ||
| 268 | return retval; | ||
| 269 | |||
| 270 | retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, | ||
| 271 | &q->gart_mqd_addr, &q->properties); | ||
| 272 | if (retval != 0) { | ||
| 273 | deallocate_hqd(dqm, q); | ||
| 274 | return retval; | ||
| 275 | } | ||
| 276 | |||
| 277 | return 0; | ||
| 278 | } | ||
| 279 | |||
| 280 | static int destroy_queue_nocpsch(struct device_queue_manager *dqm, | ||
| 281 | struct qcm_process_device *qpd, | ||
| 282 | struct queue *q) | ||
| 283 | { | ||
| 284 | int retval; | ||
| 285 | struct mqd_manager *mqd; | ||
| 286 | |||
| 287 | BUG_ON(!dqm || !q || !q->mqd || !qpd); | ||
| 288 | |||
| 289 | retval = 0; | ||
| 290 | |||
| 291 | pr_debug("kfd: In Func %s\n", __func__); | ||
| 292 | |||
| 293 | mutex_lock(&dqm->lock); | ||
| 294 | mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); | ||
| 295 | if (mqd == NULL) { | ||
| 296 | retval = -ENOMEM; | ||
| 297 | goto out; | ||
| 298 | } | ||
| 299 | |||
| 300 | retval = mqd->destroy_mqd(mqd, q->mqd, | ||
| 301 | KFD_PREEMPT_TYPE_WAVEFRONT, | ||
| 302 | QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS, | ||
| 303 | q->pipe, q->queue); | ||
| 304 | |||
| 305 | if (retval != 0) | ||
| 306 | goto out; | ||
| 307 | |||
| 308 | deallocate_hqd(dqm, q); | ||
| 309 | |||
| 310 | mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); | ||
| 311 | |||
| 312 | list_del(&q->list); | ||
| 313 | if (list_empty(&qpd->queues_list)) | ||
| 314 | deallocate_vmid(dqm, qpd, q); | ||
| 315 | dqm->queue_count--; | ||
| 316 | out: | ||
| 317 | mutex_unlock(&dqm->lock); | ||
| 318 | return retval; | ||
| 319 | } | ||
| 320 | |||
| 321 | static int update_queue(struct device_queue_manager *dqm, struct queue *q) | ||
| 322 | { | ||
| 323 | int retval; | ||
| 324 | struct mqd_manager *mqd; | ||
| 325 | |||
| 326 | BUG_ON(!dqm || !q || !q->mqd); | ||
| 327 | |||
| 328 | mutex_lock(&dqm->lock); | ||
| 329 | mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); | ||
| 330 | if (mqd == NULL) { | ||
| 331 | mutex_unlock(&dqm->lock); | ||
| 332 | return -ENOMEM; | ||
| 333 | } | ||
| 334 | |||
| 335 | retval = mqd->update_mqd(mqd, q->mqd, &q->properties); | ||
| 336 | if (q->properties.is_active == true) | ||
| 337 | dqm->queue_count++; | ||
| 338 | else | ||
| 339 | dqm->queue_count--; | ||
| 340 | |||
| 341 | if (sched_policy != KFD_SCHED_POLICY_NO_HWS) | ||
| 342 | retval = execute_queues_cpsch(dqm, false); | ||
| 343 | |||
| 344 | mutex_unlock(&dqm->lock); | ||
| 345 | return retval; | ||
| 346 | } | ||
| 347 | |||
| 348 | static struct mqd_manager *get_mqd_manager_nocpsch( | ||
| 349 | struct device_queue_manager *dqm, enum KFD_MQD_TYPE type) | ||
| 350 | { | ||
| 351 | struct mqd_manager *mqd; | ||
| 352 | |||
| 353 | BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX); | ||
| 354 | |||
| 355 | pr_debug("kfd: In func %s mqd type %d\n", __func__, type); | ||
| 356 | |||
| 357 | mqd = dqm->mqds[type]; | ||
| 358 | if (!mqd) { | ||
| 359 | mqd = mqd_manager_init(type, dqm->dev); | ||
| 360 | if (mqd == NULL) | ||
| 361 | pr_err("kfd: mqd manager is NULL"); | ||
| 362 | dqm->mqds[type] = mqd; | ||
| 363 | } | ||
| 364 | |||
| 365 | return mqd; | ||
| 366 | } | ||
| 367 | |||
| 368 | static int register_process_nocpsch(struct device_queue_manager *dqm, | ||
| 369 | struct qcm_process_device *qpd) | ||
| 370 | { | ||
| 371 | struct device_process_node *n; | ||
| 372 | |||
| 373 | BUG_ON(!dqm || !qpd); | ||
| 374 | |||
| 375 | pr_debug("kfd: In func %s\n", __func__); | ||
| 376 | |||
| 377 | n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL); | ||
| 378 | if (!n) | ||
| 379 | return -ENOMEM; | ||
| 380 | |||
| 381 | n->qpd = qpd; | ||
| 382 | |||
| 383 | mutex_lock(&dqm->lock); | ||
| 384 | list_add(&n->list, &dqm->queues); | ||
| 385 | |||
| 386 | init_process_memory(dqm, qpd); | ||
| 387 | dqm->processes_count++; | ||
| 388 | |||
| 389 | mutex_unlock(&dqm->lock); | ||
| 390 | |||
| 391 | return 0; | ||
| 392 | } | ||
| 393 | |||
| 394 | static int unregister_process_nocpsch(struct device_queue_manager *dqm, | ||
| 395 | struct qcm_process_device *qpd) | ||
| 396 | { | ||
| 397 | int retval; | ||
| 398 | struct device_process_node *cur, *next; | ||
| 399 | |||
| 400 | BUG_ON(!dqm || !qpd); | ||
| 401 | |||
| 402 | BUG_ON(!list_empty(&qpd->queues_list)); | ||
| 403 | |||
| 404 | pr_debug("kfd: In func %s\n", __func__); | ||
| 405 | |||
| 406 | retval = 0; | ||
| 407 | mutex_lock(&dqm->lock); | ||
| 408 | |||
| 409 | list_for_each_entry_safe(cur, next, &dqm->queues, list) { | ||
| 410 | if (qpd == cur->qpd) { | ||
| 411 | list_del(&cur->list); | ||
| 412 | dqm->processes_count--; | ||
| 413 | goto out; | ||
| 414 | } | ||
| 415 | } | ||
| 416 | /* qpd not found in dqm list */ | ||
| 417 | retval = 1; | ||
| 418 | out: | ||
| 419 | mutex_unlock(&dqm->lock); | ||
| 420 | return retval; | ||
| 421 | } | ||
| 422 | |||
| 423 | static int | ||
| 424 | set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid, | ||
| 425 | unsigned int vmid) | ||
| 426 | { | ||
| 427 | uint32_t pasid_mapping; | ||
| 428 | |||
| 429 | pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | | ||
| 430 | ATC_VMID_PASID_MAPPING_VALID; | ||
| 431 | return kfd2kgd->set_pasid_vmid_mapping(dqm->dev->kgd, pasid_mapping, | ||
| 432 | vmid); | ||
| 433 | } | ||
| 434 | |||
| 435 | static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble) | ||
| 436 | { | ||
| 437 | /* In 64-bit mode, we can only control the top 3 bits of the LDS, | ||
| 438 | * scratch and GPUVM apertures. | ||
| 439 | * The hardware fills in the remaining 59 bits according to the | ||
| 440 | * following pattern: | ||
| 441 | * LDS: X0000000'00000000 - X0000001'00000000 (4GB) | ||
| 442 | * Scratch: X0000001'00000000 - X0000002'00000000 (4GB) | ||
| 443 | * GPUVM: Y0010000'00000000 - Y0020000'00000000 (1TB) | ||
| 444 | * | ||
| 445 | * (where X/Y is the configurable nybble with the low-bit 0) | ||
| 446 | * | ||
| 447 | * LDS and scratch will have the same top nybble programmed in the | ||
| 448 | * top 3 bits of SH_MEM_BASES.PRIVATE_BASE. | ||
| 449 | * GPUVM can have a different top nybble programmed in the | ||
| 450 | * top 3 bits of SH_MEM_BASES.SHARED_BASE. | ||
| 451 | * We don't bother to support different top nybbles | ||
| 452 | * for LDS/Scratch and GPUVM. | ||
| 453 | */ | ||
| 454 | |||
| 455 | BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE || | ||
| 456 | top_address_nybble == 0); | ||
| 457 | |||
| 458 | return PRIVATE_BASE(top_address_nybble << 12) | | ||
| 459 | SHARED_BASE(top_address_nybble << 12); | ||
| 460 | } | ||
| 461 | |||
| 462 | static int init_memory(struct device_queue_manager *dqm) | ||
| 463 | { | ||
| 464 | int i, retval; | ||
| 465 | |||
| 466 | for (i = 8; i < 16; i++) | ||
| 467 | set_pasid_vmid_mapping(dqm, 0, i); | ||
| 468 | |||
| 469 | retval = kfd2kgd->init_memory(dqm->dev->kgd); | ||
| 470 | if (retval == 0) | ||
| 471 | is_mem_initialized = true; | ||
| 472 | return retval; | ||
| 473 | } | ||
| 474 | |||
| 475 | |||
| 476 | static int init_pipelines(struct device_queue_manager *dqm, | ||
| 477 | unsigned int pipes_num, unsigned int first_pipe) | ||
| 478 | { | ||
| 479 | void *hpdptr; | ||
| 480 | struct mqd_manager *mqd; | ||
| 481 | unsigned int i, err, inx; | ||
| 482 | uint64_t pipe_hpd_addr; | ||
| 483 | |||
| 484 | BUG_ON(!dqm || !dqm->dev); | ||
| 485 | |||
| 486 | pr_debug("kfd: In func %s\n", __func__); | ||
| 487 | |||
| 488 | /* | ||
| 489 | * Allocate memory for the HPDs. This is hardware-owned per-pipe data. | ||
| 490 | * The driver never accesses this memory after zeroing it. | ||
| 491 | * It doesn't even have to be saved/restored on suspend/resume | ||
| 492 | * because it contains no data when there are no active queues. | ||
| 493 | */ | ||
| 494 | |||
| 495 | err = kfd2kgd->allocate_mem(dqm->dev->kgd, | ||
| 496 | CIK_HPD_EOP_BYTES * pipes_num, | ||
| 497 | PAGE_SIZE, | ||
| 498 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | ||
| 499 | (struct kgd_mem **) &dqm->pipeline_mem); | ||
| 500 | |||
| 501 | if (err) { | ||
| 502 | pr_err("kfd: error allocate vidmem num pipes: %d\n", | ||
| 503 | pipes_num); | ||
| 504 | return -ENOMEM; | ||
| 505 | } | ||
| 506 | |||
| 507 | hpdptr = dqm->pipeline_mem->cpu_ptr; | ||
| 508 | dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr; | ||
| 509 | |||
| 510 | memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num); | ||
| 511 | |||
| 512 | mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); | ||
| 513 | if (mqd == NULL) { | ||
| 514 | kfd2kgd->free_mem(dqm->dev->kgd, | ||
| 515 | (struct kgd_mem *) dqm->pipeline_mem); | ||
| 516 | return -ENOMEM; | ||
| 517 | } | ||
| 518 | |||
| 519 | for (i = 0; i < pipes_num; i++) { | ||
| 520 | inx = i + first_pipe; | ||
| 521 | pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES; | ||
| 522 | pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr); | ||
| 523 | /* = log2(bytes/4)-1 */ | ||
| 524 | kfd2kgd->init_pipeline(dqm->dev->kgd, i, | ||
| 525 | CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr); | ||
| 526 | } | ||
| 527 | |||
| 528 | return 0; | ||
| 529 | } | ||
| 530 | |||
| 531 | |||
| 532 | static int init_scheduler(struct device_queue_manager *dqm) | ||
| 533 | { | ||
| 534 | int retval; | ||
| 535 | |||
| 536 | BUG_ON(!dqm); | ||
| 537 | |||
| 538 | pr_debug("kfd: In %s\n", __func__); | ||
| 539 | |||
| 540 | retval = init_pipelines(dqm, get_pipes_num(dqm), KFD_DQM_FIRST_PIPE); | ||
| 541 | if (retval != 0) | ||
| 542 | return retval; | ||
| 543 | |||
| 544 | retval = init_memory(dqm); | ||
| 545 | |||
| 546 | return retval; | ||
| 547 | } | ||
| 548 | |||
| 549 | static int initialize_nocpsch(struct device_queue_manager *dqm) | ||
| 550 | { | ||
| 551 | int i; | ||
| 552 | |||
| 553 | BUG_ON(!dqm); | ||
| 554 | |||
| 555 | pr_debug("kfd: In func %s num of pipes: %d\n", | ||
| 556 | __func__, get_pipes_num(dqm)); | ||
| 557 | |||
| 558 | mutex_init(&dqm->lock); | ||
| 559 | INIT_LIST_HEAD(&dqm->queues); | ||
| 560 | dqm->queue_count = dqm->next_pipe_to_allocate = 0; | ||
| 561 | dqm->allocated_queues = kcalloc(get_pipes_num(dqm), | ||
| 562 | sizeof(unsigned int), GFP_KERNEL); | ||
| 563 | if (!dqm->allocated_queues) { | ||
| 564 | mutex_destroy(&dqm->lock); | ||
| 565 | return -ENOMEM; | ||
| 566 | } | ||
| 567 | |||
| 568 | for (i = 0; i < get_pipes_num(dqm); i++) | ||
| 569 | dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1; | ||
| 570 | |||
| 571 | dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1; | ||
| 572 | |||
| 573 | init_scheduler(dqm); | ||
| 574 | return 0; | ||
| 575 | } | ||
| 576 | |||
| 577 | static void uninitialize_nocpsch(struct device_queue_manager *dqm) | ||
| 578 | { | ||
| 579 | BUG_ON(!dqm); | ||
| 580 | |||
| 581 | BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0); | ||
| 582 | |||
| 583 | kfree(dqm->allocated_queues); | ||
| 584 | mutex_destroy(&dqm->lock); | ||
| 585 | kfd2kgd->free_mem(dqm->dev->kgd, | ||
| 586 | (struct kgd_mem *) dqm->pipeline_mem); | ||
| 587 | } | ||
| 588 | |||
| 589 | static int start_nocpsch(struct device_queue_manager *dqm) | ||
| 590 | { | ||
| 591 | return 0; | ||
| 592 | } | ||
| 593 | |||
| 594 | static int stop_nocpsch(struct device_queue_manager *dqm) | ||
| 595 | { | ||
| 596 | return 0; | ||
| 597 | } | ||
| 598 | |||
| 599 | /* | ||
| 600 | * Device Queue Manager implementation for cp scheduler | ||
| 601 | */ | ||
| 602 | |||
| 603 | static int set_sched_resources(struct device_queue_manager *dqm) | ||
| 604 | { | ||
| 605 | struct scheduling_resources res; | ||
| 606 | unsigned int queue_num, queue_mask; | ||
| 607 | |||
| 608 | BUG_ON(!dqm); | ||
| 609 | |||
| 610 | pr_debug("kfd: In func %s\n", __func__); | ||
| 611 | |||
| 612 | queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE; | ||
| 613 | queue_mask = (1 << queue_num) - 1; | ||
| 614 | res.vmid_mask = (1 << VMID_PER_DEVICE) - 1; | ||
| 615 | res.vmid_mask <<= KFD_VMID_START_OFFSET; | ||
| 616 | res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE); | ||
| 617 | res.gws_mask = res.oac_mask = res.gds_heap_base = | ||
| 618 | res.gds_heap_size = 0; | ||
| 619 | |||
| 620 | pr_debug("kfd: scheduling resources:\n" | ||
| 621 | " vmid mask: 0x%8X\n" | ||
| 622 | " queue mask: 0x%8llX\n", | ||
| 623 | res.vmid_mask, res.queue_mask); | ||
| 624 | |||
| 625 | return pm_send_set_resources(&dqm->packets, &res); | ||
| 626 | } | ||
| 627 | |||
| 628 | static int initialize_cpsch(struct device_queue_manager *dqm) | ||
| 629 | { | ||
| 630 | int retval; | ||
| 631 | |||
| 632 | BUG_ON(!dqm); | ||
| 633 | |||
| 634 | pr_debug("kfd: In func %s num of pipes: %d\n", | ||
| 635 | __func__, get_pipes_num_cpsch()); | ||
| 636 | |||
| 637 | mutex_init(&dqm->lock); | ||
| 638 | INIT_LIST_HEAD(&dqm->queues); | ||
| 639 | dqm->queue_count = dqm->processes_count = 0; | ||
| 640 | dqm->active_runlist = false; | ||
| 641 | retval = init_pipelines(dqm, get_pipes_num(dqm), 0); | ||
| 642 | if (retval != 0) | ||
| 643 | goto fail_init_pipelines; | ||
| 644 | |||
| 645 | return 0; | ||
| 646 | |||
| 647 | fail_init_pipelines: | ||
| 648 | mutex_destroy(&dqm->lock); | ||
| 649 | return retval; | ||
| 650 | } | ||
| 651 | |||
| 652 | static int start_cpsch(struct device_queue_manager *dqm) | ||
| 653 | { | ||
| 654 | struct device_process_node *node; | ||
| 655 | int retval; | ||
| 656 | |||
| 657 | BUG_ON(!dqm); | ||
| 658 | |||
| 659 | retval = 0; | ||
| 660 | |||
| 661 | retval = pm_init(&dqm->packets, dqm); | ||
| 662 | if (retval != 0) | ||
| 663 | goto fail_packet_manager_init; | ||
| 664 | |||
| 665 | retval = set_sched_resources(dqm); | ||
| 666 | if (retval != 0) | ||
| 667 | goto fail_set_sched_resources; | ||
| 668 | |||
| 669 | pr_debug("kfd: allocating fence memory\n"); | ||
| 670 | |||
| 671 | /* allocate fence memory on the gart */ | ||
| 672 | retval = kfd2kgd->allocate_mem(dqm->dev->kgd, | ||
| 673 | sizeof(*dqm->fence_addr), | ||
| 674 | 32, | ||
| 675 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | ||
| 676 | (struct kgd_mem **) &dqm->fence_mem); | ||
| 677 | |||
| 678 | if (retval != 0) | ||
| 679 | goto fail_allocate_vidmem; | ||
| 680 | |||
| 681 | dqm->fence_addr = dqm->fence_mem->cpu_ptr; | ||
| 682 | dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; | ||
| 683 | |||
| 684 | list_for_each_entry(node, &dqm->queues, list) | ||
| 685 | if (node->qpd->pqm->process && dqm->dev) | ||
| 686 | kfd_bind_process_to_device(dqm->dev, | ||
| 687 | node->qpd->pqm->process); | ||
| 688 | |||
| 689 | execute_queues_cpsch(dqm, true); | ||
| 690 | |||
| 691 | return 0; | ||
| 692 | fail_allocate_vidmem: | ||
| 693 | fail_set_sched_resources: | ||
| 694 | pm_uninit(&dqm->packets); | ||
| 695 | fail_packet_manager_init: | ||
| 696 | return retval; | ||
| 697 | } | ||
| 698 | |||
| 699 | static int stop_cpsch(struct device_queue_manager *dqm) | ||
| 700 | { | ||
| 701 | struct device_process_node *node; | ||
| 702 | struct kfd_process_device *pdd; | ||
| 703 | |||
| 704 | BUG_ON(!dqm); | ||
| 705 | |||
| 706 | destroy_queues_cpsch(dqm, true); | ||
| 707 | |||
| 708 | list_for_each_entry(node, &dqm->queues, list) { | ||
| 709 | pdd = kfd_get_process_device_data(dqm->dev, | ||
| 710 | node->qpd->pqm->process, 1); | ||
| 711 | pdd->bound = false; | ||
| 712 | } | ||
| 713 | kfd2kgd->free_mem(dqm->dev->kgd, | ||
| 714 | (struct kgd_mem *) dqm->fence_mem); | ||
| 715 | pm_uninit(&dqm->packets); | ||
| 716 | |||
| 717 | return 0; | ||
| 718 | } | ||
| 719 | |||
| 720 | static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, | ||
| 721 | struct kernel_queue *kq, | ||
| 722 | struct qcm_process_device *qpd) | ||
| 723 | { | ||
| 724 | BUG_ON(!dqm || !kq || !qpd); | ||
| 725 | |||
| 726 | pr_debug("kfd: In func %s\n", __func__); | ||
| 727 | |||
| 728 | mutex_lock(&dqm->lock); | ||
| 729 | list_add(&kq->list, &qpd->priv_queue_list); | ||
| 730 | dqm->queue_count++; | ||
| 731 | qpd->is_debug = true; | ||
| 732 | execute_queues_cpsch(dqm, false); | ||
| 733 | mutex_unlock(&dqm->lock); | ||
| 734 | |||
| 735 | return 0; | ||
| 736 | } | ||
| 737 | |||
| 738 | static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, | ||
| 739 | struct kernel_queue *kq, | ||
| 740 | struct qcm_process_device *qpd) | ||
| 741 | { | ||
| 742 | BUG_ON(!dqm || !kq); | ||
| 743 | |||
| 744 | pr_debug("kfd: In %s\n", __func__); | ||
| 745 | |||
| 746 | mutex_lock(&dqm->lock); | ||
| 747 | destroy_queues_cpsch(dqm, false); | ||
| 748 | list_del(&kq->list); | ||
| 749 | dqm->queue_count--; | ||
| 750 | qpd->is_debug = false; | ||
| 751 | execute_queues_cpsch(dqm, false); | ||
| 752 | mutex_unlock(&dqm->lock); | ||
| 753 | } | ||
| 754 | |||
| 755 | static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, | ||
| 756 | struct qcm_process_device *qpd, int *allocate_vmid) | ||
| 757 | { | ||
| 758 | int retval; | ||
| 759 | struct mqd_manager *mqd; | ||
| 760 | |||
| 761 | BUG_ON(!dqm || !q || !qpd); | ||
| 762 | |||
| 763 | retval = 0; | ||
| 764 | |||
| 765 | if (allocate_vmid) | ||
| 766 | *allocate_vmid = 0; | ||
| 767 | |||
| 768 | mutex_lock(&dqm->lock); | ||
| 769 | |||
| 770 | mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_CP); | ||
| 771 | if (mqd == NULL) { | ||
| 772 | mutex_unlock(&dqm->lock); | ||
| 773 | return -ENOMEM; | ||
| 774 | } | ||
| 775 | |||
| 776 | retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, | ||
| 777 | &q->gart_mqd_addr, &q->properties); | ||
| 778 | if (retval != 0) | ||
| 779 | goto out; | ||
| 780 | |||
| 781 | list_add(&q->list, &qpd->queues_list); | ||
| 782 | if (q->properties.is_active) { | ||
| 783 | dqm->queue_count++; | ||
| 784 | retval = execute_queues_cpsch(dqm, false); | ||
| 785 | } | ||
| 786 | |||
| 787 | out: | ||
| 788 | mutex_unlock(&dqm->lock); | ||
| 789 | return retval; | ||
| 790 | } | ||
| 791 | |||
| 792 | int fence_wait_timeout(unsigned int *fence_addr, unsigned int fence_value, | ||
| 793 | unsigned long timeout) | ||
| 794 | { | ||
| 795 | BUG_ON(!fence_addr); | ||
| 796 | timeout += jiffies; | ||
| 797 | |||
| 798 | while (*fence_addr != fence_value) { | ||
| 799 | if (time_after(jiffies, timeout)) { | ||
| 800 | pr_err("kfd: qcm fence wait loop timeout expired\n"); | ||
| 801 | return -ETIME; | ||
| 802 | } | ||
| 803 | cpu_relax(); | ||
| 804 | } | ||
| 805 | |||
| 806 | return 0; | ||
| 807 | } | ||
| 808 | |||
| 809 | static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock) | ||
| 810 | { | ||
| 811 | int retval; | ||
| 812 | |||
| 813 | BUG_ON(!dqm); | ||
| 814 | |||
| 815 | retval = 0; | ||
| 816 | |||
| 817 | if (lock) | ||
| 818 | mutex_lock(&dqm->lock); | ||
| 819 | if (dqm->active_runlist == false) | ||
| 820 | goto out; | ||
| 821 | retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE, | ||
| 822 | KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0); | ||
| 823 | if (retval != 0) | ||
| 824 | goto out; | ||
| 825 | |||
| 826 | *dqm->fence_addr = KFD_FENCE_INIT; | ||
| 827 | pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr, | ||
| 828 | KFD_FENCE_COMPLETED); | ||
| 829 | /* should be timed out */ | ||
| 830 | fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED, | ||
| 831 | QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS); | ||
| 832 | pm_release_ib(&dqm->packets); | ||
| 833 | dqm->active_runlist = false; | ||
| 834 | |||
| 835 | out: | ||
| 836 | if (lock) | ||
| 837 | mutex_unlock(&dqm->lock); | ||
| 838 | return retval; | ||
| 839 | } | ||
| 840 | |||
| 841 | static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock) | ||
| 842 | { | ||
| 843 | int retval; | ||
| 844 | |||
| 845 | BUG_ON(!dqm); | ||
| 846 | |||
| 847 | if (lock) | ||
| 848 | mutex_lock(&dqm->lock); | ||
| 849 | |||
| 850 | retval = destroy_queues_cpsch(dqm, false); | ||
| 851 | if (retval != 0) { | ||
| 852 | pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption"); | ||
| 853 | goto out; | ||
| 854 | } | ||
| 855 | |||
| 856 | if (dqm->queue_count <= 0 || dqm->processes_count <= 0) { | ||
| 857 | retval = 0; | ||
| 858 | goto out; | ||
| 859 | } | ||
| 860 | |||
| 861 | if (dqm->active_runlist) { | ||
| 862 | retval = 0; | ||
| 863 | goto out; | ||
| 864 | } | ||
| 865 | |||
| 866 | retval = pm_send_runlist(&dqm->packets, &dqm->queues); | ||
| 867 | if (retval != 0) { | ||
| 868 | pr_err("kfd: failed to execute runlist"); | ||
| 869 | goto out; | ||
| 870 | } | ||
| 871 | dqm->active_runlist = true; | ||
| 872 | |||
| 873 | out: | ||
| 874 | if (lock) | ||
| 875 | mutex_unlock(&dqm->lock); | ||
| 876 | return retval; | ||
| 877 | } | ||
| 878 | |||
| 879 | static int destroy_queue_cpsch(struct device_queue_manager *dqm, | ||
| 880 | struct qcm_process_device *qpd, | ||
| 881 | struct queue *q) | ||
| 882 | { | ||
| 883 | int retval; | ||
| 884 | struct mqd_manager *mqd; | ||
| 885 | |||
| 886 | BUG_ON(!dqm || !qpd || !q); | ||
| 887 | |||
| 888 | retval = 0; | ||
| 889 | |||
| 890 | /* remove queue from list to prevent rescheduling after preemption */ | ||
| 891 | mutex_lock(&dqm->lock); | ||
| 892 | |||
| 893 | mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_CP); | ||
| 894 | if (!mqd) { | ||
| 895 | retval = -ENOMEM; | ||
| 896 | goto failed; | ||
| 897 | } | ||
| 898 | |||
| 899 | list_del(&q->list); | ||
| 900 | dqm->queue_count--; | ||
| 901 | |||
| 902 | execute_queues_cpsch(dqm, false); | ||
| 903 | |||
| 904 | mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); | ||
| 905 | |||
| 906 | mutex_unlock(&dqm->lock); | ||
| 907 | |||
| 908 | return 0; | ||
| 909 | |||
| 910 | failed: | ||
| 911 | mutex_unlock(&dqm->lock); | ||
| 912 | return retval; | ||
| 913 | } | ||
| 914 | |||
| 915 | /* | ||
| 916 | * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to | ||
| 917 | * stay in user mode. | ||
| 918 | */ | ||
| 919 | #define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL | ||
| 920 | /* APE1 limit is inclusive and 64K aligned. */ | ||
| 921 | #define APE1_LIMIT_ALIGNMENT 0xFFFF | ||
| 922 | |||
| 923 | static bool set_cache_memory_policy(struct device_queue_manager *dqm, | ||
| 924 | struct qcm_process_device *qpd, | ||
| 925 | enum cache_policy default_policy, | ||
| 926 | enum cache_policy alternate_policy, | ||
| 927 | void __user *alternate_aperture_base, | ||
| 928 | uint64_t alternate_aperture_size) | ||
| 929 | { | ||
| 930 | uint32_t default_mtype; | ||
| 931 | uint32_t ape1_mtype; | ||
| 932 | |||
| 933 | pr_debug("kfd: In func %s\n", __func__); | ||
| 934 | |||
| 935 | mutex_lock(&dqm->lock); | ||
| 936 | |||
| 937 | if (alternate_aperture_size == 0) { | ||
| 938 | /* base > limit disables APE1 */ | ||
| 939 | qpd->sh_mem_ape1_base = 1; | ||
| 940 | qpd->sh_mem_ape1_limit = 0; | ||
| 941 | } else { | ||
| 942 | /* | ||
| 943 | * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]}, | ||
| 944 | * SH_MEM_APE1_BASE[31:0], 0x0000 } | ||
| 945 | * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]}, | ||
| 946 | * SH_MEM_APE1_LIMIT[31:0], 0xFFFF } | ||
| 947 | * Verify that the base and size parameters can be | ||
| 948 | * represented in this format and convert them. | ||
| 949 | * Additionally restrict APE1 to user-mode addresses. | ||
| 950 | */ | ||
| 951 | |||
| 952 | uint64_t base = (uintptr_t)alternate_aperture_base; | ||
| 953 | uint64_t limit = base + alternate_aperture_size - 1; | ||
| 954 | |||
| 955 | if (limit <= base) | ||
| 956 | goto out; | ||
| 957 | |||
| 958 | if ((base & APE1_FIXED_BITS_MASK) != 0) | ||
| 959 | goto out; | ||
| 960 | |||
| 961 | if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) | ||
| 962 | goto out; | ||
| 963 | |||
| 964 | qpd->sh_mem_ape1_base = base >> 16; | ||
| 965 | qpd->sh_mem_ape1_limit = limit >> 16; | ||
| 966 | } | ||
| 967 | |||
| 968 | default_mtype = (default_policy == cache_policy_coherent) ? | ||
| 969 | MTYPE_NONCACHED : | ||
| 970 | MTYPE_CACHED; | ||
| 971 | |||
| 972 | ape1_mtype = (alternate_policy == cache_policy_coherent) ? | ||
| 973 | MTYPE_NONCACHED : | ||
| 974 | MTYPE_CACHED; | ||
| 975 | |||
| 976 | qpd->sh_mem_config = (qpd->sh_mem_config & PTR32) | ||
| 977 | | ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) | ||
| 978 | | DEFAULT_MTYPE(default_mtype) | ||
| 979 | | APE1_MTYPE(ape1_mtype); | ||
| 980 | |||
| 981 | if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) | ||
| 982 | program_sh_mem_settings(dqm, qpd); | ||
| 983 | |||
| 984 | pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", | ||
| 985 | qpd->sh_mem_config, qpd->sh_mem_ape1_base, | ||
| 986 | qpd->sh_mem_ape1_limit); | ||
| 987 | |||
| 988 | mutex_unlock(&dqm->lock); | ||
| 989 | return true; | ||
| 990 | |||
| 991 | out: | ||
| 992 | mutex_unlock(&dqm->lock); | ||
| 993 | return false; | ||
| 994 | } | ||
| 995 | |||
| 996 | struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev) | ||
| 997 | { | ||
| 998 | struct device_queue_manager *dqm; | ||
| 999 | |||
| 1000 | BUG_ON(!dev); | ||
| 1001 | |||
| 1002 | dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL); | ||
| 1003 | if (!dqm) | ||
| 1004 | return NULL; | ||
| 1005 | |||
| 1006 | dqm->dev = dev; | ||
| 1007 | switch (sched_policy) { | ||
| 1008 | case KFD_SCHED_POLICY_HWS: | ||
| 1009 | case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: | ||
| 1010 | /* initialize dqm for cp scheduling */ | ||
| 1011 | dqm->create_queue = create_queue_cpsch; | ||
| 1012 | dqm->initialize = initialize_cpsch; | ||
| 1013 | dqm->start = start_cpsch; | ||
| 1014 | dqm->stop = stop_cpsch; | ||
| 1015 | dqm->destroy_queue = destroy_queue_cpsch; | ||
| 1016 | dqm->update_queue = update_queue; | ||
| 1017 | dqm->get_mqd_manager = get_mqd_manager_nocpsch; | ||
| 1018 | dqm->register_process = register_process_nocpsch; | ||
| 1019 | dqm->unregister_process = unregister_process_nocpsch; | ||
| 1020 | dqm->uninitialize = uninitialize_nocpsch; | ||
| 1021 | dqm->create_kernel_queue = create_kernel_queue_cpsch; | ||
| 1022 | dqm->destroy_kernel_queue = destroy_kernel_queue_cpsch; | ||
| 1023 | dqm->set_cache_memory_policy = set_cache_memory_policy; | ||
| 1024 | break; | ||
| 1025 | case KFD_SCHED_POLICY_NO_HWS: | ||
| 1026 | /* initialize dqm for no cp scheduling */ | ||
| 1027 | dqm->start = start_nocpsch; | ||
| 1028 | dqm->stop = stop_nocpsch; | ||
| 1029 | dqm->create_queue = create_queue_nocpsch; | ||
| 1030 | dqm->destroy_queue = destroy_queue_nocpsch; | ||
| 1031 | dqm->update_queue = update_queue; | ||
| 1032 | dqm->get_mqd_manager = get_mqd_manager_nocpsch; | ||
| 1033 | dqm->register_process = register_process_nocpsch; | ||
| 1034 | dqm->unregister_process = unregister_process_nocpsch; | ||
| 1035 | dqm->initialize = initialize_nocpsch; | ||
| 1036 | dqm->uninitialize = uninitialize_nocpsch; | ||
| 1037 | dqm->set_cache_memory_policy = set_cache_memory_policy; | ||
| 1038 | break; | ||
| 1039 | default: | ||
| 1040 | BUG(); | ||
| 1041 | break; | ||
| 1042 | } | ||
| 1043 | |||
| 1044 | if (dqm->initialize(dqm) != 0) { | ||
| 1045 | kfree(dqm); | ||
| 1046 | return NULL; | ||
| 1047 | } | ||
| 1048 | |||
| 1049 | return dqm; | ||
| 1050 | } | ||
| 1051 | |||
| 1052 | void device_queue_manager_uninit(struct device_queue_manager *dqm) | ||
| 1053 | { | ||
| 1054 | BUG_ON(!dqm); | ||
| 1055 | |||
| 1056 | dqm->uninitialize(dqm); | ||
| 1057 | kfree(dqm); | ||
| 1058 | } | ||
| 1059 | |||
| @@ -0,0 +1,146 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | #ifndef KFD_DEVICE_QUEUE_MANAGER_H_ | ||
| 25 | #define KFD_DEVICE_QUEUE_MANAGER_H_ | ||
| 26 | |||
| 27 | #include <linux/rwsem.h> | ||
| 28 | #include <linux/list.h> | ||
| 29 | #include "kfd_priv.h" | ||
| 30 | #include "kfd_mqd_manager.h" | ||
| 31 | |||
| 32 | #define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (500) | ||
| 33 | #define QUEUES_PER_PIPE (8) | ||
| 34 | #define PIPE_PER_ME_CP_SCHEDULING (3) | ||
| 35 | #define CIK_VMID_NUM (8) | ||
| 36 | #define KFD_VMID_START_OFFSET (8) | ||
| 37 | #define VMID_PER_DEVICE CIK_VMID_NUM | ||
| 38 | #define KFD_DQM_FIRST_PIPE (0) | ||
| 39 | |||
| 40 | struct device_process_node { | ||
| 41 | struct qcm_process_device *qpd; | ||
| 42 | struct list_head list; | ||
| 43 | }; | ||
| 44 | |||
| 45 | /** | ||
| 46 | * struct device_queue_manager | ||
| 47 | * | ||
| 48 | * @create_queue: Queue creation routine. | ||
| 49 | * | ||
| 50 | * @destroy_queue: Queue destruction routine. | ||
| 51 | * | ||
| 52 | * @update_queue: Queue update routine. | ||
| 53 | * | ||
| 54 | * @get_mqd_manager: Returns the mqd manager according to the mqd type. | ||
| 55 | * | ||
| 56 | * @exeute_queues: Dispatches the queues list to the H/W. | ||
| 57 | * | ||
| 58 | * @register_process: This routine associates a specific process with device. | ||
| 59 | * | ||
| 60 | * @unregister_process: destroys the associations between process to device. | ||
| 61 | * | ||
| 62 | * @initialize: Initializes the pipelines and memory module for that device. | ||
| 63 | * | ||
| 64 | * @start: Initializes the resources/modules the the device needs for queues | ||
| 65 | * execution. This function is called on device initialization and after the | ||
| 66 | * system woke up after suspension. | ||
| 67 | * | ||
| 68 | * @stop: This routine stops execution of all the active queue running on the | ||
| 69 | * H/W and basically this function called on system suspend. | ||
| 70 | * | ||
| 71 | * @uninitialize: Destroys all the device queue manager resources allocated in | ||
| 72 | * initialize routine. | ||
| 73 | * | ||
| 74 | * @create_kernel_queue: Creates kernel queue. Used for debug queue. | ||
| 75 | * | ||
| 76 | * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue. | ||
| 77 | * | ||
| 78 | * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the | ||
| 79 | * memory apertures. | ||
| 80 | * | ||
| 81 | * This struct is a base class for the kfd queues scheduler in the | ||
| 82 | * device level. The device base class should expose the basic operations | ||
| 83 | * for queue creation and queue destruction. This base class hides the | ||
| 84 | * scheduling mode of the driver and the specific implementation of the | ||
| 85 | * concrete device. This class is the only class in the queues scheduler | ||
| 86 | * that configures the H/W. | ||
| 87 | */ | ||
| 88 | |||
| 89 | struct device_queue_manager { | ||
| 90 | int (*create_queue)(struct device_queue_manager *dqm, | ||
| 91 | struct queue *q, | ||
| 92 | struct qcm_process_device *qpd, | ||
| 93 | int *allocate_vmid); | ||
| 94 | int (*destroy_queue)(struct device_queue_manager *dqm, | ||
| 95 | struct qcm_process_device *qpd, | ||
| 96 | struct queue *q); | ||
| 97 | int (*update_queue)(struct device_queue_manager *dqm, | ||
| 98 | struct queue *q); | ||
| 99 | |||
| 100 | struct mqd_manager * (*get_mqd_manager) | ||
| 101 | (struct device_queue_manager *dqm, | ||
| 102 | enum KFD_MQD_TYPE type); | ||
| 103 | |||
| 104 | int (*register_process)(struct device_queue_manager *dqm, | ||
| 105 | struct qcm_process_device *qpd); | ||
| 106 | int (*unregister_process)(struct device_queue_manager *dqm, | ||
| 107 | struct qcm_process_device *qpd); | ||
| 108 | int (*initialize)(struct device_queue_manager *dqm); | ||
| 109 | int (*start)(struct device_queue_manager *dqm); | ||
| 110 | int (*stop)(struct device_queue_manager *dqm); | ||
| 111 | void (*uninitialize)(struct device_queue_manager *dqm); | ||
| 112 | int (*create_kernel_queue)(struct device_queue_manager *dqm, | ||
| 113 | struct kernel_queue *kq, | ||
| 114 | struct qcm_process_device *qpd); | ||
| 115 | void (*destroy_kernel_queue)(struct device_queue_manager *dqm, | ||
| 116 | struct kernel_queue *kq, | ||
| 117 | struct qcm_process_device *qpd); | ||
| 118 | bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, | ||
| 119 | struct qcm_process_device *qpd, | ||
| 120 | enum cache_policy default_policy, | ||
| 121 | enum cache_policy alternate_policy, | ||
| 122 | void __user *alternate_aperture_base, | ||
| 123 | uint64_t alternate_aperture_size); | ||
| 124 | |||
| 125 | |||
| 126 | struct mqd_manager *mqds[KFD_MQD_TYPE_MAX]; | ||
| 127 | struct packet_manager packets; | ||
| 128 | struct kfd_dev *dev; | ||
| 129 | struct mutex lock; | ||
| 130 | struct list_head queues; | ||
| 131 | unsigned int processes_count; | ||
| 132 | unsigned int queue_count; | ||
| 133 | unsigned int next_pipe_to_allocate; | ||
| 134 | unsigned int *allocated_queues; | ||
| 135 | unsigned int vmid_bitmap; | ||
| 136 | uint64_t pipelines_addr; | ||
| 137 | struct kfd_mem_obj *pipeline_mem; | ||
| 138 | uint64_t fence_gpu_addr; | ||
| 139 | unsigned int *fence_addr; | ||
| 140 | struct kfd_mem_obj *fence_mem; | ||
| 141 | bool active_runlist; | ||
| 142 | }; | ||
| 143 | |||
| 144 | |||
| 145 | |||
| 146 | #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */ | ||
| @@ -0,0 +1,255 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | #include "kfd_priv.h" | ||
| 23 | #include <linux/mm.h> | ||
| 24 | #include <linux/mman.h> | ||
| 25 | #include <linux/slab.h> | ||
| 26 | |||
| 27 | /* | ||
| 28 | * This extension supports a kernel level doorbells management for | ||
| 29 | * the kernel queues. | ||
| 30 | * Basically the last doorbells page is devoted to kernel queues | ||
| 31 | * and that's assures that any user process won't get access to the | ||
| 32 | * kernel doorbells page | ||
| 33 | */ | ||
| 34 | static DEFINE_MUTEX(doorbell_mutex); | ||
| 35 | static unsigned long doorbell_available_index[ | ||
| 36 | DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)] = { 0 }; | ||
| 37 | |||
| 38 | #define KERNEL_DOORBELL_PASID 1 | ||
| 39 | #define KFD_SIZE_OF_DOORBELL_IN_BYTES 4 | ||
| 40 | |||
| 41 | /* | ||
| 42 | * Each device exposes a doorbell aperture, a PCI MMIO aperture that | ||
| 43 | * receives 32-bit writes that are passed to queues as wptr values. | ||
| 44 | * The doorbells are intended to be written by applications as part | ||
| 45 | * of queueing work on user-mode queues. | ||
| 46 | * We assign doorbells to applications in PAGE_SIZE-sized and aligned chunks. | ||
| 47 | * We map the doorbell address space into user-mode when a process creates | ||
| 48 | * its first queue on each device. | ||
| 49 | * Although the mapping is done by KFD, it is equivalent to an mmap of | ||
| 50 | * the /dev/kfd with the particular device encoded in the mmap offset. | ||
| 51 | * There will be other uses for mmap of /dev/kfd, so only a range of | ||
| 52 | * offsets (KFD_MMAP_DOORBELL_START-END) is used for doorbells. | ||
| 53 | */ | ||
| 54 | |||
| 55 | /* # of doorbell bytes allocated for each process. */ | ||
| 56 | static inline size_t doorbell_process_allocation(void) | ||
| 57 | { | ||
| 58 | return roundup(KFD_SIZE_OF_DOORBELL_IN_BYTES * | ||
| 59 | KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, | ||
| 60 | PAGE_SIZE); | ||
| 61 | } | ||
| 62 | |||
| 63 | /* Doorbell calculations for device init. */ | ||
| 64 | void kfd_doorbell_init(struct kfd_dev *kfd) | ||
| 65 | { | ||
| 66 | size_t doorbell_start_offset; | ||
| 67 | size_t doorbell_aperture_size; | ||
| 68 | size_t doorbell_process_limit; | ||
| 69 | |||
| 70 | /* | ||
| 71 | * We start with calculations in bytes because the input data might | ||
| 72 | * only be byte-aligned. | ||
| 73 | * Only after we have done the rounding can we assume any alignment. | ||
| 74 | */ | ||
| 75 | |||
| 76 | doorbell_start_offset = | ||
| 77 | roundup(kfd->shared_resources.doorbell_start_offset, | ||
| 78 | doorbell_process_allocation()); | ||
| 79 | |||
| 80 | doorbell_aperture_size = | ||
| 81 | rounddown(kfd->shared_resources.doorbell_aperture_size, | ||
| 82 | doorbell_process_allocation()); | ||
| 83 | |||
| 84 | if (doorbell_aperture_size > doorbell_start_offset) | ||
| 85 | doorbell_process_limit = | ||
| 86 | (doorbell_aperture_size - doorbell_start_offset) / | ||
| 87 | doorbell_process_allocation(); | ||
| 88 | else | ||
| 89 | doorbell_process_limit = 0; | ||
| 90 | |||
| 91 | kfd->doorbell_base = kfd->shared_resources.doorbell_physical_address + | ||
| 92 | doorbell_start_offset; | ||
| 93 | |||
| 94 | kfd->doorbell_id_offset = doorbell_start_offset / sizeof(u32); | ||
| 95 | kfd->doorbell_process_limit = doorbell_process_limit - 1; | ||
| 96 | |||
| 97 | kfd->doorbell_kernel_ptr = ioremap(kfd->doorbell_base, | ||
| 98 | doorbell_process_allocation()); | ||
| 99 | |||
| 100 | BUG_ON(!kfd->doorbell_kernel_ptr); | ||
| 101 | |||
| 102 | pr_debug("kfd: doorbell initialization:\n"); | ||
| 103 | pr_debug("kfd: doorbell base == 0x%08lX\n", | ||
| 104 | (uintptr_t)kfd->doorbell_base); | ||
| 105 | |||
| 106 | pr_debug("kfd: doorbell_id_offset == 0x%08lX\n", | ||
| 107 | kfd->doorbell_id_offset); | ||
| 108 | |||
| 109 | pr_debug("kfd: doorbell_process_limit == 0x%08lX\n", | ||
| 110 | doorbell_process_limit); | ||
| 111 | |||
| 112 | pr_debug("kfd: doorbell_kernel_offset == 0x%08lX\n", | ||
| 113 | (uintptr_t)kfd->doorbell_base); | ||
| 114 | |||
| 115 | pr_debug("kfd: doorbell aperture size == 0x%08lX\n", | ||
| 116 | kfd->shared_resources.doorbell_aperture_size); | ||
| 117 | |||
| 118 | pr_debug("kfd: doorbell kernel address == 0x%08lX\n", | ||
| 119 | (uintptr_t)kfd->doorbell_kernel_ptr); | ||
| 120 | } | ||
| 121 | |||
| 122 | int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma) | ||
| 123 | { | ||
| 124 | phys_addr_t address; | ||
| 125 | struct kfd_dev *dev; | ||
| 126 | |||
| 127 | /* | ||
| 128 | * For simplicitly we only allow mapping of the entire doorbell | ||
| 129 | * allocation of a single device & process. | ||
| 130 | */ | ||
| 131 | if (vma->vm_end - vma->vm_start != doorbell_process_allocation()) | ||
| 132 | return -EINVAL; | ||
| 133 | |||
| 134 | /* Find kfd device according to gpu id */ | ||
| 135 | dev = kfd_device_by_id(vma->vm_pgoff); | ||
| 136 | if (dev == NULL) | ||
| 137 | return -EINVAL; | ||
| 138 | |||
| 139 | /* Find if pdd exists for combination of process and gpu id */ | ||
| 140 | if (!kfd_get_process_device_data(dev, process, 0)) | ||
| 141 | return -EINVAL; | ||
| 142 | |||
| 143 | /* Calculate physical address of doorbell */ | ||
| 144 | address = kfd_get_process_doorbells(dev, process); | ||
| 145 | |||
| 146 | vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE | | ||
| 147 | VM_DONTDUMP | VM_PFNMAP; | ||
| 148 | |||
| 149 | vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); | ||
| 150 | |||
| 151 | pr_debug("kfd: mapping doorbell page in kfd_doorbell_mmap\n" | ||
| 152 | " target user address == 0x%08llX\n" | ||
| 153 | " physical address == 0x%08llX\n" | ||
| 154 | " vm_flags == 0x%04lX\n" | ||
| 155 | " size == 0x%04lX\n", | ||
| 156 | (unsigned long long) vma->vm_start, address, vma->vm_flags, | ||
| 157 | doorbell_process_allocation()); | ||
| 158 | |||
| 159 | |||
| 160 | return io_remap_pfn_range(vma, | ||
| 161 | vma->vm_start, | ||
| 162 | address >> PAGE_SHIFT, | ||
| 163 | doorbell_process_allocation(), | ||
| 164 | vma->vm_page_prot); | ||
| 165 | } | ||
| 166 | |||
| 167 | |||
| 168 | /* get kernel iomem pointer for a doorbell */ | ||
| 169 | u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, | ||
| 170 | unsigned int *doorbell_off) | ||
| 171 | { | ||
| 172 | u32 inx; | ||
| 173 | |||
| 174 | BUG_ON(!kfd || !doorbell_off); | ||
| 175 | |||
| 176 | mutex_lock(&doorbell_mutex); | ||
| 177 | inx = find_first_zero_bit(doorbell_available_index, | ||
| 178 | KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); | ||
| 179 | |||
| 180 | __set_bit(inx, doorbell_available_index); | ||
| 181 | mutex_unlock(&doorbell_mutex); | ||
| 182 | |||
| 183 | if (inx >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) | ||
| 184 | return NULL; | ||
| 185 | |||
| 186 | /* | ||
| 187 | * Calculating the kernel doorbell offset using "faked" kernel | ||
| 188 | * pasid that allocated for kernel queues only | ||
| 189 | */ | ||
| 190 | *doorbell_off = KERNEL_DOORBELL_PASID * (doorbell_process_allocation() / | ||
| 191 | sizeof(u32)) + inx; | ||
| 192 | |||
| 193 | pr_debug("kfd: get kernel queue doorbell\n" | ||
| 194 | " doorbell offset == 0x%08d\n" | ||
| 195 | " kernel address == 0x%08lX\n", | ||
| 196 | *doorbell_off, (uintptr_t)(kfd->doorbell_kernel_ptr + inx)); | ||
| 197 | |||
| 198 | return kfd->doorbell_kernel_ptr + inx; | ||
| 199 | } | ||
| 200 | |||
| 201 | void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr) | ||
| 202 | { | ||
| 203 | unsigned int inx; | ||
| 204 | |||
| 205 | BUG_ON(!kfd || !db_addr); | ||
| 206 | |||
| 207 | inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr); | ||
| 208 | |||
| 209 | mutex_lock(&doorbell_mutex); | ||
| 210 | __clear_bit(inx, doorbell_available_index); | ||
| 211 | mutex_unlock(&doorbell_mutex); | ||
| 212 | } | ||
| 213 | |||
| 214 | inline void write_kernel_doorbell(u32 __iomem *db, u32 value) | ||
| 215 | { | ||
| 216 | if (db) { | ||
| 217 | writel(value, db); | ||
| 218 | pr_debug("writing %d to doorbell address 0x%p\n", value, db); | ||
| 219 | } | ||
| 220 | } | ||
| 221 | |||
| 222 | /* | ||
| 223 | * queue_ids are in the range [0,MAX_PROCESS_QUEUES) and are mapped 1:1 | ||
| 224 | * to doorbells with the process's doorbell page | ||
| 225 | */ | ||
| 226 | unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd, | ||
| 227 | struct kfd_process *process, | ||
| 228 | unsigned int queue_id) | ||
| 229 | { | ||
| 230 | /* | ||
| 231 | * doorbell_id_offset accounts for doorbells taken by KGD. | ||
| 232 | * pasid * doorbell_process_allocation/sizeof(u32) adjusts | ||
| 233 | * to the process's doorbells | ||
| 234 | */ | ||
| 235 | return kfd->doorbell_id_offset + | ||
| 236 | process->pasid * (doorbell_process_allocation()/sizeof(u32)) + | ||
| 237 | queue_id; | ||
| 238 | } | ||
| 239 | |||
| 240 | uint64_t kfd_get_number_elems(struct kfd_dev *kfd) | ||
| 241 | { | ||
| 242 | uint64_t num_of_elems = (kfd->shared_resources.doorbell_aperture_size - | ||
| 243 | kfd->shared_resources.doorbell_start_offset) / | ||
| 244 | doorbell_process_allocation() + 1; | ||
| 245 | |||
| 246 | return num_of_elems; | ||
| 247 | |||
| 248 | } | ||
| 249 | |||
| 250 | phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, | ||
| 251 | struct kfd_process *process) | ||
| 252 | { | ||
| 253 | return dev->doorbell_base + | ||
| 254 | process->pasid * doorbell_process_allocation(); | ||
| 255 | } | ||
| @@ -0,0 +1,355 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | #include <linux/device.h> | ||
| 25 | #include <linux/export.h> | ||
| 26 | #include <linux/err.h> | ||
| 27 | #include <linux/fs.h> | ||
| 28 | #include <linux/sched.h> | ||
| 29 | #include <linux/slab.h> | ||
| 30 | #include <linux/uaccess.h> | ||
| 31 | #include <linux/compat.h> | ||
| 32 | #include <uapi/linux/kfd_ioctl.h> | ||
| 33 | #include <linux/time.h> | ||
| 34 | #include "kfd_priv.h" | ||
| 35 | #include <linux/mm.h> | ||
| 36 | #include <uapi/asm-generic/mman-common.h> | ||
| 37 | #include <asm/processor.h> | ||
| 38 | |||
| 39 | /* | ||
| 40 | * The primary memory I/O features being added for revisions of gfxip | ||
| 41 | * beyond 7.0 (Kaveri) are: | ||
| 42 | * | ||
| 43 | * Access to ATC/IOMMU mapped memory w/ associated extension of VA to 48b | ||
| 44 | * | ||
| 45 | * “Flat” shader memory access – These are new shader vector memory | ||
| 46 | * operations that do not reference a T#/V# so a “pointer” is what is | ||
| 47 | * sourced from the vector gprs for direct access to memory. | ||
| 48 | * This pointer space has the Shared(LDS) and Private(Scratch) memory | ||
| 49 | * mapped into this pointer space as apertures. | ||
| 50 | * The hardware then determines how to direct the memory request | ||
| 51 | * based on what apertures the request falls in. | ||
| 52 | * | ||
| 53 | * Unaligned support and alignment check | ||
| 54 | * | ||
| 55 | * | ||
| 56 | * System Unified Address - SUA | ||
| 57 | * | ||
| 58 | * The standard usage for GPU virtual addresses are that they are mapped by | ||
| 59 | * a set of page tables we call GPUVM and these page tables are managed by | ||
| 60 | * a combination of vidMM/driver software components. The current virtual | ||
| 61 | * address (VA) range for GPUVM is 40b. | ||
| 62 | * | ||
| 63 | * As of gfxip7.1 and beyond we’re adding the ability for compute memory | ||
| 64 | * clients (CP/RLC, DMA, SHADER(ifetch, scalar, and vector ops)) to access | ||
| 65 | * the same page tables used by host x86 processors and that are managed by | ||
| 66 | * the operating system. This is via a technique and hardware called ATC/IOMMU. | ||
| 67 | * The GPU has the capability of accessing both the GPUVM and ATC address | ||
| 68 | * spaces for a given VMID (process) simultaneously and we call this feature | ||
| 69 | * system unified address (SUA). | ||
| 70 | * | ||
| 71 | * There are three fundamental address modes of operation for a given VMID | ||
| 72 | * (process) on the GPU: | ||
| 73 | * | ||
| 74 | * HSA64 – 64b pointers and the default address space is ATC | ||
| 75 | * HSA32 – 32b pointers and the default address space is ATC | ||
| 76 | * GPUVM – 64b pointers and the default address space is GPUVM (driver | ||
| 77 | * model mode) | ||
| 78 | * | ||
| 79 | * | ||
| 80 | * HSA64 - ATC/IOMMU 64b | ||
| 81 | * | ||
| 82 | * A 64b pointer in the AMD64/IA64 CPU architecture is not fully utilized | ||
| 83 | * by the CPU so an AMD CPU can only access the high area | ||
| 84 | * (VA[63:47] == 0x1FFFF) and low area (VA[63:47 == 0) of the address space | ||
| 85 | * so the actual VA carried to translation is 48b. There is a “hole” in | ||
| 86 | * the middle of the 64b VA space. | ||
| 87 | * | ||
| 88 | * The GPU not only has access to all of the CPU accessible address space via | ||
| 89 | * ATC/IOMMU, but it also has access to the GPUVM address space. The “system | ||
| 90 | * unified address” feature (SUA) is the mapping of GPUVM and ATC address | ||
| 91 | * spaces into a unified pointer space. The method we take for 64b mode is | ||
| 92 | * to map the full 40b GPUVM address space into the hole of the 64b address | ||
| 93 | * space. | ||
| 94 | |||
| 95 | * The GPUVM_Base/GPUVM_Limit defines the aperture in the 64b space where we | ||
| 96 | * direct requests to be translated via GPUVM page tables instead of the | ||
| 97 | * IOMMU path. | ||
| 98 | * | ||
| 99 | * | ||
| 100 | * 64b to 49b Address conversion | ||
| 101 | * | ||
| 102 | * Note that there are still significant portions of unused regions (holes) | ||
| 103 | * in the 64b address space even for the GPU. There are several places in | ||
| 104 | * the pipeline (sw and hw), we wish to compress the 64b virtual address | ||
| 105 | * to a 49b address. This 49b address is constituted of an “ATC” bit | ||
| 106 | * plus a 48b virtual address. This 49b address is what is passed to the | ||
| 107 | * translation hardware. ATC==0 means the 48b address is a GPUVM address | ||
| 108 | * (max of 2^40 – 1) intended to be translated via GPUVM page tables. | ||
| 109 | * ATC==1 means the 48b address is intended to be translated via IOMMU | ||
| 110 | * page tables. | ||
| 111 | * | ||
| 112 | * A 64b pointer is compared to the apertures that are defined (Base/Limit), in | ||
| 113 | * this case the GPUVM aperture (red) is defined and if a pointer falls in this | ||
| 114 | * aperture, we subtract the GPUVM_Base address and set the ATC bit to zero | ||
| 115 | * as part of the 64b to 49b conversion. | ||
| 116 | * | ||
| 117 | * Where this 64b to 49b conversion is done is a function of the usage. | ||
| 118 | * Most GPU memory access is via memory objects where the driver builds | ||
| 119 | * a descriptor which consists of a base address and a memory access by | ||
| 120 | * the GPU usually consists of some kind of an offset or Cartesian coordinate | ||
| 121 | * that references this memory descriptor. This is the case for shader | ||
| 122 | * instructions that reference the T# or V# constants, or for specified | ||
| 123 | * locations of assets (ex. the shader program location). In these cases | ||
| 124 | * the driver is what handles the 64b to 49b conversion and the base | ||
| 125 | * address in the descriptor (ex. V# or T# or shader program location) | ||
| 126 | * is defined as a 48b address w/ an ATC bit. For this usage a given | ||
| 127 | * memory object cannot straddle multiple apertures in the 64b address | ||
| 128 | * space. For example a shader program cannot jump in/out between ATC | ||
| 129 | * and GPUVM space. | ||
| 130 | * | ||
| 131 | * In some cases we wish to pass a 64b pointer to the GPU hardware and | ||
| 132 | * the GPU hw does the 64b to 49b conversion before passing memory | ||
| 133 | * requests to the cache/memory system. This is the case for the | ||
| 134 | * S_LOAD and FLAT_* shader memory instructions where we have 64b pointers | ||
| 135 | * in scalar and vector GPRs respectively. | ||
| 136 | * | ||
| 137 | * In all cases (no matter where the 64b -> 49b conversion is done), the gfxip | ||
| 138 | * hardware sends a 48b address along w/ an ATC bit, to the memory controller | ||
| 139 | * on the memory request interfaces. | ||
| 140 | * | ||
| 141 | * <client>_MC_rdreq_atc // read request ATC bit | ||
| 142 | * | ||
| 143 | * 0 : <client>_MC_rdreq_addr is a GPUVM VA | ||
| 144 | * | ||
| 145 | * 1 : <client>_MC_rdreq_addr is a ATC VA | ||
| 146 | * | ||
| 147 | * | ||
| 148 | * “Spare” aperture (APE1) | ||
| 149 | * | ||
| 150 | * We use the GPUVM aperture to differentiate ATC vs. GPUVM, but we also use | ||
| 151 | * apertures to set the Mtype field for S_LOAD/FLAT_* ops which is input to the | ||
| 152 | * config tables for setting cache policies. The “spare” (APE1) aperture is | ||
| 153 | * motivated by getting a different Mtype from the default. | ||
| 154 | * The default aperture isn’t an actual base/limit aperture; it is just the | ||
| 155 | * address space that doesn’t hit any defined base/limit apertures. | ||
| 156 | * The following diagram is a complete picture of the gfxip7.x SUA apertures. | ||
| 157 | * The APE1 can be placed either below or above | ||
| 158 | * the hole (cannot be in the hole). | ||
| 159 | * | ||
| 160 | * | ||
| 161 | * General Aperture definitions and rules | ||
| 162 | * | ||
| 163 | * An aperture register definition consists of a Base, Limit, Mtype, and | ||
| 164 | * usually an ATC bit indicating which translation tables that aperture uses. | ||
| 165 | * In all cases (for SUA and DUA apertures discussed later), aperture base | ||
| 166 | * and limit definitions are 64KB aligned. | ||
| 167 | * | ||
| 168 | * <ape>_Base[63:0] = { <ape>_Base_register[63:16], 0x0000 } | ||
| 169 | * | ||
| 170 | * <ape>_Limit[63:0] = { <ape>_Limit_register[63:16], 0xFFFF } | ||
| 171 | * | ||
| 172 | * The base and limit are considered inclusive to an aperture so being | ||
| 173 | * inside an aperture means (address >= Base) AND (address <= Limit). | ||
| 174 | * | ||
| 175 | * In no case is a payload that straddles multiple apertures expected to work. | ||
| 176 | * For example a load_dword_x4 that starts in one aperture and ends in another, | ||
| 177 | * does not work. For the vector FLAT_* ops we have detection capability in | ||
| 178 | * the shader for reporting a “memory violation” back to the | ||
| 179 | * SQ block for use in traps. | ||
| 180 | * A memory violation results when an op falls into the hole, | ||
| 181 | * or a payload straddles multiple apertures. The S_LOAD instruction | ||
| 182 | * does not have this detection. | ||
| 183 | * | ||
| 184 | * Apertures cannot overlap. | ||
| 185 | * | ||
| 186 | * | ||
| 187 | * | ||
| 188 | * HSA32 - ATC/IOMMU 32b | ||
| 189 | * | ||
| 190 | * For HSA32 mode, the pointers are interpreted as 32 bits and use a single GPR | ||
| 191 | * instead of two for the S_LOAD and FLAT_* ops. The entire GPUVM space of 40b | ||
| 192 | * will not fit so there is only partial visibility to the GPUVM | ||
| 193 | * space (defined by the aperture) for S_LOAD and FLAT_* ops. | ||
| 194 | * There is no spare (APE1) aperture for HSA32 mode. | ||
| 195 | * | ||
| 196 | * | ||
| 197 | * GPUVM 64b mode (driver model) | ||
| 198 | * | ||
| 199 | * This mode is related to HSA64 in that the difference really is that | ||
| 200 | * the default aperture is GPUVM (ATC==0) and not ATC space. | ||
| 201 | * We have gfxip7.x hardware that has FLAT_* and S_LOAD support for | ||
| 202 | * SUA GPUVM mode, but does not support HSA32/HSA64. | ||
| 203 | * | ||
| 204 | * | ||
| 205 | * Device Unified Address - DUA | ||
| 206 | * | ||
| 207 | * Device unified address (DUA) is the name of the feature that maps the | ||
| 208 | * Shared(LDS) memory and Private(Scratch) memory into the overall address | ||
| 209 | * space for use by the new FLAT_* vector memory ops. The Shared and | ||
| 210 | * Private memories are mapped as apertures into the address space, | ||
| 211 | * and the hardware detects when a FLAT_* memory request is to be redirected | ||
| 212 | * to the LDS or Scratch memory when it falls into one of these apertures. | ||
| 213 | * Like the SUA apertures, the Shared/Private apertures are 64KB aligned and | ||
| 214 | * the base/limit is “in” the aperture. For both HSA64 and GPUVM SUA modes, | ||
| 215 | * the Shared/Private apertures are always placed in a limited selection of | ||
| 216 | * options in the hole of the 64b address space. For HSA32 mode, the | ||
| 217 | * Shared/Private apertures can be placed anywhere in the 32b space | ||
| 218 | * except at 0. | ||
| 219 | * | ||
| 220 | * | ||
| 221 | * HSA64 Apertures for FLAT_* vector ops | ||
| 222 | * | ||
| 223 | * For HSA64 SUA mode, the Shared and Private apertures are always placed | ||
| 224 | * in the hole w/ a limited selection of possible locations. The requests | ||
| 225 | * that fall in the private aperture are expanded as a function of the | ||
| 226 | * work-item id (tid) and redirected to the location of the | ||
| 227 | * “hidden private memory”. The hidden private can be placed in either GPUVM | ||
| 228 | * or ATC space. The addresses that fall in the shared aperture are | ||
| 229 | * re-directed to the on-chip LDS memory hardware. | ||
| 230 | * | ||
| 231 | * | ||
| 232 | * HSA32 Apertures for FLAT_* vector ops | ||
| 233 | * | ||
| 234 | * In HSA32 mode, the Private and Shared apertures can be placed anywhere | ||
| 235 | * in the 32b space except at 0 (Private or Shared Base at zero disables | ||
| 236 | * the apertures). If the base address of the apertures are non-zero | ||
| 237 | * (ie apertures exists), the size is always 64KB. | ||
| 238 | * | ||
| 239 | * | ||
| 240 | * GPUVM Apertures for FLAT_* vector ops | ||
| 241 | * | ||
| 242 | * In GPUVM mode, the Shared/Private apertures are specified identically | ||
| 243 | * to HSA64 mode where they are always in the hole at a limited selection | ||
| 244 | * of locations. | ||
| 245 | * | ||
| 246 | * | ||
| 247 | * Aperture Definitions for SUA and DUA | ||
| 248 | * | ||
| 249 | * The interpretation of the aperture register definitions for a given | ||
| 250 | * VMID is a function of the “SUA Mode” which is one of HSA64, HSA32, or | ||
| 251 | * GPUVM64 discussed in previous sections. The mode is first decoded, and | ||
| 252 | * then the remaining register decode is a function of the mode. | ||
| 253 | * | ||
| 254 | * | ||
| 255 | * SUA Mode Decode | ||
| 256 | * | ||
| 257 | * For the S_LOAD and FLAT_* shader operations, the SUA mode is decoded from | ||
| 258 | * the COMPUTE_DISPATCH_INITIATOR:DATA_ATC bit and | ||
| 259 | * the SH_MEM_CONFIG:PTR32 bits. | ||
| 260 | * | ||
| 261 | * COMPUTE_DISPATCH_INITIATOR:DATA_ATC SH_MEM_CONFIG:PTR32 Mode | ||
| 262 | * | ||
| 263 | * 1 0 HSA64 | ||
| 264 | * | ||
| 265 | * 1 1 HSA32 | ||
| 266 | * | ||
| 267 | * 0 X GPUVM64 | ||
| 268 | * | ||
| 269 | * In general the hardware will ignore the PTR32 bit and treat | ||
| 270 | * as “0” whenever DATA_ATC = “0”, but sw should set PTR32=0 | ||
| 271 | * when DATA_ATC=0. | ||
| 272 | * | ||
| 273 | * The DATA_ATC bit is only set for compute dispatches. | ||
| 274 | * All “Draw” dispatches are hardcoded to GPUVM64 mode | ||
| 275 | * for FLAT_* / S_LOAD operations. | ||
| 276 | */ | ||
| 277 | |||
| 278 | #define MAKE_GPUVM_APP_BASE(gpu_num) \ | ||
| 279 | (((uint64_t)(gpu_num) << 61) + 0x1000000000000) | ||
| 280 | |||
| 281 | #define MAKE_GPUVM_APP_LIMIT(base) \ | ||
| 282 | (((uint64_t)(base) & 0xFFFFFF0000000000) | 0xFFFFFFFFFF) | ||
| 283 | |||
| 284 | #define MAKE_SCRATCH_APP_BASE(gpu_num) \ | ||
| 285 | (((uint64_t)(gpu_num) << 61) + 0x100000000) | ||
| 286 | |||
| 287 | #define MAKE_SCRATCH_APP_LIMIT(base) \ | ||
| 288 | (((uint64_t)base & 0xFFFFFFFF00000000) | 0xFFFFFFFF) | ||
| 289 | |||
| 290 | #define MAKE_LDS_APP_BASE(gpu_num) \ | ||
| 291 | (((uint64_t)(gpu_num) << 61) + 0x0) | ||
| 292 | #define MAKE_LDS_APP_LIMIT(base) \ | ||
| 293 | (((uint64_t)(base) & 0xFFFFFFFF00000000) | 0xFFFFFFFF) | ||
| 294 | |||
| 295 | int kfd_init_apertures(struct kfd_process *process) | ||
| 296 | { | ||
| 297 | uint8_t id = 0; | ||
| 298 | struct kfd_dev *dev; | ||
| 299 | struct kfd_process_device *pdd; | ||
| 300 | |||
| 301 | mutex_lock(&process->mutex); | ||
| 302 | |||
| 303 | /*Iterating over all devices*/ | ||
| 304 | while ((dev = kfd_topology_enum_kfd_devices(id)) != NULL && | ||
| 305 | id < NUM_OF_SUPPORTED_GPUS) { | ||
| 306 | |||
| 307 | pdd = kfd_get_process_device_data(dev, process, 1); | ||
| 308 | |||
| 309 | /* | ||
| 310 | * For 64 bit process aperture will be statically reserved in | ||
| 311 | * the x86_64 non canonical process address space | ||
| 312 | * amdkfd doesn't currently support apertures for 32 bit process | ||
| 313 | */ | ||
| 314 | if (process->is_32bit_user_mode) { | ||
| 315 | pdd->lds_base = pdd->lds_limit = 0; | ||
| 316 | pdd->gpuvm_base = pdd->gpuvm_limit = 0; | ||
| 317 | pdd->scratch_base = pdd->scratch_limit = 0; | ||
| 318 | } else { | ||
| 319 | /* | ||
| 320 | * node id couldn't be 0 - the three MSB bits of | ||
| 321 | * aperture shoudn't be 0 | ||
| 322 | */ | ||
| 323 | pdd->lds_base = MAKE_LDS_APP_BASE(id + 1); | ||
| 324 | |||
| 325 | pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base); | ||
| 326 | |||
| 327 | pdd->gpuvm_base = MAKE_GPUVM_APP_BASE(id + 1); | ||
| 328 | |||
| 329 | pdd->gpuvm_limit = | ||
| 330 | MAKE_GPUVM_APP_LIMIT(pdd->gpuvm_base); | ||
| 331 | |||
| 332 | pdd->scratch_base = MAKE_SCRATCH_APP_BASE(id + 1); | ||
| 333 | |||
| 334 | pdd->scratch_limit = | ||
| 335 | MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base); | ||
| 336 | } | ||
| 337 | |||
| 338 | dev_dbg(kfd_device, "node id %u\n", id); | ||
| 339 | dev_dbg(kfd_device, "gpu id %u\n", pdd->dev->id); | ||
| 340 | dev_dbg(kfd_device, "lds_base %llX\n", pdd->lds_base); | ||
| 341 | dev_dbg(kfd_device, "lds_limit %llX\n", pdd->lds_limit); | ||
| 342 | dev_dbg(kfd_device, "gpuvm_base %llX\n", pdd->gpuvm_base); | ||
| 343 | dev_dbg(kfd_device, "gpuvm_limit %llX\n", pdd->gpuvm_limit); | ||
| 344 | dev_dbg(kfd_device, "scratch_base %llX\n", pdd->scratch_base); | ||
| 345 | dev_dbg(kfd_device, "scratch_limit %llX\n", pdd->scratch_limit); | ||
| 346 | |||
| 347 | id++; | ||
| 348 | } | ||
| 349 | |||
| 350 | mutex_unlock(&process->mutex); | ||
| 351 | |||
| 352 | return 0; | ||
| 353 | } | ||
| 354 | |||
| 355 | |||
| @@ -0,0 +1,176 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | /* | ||
| 24 | * KFD Interrupts. | ||
| 25 | * | ||
| 26 | * AMD GPUs deliver interrupts by pushing an interrupt description onto the | ||
| 27 | * interrupt ring and then sending an interrupt. KGD receives the interrupt | ||
| 28 | * in ISR and sends us a pointer to each new entry on the interrupt ring. | ||
| 29 | * | ||
| 30 | * We generally can't process interrupt-signaled events from ISR, so we call | ||
| 31 | * out to each interrupt client module (currently only the scheduler) to ask if | ||
| 32 | * each interrupt is interesting. If they return true, then it requires further | ||
| 33 | * processing so we copy it to an internal interrupt ring and call each | ||
| 34 | * interrupt client again from a work-queue. | ||
| 35 | * | ||
| 36 | * There's no acknowledgment for the interrupts we use. The hardware simply | ||
| 37 | * queues a new interrupt each time without waiting. | ||
| 38 | * | ||
| 39 | * The fixed-size internal queue means that it's possible for us to lose | ||
| 40 | * interrupts because we have no back-pressure to the hardware. | ||
| 41 | */ | ||
| 42 | |||
| 43 | #include <linux/slab.h> | ||
| 44 | #include <linux/device.h> | ||
| 45 | #include "kfd_priv.h" | ||
| 46 | |||
| 47 | #define KFD_INTERRUPT_RING_SIZE 256 | ||
| 48 | |||
| 49 | static void interrupt_wq(struct work_struct *); | ||
| 50 | |||
| 51 | int kfd_interrupt_init(struct kfd_dev *kfd) | ||
| 52 | { | ||
| 53 | void *interrupt_ring = kmalloc_array(KFD_INTERRUPT_RING_SIZE, | ||
| 54 | kfd->device_info->ih_ring_entry_size, | ||
| 55 | GFP_KERNEL); | ||
| 56 | if (!interrupt_ring) | ||
| 57 | return -ENOMEM; | ||
| 58 | |||
| 59 | kfd->interrupt_ring = interrupt_ring; | ||
| 60 | kfd->interrupt_ring_size = | ||
| 61 | KFD_INTERRUPT_RING_SIZE * kfd->device_info->ih_ring_entry_size; | ||
| 62 | atomic_set(&kfd->interrupt_ring_wptr, 0); | ||
| 63 | atomic_set(&kfd->interrupt_ring_rptr, 0); | ||
| 64 | |||
| 65 | spin_lock_init(&kfd->interrupt_lock); | ||
| 66 | |||
| 67 | INIT_WORK(&kfd->interrupt_work, interrupt_wq); | ||
| 68 | |||
| 69 | kfd->interrupts_active = true; | ||
| 70 | |||
| 71 | /* | ||
| 72 | * After this function returns, the interrupt will be enabled. This | ||
| 73 | * barrier ensures that the interrupt running on a different processor | ||
| 74 | * sees all the above writes. | ||
| 75 | */ | ||
| 76 | smp_wmb(); | ||
| 77 | |||
| 78 | return 0; | ||
| 79 | } | ||
| 80 | |||
| 81 | void kfd_interrupt_exit(struct kfd_dev *kfd) | ||
| 82 | { | ||
| 83 | /* | ||
| 84 | * Stop the interrupt handler from writing to the ring and scheduling | ||
| 85 | * workqueue items. The spinlock ensures that any interrupt running | ||
| 86 | * after we have unlocked sees interrupts_active = false. | ||
| 87 | */ | ||
| 88 | unsigned long flags; | ||
| 89 | |||
| 90 | spin_lock_irqsave(&kfd->interrupt_lock, flags); | ||
| 91 | kfd->interrupts_active = false; | ||
| 92 | spin_unlock_irqrestore(&kfd->interrupt_lock, flags); | ||
| 93 | |||
| 94 | /* | ||
| 95 | * Flush_scheduled_work ensures that there are no outstanding | ||
| 96 | * work-queue items that will access interrupt_ring. New work items | ||
| 97 | * can't be created because we stopped interrupt handling above. | ||
| 98 | */ | ||
| 99 | flush_scheduled_work(); | ||
| 100 | |||
| 101 | kfree(kfd->interrupt_ring); | ||
| 102 | } | ||
| 103 | |||
| 104 | /* | ||
| 105 | * This assumes that it can't be called concurrently with itself | ||
| 106 | * but only with dequeue_ih_ring_entry. | ||
| 107 | */ | ||
| 108 | bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry) | ||
| 109 | { | ||
| 110 | unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr); | ||
| 111 | unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr); | ||
| 112 | |||
| 113 | if ((rptr - wptr) % kfd->interrupt_ring_size == | ||
| 114 | kfd->device_info->ih_ring_entry_size) { | ||
| 115 | /* This is very bad, the system is likely to hang. */ | ||
| 116 | dev_err_ratelimited(kfd_chardev(), | ||
| 117 | "Interrupt ring overflow, dropping interrupt.\n"); | ||
| 118 | return false; | ||
| 119 | } | ||
| 120 | |||
| 121 | memcpy(kfd->interrupt_ring + wptr, ih_ring_entry, | ||
| 122 | kfd->device_info->ih_ring_entry_size); | ||
| 123 | |||
| 124 | wptr = (wptr + kfd->device_info->ih_ring_entry_size) % | ||
| 125 | kfd->interrupt_ring_size; | ||
| 126 | smp_wmb(); /* Ensure memcpy'd data is visible before wptr update. */ | ||
| 127 | atomic_set(&kfd->interrupt_ring_wptr, wptr); | ||
| 128 | |||
| 129 | return true; | ||
| 130 | } | ||
| 131 | |||
| 132 | /* | ||
| 133 | * This assumes that it can't be called concurrently with itself | ||
| 134 | * but only with enqueue_ih_ring_entry. | ||
| 135 | */ | ||
| 136 | static bool dequeue_ih_ring_entry(struct kfd_dev *kfd, void *ih_ring_entry) | ||
| 137 | { | ||
| 138 | /* | ||
| 139 | * Assume that wait queues have an implicit barrier, i.e. anything that | ||
| 140 | * happened in the ISR before it queued work is visible. | ||
| 141 | */ | ||
| 142 | |||
| 143 | unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr); | ||
| 144 | unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr); | ||
| 145 | |||
| 146 | if (rptr == wptr) | ||
| 147 | return false; | ||
| 148 | |||
| 149 | memcpy(ih_ring_entry, kfd->interrupt_ring + rptr, | ||
| 150 | kfd->device_info->ih_ring_entry_size); | ||
| 151 | |||
| 152 | rptr = (rptr + kfd->device_info->ih_ring_entry_size) % | ||
| 153 | kfd->interrupt_ring_size; | ||
| 154 | |||
| 155 | /* | ||
| 156 | * Ensure the rptr write update is not visible until | ||
| 157 | * memcpy has finished reading. | ||
| 158 | */ | ||
| 159 | smp_mb(); | ||
| 160 | atomic_set(&kfd->interrupt_ring_rptr, rptr); | ||
| 161 | |||
| 162 | return true; | ||
| 163 | } | ||
| 164 | |||
| 165 | static void interrupt_wq(struct work_struct *work) | ||
| 166 | { | ||
| 167 | struct kfd_dev *dev = container_of(work, struct kfd_dev, | ||
| 168 | interrupt_work); | ||
| 169 | |||
| 170 | uint32_t ih_ring_entry[DIV_ROUND_UP( | ||
| 171 | dev->device_info->ih_ring_entry_size, | ||
| 172 | sizeof(uint32_t))]; | ||
| 173 | |||
| 174 | while (dequeue_ih_ring_entry(dev, ih_ring_entry)) | ||
| 175 | ; | ||
| 176 | } | ||
| @@ -0,0 +1,347 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | #include <linux/types.h> | ||
| 25 | #include <linux/mutex.h> | ||
| 26 | #include <linux/slab.h> | ||
| 27 | #include <linux/printk.h> | ||
| 28 | #include "kfd_kernel_queue.h" | ||
| 29 | #include "kfd_priv.h" | ||
| 30 | #include "kfd_device_queue_manager.h" | ||
| 31 | #include "kfd_pm4_headers.h" | ||
| 32 | #include "kfd_pm4_opcodes.h" | ||
| 33 | |||
| 34 | #define PM4_COUNT_ZERO (((1 << 15) - 1) << 16) | ||
| 35 | |||
| 36 | static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev, | ||
| 37 | enum kfd_queue_type type, unsigned int queue_size) | ||
| 38 | { | ||
| 39 | struct queue_properties prop; | ||
| 40 | int retval; | ||
| 41 | union PM4_MES_TYPE_3_HEADER nop; | ||
| 42 | |||
| 43 | BUG_ON(!kq || !dev); | ||
| 44 | BUG_ON(type != KFD_QUEUE_TYPE_DIQ && type != KFD_QUEUE_TYPE_HIQ); | ||
| 45 | |||
| 46 | pr_debug("kfd: In func %s initializing queue type %d size %d\n", | ||
| 47 | __func__, KFD_QUEUE_TYPE_HIQ, queue_size); | ||
| 48 | |||
| 49 | nop.opcode = IT_NOP; | ||
| 50 | nop.type = PM4_TYPE_3; | ||
| 51 | nop.u32all |= PM4_COUNT_ZERO; | ||
| 52 | |||
| 53 | kq->dev = dev; | ||
| 54 | kq->nop_packet = nop.u32all; | ||
| 55 | switch (type) { | ||
| 56 | case KFD_QUEUE_TYPE_DIQ: | ||
| 57 | case KFD_QUEUE_TYPE_HIQ: | ||
| 58 | kq->mqd = dev->dqm->get_mqd_manager(dev->dqm, | ||
| 59 | KFD_MQD_TYPE_CIK_HIQ); | ||
| 60 | break; | ||
| 61 | default: | ||
| 62 | BUG(); | ||
| 63 | break; | ||
| 64 | } | ||
| 65 | |||
| 66 | if (kq->mqd == NULL) | ||
| 67 | return false; | ||
| 68 | |||
| 69 | prop.doorbell_ptr = | ||
| 70 | (uint32_t *)kfd_get_kernel_doorbell(dev, &prop.doorbell_off); | ||
| 71 | |||
| 72 | if (prop.doorbell_ptr == NULL) | ||
| 73 | goto err_get_kernel_doorbell; | ||
| 74 | |||
| 75 | retval = kfd2kgd->allocate_mem(dev->kgd, | ||
| 76 | queue_size, | ||
| 77 | PAGE_SIZE, | ||
| 78 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | ||
| 79 | (struct kgd_mem **) &kq->pq); | ||
| 80 | |||
| 81 | if (retval != 0) | ||
| 82 | goto err_pq_allocate_vidmem; | ||
| 83 | |||
| 84 | kq->pq_kernel_addr = kq->pq->cpu_ptr; | ||
| 85 | kq->pq_gpu_addr = kq->pq->gpu_addr; | ||
| 86 | |||
| 87 | retval = kfd2kgd->allocate_mem(dev->kgd, | ||
| 88 | sizeof(*kq->rptr_kernel), | ||
| 89 | 32, | ||
| 90 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | ||
| 91 | (struct kgd_mem **) &kq->rptr_mem); | ||
| 92 | |||
| 93 | if (retval != 0) | ||
| 94 | goto err_rptr_allocate_vidmem; | ||
| 95 | |||
| 96 | kq->rptr_kernel = kq->rptr_mem->cpu_ptr; | ||
| 97 | kq->rptr_gpu_addr = kq->rptr_mem->gpu_addr; | ||
| 98 | |||
| 99 | retval = kfd2kgd->allocate_mem(dev->kgd, | ||
| 100 | sizeof(*kq->wptr_kernel), | ||
| 101 | 32, | ||
| 102 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | ||
| 103 | (struct kgd_mem **) &kq->wptr_mem); | ||
| 104 | |||
| 105 | if (retval != 0) | ||
| 106 | goto err_wptr_allocate_vidmem; | ||
| 107 | |||
| 108 | kq->wptr_kernel = kq->wptr_mem->cpu_ptr; | ||
| 109 | kq->wptr_gpu_addr = kq->wptr_mem->gpu_addr; | ||
| 110 | |||
| 111 | memset(kq->pq_kernel_addr, 0, queue_size); | ||
| 112 | memset(kq->rptr_kernel, 0, sizeof(*kq->rptr_kernel)); | ||
| 113 | memset(kq->wptr_kernel, 0, sizeof(*kq->wptr_kernel)); | ||
| 114 | |||
| 115 | prop.queue_size = queue_size; | ||
| 116 | prop.is_interop = false; | ||
| 117 | prop.priority = 1; | ||
| 118 | prop.queue_percent = 100; | ||
| 119 | prop.type = type; | ||
| 120 | prop.vmid = 0; | ||
| 121 | prop.queue_address = kq->pq_gpu_addr; | ||
| 122 | prop.read_ptr = (uint32_t *) kq->rptr_gpu_addr; | ||
| 123 | prop.write_ptr = (uint32_t *) kq->wptr_gpu_addr; | ||
| 124 | |||
| 125 | if (init_queue(&kq->queue, prop) != 0) | ||
| 126 | goto err_init_queue; | ||
| 127 | |||
| 128 | kq->queue->device = dev; | ||
| 129 | kq->queue->process = kfd_get_process(current); | ||
| 130 | |||
| 131 | retval = kq->mqd->init_mqd(kq->mqd, &kq->queue->mqd, | ||
| 132 | &kq->queue->mqd_mem_obj, | ||
| 133 | &kq->queue->gart_mqd_addr, | ||
| 134 | &kq->queue->properties); | ||
| 135 | if (retval != 0) | ||
| 136 | goto err_init_mqd; | ||
| 137 | |||
| 138 | /* assign HIQ to HQD */ | ||
| 139 | if (type == KFD_QUEUE_TYPE_HIQ) { | ||
| 140 | pr_debug("assigning hiq to hqd\n"); | ||
| 141 | kq->queue->pipe = KFD_CIK_HIQ_PIPE; | ||
| 142 | kq->queue->queue = KFD_CIK_HIQ_QUEUE; | ||
| 143 | kq->mqd->load_mqd(kq->mqd, kq->queue->mqd, kq->queue->pipe, | ||
| 144 | kq->queue->queue, NULL); | ||
| 145 | } else { | ||
| 146 | /* allocate fence for DIQ */ | ||
| 147 | |||
| 148 | retval = kfd2kgd->allocate_mem(dev->kgd, | ||
| 149 | sizeof(uint32_t), | ||
| 150 | 32, | ||
| 151 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | ||
| 152 | (struct kgd_mem **) &kq->fence_mem_obj); | ||
| 153 | |||
| 154 | if (retval != 0) | ||
| 155 | goto err_alloc_fence; | ||
| 156 | |||
| 157 | kq->fence_kernel_address = kq->fence_mem_obj->cpu_ptr; | ||
| 158 | kq->fence_gpu_addr = kq->fence_mem_obj->gpu_addr; | ||
| 159 | } | ||
| 160 | |||
| 161 | print_queue(kq->queue); | ||
| 162 | |||
| 163 | return true; | ||
| 164 | err_alloc_fence: | ||
| 165 | err_init_mqd: | ||
| 166 | uninit_queue(kq->queue); | ||
| 167 | err_init_queue: | ||
| 168 | kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->wptr_mem); | ||
| 169 | err_wptr_allocate_vidmem: | ||
| 170 | kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->rptr_mem); | ||
| 171 | err_rptr_allocate_vidmem: | ||
| 172 | kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->pq); | ||
| 173 | err_pq_allocate_vidmem: | ||
| 174 | pr_err("kfd: error init pq\n"); | ||
| 175 | kfd_release_kernel_doorbell(dev, (u32 *)prop.doorbell_ptr); | ||
| 176 | err_get_kernel_doorbell: | ||
| 177 | pr_err("kfd: error init doorbell"); | ||
| 178 | return false; | ||
| 179 | |||
| 180 | } | ||
| 181 | |||
| 182 | static void uninitialize(struct kernel_queue *kq) | ||
| 183 | { | ||
| 184 | BUG_ON(!kq); | ||
| 185 | |||
| 186 | if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ) | ||
| 187 | kq->mqd->destroy_mqd(kq->mqd, | ||
| 188 | NULL, | ||
| 189 | false, | ||
| 190 | QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS, | ||
| 191 | kq->queue->pipe, | ||
| 192 | kq->queue->queue); | ||
| 193 | |||
| 194 | kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->rptr_mem); | ||
| 195 | kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->wptr_mem); | ||
| 196 | kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->pq); | ||
| 197 | kfd_release_kernel_doorbell(kq->dev, | ||
| 198 | (u32 *)kq->queue->properties.doorbell_ptr); | ||
| 199 | uninit_queue(kq->queue); | ||
| 200 | } | ||
| 201 | |||
| 202 | static int acquire_packet_buffer(struct kernel_queue *kq, | ||
| 203 | size_t packet_size_in_dwords, unsigned int **buffer_ptr) | ||
| 204 | { | ||
| 205 | size_t available_size; | ||
| 206 | size_t queue_size_dwords; | ||
| 207 | uint32_t wptr, rptr; | ||
| 208 | unsigned int *queue_address; | ||
| 209 | |||
| 210 | BUG_ON(!kq || !buffer_ptr); | ||
| 211 | |||
| 212 | rptr = *kq->rptr_kernel; | ||
| 213 | wptr = *kq->wptr_kernel; | ||
| 214 | queue_address = (unsigned int *)kq->pq_kernel_addr; | ||
| 215 | queue_size_dwords = kq->queue->properties.queue_size / sizeof(uint32_t); | ||
| 216 | |||
| 217 | pr_debug("kfd: In func %s\nrptr: %d\nwptr: %d\nqueue_address 0x%p\n", | ||
| 218 | __func__, rptr, wptr, queue_address); | ||
| 219 | |||
| 220 | available_size = (rptr - 1 - wptr + queue_size_dwords) % | ||
| 221 | queue_size_dwords; | ||
| 222 | |||
| 223 | if (packet_size_in_dwords >= queue_size_dwords || | ||
| 224 | packet_size_in_dwords >= available_size) | ||
| 225 | return -ENOMEM; | ||
| 226 | |||
| 227 | if (wptr + packet_size_in_dwords >= queue_size_dwords) { | ||
| 228 | while (wptr > 0) { | ||
| 229 | queue_address[wptr] = kq->nop_packet; | ||
| 230 | wptr = (wptr + 1) % queue_size_dwords; | ||
| 231 | } | ||
| 232 | } | ||
| 233 | |||
| 234 | *buffer_ptr = &queue_address[wptr]; | ||
| 235 | kq->pending_wptr = wptr + packet_size_in_dwords; | ||
| 236 | |||
| 237 | return 0; | ||
| 238 | } | ||
| 239 | |||
| 240 | static void submit_packet(struct kernel_queue *kq) | ||
| 241 | { | ||
| 242 | #ifdef DEBUG | ||
| 243 | int i; | ||
| 244 | #endif | ||
| 245 | |||
| 246 | BUG_ON(!kq); | ||
| 247 | |||
| 248 | #ifdef DEBUG | ||
| 249 | for (i = *kq->wptr_kernel; i < kq->pending_wptr; i++) { | ||
| 250 | pr_debug("0x%2X ", kq->pq_kernel_addr[i]); | ||
| 251 | if (i % 15 == 0) | ||
| 252 | pr_debug("\n"); | ||
| 253 | } | ||
| 254 | pr_debug("\n"); | ||
| 255 | #endif | ||
| 256 | |||
| 257 | *kq->wptr_kernel = kq->pending_wptr; | ||
| 258 | write_kernel_doorbell((u32 *)kq->queue->properties.doorbell_ptr, | ||
| 259 | kq->pending_wptr); | ||
| 260 | } | ||
| 261 | |||
| 262 | static int sync_with_hw(struct kernel_queue *kq, unsigned long timeout_ms) | ||
| 263 | { | ||
| 264 | unsigned long org_timeout_ms; | ||
| 265 | |||
| 266 | BUG_ON(!kq); | ||
| 267 | |||
| 268 | org_timeout_ms = timeout_ms; | ||
| 269 | timeout_ms += jiffies * 1000 / HZ; | ||
| 270 | while (*kq->wptr_kernel != *kq->rptr_kernel) { | ||
| 271 | if (time_after(jiffies * 1000 / HZ, timeout_ms)) { | ||
| 272 | pr_err("kfd: kernel_queue %s timeout expired %lu\n", | ||
| 273 | __func__, org_timeout_ms); | ||
| 274 | pr_err("kfd: wptr: %d rptr: %d\n", | ||
| 275 | *kq->wptr_kernel, *kq->rptr_kernel); | ||
| 276 | return -ETIME; | ||
| 277 | } | ||
| 278 | cpu_relax(); | ||
| 279 | } | ||
| 280 | |||
| 281 | return 0; | ||
| 282 | } | ||
| 283 | |||
| 284 | static void rollback_packet(struct kernel_queue *kq) | ||
| 285 | { | ||
| 286 | BUG_ON(!kq); | ||
| 287 | kq->pending_wptr = *kq->queue->properties.write_ptr; | ||
| 288 | } | ||
| 289 | |||
| 290 | struct kernel_queue *kernel_queue_init(struct kfd_dev *dev, | ||
| 291 | enum kfd_queue_type type) | ||
| 292 | { | ||
| 293 | struct kernel_queue *kq; | ||
| 294 | |||
| 295 | BUG_ON(!dev); | ||
| 296 | |||
| 297 | kq = kzalloc(sizeof(struct kernel_queue), GFP_KERNEL); | ||
| 298 | if (!kq) | ||
| 299 | return NULL; | ||
| 300 | |||
| 301 | kq->initialize = initialize; | ||
| 302 | kq->uninitialize = uninitialize; | ||
| 303 | kq->acquire_packet_buffer = acquire_packet_buffer; | ||
| 304 | kq->submit_packet = submit_packet; | ||
| 305 | kq->sync_with_hw = sync_with_hw; | ||
| 306 | kq->rollback_packet = rollback_packet; | ||
| 307 | |||
| 308 | if (kq->initialize(kq, dev, type, KFD_KERNEL_QUEUE_SIZE) == false) { | ||
| 309 | pr_err("kfd: failed to init kernel queue\n"); | ||
| 310 | kfree(kq); | ||
| 311 | return NULL; | ||
| 312 | } | ||
| 313 | return kq; | ||
| 314 | } | ||
| 315 | |||
| 316 | void kernel_queue_uninit(struct kernel_queue *kq) | ||
| 317 | { | ||
| 318 | BUG_ON(!kq); | ||
| 319 | |||
| 320 | kq->uninitialize(kq); | ||
| 321 | kfree(kq); | ||
| 322 | } | ||
| 323 | |||
| 324 | void test_kq(struct kfd_dev *dev) | ||
| 325 | { | ||
| 326 | struct kernel_queue *kq; | ||
| 327 | uint32_t *buffer, i; | ||
| 328 | int retval; | ||
| 329 | |||
| 330 | BUG_ON(!dev); | ||
| 331 | |||
| 332 | pr_debug("kfd: starting kernel queue test\n"); | ||
| 333 | |||
| 334 | kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_HIQ); | ||
| 335 | BUG_ON(!kq); | ||
| 336 | |||
| 337 | retval = kq->acquire_packet_buffer(kq, 5, &buffer); | ||
| 338 | BUG_ON(retval != 0); | ||
| 339 | for (i = 0; i < 5; i++) | ||
| 340 | buffer[i] = kq->nop_packet; | ||
| 341 | kq->submit_packet(kq); | ||
| 342 | kq->sync_with_hw(kq, 1000); | ||
| 343 | |||
| 344 | pr_debug("kfd: ending kernel queue test\n"); | ||
| 345 | } | ||
| 346 | |||
| 347 | |||
| @@ -0,0 +1,69 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | #ifndef KFD_KERNEL_QUEUE_H_ | ||
| 25 | #define KFD_KERNEL_QUEUE_H_ | ||
| 26 | |||
| 27 | #include <linux/list.h> | ||
| 28 | #include <linux/types.h> | ||
| 29 | #include "kfd_priv.h" | ||
| 30 | |||
| 31 | struct kernel_queue { | ||
| 32 | /* interface */ | ||
| 33 | bool (*initialize)(struct kernel_queue *kq, struct kfd_dev *dev, | ||
| 34 | enum kfd_queue_type type, unsigned int queue_size); | ||
| 35 | void (*uninitialize)(struct kernel_queue *kq); | ||
| 36 | int (*acquire_packet_buffer)(struct kernel_queue *kq, | ||
| 37 | size_t packet_size_in_dwords, | ||
| 38 | unsigned int **buffer_ptr); | ||
| 39 | |||
| 40 | void (*submit_packet)(struct kernel_queue *kq); | ||
| 41 | int (*sync_with_hw)(struct kernel_queue *kq, | ||
| 42 | unsigned long timeout_ms); | ||
| 43 | void (*rollback_packet)(struct kernel_queue *kq); | ||
| 44 | |||
| 45 | /* data */ | ||
| 46 | struct kfd_dev *dev; | ||
| 47 | struct mqd_manager *mqd; | ||
| 48 | struct queue *queue; | ||
| 49 | uint32_t pending_wptr; | ||
| 50 | unsigned int nop_packet; | ||
| 51 | |||
| 52 | struct kfd_mem_obj *rptr_mem; | ||
| 53 | uint32_t *rptr_kernel; | ||
| 54 | uint64_t rptr_gpu_addr; | ||
| 55 | struct kfd_mem_obj *wptr_mem; | ||
| 56 | uint32_t *wptr_kernel; | ||
| 57 | uint64_t wptr_gpu_addr; | ||
| 58 | struct kfd_mem_obj *pq; | ||
| 59 | uint64_t pq_gpu_addr; | ||
| 60 | uint32_t *pq_kernel_addr; | ||
| 61 | |||
| 62 | struct kfd_mem_obj *fence_mem_obj; | ||
| 63 | uint64_t fence_gpu_addr; | ||
| 64 | void *fence_kernel_address; | ||
| 65 | |||
| 66 | struct list_head list; | ||
| 67 | }; | ||
| 68 | |||
| 69 | #endif /* KFD_KERNEL_QUEUE_H_ */ | ||
| @@ -0,0 +1,159 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | #include <linux/module.h> | ||
| 24 | #include <linux/sched.h> | ||
| 25 | #include <linux/moduleparam.h> | ||
| 26 | #include <linux/device.h> | ||
| 27 | #include "kfd_priv.h" | ||
| 28 | |||
| 29 | #define KFD_DRIVER_AUTHOR "AMD Inc. and others" | ||
| 30 | |||
| 31 | #define KFD_DRIVER_DESC "Standalone HSA driver for AMD's GPUs" | ||
| 32 | #define KFD_DRIVER_DATE "20141113" | ||
| 33 | #define KFD_DRIVER_MAJOR 0 | ||
| 34 | #define KFD_DRIVER_MINOR 7 | ||
| 35 | #define KFD_DRIVER_PATCHLEVEL 0 | ||
| 36 | |||
| 37 | const struct kfd2kgd_calls *kfd2kgd; | ||
| 38 | static const struct kgd2kfd_calls kgd2kfd = { | ||
| 39 | .exit = kgd2kfd_exit, | ||
| 40 | .probe = kgd2kfd_probe, | ||
| 41 | .device_init = kgd2kfd_device_init, | ||
| 42 | .device_exit = kgd2kfd_device_exit, | ||
| 43 | .interrupt = kgd2kfd_interrupt, | ||
| 44 | .suspend = kgd2kfd_suspend, | ||
| 45 | .resume = kgd2kfd_resume, | ||
| 46 | }; | ||
| 47 | |||
| 48 | int sched_policy = KFD_SCHED_POLICY_HWS; | ||
| 49 | module_param(sched_policy, int, 0444); | ||
| 50 | MODULE_PARM_DESC(sched_policy, | ||
| 51 | "Kernel cmdline parameter that defines the amdkfd scheduling policy"); | ||
| 52 | |||
| 53 | int max_num_of_processes = KFD_MAX_NUM_OF_PROCESSES_DEFAULT; | ||
| 54 | module_param(max_num_of_processes, int, 0444); | ||
| 55 | MODULE_PARM_DESC(max_num_of_processes, | ||
| 56 | "Kernel cmdline parameter that defines the amdkfd maximum number of supported processes"); | ||
| 57 | |||
| 58 | int max_num_of_queues_per_process = KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT; | ||
| 59 | module_param(max_num_of_queues_per_process, int, 0444); | ||
| 60 | MODULE_PARM_DESC(max_num_of_queues_per_process, | ||
| 61 | "Kernel cmdline parameter that defines the amdkfd maximum number of supported queues per process"); | ||
| 62 | |||
| 63 | bool kgd2kfd_init(unsigned interface_version, | ||
| 64 | const struct kfd2kgd_calls *f2g, | ||
| 65 | const struct kgd2kfd_calls **g2f) | ||
| 66 | { | ||
| 67 | /* | ||
| 68 | * Only one interface version is supported, | ||
| 69 | * no kfd/kgd version skew allowed. | ||
| 70 | */ | ||
| 71 | if (interface_version != KFD_INTERFACE_VERSION) | ||
| 72 | return false; | ||
| 73 | |||
| 74 | /* Protection against multiple amd kgd loads */ | ||
| 75 | if (kfd2kgd) | ||
| 76 | return true; | ||
| 77 | |||
| 78 | kfd2kgd = f2g; | ||
| 79 | *g2f = &kgd2kfd; | ||
| 80 | |||
| 81 | return true; | ||
| 82 | } | ||
| 83 | EXPORT_SYMBOL(kgd2kfd_init); | ||
| 84 | |||
| 85 | void kgd2kfd_exit(void) | ||
| 86 | { | ||
| 87 | } | ||
| 88 | |||
| 89 | static int __init kfd_module_init(void) | ||
| 90 | { | ||
| 91 | int err; | ||
| 92 | |||
| 93 | kfd2kgd = NULL; | ||
| 94 | |||
| 95 | /* Verify module parameters */ | ||
| 96 | if ((sched_policy < KFD_SCHED_POLICY_HWS) || | ||
| 97 | (sched_policy > KFD_SCHED_POLICY_NO_HWS)) { | ||
| 98 | pr_err("kfd: sched_policy has invalid value\n"); | ||
| 99 | return -1; | ||
| 100 | } | ||
| 101 | |||
| 102 | /* Verify module parameters */ | ||
| 103 | if ((max_num_of_processes < 0) || | ||
| 104 | (max_num_of_processes > KFD_MAX_NUM_OF_PROCESSES)) { | ||
| 105 | pr_err("kfd: max_num_of_processes must be between 0 to KFD_MAX_NUM_OF_PROCESSES\n"); | ||
| 106 | return -1; | ||
| 107 | } | ||
| 108 | |||
| 109 | if ((max_num_of_queues_per_process < 0) || | ||
| 110 | (max_num_of_queues_per_process > | ||
| 111 | KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)) { | ||
| 112 | pr_err("kfd: max_num_of_queues_per_process must be between 0 to KFD_MAX_NUM_OF_QUEUES_PER_PROCESS\n"); | ||
| 113 | return -1; | ||
| 114 | } | ||
| 115 | |||
| 116 | err = kfd_pasid_init(); | ||
| 117 | if (err < 0) | ||
| 118 | goto err_pasid; | ||
| 119 | |||
| 120 | err = kfd_chardev_init(); | ||
| 121 | if (err < 0) | ||
| 122 | goto err_ioctl; | ||
| 123 | |||
| 124 | err = kfd_topology_init(); | ||
| 125 | if (err < 0) | ||
| 126 | goto err_topology; | ||
| 127 | |||
| 128 | kfd_process_create_wq(); | ||
| 129 | |||
| 130 | dev_info(kfd_device, "Initialized module\n"); | ||
| 131 | |||
| 132 | return 0; | ||
| 133 | |||
| 134 | err_topology: | ||
| 135 | kfd_chardev_exit(); | ||
| 136 | err_ioctl: | ||
| 137 | kfd_pasid_exit(); | ||
| 138 | err_pasid: | ||
| 139 | return err; | ||
| 140 | } | ||
| 141 | |||
| 142 | static void __exit kfd_module_exit(void) | ||
| 143 | { | ||
| 144 | kfd_process_destroy_wq(); | ||
| 145 | kfd_topology_shutdown(); | ||
| 146 | kfd_chardev_exit(); | ||
| 147 | kfd_pasid_exit(); | ||
| 148 | dev_info(kfd_device, "Removed module\n"); | ||
| 149 | } | ||
| 150 | |||
| 151 | module_init(kfd_module_init); | ||
| 152 | module_exit(kfd_module_exit); | ||
| 153 | |||
| 154 | MODULE_AUTHOR(KFD_DRIVER_AUTHOR); | ||
| 155 | MODULE_DESCRIPTION(KFD_DRIVER_DESC); | ||
| 156 | MODULE_LICENSE("GPL and additional rights"); | ||
| 157 | MODULE_VERSION(__stringify(KFD_DRIVER_MAJOR) "." | ||
| 158 | __stringify(KFD_DRIVER_MINOR) "." | ||
| 159 | __stringify(KFD_DRIVER_PATCHLEVEL)); | ||
| @@ -0,0 +1,346 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | #include <linux/printk.h> | ||
| 25 | #include <linux/slab.h> | ||
| 26 | #include "kfd_priv.h" | ||
| 27 | #include "kfd_mqd_manager.h" | ||
| 28 | #include "cik_regs.h" | ||
| 29 | #include "../../radeon/cik_reg.h" | ||
| 30 | |||
| 31 | inline void busy_wait(unsigned long ms) | ||
| 32 | { | ||
| 33 | while (time_before(jiffies, ms)) | ||
| 34 | cpu_relax(); | ||
| 35 | } | ||
| 36 | |||
| 37 | static inline struct cik_mqd *get_mqd(void *mqd) | ||
| 38 | { | ||
| 39 | return (struct cik_mqd *)mqd; | ||
| 40 | } | ||
| 41 | |||
| 42 | static int init_mqd(struct mqd_manager *mm, void **mqd, | ||
| 43 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | ||
| 44 | struct queue_properties *q) | ||
| 45 | { | ||
| 46 | uint64_t addr; | ||
| 47 | struct cik_mqd *m; | ||
| 48 | int retval; | ||
| 49 | |||
| 50 | BUG_ON(!mm || !q || !mqd); | ||
| 51 | |||
| 52 | pr_debug("kfd: In func %s\n", __func__); | ||
| 53 | |||
| 54 | retval = kfd2kgd->allocate_mem(mm->dev->kgd, | ||
| 55 | sizeof(struct cik_mqd), | ||
| 56 | 256, | ||
| 57 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | ||
| 58 | (struct kgd_mem **) mqd_mem_obj); | ||
| 59 | |||
| 60 | if (retval != 0) | ||
| 61 | return -ENOMEM; | ||
| 62 | |||
| 63 | m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; | ||
| 64 | addr = (*mqd_mem_obj)->gpu_addr; | ||
| 65 | |||
| 66 | memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); | ||
| 67 | |||
| 68 | m->header = 0xC0310800; | ||
| 69 | m->compute_pipelinestat_enable = 1; | ||
| 70 | m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | ||
| 71 | m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | ||
| 72 | m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | ||
| 73 | m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | ||
| 74 | |||
| 75 | /* | ||
| 76 | * Make sure to use the last queue state saved on mqd when the cp | ||
| 77 | * reassigns the queue, so when queue is switched on/off (e.g over | ||
| 78 | * subscription or quantum timeout) the context will be consistent | ||
| 79 | */ | ||
| 80 | m->cp_hqd_persistent_state = | ||
| 81 | DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ; | ||
| 82 | |||
| 83 | m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; | ||
| 84 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); | ||
| 85 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); | ||
| 86 | |||
| 87 | m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN; | ||
| 88 | /* Although WinKFD writes this, I suspect it should not be necessary */ | ||
| 89 | m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE; | ||
| 90 | |||
| 91 | m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | | ||
| 92 | QUANTUM_DURATION(10); | ||
| 93 | |||
| 94 | /* | ||
| 95 | * Pipe Priority | ||
| 96 | * Identifies the pipe relative priority when this queue is connected | ||
| 97 | * to the pipeline. The pipe priority is against the GFX pipe and HP3D. | ||
| 98 | * In KFD we are using a fixed pipe priority set to CS_MEDIUM. | ||
| 99 | * 0 = CS_LOW (typically below GFX) | ||
| 100 | * 1 = CS_MEDIUM (typically between HP3D and GFX | ||
| 101 | * 2 = CS_HIGH (typically above HP3D) | ||
| 102 | */ | ||
| 103 | m->cp_hqd_pipe_priority = 1; | ||
| 104 | m->cp_hqd_queue_priority = 15; | ||
| 105 | |||
| 106 | *mqd = m; | ||
| 107 | if (gart_addr != NULL) | ||
| 108 | *gart_addr = addr; | ||
| 109 | retval = mm->update_mqd(mm, m, q); | ||
| 110 | |||
| 111 | return retval; | ||
| 112 | } | ||
| 113 | |||
| 114 | static void uninit_mqd(struct mqd_manager *mm, void *mqd, | ||
| 115 | struct kfd_mem_obj *mqd_mem_obj) | ||
| 116 | { | ||
| 117 | BUG_ON(!mm || !mqd); | ||
| 118 | kfd2kgd->free_mem(mm->dev->kgd, (struct kgd_mem *) mqd_mem_obj); | ||
| 119 | } | ||
| 120 | |||
| 121 | static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id, | ||
| 122 | uint32_t queue_id, uint32_t __user *wptr) | ||
| 123 | { | ||
| 124 | return kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id, wptr); | ||
| 125 | |||
| 126 | } | ||
| 127 | |||
| 128 | static int update_mqd(struct mqd_manager *mm, void *mqd, | ||
| 129 | struct queue_properties *q) | ||
| 130 | { | ||
| 131 | struct cik_mqd *m; | ||
| 132 | |||
| 133 | BUG_ON(!mm || !q || !mqd); | ||
| 134 | |||
| 135 | pr_debug("kfd: In func %s\n", __func__); | ||
| 136 | |||
| 137 | m = get_mqd(mqd); | ||
| 138 | m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | ||
| 139 | DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN; | ||
| 140 | |||
| 141 | /* | ||
| 142 | * Calculating queue size which is log base 2 of actual queue size -1 | ||
| 143 | * dwords and another -1 for ffs | ||
| 144 | */ | ||
| 145 | m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) | ||
| 146 | - 1 - 1; | ||
| 147 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); | ||
| 148 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | ||
| 149 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | ||
| 150 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | ||
| 151 | m->cp_hqd_pq_doorbell_control = DOORBELL_EN | | ||
| 152 | DOORBELL_OFFSET(q->doorbell_off); | ||
| 153 | |||
| 154 | m->cp_hqd_vmid = q->vmid; | ||
| 155 | |||
| 156 | if (q->format == KFD_QUEUE_FORMAT_AQL) { | ||
| 157 | m->cp_hqd_iq_rptr = AQL_ENABLE; | ||
| 158 | m->cp_hqd_pq_control |= NO_UPDATE_RPTR; | ||
| 159 | } | ||
| 160 | |||
| 161 | m->cp_hqd_active = 0; | ||
| 162 | q->is_active = false; | ||
| 163 | if (q->queue_size > 0 && | ||
| 164 | q->queue_address != 0 && | ||
| 165 | q->queue_percent > 0) { | ||
| 166 | m->cp_hqd_active = 1; | ||
| 167 | q->is_active = true; | ||
| 168 | } | ||
| 169 | |||
| 170 | return 0; | ||
| 171 | } | ||
| 172 | |||
| 173 | static int destroy_mqd(struct mqd_manager *mm, void *mqd, | ||
| 174 | enum kfd_preempt_type type, | ||
| 175 | unsigned int timeout, uint32_t pipe_id, | ||
| 176 | uint32_t queue_id) | ||
| 177 | { | ||
| 178 | return kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout, | ||
| 179 | pipe_id, queue_id); | ||
| 180 | } | ||
| 181 | |||
| 182 | bool is_occupied(struct mqd_manager *mm, void *mqd, | ||
| 183 | uint64_t queue_address, uint32_t pipe_id, | ||
| 184 | uint32_t queue_id) | ||
| 185 | { | ||
| 186 | |||
| 187 | return kfd2kgd->hqd_is_occupies(mm->dev->kgd, queue_address, | ||
| 188 | pipe_id, queue_id); | ||
| 189 | |||
| 190 | } | ||
| 191 | |||
| 192 | /* | ||
| 193 | * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation. | ||
| 194 | * The HIQ queue in Kaveri is using the same MQD structure as all the user mode | ||
| 195 | * queues but with different initial values. | ||
| 196 | */ | ||
| 197 | |||
| 198 | static int init_mqd_hiq(struct mqd_manager *mm, void **mqd, | ||
| 199 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | ||
| 200 | struct queue_properties *q) | ||
| 201 | { | ||
| 202 | uint64_t addr; | ||
| 203 | struct cik_mqd *m; | ||
| 204 | int retval; | ||
| 205 | |||
| 206 | BUG_ON(!mm || !q || !mqd || !mqd_mem_obj); | ||
| 207 | |||
| 208 | pr_debug("kfd: In func %s\n", __func__); | ||
| 209 | |||
| 210 | retval = kfd2kgd->allocate_mem(mm->dev->kgd, | ||
| 211 | sizeof(struct cik_mqd), | ||
| 212 | 256, | ||
| 213 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | ||
| 214 | (struct kgd_mem **) mqd_mem_obj); | ||
| 215 | |||
| 216 | if (retval != 0) | ||
| 217 | return -ENOMEM; | ||
| 218 | |||
| 219 | m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; | ||
| 220 | addr = (*mqd_mem_obj)->gpu_addr; | ||
| 221 | |||
| 222 | memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); | ||
| 223 | |||
| 224 | m->header = 0xC0310800; | ||
| 225 | m->compute_pipelinestat_enable = 1; | ||
| 226 | m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | ||
| 227 | m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | ||
| 228 | m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | ||
| 229 | m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | ||
| 230 | |||
| 231 | m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE | | ||
| 232 | PRELOAD_REQ; | ||
| 233 | m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | | ||
| 234 | QUANTUM_DURATION(10); | ||
| 235 | |||
| 236 | m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; | ||
| 237 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); | ||
| 238 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); | ||
| 239 | |||
| 240 | m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE; | ||
| 241 | |||
| 242 | /* | ||
| 243 | * Pipe Priority | ||
| 244 | * Identifies the pipe relative priority when this queue is connected | ||
| 245 | * to the pipeline. The pipe priority is against the GFX pipe and HP3D. | ||
| 246 | * In KFD we are using a fixed pipe priority set to CS_MEDIUM. | ||
| 247 | * 0 = CS_LOW (typically below GFX) | ||
| 248 | * 1 = CS_MEDIUM (typically between HP3D and GFX | ||
| 249 | * 2 = CS_HIGH (typically above HP3D) | ||
| 250 | */ | ||
| 251 | m->cp_hqd_pipe_priority = 1; | ||
| 252 | m->cp_hqd_queue_priority = 15; | ||
| 253 | |||
| 254 | *mqd = m; | ||
| 255 | if (gart_addr) | ||
| 256 | *gart_addr = addr; | ||
| 257 | retval = mm->update_mqd(mm, m, q); | ||
| 258 | |||
| 259 | return retval; | ||
| 260 | } | ||
| 261 | |||
| 262 | static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, | ||
| 263 | struct queue_properties *q) | ||
| 264 | { | ||
| 265 | struct cik_mqd *m; | ||
| 266 | |||
| 267 | BUG_ON(!mm || !q || !mqd); | ||
| 268 | |||
| 269 | pr_debug("kfd: In func %s\n", __func__); | ||
| 270 | |||
| 271 | m = get_mqd(mqd); | ||
| 272 | m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | ||
| 273 | DEFAULT_MIN_AVAIL_SIZE | | ||
| 274 | PRIV_STATE | | ||
| 275 | KMD_QUEUE; | ||
| 276 | |||
| 277 | /* | ||
| 278 | * Calculating queue size which is log base 2 of actual queue | ||
| 279 | * size -1 dwords | ||
| 280 | */ | ||
| 281 | m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) | ||
| 282 | - 1 - 1; | ||
| 283 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); | ||
| 284 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | ||
| 285 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | ||
| 286 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | ||
| 287 | m->cp_hqd_pq_doorbell_control = DOORBELL_EN | | ||
| 288 | DOORBELL_OFFSET(q->doorbell_off); | ||
| 289 | |||
| 290 | m->cp_hqd_vmid = q->vmid; | ||
| 291 | |||
| 292 | m->cp_hqd_active = 0; | ||
| 293 | q->is_active = false; | ||
| 294 | if (q->queue_size > 0 && | ||
| 295 | q->queue_address != 0 && | ||
| 296 | q->queue_percent > 0) { | ||
| 297 | m->cp_hqd_active = 1; | ||
| 298 | q->is_active = true; | ||
| 299 | } | ||
| 300 | |||
| 301 | return 0; | ||
| 302 | } | ||
| 303 | |||
| 304 | struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, | ||
| 305 | struct kfd_dev *dev) | ||
| 306 | { | ||
| 307 | struct mqd_manager *mqd; | ||
| 308 | |||
| 309 | BUG_ON(!dev); | ||
| 310 | BUG_ON(type >= KFD_MQD_TYPE_MAX); | ||
| 311 | |||
| 312 | pr_debug("kfd: In func %s\n", __func__); | ||
| 313 | |||
| 314 | mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL); | ||
| 315 | if (!mqd) | ||
| 316 | return NULL; | ||
| 317 | |||
| 318 | mqd->dev = dev; | ||
| 319 | |||
| 320 | switch (type) { | ||
| 321 | case KFD_MQD_TYPE_CIK_CP: | ||
| 322 | case KFD_MQD_TYPE_CIK_COMPUTE: | ||
| 323 | mqd->init_mqd = init_mqd; | ||
| 324 | mqd->uninit_mqd = uninit_mqd; | ||
| 325 | mqd->load_mqd = load_mqd; | ||
| 326 | mqd->update_mqd = update_mqd; | ||
| 327 | mqd->destroy_mqd = destroy_mqd; | ||
| 328 | mqd->is_occupied = is_occupied; | ||
| 329 | break; | ||
| 330 | case KFD_MQD_TYPE_CIK_HIQ: | ||
| 331 | mqd->init_mqd = init_mqd_hiq; | ||
| 332 | mqd->uninit_mqd = uninit_mqd; | ||
| 333 | mqd->load_mqd = load_mqd; | ||
| 334 | mqd->update_mqd = update_mqd_hiq; | ||
| 335 | mqd->destroy_mqd = destroy_mqd; | ||
| 336 | mqd->is_occupied = is_occupied; | ||
| 337 | break; | ||
| 338 | default: | ||
| 339 | kfree(mqd); | ||
| 340 | return NULL; | ||
| 341 | } | ||
| 342 | |||
| 343 | return mqd; | ||
| 344 | } | ||
| 345 | |||
| 346 | /* SDMA queues should be implemented here when the cp will supports them */ | ||
| @@ -0,0 +1,91 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | #ifndef KFD_MQD_MANAGER_H_ | ||
| 25 | #define KFD_MQD_MANAGER_H_ | ||
| 26 | |||
| 27 | #include "kfd_priv.h" | ||
| 28 | |||
| 29 | /** | ||
| 30 | * struct mqd_manager | ||
| 31 | * | ||
| 32 | * @init_mqd: Allocates the mqd buffer on local gpu memory and initialize it. | ||
| 33 | * | ||
| 34 | * @load_mqd: Loads the mqd to a concrete hqd slot. Used only for no cp | ||
| 35 | * scheduling mode. | ||
| 36 | * | ||
| 37 | * @update_mqd: Handles a update call for the MQD | ||
| 38 | * | ||
| 39 | * @destroy_mqd: Destroys the HQD slot and by that preempt the relevant queue. | ||
| 40 | * Used only for no cp scheduling. | ||
| 41 | * | ||
| 42 | * @uninit_mqd: Releases the mqd buffer from local gpu memory. | ||
| 43 | * | ||
| 44 | * @is_occupied: Checks if the relevant HQD slot is occupied. | ||
| 45 | * | ||
| 46 | * @mqd_mutex: Mqd manager mutex. | ||
| 47 | * | ||
| 48 | * @dev: The kfd device structure coupled with this module. | ||
| 49 | * | ||
| 50 | * MQD stands for Memory Queue Descriptor which represents the current queue | ||
| 51 | * state in the memory and initiate the HQD (Hardware Queue Descriptor) state. | ||
| 52 | * This structure is actually a base class for the different types of MQDs | ||
| 53 | * structures for the variant ASICs that should be supported in the future. | ||
| 54 | * This base class is also contains all the MQD specific operations. | ||
| 55 | * Another important thing to mention is that each queue has a MQD that keeps | ||
| 56 | * his state (or context) after each preemption or reassignment. | ||
| 57 | * Basically there are a instances of the mqd manager class per MQD type per | ||
| 58 | * ASIC. Currently the kfd driver supports only Kaveri so there are instances | ||
| 59 | * per KFD_MQD_TYPE for each device. | ||
| 60 | * | ||
| 61 | */ | ||
| 62 | |||
| 63 | struct mqd_manager { | ||
| 64 | int (*init_mqd)(struct mqd_manager *mm, void **mqd, | ||
| 65 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | ||
| 66 | struct queue_properties *q); | ||
| 67 | |||
| 68 | int (*load_mqd)(struct mqd_manager *mm, void *mqd, | ||
| 69 | uint32_t pipe_id, uint32_t queue_id, | ||
| 70 | uint32_t __user *wptr); | ||
| 71 | |||
| 72 | int (*update_mqd)(struct mqd_manager *mm, void *mqd, | ||
| 73 | struct queue_properties *q); | ||
| 74 | |||
| 75 | int (*destroy_mqd)(struct mqd_manager *mm, void *mqd, | ||
| 76 | enum kfd_preempt_type type, | ||
| 77 | unsigned int timeout, uint32_t pipe_id, | ||
| 78 | uint32_t queue_id); | ||
| 79 | |||
| 80 | void (*uninit_mqd)(struct mqd_manager *mm, void *mqd, | ||
| 81 | struct kfd_mem_obj *mqd_mem_obj); | ||
| 82 | |||
| 83 | bool (*is_occupied)(struct mqd_manager *mm, void *mqd, | ||
| 84 | uint64_t queue_address, uint32_t pipe_id, | ||
| 85 | uint32_t queue_id); | ||
| 86 | |||
| 87 | struct mutex mqd_mutex; | ||
| 88 | struct kfd_dev *dev; | ||
| 89 | }; | ||
| 90 | |||
| 91 | #endif /* KFD_MQD_MANAGER_H_ */ | ||
| @@ -0,0 +1,565 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | #include <linux/slab.h> | ||
| 25 | #include <linux/mutex.h> | ||
| 26 | #include "kfd_device_queue_manager.h" | ||
| 27 | #include "kfd_kernel_queue.h" | ||
| 28 | #include "kfd_priv.h" | ||
| 29 | #include "kfd_pm4_headers.h" | ||
| 30 | #include "kfd_pm4_opcodes.h" | ||
| 31 | |||
| 32 | static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes, | ||
| 33 | unsigned int buffer_size_bytes) | ||
| 34 | { | ||
| 35 | unsigned int temp = *wptr + increment_bytes / sizeof(uint32_t); | ||
| 36 | |||
| 37 | BUG_ON((temp * sizeof(uint32_t)) > buffer_size_bytes); | ||
| 38 | *wptr = temp; | ||
| 39 | } | ||
| 40 | |||
| 41 | static unsigned int build_pm4_header(unsigned int opcode, size_t packet_size) | ||
| 42 | { | ||
| 43 | union PM4_MES_TYPE_3_HEADER header; | ||
| 44 | |||
| 45 | header.u32all = 0; | ||
| 46 | header.opcode = opcode; | ||
| 47 | header.count = packet_size/sizeof(uint32_t) - 2; | ||
| 48 | header.type = PM4_TYPE_3; | ||
| 49 | |||
| 50 | return header.u32all; | ||
| 51 | } | ||
| 52 | |||
| 53 | static void pm_calc_rlib_size(struct packet_manager *pm, | ||
| 54 | unsigned int *rlib_size, | ||
| 55 | bool *over_subscription) | ||
| 56 | { | ||
| 57 | unsigned int process_count, queue_count; | ||
| 58 | |||
| 59 | BUG_ON(!pm || !rlib_size || !over_subscription); | ||
| 60 | |||
| 61 | process_count = pm->dqm->processes_count; | ||
| 62 | queue_count = pm->dqm->queue_count; | ||
| 63 | |||
| 64 | /* check if there is over subscription*/ | ||
| 65 | *over_subscription = false; | ||
| 66 | if ((process_count > 1) || | ||
| 67 | queue_count > PIPE_PER_ME_CP_SCHEDULING * QUEUES_PER_PIPE) { | ||
| 68 | *over_subscription = true; | ||
| 69 | pr_debug("kfd: over subscribed runlist\n"); | ||
| 70 | } | ||
| 71 | |||
| 72 | /* calculate run list ib allocation size */ | ||
| 73 | *rlib_size = process_count * sizeof(struct pm4_map_process) + | ||
| 74 | queue_count * sizeof(struct pm4_map_queues); | ||
| 75 | |||
| 76 | /* | ||
| 77 | * Increase the allocation size in case we need a chained run list | ||
| 78 | * when over subscription | ||
| 79 | */ | ||
| 80 | if (*over_subscription) | ||
| 81 | *rlib_size += sizeof(struct pm4_runlist); | ||
| 82 | |||
| 83 | pr_debug("kfd: runlist ib size %d\n", *rlib_size); | ||
| 84 | } | ||
| 85 | |||
| 86 | static int pm_allocate_runlist_ib(struct packet_manager *pm, | ||
| 87 | unsigned int **rl_buffer, | ||
| 88 | uint64_t *rl_gpu_buffer, | ||
| 89 | unsigned int *rl_buffer_size, | ||
| 90 | bool *is_over_subscription) | ||
| 91 | { | ||
| 92 | int retval; | ||
| 93 | |||
| 94 | BUG_ON(!pm); | ||
| 95 | BUG_ON(pm->allocated == true); | ||
| 96 | BUG_ON(is_over_subscription == NULL); | ||
| 97 | |||
| 98 | pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription); | ||
| 99 | |||
| 100 | retval = kfd2kgd->allocate_mem(pm->dqm->dev->kgd, | ||
| 101 | *rl_buffer_size, | ||
| 102 | PAGE_SIZE, | ||
| 103 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | ||
| 104 | (struct kgd_mem **) &pm->ib_buffer_obj); | ||
| 105 | |||
| 106 | if (retval != 0) { | ||
| 107 | pr_err("kfd: failed to allocate runlist IB\n"); | ||
| 108 | return retval; | ||
| 109 | } | ||
| 110 | |||
| 111 | *(void **)rl_buffer = pm->ib_buffer_obj->cpu_ptr; | ||
| 112 | *rl_gpu_buffer = pm->ib_buffer_obj->gpu_addr; | ||
| 113 | |||
| 114 | memset(*rl_buffer, 0, *rl_buffer_size); | ||
| 115 | pm->allocated = true; | ||
| 116 | return retval; | ||
| 117 | } | ||
| 118 | |||
| 119 | static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer, | ||
| 120 | uint64_t ib, size_t ib_size_in_dwords, bool chain) | ||
| 121 | { | ||
| 122 | struct pm4_runlist *packet; | ||
| 123 | |||
| 124 | BUG_ON(!pm || !buffer || !ib); | ||
| 125 | |||
| 126 | packet = (struct pm4_runlist *)buffer; | ||
| 127 | |||
| 128 | memset(buffer, 0, sizeof(struct pm4_runlist)); | ||
| 129 | packet->header.u32all = build_pm4_header(IT_RUN_LIST, | ||
| 130 | sizeof(struct pm4_runlist)); | ||
| 131 | |||
| 132 | packet->bitfields4.ib_size = ib_size_in_dwords; | ||
| 133 | packet->bitfields4.chain = chain ? 1 : 0; | ||
| 134 | packet->bitfields4.offload_polling = 0; | ||
| 135 | packet->bitfields4.valid = 1; | ||
| 136 | packet->ordinal2 = lower_32_bits(ib); | ||
| 137 | packet->bitfields3.ib_base_hi = upper_32_bits(ib); | ||
| 138 | |||
| 139 | return 0; | ||
| 140 | } | ||
| 141 | |||
| 142 | static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer, | ||
| 143 | struct qcm_process_device *qpd) | ||
| 144 | { | ||
| 145 | struct pm4_map_process *packet; | ||
| 146 | struct queue *cur; | ||
| 147 | uint32_t num_queues; | ||
| 148 | |||
| 149 | BUG_ON(!pm || !buffer || !qpd); | ||
| 150 | |||
| 151 | packet = (struct pm4_map_process *)buffer; | ||
| 152 | |||
| 153 | pr_debug("kfd: In func %s\n", __func__); | ||
| 154 | |||
| 155 | memset(buffer, 0, sizeof(struct pm4_map_process)); | ||
| 156 | |||
| 157 | packet->header.u32all = build_pm4_header(IT_MAP_PROCESS, | ||
| 158 | sizeof(struct pm4_map_process)); | ||
| 159 | packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0; | ||
| 160 | packet->bitfields2.process_quantum = 1; | ||
| 161 | packet->bitfields2.pasid = qpd->pqm->process->pasid; | ||
| 162 | packet->bitfields3.page_table_base = qpd->page_table_base; | ||
| 163 | packet->bitfields10.gds_size = qpd->gds_size; | ||
| 164 | packet->bitfields10.num_gws = qpd->num_gws; | ||
| 165 | packet->bitfields10.num_oac = qpd->num_oac; | ||
| 166 | num_queues = 0; | ||
| 167 | list_for_each_entry(cur, &qpd->queues_list, list) | ||
| 168 | num_queues++; | ||
| 169 | packet->bitfields10.num_queues = num_queues; | ||
| 170 | |||
| 171 | packet->sh_mem_config = qpd->sh_mem_config; | ||
| 172 | packet->sh_mem_bases = qpd->sh_mem_bases; | ||
| 173 | packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base; | ||
| 174 | packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit; | ||
| 175 | |||
| 176 | packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area); | ||
| 177 | packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area); | ||
| 178 | |||
| 179 | return 0; | ||
| 180 | } | ||
| 181 | |||
| 182 | static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer, | ||
| 183 | struct queue *q) | ||
| 184 | { | ||
| 185 | struct pm4_map_queues *packet; | ||
| 186 | |||
| 187 | BUG_ON(!pm || !buffer || !q); | ||
| 188 | |||
| 189 | pr_debug("kfd: In func %s\n", __func__); | ||
| 190 | |||
| 191 | packet = (struct pm4_map_queues *)buffer; | ||
| 192 | memset(buffer, 0, sizeof(struct pm4_map_queues)); | ||
| 193 | |||
| 194 | packet->header.u32all = build_pm4_header(IT_MAP_QUEUES, | ||
| 195 | sizeof(struct pm4_map_queues)); | ||
| 196 | packet->bitfields2.alloc_format = | ||
| 197 | alloc_format__mes_map_queues__one_per_pipe; | ||
| 198 | packet->bitfields2.num_queues = 1; | ||
| 199 | packet->bitfields2.queue_sel = | ||
| 200 | queue_sel__mes_map_queues__map_to_hws_determined_queue_slots; | ||
| 201 | |||
| 202 | packet->bitfields2.vidmem = (q->properties.is_interop) ? | ||
| 203 | vidmem__mes_map_queues__uses_video_memory : | ||
| 204 | vidmem__mes_map_queues__uses_no_video_memory; | ||
| 205 | |||
| 206 | switch (q->properties.type) { | ||
| 207 | case KFD_QUEUE_TYPE_COMPUTE: | ||
| 208 | case KFD_QUEUE_TYPE_DIQ: | ||
| 209 | packet->bitfields2.engine_sel = | ||
| 210 | engine_sel__mes_map_queues__compute; | ||
| 211 | break; | ||
| 212 | case KFD_QUEUE_TYPE_SDMA: | ||
| 213 | packet->bitfields2.engine_sel = | ||
| 214 | engine_sel__mes_map_queues__sdma0; | ||
| 215 | break; | ||
| 216 | default: | ||
| 217 | BUG(); | ||
| 218 | break; | ||
| 219 | } | ||
| 220 | |||
| 221 | packet->mes_map_queues_ordinals[0].bitfields3.doorbell_offset = | ||
| 222 | q->properties.doorbell_off; | ||
| 223 | |||
| 224 | packet->mes_map_queues_ordinals[0].mqd_addr_lo = | ||
| 225 | lower_32_bits(q->gart_mqd_addr); | ||
| 226 | |||
| 227 | packet->mes_map_queues_ordinals[0].mqd_addr_hi = | ||
| 228 | upper_32_bits(q->gart_mqd_addr); | ||
| 229 | |||
| 230 | packet->mes_map_queues_ordinals[0].wptr_addr_lo = | ||
| 231 | lower_32_bits((uint64_t)q->properties.write_ptr); | ||
| 232 | |||
| 233 | packet->mes_map_queues_ordinals[0].wptr_addr_hi = | ||
| 234 | upper_32_bits((uint64_t)q->properties.write_ptr); | ||
| 235 | |||
| 236 | return 0; | ||
| 237 | } | ||
| 238 | |||
| 239 | static int pm_create_runlist_ib(struct packet_manager *pm, | ||
| 240 | struct list_head *queues, | ||
| 241 | uint64_t *rl_gpu_addr, | ||
| 242 | size_t *rl_size_bytes) | ||
| 243 | { | ||
| 244 | unsigned int alloc_size_bytes; | ||
| 245 | unsigned int *rl_buffer, rl_wptr, i; | ||
| 246 | int retval, proccesses_mapped; | ||
| 247 | struct device_process_node *cur; | ||
| 248 | struct qcm_process_device *qpd; | ||
| 249 | struct queue *q; | ||
| 250 | struct kernel_queue *kq; | ||
| 251 | bool is_over_subscription; | ||
| 252 | |||
| 253 | BUG_ON(!pm || !queues || !rl_size_bytes || !rl_gpu_addr); | ||
| 254 | |||
| 255 | rl_wptr = retval = proccesses_mapped = 0; | ||
| 256 | |||
| 257 | retval = pm_allocate_runlist_ib(pm, &rl_buffer, rl_gpu_addr, | ||
| 258 | &alloc_size_bytes, &is_over_subscription); | ||
| 259 | if (retval != 0) | ||
| 260 | return retval; | ||
| 261 | |||
| 262 | *rl_size_bytes = alloc_size_bytes; | ||
| 263 | |||
| 264 | pr_debug("kfd: In func %s\n", __func__); | ||
| 265 | pr_debug("kfd: building runlist ib process count: %d queues count %d\n", | ||
| 266 | pm->dqm->processes_count, pm->dqm->queue_count); | ||
| 267 | |||
| 268 | /* build the run list ib packet */ | ||
| 269 | list_for_each_entry(cur, queues, list) { | ||
| 270 | qpd = cur->qpd; | ||
| 271 | /* build map process packet */ | ||
| 272 | if (proccesses_mapped >= pm->dqm->processes_count) { | ||
| 273 | pr_debug("kfd: not enough space left in runlist IB\n"); | ||
| 274 | pm_release_ib(pm); | ||
| 275 | return -ENOMEM; | ||
| 276 | } | ||
| 277 | retval = pm_create_map_process(pm, &rl_buffer[rl_wptr], qpd); | ||
| 278 | if (retval != 0) | ||
| 279 | return retval; | ||
| 280 | proccesses_mapped++; | ||
| 281 | inc_wptr(&rl_wptr, sizeof(struct pm4_map_process), | ||
| 282 | alloc_size_bytes); | ||
| 283 | |||
| 284 | list_for_each_entry(kq, &qpd->priv_queue_list, list) { | ||
| 285 | if (kq->queue->properties.is_active != true) | ||
| 286 | continue; | ||
| 287 | retval = pm_create_map_queue(pm, &rl_buffer[rl_wptr], | ||
| 288 | kq->queue); | ||
| 289 | if (retval != 0) | ||
| 290 | return retval; | ||
| 291 | inc_wptr(&rl_wptr, sizeof(struct pm4_map_queues), | ||
| 292 | alloc_size_bytes); | ||
| 293 | } | ||
| 294 | |||
| 295 | list_for_each_entry(q, &qpd->queues_list, list) { | ||
| 296 | if (q->properties.is_active != true) | ||
| 297 | continue; | ||
| 298 | retval = pm_create_map_queue(pm, | ||
| 299 | &rl_buffer[rl_wptr], q); | ||
| 300 | if (retval != 0) | ||
| 301 | return retval; | ||
| 302 | inc_wptr(&rl_wptr, sizeof(struct pm4_map_queues), | ||
| 303 | alloc_size_bytes); | ||
| 304 | } | ||
| 305 | } | ||
| 306 | |||
| 307 | pr_debug("kfd: finished map process and queues to runlist\n"); | ||
| 308 | |||
| 309 | if (is_over_subscription) | ||
| 310 | pm_create_runlist(pm, &rl_buffer[rl_wptr], *rl_gpu_addr, | ||
| 311 | alloc_size_bytes / sizeof(uint32_t), true); | ||
| 312 | |||
| 313 | for (i = 0; i < alloc_size_bytes / sizeof(uint32_t); i++) | ||
| 314 | pr_debug("0x%2X ", rl_buffer[i]); | ||
| 315 | pr_debug("\n"); | ||
| 316 | |||
| 317 | return 0; | ||
| 318 | } | ||
| 319 | |||
| 320 | int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm) | ||
| 321 | { | ||
| 322 | BUG_ON(!dqm); | ||
| 323 | |||
| 324 | pm->dqm = dqm; | ||
| 325 | mutex_init(&pm->lock); | ||
| 326 | pm->priv_queue = kernel_queue_init(dqm->dev, KFD_QUEUE_TYPE_HIQ); | ||
| 327 | if (pm->priv_queue == NULL) { | ||
| 328 | mutex_destroy(&pm->lock); | ||
| 329 | return -ENOMEM; | ||
| 330 | } | ||
| 331 | pm->allocated = false; | ||
| 332 | |||
| 333 | return 0; | ||
| 334 | } | ||
| 335 | |||
| 336 | void pm_uninit(struct packet_manager *pm) | ||
| 337 | { | ||
| 338 | BUG_ON(!pm); | ||
| 339 | |||
| 340 | mutex_destroy(&pm->lock); | ||
| 341 | kernel_queue_uninit(pm->priv_queue); | ||
| 342 | } | ||
| 343 | |||
| 344 | int pm_send_set_resources(struct packet_manager *pm, | ||
| 345 | struct scheduling_resources *res) | ||
| 346 | { | ||
| 347 | struct pm4_set_resources *packet; | ||
| 348 | |||
| 349 | BUG_ON(!pm || !res); | ||
| 350 | |||
| 351 | pr_debug("kfd: In func %s\n", __func__); | ||
| 352 | |||
| 353 | mutex_lock(&pm->lock); | ||
| 354 | pm->priv_queue->acquire_packet_buffer(pm->priv_queue, | ||
| 355 | sizeof(*packet) / sizeof(uint32_t), | ||
| 356 | (unsigned int **)&packet); | ||
| 357 | if (packet == NULL) { | ||
| 358 | mutex_unlock(&pm->lock); | ||
| 359 | pr_err("kfd: failed to allocate buffer on kernel queue\n"); | ||
| 360 | return -ENOMEM; | ||
| 361 | } | ||
| 362 | |||
| 363 | memset(packet, 0, sizeof(struct pm4_set_resources)); | ||
| 364 | packet->header.u32all = build_pm4_header(IT_SET_RESOURCES, | ||
| 365 | sizeof(struct pm4_set_resources)); | ||
| 366 | |||
| 367 | packet->bitfields2.queue_type = | ||
| 368 | queue_type__mes_set_resources__hsa_interface_queue_hiq; | ||
| 369 | packet->bitfields2.vmid_mask = res->vmid_mask; | ||
| 370 | packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY; | ||
| 371 | packet->bitfields7.oac_mask = res->oac_mask; | ||
| 372 | packet->bitfields8.gds_heap_base = res->gds_heap_base; | ||
| 373 | packet->bitfields8.gds_heap_size = res->gds_heap_size; | ||
| 374 | |||
| 375 | packet->gws_mask_lo = lower_32_bits(res->gws_mask); | ||
| 376 | packet->gws_mask_hi = upper_32_bits(res->gws_mask); | ||
| 377 | |||
| 378 | packet->queue_mask_lo = lower_32_bits(res->queue_mask); | ||
| 379 | packet->queue_mask_hi = upper_32_bits(res->queue_mask); | ||
| 380 | |||
| 381 | pm->priv_queue->submit_packet(pm->priv_queue); | ||
| 382 | pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); | ||
| 383 | |||
| 384 | mutex_unlock(&pm->lock); | ||
| 385 | |||
| 386 | return 0; | ||
| 387 | } | ||
| 388 | |||
| 389 | int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues) | ||
| 390 | { | ||
| 391 | uint64_t rl_gpu_ib_addr; | ||
| 392 | uint32_t *rl_buffer; | ||
| 393 | size_t rl_ib_size, packet_size_dwords; | ||
| 394 | int retval; | ||
| 395 | |||
| 396 | BUG_ON(!pm || !dqm_queues); | ||
| 397 | |||
| 398 | retval = pm_create_runlist_ib(pm, dqm_queues, &rl_gpu_ib_addr, | ||
| 399 | &rl_ib_size); | ||
| 400 | if (retval != 0) | ||
| 401 | goto fail_create_runlist_ib; | ||
| 402 | |||
| 403 | pr_debug("kfd: runlist IB address: 0x%llX\n", rl_gpu_ib_addr); | ||
| 404 | |||
| 405 | packet_size_dwords = sizeof(struct pm4_runlist) / sizeof(uint32_t); | ||
| 406 | mutex_lock(&pm->lock); | ||
| 407 | |||
| 408 | retval = pm->priv_queue->acquire_packet_buffer(pm->priv_queue, | ||
| 409 | packet_size_dwords, &rl_buffer); | ||
| 410 | if (retval != 0) | ||
| 411 | goto fail_acquire_packet_buffer; | ||
| 412 | |||
| 413 | retval = pm_create_runlist(pm, rl_buffer, rl_gpu_ib_addr, | ||
| 414 | rl_ib_size / sizeof(uint32_t), false); | ||
| 415 | if (retval != 0) | ||
| 416 | goto fail_create_runlist; | ||
| 417 | |||
| 418 | pm->priv_queue->submit_packet(pm->priv_queue); | ||
| 419 | pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); | ||
| 420 | |||
| 421 | mutex_unlock(&pm->lock); | ||
| 422 | |||
| 423 | return retval; | ||
| 424 | |||
| 425 | fail_create_runlist: | ||
| 426 | pm->priv_queue->rollback_packet(pm->priv_queue); | ||
| 427 | fail_acquire_packet_buffer: | ||
| 428 | mutex_unlock(&pm->lock); | ||
| 429 | fail_create_runlist_ib: | ||
| 430 | if (pm->allocated == true) | ||
| 431 | pm_release_ib(pm); | ||
| 432 | return retval; | ||
| 433 | } | ||
| 434 | |||
| 435 | int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address, | ||
| 436 | uint32_t fence_value) | ||
| 437 | { | ||
| 438 | int retval; | ||
| 439 | struct pm4_query_status *packet; | ||
| 440 | |||
| 441 | BUG_ON(!pm || !fence_address); | ||
| 442 | |||
| 443 | mutex_lock(&pm->lock); | ||
| 444 | retval = pm->priv_queue->acquire_packet_buffer( | ||
| 445 | pm->priv_queue, | ||
| 446 | sizeof(struct pm4_query_status) / sizeof(uint32_t), | ||
| 447 | (unsigned int **)&packet); | ||
| 448 | if (retval != 0) | ||
| 449 | goto fail_acquire_packet_buffer; | ||
| 450 | |||
| 451 | packet->header.u32all = build_pm4_header(IT_QUERY_STATUS, | ||
| 452 | sizeof(struct pm4_query_status)); | ||
| 453 | |||
| 454 | packet->bitfields2.context_id = 0; | ||
| 455 | packet->bitfields2.interrupt_sel = | ||
| 456 | interrupt_sel__mes_query_status__completion_status; | ||
| 457 | packet->bitfields2.command = | ||
| 458 | command__mes_query_status__fence_only_after_write_ack; | ||
| 459 | |||
| 460 | packet->addr_hi = upper_32_bits((uint64_t)fence_address); | ||
| 461 | packet->addr_lo = lower_32_bits((uint64_t)fence_address); | ||
| 462 | packet->data_hi = upper_32_bits((uint64_t)fence_value); | ||
| 463 | packet->data_lo = lower_32_bits((uint64_t)fence_value); | ||
| 464 | |||
| 465 | pm->priv_queue->submit_packet(pm->priv_queue); | ||
| 466 | pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); | ||
| 467 | mutex_unlock(&pm->lock); | ||
| 468 | |||
| 469 | return 0; | ||
| 470 | |||
| 471 | fail_acquire_packet_buffer: | ||
| 472 | mutex_unlock(&pm->lock); | ||
| 473 | return retval; | ||
| 474 | } | ||
| 475 | |||
| 476 | int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type, | ||
| 477 | enum kfd_preempt_type_filter mode, | ||
| 478 | uint32_t filter_param, bool reset, | ||
| 479 | unsigned int sdma_engine) | ||
| 480 | { | ||
| 481 | int retval; | ||
| 482 | uint32_t *buffer; | ||
| 483 | struct pm4_unmap_queues *packet; | ||
| 484 | |||
| 485 | BUG_ON(!pm); | ||
| 486 | |||
| 487 | mutex_lock(&pm->lock); | ||
| 488 | retval = pm->priv_queue->acquire_packet_buffer( | ||
| 489 | pm->priv_queue, | ||
| 490 | sizeof(struct pm4_unmap_queues) / sizeof(uint32_t), | ||
| 491 | &buffer); | ||
| 492 | if (retval != 0) | ||
| 493 | goto err_acquire_packet_buffer; | ||
| 494 | |||
| 495 | packet = (struct pm4_unmap_queues *)buffer; | ||
| 496 | memset(buffer, 0, sizeof(struct pm4_unmap_queues)); | ||
| 497 | |||
| 498 | packet->header.u32all = build_pm4_header(IT_UNMAP_QUEUES, | ||
| 499 | sizeof(struct pm4_unmap_queues)); | ||
| 500 | switch (type) { | ||
| 501 | case KFD_QUEUE_TYPE_COMPUTE: | ||
| 502 | case KFD_QUEUE_TYPE_DIQ: | ||
| 503 | packet->bitfields2.engine_sel = | ||
| 504 | engine_sel__mes_unmap_queues__compute; | ||
| 505 | break; | ||
| 506 | case KFD_QUEUE_TYPE_SDMA: | ||
| 507 | packet->bitfields2.engine_sel = | ||
| 508 | engine_sel__mes_unmap_queues__sdma0 + sdma_engine; | ||
| 509 | break; | ||
| 510 | default: | ||
| 511 | BUG(); | ||
| 512 | break; | ||
| 513 | } | ||
| 514 | |||
| 515 | if (reset) | ||
| 516 | packet->bitfields2.action = | ||
| 517 | action__mes_unmap_queues__reset_queues; | ||
| 518 | else | ||
| 519 | packet->bitfields2.action = | ||
| 520 | action__mes_unmap_queues__preempt_queues; | ||
| 521 | |||
| 522 | switch (mode) { | ||
| 523 | case KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE: | ||
| 524 | packet->bitfields2.queue_sel = | ||
| 525 | queue_sel__mes_unmap_queues__perform_request_on_specified_queues; | ||
| 526 | packet->bitfields2.num_queues = 1; | ||
| 527 | packet->bitfields3b.doorbell_offset0 = filter_param; | ||
| 528 | break; | ||
| 529 | case KFD_PREEMPT_TYPE_FILTER_BY_PASID: | ||
| 530 | packet->bitfields2.queue_sel = | ||
| 531 | queue_sel__mes_unmap_queues__perform_request_on_pasid_queues; | ||
| 532 | packet->bitfields3a.pasid = filter_param; | ||
| 533 | break; | ||
| 534 | case KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES: | ||
| 535 | packet->bitfields2.queue_sel = | ||
| 536 | queue_sel__mes_unmap_queues__perform_request_on_all_active_queues; | ||
| 537 | break; | ||
| 538 | default: | ||
| 539 | BUG(); | ||
| 540 | break; | ||
| 541 | }; | ||
| 542 | |||
| 543 | pm->priv_queue->submit_packet(pm->priv_queue); | ||
| 544 | pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); | ||
| 545 | |||
| 546 | mutex_unlock(&pm->lock); | ||
| 547 | return 0; | ||
| 548 | |||
| 549 | err_acquire_packet_buffer: | ||
| 550 | mutex_unlock(&pm->lock); | ||
| 551 | return retval; | ||
| 552 | } | ||
| 553 | |||
| 554 | void pm_release_ib(struct packet_manager *pm) | ||
| 555 | { | ||
| 556 | BUG_ON(!pm); | ||
| 557 | |||
| 558 | mutex_lock(&pm->lock); | ||
| 559 | if (pm->allocated) { | ||
| 560 | kfd2kgd->free_mem(pm->dqm->dev->kgd, | ||
| 561 | (struct kgd_mem *) pm->ib_buffer_obj); | ||
| 562 | pm->allocated = false; | ||
| 563 | } | ||
| 564 | mutex_unlock(&pm->lock); | ||
| 565 | } | ||
| @@ -0,0 +1,97 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | #include <linux/slab.h> | ||
| 24 | #include <linux/types.h> | ||
| 25 | #include "kfd_priv.h" | ||
| 26 | |||
| 27 | static unsigned long *pasid_bitmap; | ||
| 28 | static unsigned int pasid_limit; | ||
| 29 | static DEFINE_MUTEX(pasid_mutex); | ||
| 30 | |||
| 31 | int kfd_pasid_init(void) | ||
| 32 | { | ||
| 33 | pasid_limit = max_num_of_processes; | ||
| 34 | |||
| 35 | pasid_bitmap = kzalloc(DIV_ROUND_UP(pasid_limit, BITS_PER_BYTE), | ||
| 36 | GFP_KERNEL); | ||
| 37 | if (!pasid_bitmap) | ||
| 38 | return -ENOMEM; | ||
| 39 | |||
| 40 | set_bit(0, pasid_bitmap); /* PASID 0 is reserved. */ | ||
| 41 | |||
| 42 | return 0; | ||
| 43 | } | ||
| 44 | |||
| 45 | void kfd_pasid_exit(void) | ||
| 46 | { | ||
| 47 | kfree(pasid_bitmap); | ||
| 48 | } | ||
| 49 | |||
| 50 | bool kfd_set_pasid_limit(unsigned int new_limit) | ||
| 51 | { | ||
| 52 | if (new_limit < pasid_limit) { | ||
| 53 | bool ok; | ||
| 54 | |||
| 55 | mutex_lock(&pasid_mutex); | ||
| 56 | |||
| 57 | /* ensure that no pasids >= new_limit are in-use */ | ||
| 58 | ok = (find_next_bit(pasid_bitmap, pasid_limit, new_limit) == | ||
| 59 | pasid_limit); | ||
| 60 | if (ok) | ||
| 61 | pasid_limit = new_limit; | ||
| 62 | |||
| 63 | mutex_unlock(&pasid_mutex); | ||
| 64 | |||
| 65 | return ok; | ||
| 66 | } | ||
| 67 | |||
| 68 | return true; | ||
| 69 | } | ||
| 70 | |||
| 71 | inline unsigned int kfd_get_pasid_limit(void) | ||
| 72 | { | ||
| 73 | return pasid_limit; | ||
| 74 | } | ||
| 75 | |||
| 76 | unsigned int kfd_pasid_alloc(void) | ||
| 77 | { | ||
| 78 | unsigned int found; | ||
| 79 | |||
| 80 | mutex_lock(&pasid_mutex); | ||
| 81 | |||
| 82 | found = find_first_zero_bit(pasid_bitmap, pasid_limit); | ||
| 83 | if (found == pasid_limit) | ||
| 84 | found = 0; | ||
| 85 | else | ||
| 86 | set_bit(found, pasid_bitmap); | ||
| 87 | |||
| 88 | mutex_unlock(&pasid_mutex); | ||
| 89 | |||
| 90 | return found; | ||
| 91 | } | ||
| 92 | |||
| 93 | void kfd_pasid_free(unsigned int pasid) | ||
| 94 | { | ||
| 95 | BUG_ON(pasid == 0 || pasid >= pasid_limit); | ||
| 96 | clear_bit(pasid, pasid_bitmap); | ||
| 97 | } | ||
| @@ -0,0 +1,405 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | #ifndef KFD_PM4_HEADERS_H_ | ||
| 25 | #define KFD_PM4_HEADERS_H_ | ||
| 26 | |||
| 27 | #ifndef PM4_MES_HEADER_DEFINED | ||
| 28 | #define PM4_MES_HEADER_DEFINED | ||
| 29 | union PM4_MES_TYPE_3_HEADER { | ||
| 30 | struct { | ||
| 31 | uint32_t reserved1:8; /* < reserved */ | ||
| 32 | uint32_t opcode:8; /* < IT opcode */ | ||
| 33 | uint32_t count:14; /* < number of DWORDs - 1 | ||
| 34 | * in the information body. | ||
| 35 | */ | ||
| 36 | uint32_t type:2; /* < packet identifier. | ||
| 37 | * It should be 3 for type 3 packets | ||
| 38 | */ | ||
| 39 | }; | ||
| 40 | uint32_t u32all; | ||
| 41 | }; | ||
| 42 | #endif /* PM4_MES_HEADER_DEFINED */ | ||
| 43 | |||
| 44 | /* --------------------MES_SET_RESOURCES-------------------- */ | ||
| 45 | |||
| 46 | #ifndef PM4_MES_SET_RESOURCES_DEFINED | ||
| 47 | #define PM4_MES_SET_RESOURCES_DEFINED | ||
| 48 | enum set_resources_queue_type_enum { | ||
| 49 | queue_type__mes_set_resources__kernel_interface_queue_kiq = 0, | ||
| 50 | queue_type__mes_set_resources__hsa_interface_queue_hiq = 1, | ||
| 51 | queue_type__mes_set_resources__hsa_debug_interface_queue = 4 | ||
| 52 | }; | ||
| 53 | |||
| 54 | struct pm4_set_resources { | ||
| 55 | union { | ||
| 56 | union PM4_MES_TYPE_3_HEADER header; /* header */ | ||
| 57 | uint32_t ordinal1; | ||
| 58 | }; | ||
| 59 | |||
| 60 | union { | ||
| 61 | struct { | ||
| 62 | uint32_t vmid_mask:16; | ||
| 63 | uint32_t unmap_latency:8; | ||
| 64 | uint32_t reserved1:5; | ||
| 65 | enum set_resources_queue_type_enum queue_type:3; | ||
| 66 | } bitfields2; | ||
| 67 | uint32_t ordinal2; | ||
| 68 | }; | ||
| 69 | |||
| 70 | uint32_t queue_mask_lo; | ||
| 71 | uint32_t queue_mask_hi; | ||
| 72 | uint32_t gws_mask_lo; | ||
| 73 | uint32_t gws_mask_hi; | ||
| 74 | |||
| 75 | union { | ||
| 76 | struct { | ||
| 77 | uint32_t oac_mask:16; | ||
| 78 | uint32_t reserved2:16; | ||
| 79 | } bitfields7; | ||
| 80 | uint32_t ordinal7; | ||
| 81 | }; | ||
| 82 | |||
| 83 | union { | ||
| 84 | struct { | ||
| 85 | uint32_t gds_heap_base:6; | ||
| 86 | uint32_t reserved3:5; | ||
| 87 | uint32_t gds_heap_size:6; | ||
| 88 | uint32_t reserved4:15; | ||
| 89 | } bitfields8; | ||
| 90 | uint32_t ordinal8; | ||
| 91 | }; | ||
| 92 | |||
| 93 | }; | ||
| 94 | #endif | ||
| 95 | |||
| 96 | /*--------------------MES_RUN_LIST-------------------- */ | ||
| 97 | |||
| 98 | #ifndef PM4_MES_RUN_LIST_DEFINED | ||
| 99 | #define PM4_MES_RUN_LIST_DEFINED | ||
| 100 | |||
| 101 | struct pm4_runlist { | ||
| 102 | union { | ||
| 103 | union PM4_MES_TYPE_3_HEADER header; /* header */ | ||
| 104 | uint32_t ordinal1; | ||
| 105 | }; | ||
| 106 | |||
| 107 | union { | ||
| 108 | struct { | ||
| 109 | uint32_t reserved1:2; | ||
| 110 | uint32_t ib_base_lo:30; | ||
| 111 | } bitfields2; | ||
| 112 | uint32_t ordinal2; | ||
| 113 | }; | ||
| 114 | |||
| 115 | union { | ||
| 116 | struct { | ||
| 117 | uint32_t ib_base_hi:16; | ||
| 118 | uint32_t reserved2:16; | ||
| 119 | } bitfields3; | ||
| 120 | uint32_t ordinal3; | ||
| 121 | }; | ||
| 122 | |||
| 123 | union { | ||
| 124 | struct { | ||
| 125 | uint32_t ib_size:20; | ||
| 126 | uint32_t chain:1; | ||
| 127 | uint32_t offload_polling:1; | ||
| 128 | uint32_t reserved3:1; | ||
| 129 | uint32_t valid:1; | ||
| 130 | uint32_t reserved4:8; | ||
| 131 | } bitfields4; | ||
| 132 | uint32_t ordinal4; | ||
| 133 | }; | ||
| 134 | |||
| 135 | }; | ||
| 136 | #endif | ||
| 137 | |||
| 138 | /*--------------------MES_MAP_PROCESS-------------------- */ | ||
| 139 | |||
| 140 | #ifndef PM4_MES_MAP_PROCESS_DEFINED | ||
| 141 | #define PM4_MES_MAP_PROCESS_DEFINED | ||
| 142 | |||
| 143 | struct pm4_map_process { | ||
| 144 | union { | ||
| 145 | union PM4_MES_TYPE_3_HEADER header; /* header */ | ||
| 146 | uint32_t ordinal1; | ||
| 147 | }; | ||
| 148 | |||
| 149 | union { | ||
| 150 | struct { | ||
| 151 | uint32_t pasid:16; | ||
| 152 | uint32_t reserved1:8; | ||
| 153 | uint32_t diq_enable:1; | ||
| 154 | uint32_t process_quantum:7; | ||
| 155 | } bitfields2; | ||
| 156 | uint32_t ordinal2; | ||
| 157 | }; | ||
| 158 | |||
| 159 | union { | ||
| 160 | struct { | ||
| 161 | uint32_t page_table_base:28; | ||
| 162 | uint32_t reserved3:4; | ||
| 163 | } bitfields3; | ||
| 164 | uint32_t ordinal3; | ||
| 165 | }; | ||
| 166 | |||
| 167 | uint32_t sh_mem_bases; | ||
| 168 | uint32_t sh_mem_ape1_base; | ||
| 169 | uint32_t sh_mem_ape1_limit; | ||
| 170 | uint32_t sh_mem_config; | ||
| 171 | uint32_t gds_addr_lo; | ||
| 172 | uint32_t gds_addr_hi; | ||
| 173 | |||
| 174 | union { | ||
| 175 | struct { | ||
| 176 | uint32_t num_gws:6; | ||
| 177 | uint32_t reserved4:2; | ||
| 178 | uint32_t num_oac:4; | ||
| 179 | uint32_t reserved5:4; | ||
| 180 | uint32_t gds_size:6; | ||
| 181 | uint32_t num_queues:10; | ||
| 182 | } bitfields10; | ||
| 183 | uint32_t ordinal10; | ||
| 184 | }; | ||
| 185 | |||
| 186 | }; | ||
| 187 | #endif | ||
| 188 | |||
| 189 | /*--------------------MES_MAP_QUEUES--------------------*/ | ||
| 190 | |||
| 191 | #ifndef PM4_MES_MAP_QUEUES_DEFINED | ||
| 192 | #define PM4_MES_MAP_QUEUES_DEFINED | ||
| 193 | enum map_queues_queue_sel_enum { | ||
| 194 | queue_sel__mes_map_queues__map_to_specified_queue_slots = 0, | ||
| 195 | queue_sel__mes_map_queues__map_to_hws_determined_queue_slots = 1, | ||
| 196 | queue_sel__mes_map_queues__enable_process_queues = 2 | ||
| 197 | }; | ||
| 198 | |||
| 199 | enum map_queues_vidmem_enum { | ||
| 200 | vidmem__mes_map_queues__uses_no_video_memory = 0, | ||
| 201 | vidmem__mes_map_queues__uses_video_memory = 1 | ||
| 202 | }; | ||
| 203 | |||
| 204 | enum map_queues_alloc_format_enum { | ||
| 205 | alloc_format__mes_map_queues__one_per_pipe = 0, | ||
| 206 | alloc_format__mes_map_queues__all_on_one_pipe = 1 | ||
| 207 | }; | ||
| 208 | |||
| 209 | enum map_queues_engine_sel_enum { | ||
| 210 | engine_sel__mes_map_queues__compute = 0, | ||
| 211 | engine_sel__mes_map_queues__sdma0 = 2, | ||
| 212 | engine_sel__mes_map_queues__sdma1 = 3 | ||
| 213 | }; | ||
| 214 | |||
| 215 | struct pm4_map_queues { | ||
| 216 | union { | ||
| 217 | union PM4_MES_TYPE_3_HEADER header; /* header */ | ||
| 218 | uint32_t ordinal1; | ||
| 219 | }; | ||
| 220 | |||
| 221 | union { | ||
| 222 | struct { | ||
| 223 | uint32_t reserved1:4; | ||
| 224 | enum map_queues_queue_sel_enum queue_sel:2; | ||
| 225 | uint32_t reserved2:2; | ||
| 226 | uint32_t vmid:4; | ||
| 227 | uint32_t reserved3:4; | ||
| 228 | enum map_queues_vidmem_enum vidmem:2; | ||
| 229 | uint32_t reserved4:6; | ||
| 230 | enum map_queues_alloc_format_enum alloc_format:2; | ||
| 231 | enum map_queues_engine_sel_enum engine_sel:3; | ||
| 232 | uint32_t num_queues:3; | ||
| 233 | } bitfields2; | ||
| 234 | uint32_t ordinal2; | ||
| 235 | }; | ||
| 236 | |||
| 237 | struct { | ||
| 238 | union { | ||
| 239 | struct { | ||
| 240 | uint32_t reserved5:2; | ||
| 241 | uint32_t doorbell_offset:21; | ||
| 242 | uint32_t reserved6:3; | ||
| 243 | uint32_t queue:6; | ||
| 244 | } bitfields3; | ||
| 245 | uint32_t ordinal3; | ||
| 246 | }; | ||
| 247 | |||
| 248 | uint32_t mqd_addr_lo; | ||
| 249 | uint32_t mqd_addr_hi; | ||
| 250 | uint32_t wptr_addr_lo; | ||
| 251 | uint32_t wptr_addr_hi; | ||
| 252 | |||
| 253 | } mes_map_queues_ordinals[1]; /* 1..N of these ordinal groups */ | ||
| 254 | |||
| 255 | }; | ||
| 256 | #endif | ||
| 257 | |||
| 258 | /*--------------------MES_QUERY_STATUS--------------------*/ | ||
| 259 | |||
| 260 | #ifndef PM4_MES_QUERY_STATUS_DEFINED | ||
| 261 | #define PM4_MES_QUERY_STATUS_DEFINED | ||
| 262 | enum query_status_interrupt_sel_enum { | ||
| 263 | interrupt_sel__mes_query_status__completion_status = 0, | ||
| 264 | interrupt_sel__mes_query_status__process_status = 1, | ||
| 265 | interrupt_sel__mes_query_status__queue_status = 2 | ||
| 266 | }; | ||
| 267 | |||
| 268 | enum query_status_command_enum { | ||
| 269 | command__mes_query_status__interrupt_only = 0, | ||
| 270 | command__mes_query_status__fence_only_immediate = 1, | ||
| 271 | command__mes_query_status__fence_only_after_write_ack = 2, | ||
| 272 | command__mes_query_status__fence_wait_for_write_ack_send_interrupt = 3 | ||
| 273 | }; | ||
| 274 | |||
| 275 | enum query_status_engine_sel_enum { | ||
| 276 | engine_sel__mes_query_status__compute = 0, | ||
| 277 | engine_sel__mes_query_status__sdma0_queue = 2, | ||
| 278 | engine_sel__mes_query_status__sdma1_queue = 3 | ||
| 279 | }; | ||
| 280 | |||
| 281 | struct pm4_query_status { | ||
| 282 | union { | ||
| 283 | union PM4_MES_TYPE_3_HEADER header; /* header */ | ||
| 284 | uint32_t ordinal1; | ||
| 285 | }; | ||
| 286 | |||
| 287 | union { | ||
| 288 | struct { | ||
| 289 | uint32_t context_id:28; | ||
| 290 | enum query_status_interrupt_sel_enum interrupt_sel:2; | ||
| 291 | enum query_status_command_enum command:2; | ||
| 292 | } bitfields2; | ||
| 293 | uint32_t ordinal2; | ||
| 294 | }; | ||
| 295 | |||
| 296 | union { | ||
| 297 | struct { | ||
| 298 | uint32_t pasid:16; | ||
| 299 | uint32_t reserved1:16; | ||
| 300 | } bitfields3a; | ||
| 301 | struct { | ||
| 302 | uint32_t reserved2:2; | ||
| 303 | uint32_t doorbell_offset:21; | ||
| 304 | uint32_t reserved3:3; | ||
| 305 | enum query_status_engine_sel_enum engine_sel:3; | ||
| 306 | uint32_t reserved4:3; | ||
| 307 | } bitfields3b; | ||
| 308 | uint32_t ordinal3; | ||
| 309 | }; | ||
| 310 | |||
| 311 | uint32_t addr_lo; | ||
| 312 | uint32_t addr_hi; | ||
| 313 | uint32_t data_lo; | ||
| 314 | uint32_t data_hi; | ||
| 315 | }; | ||
| 316 | #endif | ||
| 317 | |||
| 318 | /*--------------------MES_UNMAP_QUEUES--------------------*/ | ||
| 319 | |||
| 320 | #ifndef PM4_MES_UNMAP_QUEUES_DEFINED | ||
| 321 | #define PM4_MES_UNMAP_QUEUES_DEFINED | ||
| 322 | enum unmap_queues_action_enum { | ||
| 323 | action__mes_unmap_queues__preempt_queues = 0, | ||
| 324 | action__mes_unmap_queues__reset_queues = 1, | ||
| 325 | action__mes_unmap_queues__disable_process_queues = 2 | ||
| 326 | }; | ||
| 327 | |||
| 328 | enum unmap_queues_queue_sel_enum { | ||
| 329 | queue_sel__mes_unmap_queues__perform_request_on_specified_queues = 0, | ||
| 330 | queue_sel__mes_unmap_queues__perform_request_on_pasid_queues = 1, | ||
| 331 | queue_sel__mes_unmap_queues__perform_request_on_all_active_queues = 2 | ||
| 332 | }; | ||
| 333 | |||
| 334 | enum unmap_queues_engine_sel_enum { | ||
| 335 | engine_sel__mes_unmap_queues__compute = 0, | ||
| 336 | engine_sel__mes_unmap_queues__sdma0 = 2, | ||
| 337 | engine_sel__mes_unmap_queues__sdma1 = 3 | ||
| 338 | }; | ||
| 339 | |||
| 340 | struct pm4_unmap_queues { | ||
| 341 | union { | ||
| 342 | union PM4_MES_TYPE_3_HEADER header; /* header */ | ||
| 343 | uint32_t ordinal1; | ||
| 344 | }; | ||
| 345 | |||
| 346 | union { | ||
| 347 | struct { | ||
| 348 | enum unmap_queues_action_enum action:2; | ||
| 349 | uint32_t reserved1:2; | ||
| 350 | enum unmap_queues_queue_sel_enum queue_sel:2; | ||
| 351 | uint32_t reserved2:20; | ||
| 352 | enum unmap_queues_engine_sel_enum engine_sel:3; | ||
| 353 | uint32_t num_queues:3; | ||
| 354 | } bitfields2; | ||
| 355 | uint32_t ordinal2; | ||
| 356 | }; | ||
| 357 | |||
| 358 | union { | ||
| 359 | struct { | ||
| 360 | uint32_t pasid:16; | ||
| 361 | uint32_t reserved3:16; | ||
| 362 | } bitfields3a; | ||
| 363 | struct { | ||
| 364 | uint32_t reserved4:2; | ||
| 365 | uint32_t doorbell_offset0:21; | ||
| 366 | uint32_t reserved5:9; | ||
| 367 | } bitfields3b; | ||
| 368 | uint32_t ordinal3; | ||
| 369 | }; | ||
| 370 | |||
| 371 | union { | ||
| 372 | struct { | ||
| 373 | uint32_t reserved6:2; | ||
| 374 | uint32_t doorbell_offset1:21; | ||
| 375 | uint32_t reserved7:9; | ||
| 376 | } bitfields4; | ||
| 377 | uint32_t ordinal4; | ||
| 378 | }; | ||
| 379 | |||
| 380 | union { | ||
| 381 | struct { | ||
| 382 | uint32_t reserved8:2; | ||
| 383 | uint32_t doorbell_offset2:21; | ||
| 384 | uint32_t reserved9:9; | ||
| 385 | } bitfields5; | ||
| 386 | uint32_t ordinal5; | ||
| 387 | }; | ||
| 388 | |||
| 389 | union { | ||
| 390 | struct { | ||
| 391 | uint32_t reserved10:2; | ||
| 392 | uint32_t doorbell_offset3:21; | ||
| 393 | uint32_t reserved11:9; | ||
| 394 | } bitfields6; | ||
| 395 | uint32_t ordinal6; | ||
| 396 | }; | ||
| 397 | |||
| 398 | }; | ||
| 399 | #endif | ||
| 400 | |||
| 401 | enum { | ||
| 402 | CACHE_FLUSH_AND_INV_TS_EVENT = 0x00000014 | ||
| 403 | }; | ||
| 404 | |||
| 405 | #endif /* KFD_PM4_HEADERS_H_ */ | ||
| @@ -0,0 +1,107 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | |||
| 25 | #ifndef KFD_PM4_OPCODES_H | ||
| 26 | #define KFD_PM4_OPCODES_H | ||
| 27 | |||
| 28 | enum it_opcode_type { | ||
| 29 | IT_NOP = 0x10, | ||
| 30 | IT_SET_BASE = 0x11, | ||
| 31 | IT_CLEAR_STATE = 0x12, | ||
| 32 | IT_INDEX_BUFFER_SIZE = 0x13, | ||
| 33 | IT_DISPATCH_DIRECT = 0x15, | ||
| 34 | IT_DISPATCH_INDIRECT = 0x16, | ||
| 35 | IT_ATOMIC_GDS = 0x1D, | ||
| 36 | IT_OCCLUSION_QUERY = 0x1F, | ||
| 37 | IT_SET_PREDICATION = 0x20, | ||
| 38 | IT_REG_RMW = 0x21, | ||
| 39 | IT_COND_EXEC = 0x22, | ||
| 40 | IT_PRED_EXEC = 0x23, | ||
| 41 | IT_DRAW_INDIRECT = 0x24, | ||
| 42 | IT_DRAW_INDEX_INDIRECT = 0x25, | ||
| 43 | IT_INDEX_BASE = 0x26, | ||
| 44 | IT_DRAW_INDEX_2 = 0x27, | ||
| 45 | IT_CONTEXT_CONTROL = 0x28, | ||
| 46 | IT_INDEX_TYPE = 0x2A, | ||
| 47 | IT_DRAW_INDIRECT_MULTI = 0x2C, | ||
| 48 | IT_DRAW_INDEX_AUTO = 0x2D, | ||
| 49 | IT_NUM_INSTANCES = 0x2F, | ||
| 50 | IT_DRAW_INDEX_MULTI_AUTO = 0x30, | ||
| 51 | IT_INDIRECT_BUFFER_CNST = 0x33, | ||
| 52 | IT_STRMOUT_BUFFER_UPDATE = 0x34, | ||
| 53 | IT_DRAW_INDEX_OFFSET_2 = 0x35, | ||
| 54 | IT_DRAW_PREAMBLE = 0x36, | ||
| 55 | IT_WRITE_DATA = 0x37, | ||
| 56 | IT_DRAW_INDEX_INDIRECT_MULTI = 0x38, | ||
| 57 | IT_MEM_SEMAPHORE = 0x39, | ||
| 58 | IT_COPY_DW = 0x3B, | ||
| 59 | IT_WAIT_REG_MEM = 0x3C, | ||
| 60 | IT_INDIRECT_BUFFER = 0x3F, | ||
| 61 | IT_COPY_DATA = 0x40, | ||
| 62 | IT_PFP_SYNC_ME = 0x42, | ||
| 63 | IT_SURFACE_SYNC = 0x43, | ||
| 64 | IT_COND_WRITE = 0x45, | ||
| 65 | IT_EVENT_WRITE = 0x46, | ||
| 66 | IT_EVENT_WRITE_EOP = 0x47, | ||
| 67 | IT_EVENT_WRITE_EOS = 0x48, | ||
| 68 | IT_RELEASE_MEM = 0x49, | ||
| 69 | IT_PREAMBLE_CNTL = 0x4A, | ||
| 70 | IT_DMA_DATA = 0x50, | ||
| 71 | IT_ACQUIRE_MEM = 0x58, | ||
| 72 | IT_REWIND = 0x59, | ||
| 73 | IT_LOAD_UCONFIG_REG = 0x5E, | ||
| 74 | IT_LOAD_SH_REG = 0x5F, | ||
| 75 | IT_LOAD_CONFIG_REG = 0x60, | ||
| 76 | IT_LOAD_CONTEXT_REG = 0x61, | ||
| 77 | IT_SET_CONFIG_REG = 0x68, | ||
| 78 | IT_SET_CONTEXT_REG = 0x69, | ||
| 79 | IT_SET_CONTEXT_REG_INDIRECT = 0x73, | ||
| 80 | IT_SET_SH_REG = 0x76, | ||
| 81 | IT_SET_SH_REG_OFFSET = 0x77, | ||
| 82 | IT_SET_QUEUE_REG = 0x78, | ||
| 83 | IT_SET_UCONFIG_REG = 0x79, | ||
| 84 | IT_SCRATCH_RAM_WRITE = 0x7D, | ||
| 85 | IT_SCRATCH_RAM_READ = 0x7E, | ||
| 86 | IT_LOAD_CONST_RAM = 0x80, | ||
| 87 | IT_WRITE_CONST_RAM = 0x81, | ||
| 88 | IT_DUMP_CONST_RAM = 0x83, | ||
| 89 | IT_INCREMENT_CE_COUNTER = 0x84, | ||
| 90 | IT_INCREMENT_DE_COUNTER = 0x85, | ||
| 91 | IT_WAIT_ON_CE_COUNTER = 0x86, | ||
| 92 | IT_WAIT_ON_DE_COUNTER_DIFF = 0x88, | ||
| 93 | IT_SWITCH_BUFFER = 0x8B, | ||
| 94 | IT_SET_RESOURCES = 0xA0, | ||
| 95 | IT_MAP_PROCESS = 0xA1, | ||
| 96 | IT_MAP_QUEUES = 0xA2, | ||
| 97 | IT_UNMAP_QUEUES = 0xA3, | ||
| 98 | IT_QUERY_STATUS = 0xA4, | ||
| 99 | IT_RUN_LIST = 0xA5, | ||
| 100 | }; | ||
| 101 | |||
| 102 | #define PM4_TYPE_0 0 | ||
| 103 | #define PM4_TYPE_2 2 | ||
| 104 | #define PM4_TYPE_3 3 | ||
| 105 | |||
| 106 | #endif /* KFD_PM4_OPCODES_H */ | ||
| 107 | |||
| @@ -0,0 +1,598 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | #ifndef KFD_PRIV_H_INCLUDED | ||
| 24 | #define KFD_PRIV_H_INCLUDED | ||
| 25 | |||
| 26 | #include <linux/hashtable.h> | ||
| 27 | #include <linux/mmu_notifier.h> | ||
| 28 | #include <linux/mutex.h> | ||
| 29 | #include <linux/types.h> | ||
| 30 | #include <linux/atomic.h> | ||
| 31 | #include <linux/workqueue.h> | ||
| 32 | #include <linux/spinlock.h> | ||
| 33 | #include <linux/kfd_ioctl.h> | ||
| 34 | #include <kgd_kfd_interface.h> | ||
| 35 | |||
| 36 | #define KFD_SYSFS_FILE_MODE 0444 | ||
| 37 | |||
| 38 | /* | ||
| 39 | * When working with cp scheduler we should assign the HIQ manually or via | ||
| 40 | * the radeon driver to a fixed hqd slot, here are the fixed HIQ hqd slot | ||
| 41 | * definitions for Kaveri. In Kaveri only the first ME queues participates | ||
| 42 | * in the cp scheduling taking that in mind we set the HIQ slot in the | ||
| 43 | * second ME. | ||
| 44 | */ | ||
| 45 | #define KFD_CIK_HIQ_PIPE 4 | ||
| 46 | #define KFD_CIK_HIQ_QUEUE 0 | ||
| 47 | |||
| 48 | /* GPU ID hash width in bits */ | ||
| 49 | #define KFD_GPU_ID_HASH_WIDTH 16 | ||
| 50 | |||
| 51 | /* Macro for allocating structures */ | ||
| 52 | #define kfd_alloc_struct(ptr_to_struct) \ | ||
| 53 | ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL)) | ||
| 54 | |||
| 55 | /* Kernel module parameter to specify maximum number of supported processes */ | ||
| 56 | extern int max_num_of_processes; | ||
| 57 | |||
| 58 | #define KFD_MAX_NUM_OF_PROCESSES_DEFAULT 32 | ||
| 59 | #define KFD_MAX_NUM_OF_PROCESSES 512 | ||
| 60 | |||
| 61 | /* | ||
| 62 | * Kernel module parameter to specify maximum number of supported queues | ||
| 63 | * per process | ||
| 64 | */ | ||
| 65 | extern int max_num_of_queues_per_process; | ||
| 66 | |||
| 67 | #define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT 128 | ||
| 68 | #define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024 | ||
| 69 | |||
| 70 | #define KFD_KERNEL_QUEUE_SIZE 2048 | ||
| 71 | |||
| 72 | /* Kernel module parameter to specify the scheduling policy */ | ||
| 73 | extern int sched_policy; | ||
| 74 | |||
| 75 | /** | ||
| 76 | * enum kfd_sched_policy | ||
| 77 | * | ||
| 78 | * @KFD_SCHED_POLICY_HWS: H/W scheduling policy known as command processor (cp) | ||
| 79 | * scheduling. In this scheduling mode we're using the firmware code to | ||
| 80 | * schedule the user mode queues and kernel queues such as HIQ and DIQ. | ||
| 81 | * the HIQ queue is used as a special queue that dispatches the configuration | ||
| 82 | * to the cp and the user mode queues list that are currently running. | ||
| 83 | * the DIQ queue is a debugging queue that dispatches debugging commands to the | ||
| 84 | * firmware. | ||
| 85 | * in this scheduling mode user mode queues over subscription feature is | ||
| 86 | * enabled. | ||
| 87 | * | ||
| 88 | * @KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: The same as above but the over | ||
| 89 | * subscription feature disabled. | ||
| 90 | * | ||
| 91 | * @KFD_SCHED_POLICY_NO_HWS: no H/W scheduling policy is a mode which directly | ||
| 92 | * set the command processor registers and sets the queues "manually". This | ||
| 93 | * mode is used *ONLY* for debugging proposes. | ||
| 94 | * | ||
| 95 | */ | ||
| 96 | enum kfd_sched_policy { | ||
| 97 | KFD_SCHED_POLICY_HWS = 0, | ||
| 98 | KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION, | ||
| 99 | KFD_SCHED_POLICY_NO_HWS | ||
| 100 | }; | ||
| 101 | |||
| 102 | enum cache_policy { | ||
| 103 | cache_policy_coherent, | ||
| 104 | cache_policy_noncoherent | ||
| 105 | }; | ||
| 106 | |||
| 107 | struct kfd_device_info { | ||
| 108 | unsigned int max_pasid_bits; | ||
| 109 | size_t ih_ring_entry_size; | ||
| 110 | uint16_t mqd_size_aligned; | ||
| 111 | }; | ||
| 112 | |||
| 113 | struct kfd_dev { | ||
| 114 | struct kgd_dev *kgd; | ||
| 115 | |||
| 116 | const struct kfd_device_info *device_info; | ||
| 117 | struct pci_dev *pdev; | ||
| 118 | |||
| 119 | unsigned int id; /* topology stub index */ | ||
| 120 | |||
| 121 | phys_addr_t doorbell_base; /* Start of actual doorbells used by | ||
| 122 | * KFD. It is aligned for mapping | ||
| 123 | * into user mode | ||
| 124 | */ | ||
| 125 | size_t doorbell_id_offset; /* Doorbell offset (from KFD doorbell | ||
| 126 | * to HW doorbell, GFX reserved some | ||
| 127 | * at the start) | ||
| 128 | */ | ||
| 129 | size_t doorbell_process_limit; /* Number of processes we have doorbell | ||
| 130 | * space for. | ||
| 131 | */ | ||
| 132 | u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells | ||
| 133 | * page used by kernel queue | ||
| 134 | */ | ||
| 135 | |||
| 136 | struct kgd2kfd_shared_resources shared_resources; | ||
| 137 | |||
| 138 | void *interrupt_ring; | ||
| 139 | size_t interrupt_ring_size; | ||
| 140 | atomic_t interrupt_ring_rptr; | ||
| 141 | atomic_t interrupt_ring_wptr; | ||
| 142 | struct work_struct interrupt_work; | ||
| 143 | spinlock_t interrupt_lock; | ||
| 144 | |||
| 145 | /* QCM Device instance */ | ||
| 146 | struct device_queue_manager *dqm; | ||
| 147 | |||
| 148 | bool init_complete; | ||
| 149 | /* | ||
| 150 | * Interrupts of interest to KFD are copied | ||
| 151 | * from the HW ring into a SW ring. | ||
| 152 | */ | ||
| 153 | bool interrupts_active; | ||
| 154 | }; | ||
| 155 | |||
| 156 | /* KGD2KFD callbacks */ | ||
| 157 | void kgd2kfd_exit(void); | ||
| 158 | struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev); | ||
| 159 | bool kgd2kfd_device_init(struct kfd_dev *kfd, | ||
| 160 | const struct kgd2kfd_shared_resources *gpu_resources); | ||
| 161 | void kgd2kfd_device_exit(struct kfd_dev *kfd); | ||
| 162 | |||
| 163 | extern const struct kfd2kgd_calls *kfd2kgd; | ||
| 164 | |||
| 165 | struct kfd_mem_obj { | ||
| 166 | void *bo; | ||
| 167 | uint64_t gpu_addr; | ||
| 168 | uint32_t *cpu_ptr; | ||
| 169 | }; | ||
| 170 | |||
| 171 | enum kfd_mempool { | ||
| 172 | KFD_MEMPOOL_SYSTEM_CACHEABLE = 1, | ||
| 173 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2, | ||
| 174 | KFD_MEMPOOL_FRAMEBUFFER = 3, | ||
| 175 | }; | ||
| 176 | |||
| 177 | /* Character device interface */ | ||
| 178 | int kfd_chardev_init(void); | ||
| 179 | void kfd_chardev_exit(void); | ||
| 180 | struct device *kfd_chardev(void); | ||
| 181 | |||
| 182 | /** | ||
| 183 | * enum kfd_preempt_type_filter | ||
| 184 | * | ||
| 185 | * @KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE: Preempts single queue. | ||
| 186 | * | ||
| 187 | * @KFD_PRERMPT_TYPE_FILTER_ALL_QUEUES: Preempts all queues in the | ||
| 188 | * running queues list. | ||
| 189 | * | ||
| 190 | * @KFD_PRERMPT_TYPE_FILTER_BY_PASID: Preempts queues that belongs to | ||
| 191 | * specific process. | ||
| 192 | * | ||
| 193 | */ | ||
| 194 | enum kfd_preempt_type_filter { | ||
| 195 | KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE, | ||
| 196 | KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, | ||
| 197 | KFD_PREEMPT_TYPE_FILTER_BY_PASID | ||
| 198 | }; | ||
| 199 | |||
| 200 | enum kfd_preempt_type { | ||
| 201 | KFD_PREEMPT_TYPE_WAVEFRONT, | ||
| 202 | KFD_PREEMPT_TYPE_WAVEFRONT_RESET | ||
| 203 | }; | ||
| 204 | |||
| 205 | /** | ||
| 206 | * enum kfd_queue_type | ||
| 207 | * | ||
| 208 | * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type. | ||
| 209 | * | ||
| 210 | * @KFD_QUEUE_TYPE_SDMA: Sdma user mode queue type. | ||
| 211 | * | ||
| 212 | * @KFD_QUEUE_TYPE_HIQ: HIQ queue type. | ||
| 213 | * | ||
| 214 | * @KFD_QUEUE_TYPE_DIQ: DIQ queue type. | ||
| 215 | */ | ||
| 216 | enum kfd_queue_type { | ||
| 217 | KFD_QUEUE_TYPE_COMPUTE, | ||
| 218 | KFD_QUEUE_TYPE_SDMA, | ||
| 219 | KFD_QUEUE_TYPE_HIQ, | ||
| 220 | KFD_QUEUE_TYPE_DIQ | ||
| 221 | }; | ||
| 222 | |||
| 223 | enum kfd_queue_format { | ||
| 224 | KFD_QUEUE_FORMAT_PM4, | ||
| 225 | KFD_QUEUE_FORMAT_AQL | ||
| 226 | }; | ||
| 227 | |||
| 228 | /** | ||
| 229 | * struct queue_properties | ||
| 230 | * | ||
| 231 | * @type: The queue type. | ||
| 232 | * | ||
| 233 | * @queue_id: Queue identifier. | ||
| 234 | * | ||
| 235 | * @queue_address: Queue ring buffer address. | ||
| 236 | * | ||
| 237 | * @queue_size: Queue ring buffer size. | ||
| 238 | * | ||
| 239 | * @priority: Defines the queue priority relative to other queues in the | ||
| 240 | * process. | ||
| 241 | * This is just an indication and HW scheduling may override the priority as | ||
| 242 | * necessary while keeping the relative prioritization. | ||
| 243 | * the priority granularity is from 0 to f which f is the highest priority. | ||
| 244 | * currently all queues are initialized with the highest priority. | ||
| 245 | * | ||
| 246 | * @queue_percent: This field is partially implemented and currently a zero in | ||
| 247 | * this field defines that the queue is non active. | ||
| 248 | * | ||
| 249 | * @read_ptr: User space address which points to the number of dwords the | ||
| 250 | * cp read from the ring buffer. This field updates automatically by the H/W. | ||
| 251 | * | ||
| 252 | * @write_ptr: Defines the number of dwords written to the ring buffer. | ||
| 253 | * | ||
| 254 | * @doorbell_ptr: This field aim is to notify the H/W of new packet written to | ||
| 255 | * the queue ring buffer. This field should be similar to write_ptr and the user | ||
| 256 | * should update this field after he updated the write_ptr. | ||
| 257 | * | ||
| 258 | * @doorbell_off: The doorbell offset in the doorbell pci-bar. | ||
| 259 | * | ||
| 260 | * @is_interop: Defines if this is a interop queue. Interop queue means that the | ||
| 261 | * queue can access both graphics and compute resources. | ||
| 262 | * | ||
| 263 | * @is_active: Defines if the queue is active or not. | ||
| 264 | * | ||
| 265 | * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid | ||
| 266 | * of the queue. | ||
| 267 | * | ||
| 268 | * This structure represents the queue properties for each queue no matter if | ||
| 269 | * it's user mode or kernel mode queue. | ||
| 270 | * | ||
| 271 | */ | ||
| 272 | struct queue_properties { | ||
| 273 | enum kfd_queue_type type; | ||
| 274 | enum kfd_queue_format format; | ||
| 275 | unsigned int queue_id; | ||
| 276 | uint64_t queue_address; | ||
| 277 | uint64_t queue_size; | ||
| 278 | uint32_t priority; | ||
| 279 | uint32_t queue_percent; | ||
| 280 | uint32_t *read_ptr; | ||
| 281 | uint32_t *write_ptr; | ||
| 282 | uint32_t *doorbell_ptr; | ||
| 283 | uint32_t doorbell_off; | ||
| 284 | bool is_interop; | ||
| 285 | bool is_active; | ||
| 286 | /* Not relevant for user mode queues in cp scheduling */ | ||
| 287 | unsigned int vmid; | ||
| 288 | }; | ||
| 289 | |||
| 290 | /** | ||
| 291 | * struct queue | ||
| 292 | * | ||
| 293 | * @list: Queue linked list. | ||
| 294 | * | ||
| 295 | * @mqd: The queue MQD. | ||
| 296 | * | ||
| 297 | * @mqd_mem_obj: The MQD local gpu memory object. | ||
| 298 | * | ||
| 299 | * @gart_mqd_addr: The MQD gart mc address. | ||
| 300 | * | ||
| 301 | * @properties: The queue properties. | ||
| 302 | * | ||
| 303 | * @mec: Used only in no cp scheduling mode and identifies to micro engine id | ||
| 304 | * that the queue should be execute on. | ||
| 305 | * | ||
| 306 | * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe id. | ||
| 307 | * | ||
| 308 | * @queue: Used only in no cp scheduliong mode and identifies the queue's slot. | ||
| 309 | * | ||
| 310 | * @process: The kfd process that created this queue. | ||
| 311 | * | ||
| 312 | * @device: The kfd device that created this queue. | ||
| 313 | * | ||
| 314 | * This structure represents user mode compute queues. | ||
| 315 | * It contains all the necessary data to handle such queues. | ||
| 316 | * | ||
| 317 | */ | ||
| 318 | |||
| 319 | struct queue { | ||
| 320 | struct list_head list; | ||
| 321 | void *mqd; | ||
| 322 | struct kfd_mem_obj *mqd_mem_obj; | ||
| 323 | uint64_t gart_mqd_addr; | ||
| 324 | struct queue_properties properties; | ||
| 325 | |||
| 326 | uint32_t mec; | ||
| 327 | uint32_t pipe; | ||
| 328 | uint32_t queue; | ||
| 329 | |||
| 330 | struct kfd_process *process; | ||
| 331 | struct kfd_dev *device; | ||
| 332 | }; | ||
| 333 | |||
| 334 | /* | ||
| 335 | * Please read the kfd_mqd_manager.h description. | ||
| 336 | */ | ||
| 337 | enum KFD_MQD_TYPE { | ||
| 338 | KFD_MQD_TYPE_CIK_COMPUTE = 0, /* for no cp scheduling */ | ||
| 339 | KFD_MQD_TYPE_CIK_HIQ, /* for hiq */ | ||
| 340 | KFD_MQD_TYPE_CIK_CP, /* for cp queues and diq */ | ||
| 341 | KFD_MQD_TYPE_CIK_SDMA, /* for sdma queues */ | ||
| 342 | KFD_MQD_TYPE_MAX | ||
| 343 | }; | ||
| 344 | |||
| 345 | struct scheduling_resources { | ||
| 346 | unsigned int vmid_mask; | ||
| 347 | enum kfd_queue_type type; | ||
| 348 | uint64_t queue_mask; | ||
| 349 | uint64_t gws_mask; | ||
| 350 | uint32_t oac_mask; | ||
| 351 | uint32_t gds_heap_base; | ||
| 352 | uint32_t gds_heap_size; | ||
| 353 | }; | ||
| 354 | |||
| 355 | struct process_queue_manager { | ||
| 356 | /* data */ | ||
| 357 | struct kfd_process *process; | ||
| 358 | unsigned int num_concurrent_processes; | ||
| 359 | struct list_head queues; | ||
| 360 | unsigned long *queue_slot_bitmap; | ||
| 361 | }; | ||
| 362 | |||
| 363 | struct qcm_process_device { | ||
| 364 | /* The Device Queue Manager that owns this data */ | ||
| 365 | struct device_queue_manager *dqm; | ||
| 366 | struct process_queue_manager *pqm; | ||
| 367 | /* Device Queue Manager lock */ | ||
| 368 | struct mutex *lock; | ||
| 369 | /* Queues list */ | ||
| 370 | struct list_head queues_list; | ||
| 371 | struct list_head priv_queue_list; | ||
| 372 | |||
| 373 | unsigned int queue_count; | ||
| 374 | unsigned int vmid; | ||
| 375 | bool is_debug; | ||
| 376 | /* | ||
| 377 | * All the memory management data should be here too | ||
| 378 | */ | ||
| 379 | uint64_t gds_context_area; | ||
| 380 | uint32_t sh_mem_config; | ||
| 381 | uint32_t sh_mem_bases; | ||
| 382 | uint32_t sh_mem_ape1_base; | ||
| 383 | uint32_t sh_mem_ape1_limit; | ||
| 384 | uint32_t page_table_base; | ||
| 385 | uint32_t gds_size; | ||
| 386 | uint32_t num_gws; | ||
| 387 | uint32_t num_oac; | ||
| 388 | }; | ||
| 389 | |||
| 390 | /* Data that is per-process-per device. */ | ||
| 391 | struct kfd_process_device { | ||
| 392 | /* | ||
| 393 | * List of all per-device data for a process. | ||
| 394 | * Starts from kfd_process.per_device_data. | ||
| 395 | */ | ||
| 396 | struct list_head per_device_list; | ||
| 397 | |||
| 398 | /* The device that owns this data. */ | ||
| 399 | struct kfd_dev *dev; | ||
| 400 | |||
| 401 | |||
| 402 | /* per-process-per device QCM data structure */ | ||
| 403 | struct qcm_process_device qpd; | ||
| 404 | |||
| 405 | /*Apertures*/ | ||
| 406 | uint64_t lds_base; | ||
| 407 | uint64_t lds_limit; | ||
| 408 | uint64_t gpuvm_base; | ||
| 409 | uint64_t gpuvm_limit; | ||
| 410 | uint64_t scratch_base; | ||
| 411 | uint64_t scratch_limit; | ||
| 412 | |||
| 413 | /* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */ | ||
| 414 | bool bound; | ||
| 415 | }; | ||
| 416 | |||
| 417 | /* Process data */ | ||
| 418 | struct kfd_process { | ||
| 419 | /* | ||
| 420 | * kfd_process are stored in an mm_struct*->kfd_process* | ||
| 421 | * hash table (kfd_processes in kfd_process.c) | ||
| 422 | */ | ||
| 423 | struct hlist_node kfd_processes; | ||
| 424 | |||
| 425 | struct mm_struct *mm; | ||
| 426 | |||
| 427 | struct mutex mutex; | ||
| 428 | |||
| 429 | /* | ||
| 430 | * In any process, the thread that started main() is the lead | ||
| 431 | * thread and outlives the rest. | ||
| 432 | * It is here because amd_iommu_bind_pasid wants a task_struct. | ||
| 433 | */ | ||
| 434 | struct task_struct *lead_thread; | ||
| 435 | |||
| 436 | /* We want to receive a notification when the mm_struct is destroyed */ | ||
| 437 | struct mmu_notifier mmu_notifier; | ||
| 438 | |||
| 439 | /* Use for delayed freeing of kfd_process structure */ | ||
| 440 | struct rcu_head rcu; | ||
| 441 | |||
| 442 | unsigned int pasid; | ||
| 443 | |||
| 444 | /* | ||
| 445 | * List of kfd_process_device structures, | ||
| 446 | * one for each device the process is using. | ||
| 447 | */ | ||
| 448 | struct list_head per_device_data; | ||
| 449 | |||
| 450 | struct process_queue_manager pqm; | ||
| 451 | |||
| 452 | /* The process's queues. */ | ||
| 453 | size_t queue_array_size; | ||
| 454 | |||
| 455 | /* Size is queue_array_size, up to MAX_PROCESS_QUEUES. */ | ||
| 456 | struct kfd_queue **queues; | ||
| 457 | |||
| 458 | unsigned long allocated_queue_bitmap[DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)]; | ||
| 459 | |||
| 460 | /*Is the user space process 32 bit?*/ | ||
| 461 | bool is_32bit_user_mode; | ||
| 462 | }; | ||
| 463 | |||
| 464 | void kfd_process_create_wq(void); | ||
| 465 | void kfd_process_destroy_wq(void); | ||
| 466 | struct kfd_process *kfd_create_process(const struct task_struct *); | ||
| 467 | struct kfd_process *kfd_get_process(const struct task_struct *); | ||
| 468 | |||
| 469 | struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, | ||
| 470 | struct kfd_process *p); | ||
| 471 | void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid); | ||
| 472 | struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, | ||
| 473 | struct kfd_process *p, | ||
| 474 | int create_pdd); | ||
| 475 | |||
| 476 | /* Process device data iterator */ | ||
| 477 | struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p); | ||
| 478 | struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p, | ||
| 479 | struct kfd_process_device *pdd); | ||
| 480 | bool kfd_has_process_device_data(struct kfd_process *p); | ||
| 481 | |||
| 482 | /* PASIDs */ | ||
| 483 | int kfd_pasid_init(void); | ||
| 484 | void kfd_pasid_exit(void); | ||
| 485 | bool kfd_set_pasid_limit(unsigned int new_limit); | ||
| 486 | unsigned int kfd_get_pasid_limit(void); | ||
| 487 | unsigned int kfd_pasid_alloc(void); | ||
| 488 | void kfd_pasid_free(unsigned int pasid); | ||
| 489 | |||
| 490 | /* Doorbells */ | ||
| 491 | void kfd_doorbell_init(struct kfd_dev *kfd); | ||
| 492 | int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma); | ||
| 493 | u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, | ||
| 494 | unsigned int *doorbell_off); | ||
| 495 | void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr); | ||
| 496 | u32 read_kernel_doorbell(u32 __iomem *db); | ||
| 497 | void write_kernel_doorbell(u32 __iomem *db, u32 value); | ||
| 498 | unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd, | ||
| 499 | struct kfd_process *process, | ||
| 500 | unsigned int queue_id); | ||
| 501 | |||
| 502 | extern struct device *kfd_device; | ||
| 503 | |||
| 504 | /* Topology */ | ||
| 505 | int kfd_topology_init(void); | ||
| 506 | void kfd_topology_shutdown(void); | ||
| 507 | int kfd_topology_add_device(struct kfd_dev *gpu); | ||
| 508 | int kfd_topology_remove_device(struct kfd_dev *gpu); | ||
| 509 | struct kfd_dev *kfd_device_by_id(uint32_t gpu_id); | ||
| 510 | struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev); | ||
| 511 | struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx); | ||
| 512 | |||
| 513 | /* Interrupts */ | ||
| 514 | int kfd_interrupt_init(struct kfd_dev *dev); | ||
| 515 | void kfd_interrupt_exit(struct kfd_dev *dev); | ||
| 516 | void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry); | ||
| 517 | bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry); | ||
| 518 | |||
| 519 | /* Power Management */ | ||
| 520 | void kgd2kfd_suspend(struct kfd_dev *kfd); | ||
| 521 | int kgd2kfd_resume(struct kfd_dev *kfd); | ||
| 522 | |||
| 523 | /* amdkfd Apertures */ | ||
| 524 | int kfd_init_apertures(struct kfd_process *process); | ||
| 525 | |||
| 526 | /* Queue Context Management */ | ||
| 527 | inline uint32_t lower_32(uint64_t x); | ||
| 528 | inline uint32_t upper_32(uint64_t x); | ||
| 529 | |||
| 530 | int init_queue(struct queue **q, struct queue_properties properties); | ||
| 531 | void uninit_queue(struct queue *q); | ||
| 532 | void print_queue_properties(struct queue_properties *q); | ||
| 533 | void print_queue(struct queue *q); | ||
| 534 | |||
| 535 | struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, | ||
| 536 | struct kfd_dev *dev); | ||
| 537 | struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev); | ||
| 538 | void device_queue_manager_uninit(struct device_queue_manager *dqm); | ||
| 539 | struct kernel_queue *kernel_queue_init(struct kfd_dev *dev, | ||
| 540 | enum kfd_queue_type type); | ||
| 541 | void kernel_queue_uninit(struct kernel_queue *kq); | ||
| 542 | |||
| 543 | /* Process Queue Manager */ | ||
| 544 | struct process_queue_node { | ||
| 545 | struct queue *q; | ||
| 546 | struct kernel_queue *kq; | ||
| 547 | struct list_head process_queue_list; | ||
| 548 | }; | ||
| 549 | |||
| 550 | int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p); | ||
| 551 | void pqm_uninit(struct process_queue_manager *pqm); | ||
| 552 | int pqm_create_queue(struct process_queue_manager *pqm, | ||
| 553 | struct kfd_dev *dev, | ||
| 554 | struct file *f, | ||
| 555 | struct queue_properties *properties, | ||
| 556 | unsigned int flags, | ||
| 557 | enum kfd_queue_type type, | ||
| 558 | unsigned int *qid); | ||
| 559 | int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid); | ||
| 560 | int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid, | ||
| 561 | struct queue_properties *p); | ||
| 562 | |||
| 563 | /* Packet Manager */ | ||
| 564 | |||
| 565 | #define KFD_HIQ_TIMEOUT (500) | ||
| 566 | |||
| 567 | #define KFD_FENCE_COMPLETED (100) | ||
| 568 | #define KFD_FENCE_INIT (10) | ||
| 569 | #define KFD_UNMAP_LATENCY (150) | ||
| 570 | |||
| 571 | struct packet_manager { | ||
| 572 | struct device_queue_manager *dqm; | ||
| 573 | struct kernel_queue *priv_queue; | ||
| 574 | struct mutex lock; | ||
| 575 | bool allocated; | ||
| 576 | struct kfd_mem_obj *ib_buffer_obj; | ||
| 577 | }; | ||
| 578 | |||
| 579 | int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm); | ||
| 580 | void pm_uninit(struct packet_manager *pm); | ||
| 581 | int pm_send_set_resources(struct packet_manager *pm, | ||
| 582 | struct scheduling_resources *res); | ||
| 583 | int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues); | ||
| 584 | int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address, | ||
| 585 | uint32_t fence_value); | ||
| 586 | |||
| 587 | int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type, | ||
| 588 | enum kfd_preempt_type_filter mode, | ||
| 589 | uint32_t filter_param, bool reset, | ||
| 590 | unsigned int sdma_engine); | ||
| 591 | |||
| 592 | void pm_release_ib(struct packet_manager *pm); | ||
| 593 | |||
| 594 | uint64_t kfd_get_number_elems(struct kfd_dev *kfd); | ||
| 595 | phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, | ||
| 596 | struct kfd_process *process); | ||
| 597 | |||
| 598 | #endif | ||
| @@ -0,0 +1,415 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | #include <linux/mutex.h> | ||
| 24 | #include <linux/log2.h> | ||
| 25 | #include <linux/sched.h> | ||
| 26 | #include <linux/slab.h> | ||
| 27 | #include <linux/amd-iommu.h> | ||
| 28 | #include <linux/notifier.h> | ||
| 29 | struct mm_struct; | ||
| 30 | |||
| 31 | #include "kfd_priv.h" | ||
| 32 | |||
| 33 | /* | ||
| 34 | * Initial size for the array of queues. | ||
| 35 | * The allocated size is doubled each time | ||
| 36 | * it is exceeded up to MAX_PROCESS_QUEUES. | ||
| 37 | */ | ||
| 38 | #define INITIAL_QUEUE_ARRAY_SIZE 16 | ||
| 39 | |||
| 40 | /* | ||
| 41 | * List of struct kfd_process (field kfd_process). | ||
| 42 | * Unique/indexed by mm_struct* | ||
| 43 | */ | ||
| 44 | #define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */ | ||
| 45 | static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE); | ||
| 46 | static DEFINE_MUTEX(kfd_processes_mutex); | ||
| 47 | |||
| 48 | DEFINE_STATIC_SRCU(kfd_processes_srcu); | ||
| 49 | |||
| 50 | static struct workqueue_struct *kfd_process_wq; | ||
| 51 | |||
| 52 | struct kfd_process_release_work { | ||
| 53 | struct work_struct kfd_work; | ||
| 54 | struct kfd_process *p; | ||
| 55 | }; | ||
| 56 | |||
| 57 | static struct kfd_process *find_process(const struct task_struct *thread); | ||
| 58 | static struct kfd_process *create_process(const struct task_struct *thread); | ||
| 59 | |||
| 60 | void kfd_process_create_wq(void) | ||
| 61 | { | ||
| 62 | if (!kfd_process_wq) | ||
| 63 | kfd_process_wq = create_workqueue("kfd_process_wq"); | ||
| 64 | } | ||
| 65 | |||
| 66 | void kfd_process_destroy_wq(void) | ||
| 67 | { | ||
| 68 | if (kfd_process_wq) { | ||
| 69 | flush_workqueue(kfd_process_wq); | ||
| 70 | destroy_workqueue(kfd_process_wq); | ||
| 71 | kfd_process_wq = NULL; | ||
| 72 | } | ||
| 73 | } | ||
| 74 | |||
| 75 | struct kfd_process *kfd_create_process(const struct task_struct *thread) | ||
| 76 | { | ||
| 77 | struct kfd_process *process; | ||
| 78 | |||
| 79 | BUG_ON(!kfd_process_wq); | ||
| 80 | |||
| 81 | if (thread->mm == NULL) | ||
| 82 | return ERR_PTR(-EINVAL); | ||
| 83 | |||
| 84 | /* Only the pthreads threading model is supported. */ | ||
| 85 | if (thread->group_leader->mm != thread->mm) | ||
| 86 | return ERR_PTR(-EINVAL); | ||
| 87 | |||
| 88 | /* Take mmap_sem because we call __mmu_notifier_register inside */ | ||
| 89 | down_write(&thread->mm->mmap_sem); | ||
| 90 | |||
| 91 | /* | ||
| 92 | * take kfd processes mutex before starting of process creation | ||
| 93 | * so there won't be a case where two threads of the same process | ||
| 94 | * create two kfd_process structures | ||
| 95 | */ | ||
| 96 | mutex_lock(&kfd_processes_mutex); | ||
| 97 | |||
| 98 | /* A prior open of /dev/kfd could have already created the process. */ | ||
| 99 | process = find_process(thread); | ||
| 100 | if (process) | ||
| 101 | pr_debug("kfd: process already found\n"); | ||
| 102 | |||
| 103 | if (!process) | ||
| 104 | process = create_process(thread); | ||
| 105 | |||
| 106 | mutex_unlock(&kfd_processes_mutex); | ||
| 107 | |||
| 108 | up_write(&thread->mm->mmap_sem); | ||
| 109 | |||
| 110 | return process; | ||
| 111 | } | ||
| 112 | |||
| 113 | struct kfd_process *kfd_get_process(const struct task_struct *thread) | ||
| 114 | { | ||
| 115 | struct kfd_process *process; | ||
| 116 | |||
| 117 | if (thread->mm == NULL) | ||
| 118 | return ERR_PTR(-EINVAL); | ||
| 119 | |||
| 120 | /* Only the pthreads threading model is supported. */ | ||
| 121 | if (thread->group_leader->mm != thread->mm) | ||
| 122 | return ERR_PTR(-EINVAL); | ||
| 123 | |||
| 124 | process = find_process(thread); | ||
| 125 | |||
| 126 | return process; | ||
| 127 | } | ||
| 128 | |||
| 129 | static struct kfd_process *find_process_by_mm(const struct mm_struct *mm) | ||
| 130 | { | ||
| 131 | struct kfd_process *process; | ||
| 132 | |||
| 133 | hash_for_each_possible_rcu(kfd_processes_table, process, | ||
| 134 | kfd_processes, (uintptr_t)mm) | ||
| 135 | if (process->mm == mm) | ||
| 136 | return process; | ||
| 137 | |||
| 138 | return NULL; | ||
| 139 | } | ||
| 140 | |||
| 141 | static struct kfd_process *find_process(const struct task_struct *thread) | ||
| 142 | { | ||
| 143 | struct kfd_process *p; | ||
| 144 | int idx; | ||
| 145 | |||
| 146 | idx = srcu_read_lock(&kfd_processes_srcu); | ||
| 147 | p = find_process_by_mm(thread->mm); | ||
| 148 | srcu_read_unlock(&kfd_processes_srcu, idx); | ||
| 149 | |||
| 150 | return p; | ||
| 151 | } | ||
| 152 | |||
| 153 | static void kfd_process_wq_release(struct work_struct *work) | ||
| 154 | { | ||
| 155 | struct kfd_process_release_work *my_work; | ||
| 156 | struct kfd_process_device *pdd, *temp; | ||
| 157 | struct kfd_process *p; | ||
| 158 | |||
| 159 | my_work = (struct kfd_process_release_work *) work; | ||
| 160 | |||
| 161 | p = my_work->p; | ||
| 162 | |||
| 163 | mutex_lock(&p->mutex); | ||
| 164 | |||
| 165 | list_for_each_entry_safe(pdd, temp, &p->per_device_data, | ||
| 166 | per_device_list) { | ||
| 167 | amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid); | ||
| 168 | list_del(&pdd->per_device_list); | ||
| 169 | |||
| 170 | kfree(pdd); | ||
| 171 | } | ||
| 172 | |||
| 173 | kfd_pasid_free(p->pasid); | ||
| 174 | |||
| 175 | mutex_unlock(&p->mutex); | ||
| 176 | |||
| 177 | mutex_destroy(&p->mutex); | ||
| 178 | |||
| 179 | kfree(p->queues); | ||
| 180 | |||
| 181 | kfree(p); | ||
| 182 | |||
| 183 | kfree((void *)work); | ||
| 184 | } | ||
| 185 | |||
| 186 | static void kfd_process_destroy_delayed(struct rcu_head *rcu) | ||
| 187 | { | ||
| 188 | struct kfd_process_release_work *work; | ||
| 189 | struct kfd_process *p; | ||
| 190 | |||
| 191 | BUG_ON(!kfd_process_wq); | ||
| 192 | |||
| 193 | p = container_of(rcu, struct kfd_process, rcu); | ||
| 194 | BUG_ON(atomic_read(&p->mm->mm_count) <= 0); | ||
| 195 | |||
| 196 | mmdrop(p->mm); | ||
| 197 | |||
| 198 | work = (struct kfd_process_release_work *) | ||
| 199 | kmalloc(sizeof(struct kfd_process_release_work), GFP_KERNEL); | ||
| 200 | |||
| 201 | if (work) { | ||
| 202 | INIT_WORK((struct work_struct *) work, kfd_process_wq_release); | ||
| 203 | work->p = p; | ||
| 204 | queue_work(kfd_process_wq, (struct work_struct *) work); | ||
| 205 | } | ||
| 206 | } | ||
| 207 | |||
| 208 | static void kfd_process_notifier_release(struct mmu_notifier *mn, | ||
| 209 | struct mm_struct *mm) | ||
| 210 | { | ||
| 211 | struct kfd_process *p; | ||
| 212 | |||
| 213 | /* | ||
| 214 | * The kfd_process structure can not be free because the | ||
| 215 | * mmu_notifier srcu is read locked | ||
| 216 | */ | ||
| 217 | p = container_of(mn, struct kfd_process, mmu_notifier); | ||
| 218 | BUG_ON(p->mm != mm); | ||
| 219 | |||
| 220 | mutex_lock(&kfd_processes_mutex); | ||
| 221 | hash_del_rcu(&p->kfd_processes); | ||
| 222 | mutex_unlock(&kfd_processes_mutex); | ||
| 223 | synchronize_srcu(&kfd_processes_srcu); | ||
| 224 | |||
| 225 | mutex_lock(&p->mutex); | ||
| 226 | |||
| 227 | /* In case our notifier is called before IOMMU notifier */ | ||
| 228 | pqm_uninit(&p->pqm); | ||
| 229 | |||
| 230 | mutex_unlock(&p->mutex); | ||
| 231 | |||
| 232 | /* | ||
| 233 | * Because we drop mm_count inside kfd_process_destroy_delayed | ||
| 234 | * and because the mmu_notifier_unregister function also drop | ||
| 235 | * mm_count we need to take an extra count here. | ||
| 236 | */ | ||
| 237 | atomic_inc(&p->mm->mm_count); | ||
| 238 | mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm); | ||
| 239 | mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed); | ||
| 240 | } | ||
| 241 | |||
| 242 | static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = { | ||
| 243 | .release = kfd_process_notifier_release, | ||
| 244 | }; | ||
| 245 | |||
| 246 | static struct kfd_process *create_process(const struct task_struct *thread) | ||
| 247 | { | ||
| 248 | struct kfd_process *process; | ||
| 249 | int err = -ENOMEM; | ||
| 250 | |||
| 251 | process = kzalloc(sizeof(*process), GFP_KERNEL); | ||
| 252 | |||
| 253 | if (!process) | ||
| 254 | goto err_alloc_process; | ||
| 255 | |||
| 256 | process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE, | ||
| 257 | sizeof(process->queues[0]), GFP_KERNEL); | ||
| 258 | if (!process->queues) | ||
| 259 | goto err_alloc_queues; | ||
| 260 | |||
| 261 | process->pasid = kfd_pasid_alloc(); | ||
| 262 | if (process->pasid == 0) | ||
| 263 | goto err_alloc_pasid; | ||
| 264 | |||
| 265 | mutex_init(&process->mutex); | ||
| 266 | |||
| 267 | process->mm = thread->mm; | ||
| 268 | |||
| 269 | /* register notifier */ | ||
| 270 | process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops; | ||
| 271 | err = __mmu_notifier_register(&process->mmu_notifier, process->mm); | ||
| 272 | if (err) | ||
| 273 | goto err_mmu_notifier; | ||
| 274 | |||
| 275 | hash_add_rcu(kfd_processes_table, &process->kfd_processes, | ||
| 276 | (uintptr_t)process->mm); | ||
| 277 | |||
| 278 | process->lead_thread = thread->group_leader; | ||
| 279 | |||
| 280 | process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE; | ||
| 281 | |||
| 282 | INIT_LIST_HEAD(&process->per_device_data); | ||
| 283 | |||
| 284 | err = pqm_init(&process->pqm, process); | ||
| 285 | if (err != 0) | ||
| 286 | goto err_process_pqm_init; | ||
| 287 | |||
| 288 | return process; | ||
| 289 | |||
| 290 | err_process_pqm_init: | ||
| 291 | hash_del_rcu(&process->kfd_processes); | ||
| 292 | synchronize_rcu(); | ||
| 293 | mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm); | ||
| 294 | err_mmu_notifier: | ||
| 295 | kfd_pasid_free(process->pasid); | ||
| 296 | err_alloc_pasid: | ||
| 297 | kfree(process->queues); | ||
| 298 | err_alloc_queues: | ||
| 299 | kfree(process); | ||
| 300 | err_alloc_process: | ||
| 301 | return ERR_PTR(err); | ||
| 302 | } | ||
| 303 | |||
| 304 | struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, | ||
| 305 | struct kfd_process *p, | ||
| 306 | int create_pdd) | ||
| 307 | { | ||
| 308 | struct kfd_process_device *pdd = NULL; | ||
| 309 | |||
| 310 | list_for_each_entry(pdd, &p->per_device_data, per_device_list) | ||
| 311 | if (pdd->dev == dev) | ||
| 312 | return pdd; | ||
| 313 | |||
| 314 | if (create_pdd) { | ||
| 315 | pdd = kzalloc(sizeof(*pdd), GFP_KERNEL); | ||
| 316 | if (pdd != NULL) { | ||
| 317 | pdd->dev = dev; | ||
| 318 | INIT_LIST_HEAD(&pdd->qpd.queues_list); | ||
| 319 | INIT_LIST_HEAD(&pdd->qpd.priv_queue_list); | ||
| 320 | pdd->qpd.dqm = dev->dqm; | ||
| 321 | list_add(&pdd->per_device_list, &p->per_device_data); | ||
| 322 | } | ||
| 323 | } | ||
| 324 | |||
| 325 | return pdd; | ||
| 326 | } | ||
| 327 | |||
| 328 | /* | ||
| 329 | * Direct the IOMMU to bind the process (specifically the pasid->mm) | ||
| 330 | * to the device. | ||
| 331 | * Unbinding occurs when the process dies or the device is removed. | ||
| 332 | * | ||
| 333 | * Assumes that the process lock is held. | ||
| 334 | */ | ||
| 335 | struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, | ||
| 336 | struct kfd_process *p) | ||
| 337 | { | ||
| 338 | struct kfd_process_device *pdd = kfd_get_process_device_data(dev, p, 1); | ||
| 339 | int err; | ||
| 340 | |||
| 341 | if (pdd == NULL) | ||
| 342 | return ERR_PTR(-ENOMEM); | ||
| 343 | |||
| 344 | if (pdd->bound) | ||
| 345 | return pdd; | ||
| 346 | |||
| 347 | err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread); | ||
| 348 | if (err < 0) | ||
| 349 | return ERR_PTR(err); | ||
| 350 | |||
| 351 | if (err < 0) { | ||
| 352 | amd_iommu_unbind_pasid(dev->pdev, p->pasid); | ||
| 353 | return ERR_PTR(err); | ||
| 354 | } | ||
| 355 | |||
| 356 | pdd->bound = true; | ||
| 357 | |||
| 358 | return pdd; | ||
| 359 | } | ||
| 360 | |||
| 361 | void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid) | ||
| 362 | { | ||
| 363 | struct kfd_process *p; | ||
| 364 | struct kfd_process_device *pdd; | ||
| 365 | int idx, i; | ||
| 366 | |||
| 367 | BUG_ON(dev == NULL); | ||
| 368 | |||
| 369 | idx = srcu_read_lock(&kfd_processes_srcu); | ||
| 370 | |||
| 371 | hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes) | ||
| 372 | if (p->pasid == pasid) | ||
| 373 | break; | ||
| 374 | |||
| 375 | srcu_read_unlock(&kfd_processes_srcu, idx); | ||
| 376 | |||
| 377 | BUG_ON(p->pasid != pasid); | ||
| 378 | |||
| 379 | mutex_lock(&p->mutex); | ||
| 380 | |||
| 381 | pqm_uninit(&p->pqm); | ||
| 382 | |||
| 383 | pdd = kfd_get_process_device_data(dev, p, 0); | ||
| 384 | |||
| 385 | /* | ||
| 386 | * Just mark pdd as unbound, because we still need it to call | ||
| 387 | * amd_iommu_unbind_pasid() in when the process exits. | ||
| 388 | * We don't call amd_iommu_unbind_pasid() here | ||
| 389 | * because the IOMMU called us. | ||
| 390 | */ | ||
| 391 | if (pdd) | ||
| 392 | pdd->bound = false; | ||
| 393 | |||
| 394 | mutex_unlock(&p->mutex); | ||
| 395 | } | ||
| 396 | |||
| 397 | struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p) | ||
| 398 | { | ||
| 399 | return list_first_entry(&p->per_device_data, | ||
| 400 | struct kfd_process_device, | ||
| 401 | per_device_list); | ||
| 402 | } | ||
| 403 | |||
| 404 | struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p, | ||
| 405 | struct kfd_process_device *pdd) | ||
| 406 | { | ||
| 407 | if (list_is_last(&pdd->per_device_list, &p->per_device_data)) | ||
| 408 | return NULL; | ||
| 409 | return list_next_entry(pdd, per_device_list); | ||
| 410 | } | ||
| 411 | |||
| 412 | bool kfd_has_process_device_data(struct kfd_process *p) | ||
| 413 | { | ||
| 414 | return !(list_empty(&p->per_device_data)); | ||
| 415 | } | ||
| @@ -0,0 +1,342 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | #include <linux/slab.h> | ||
| 25 | #include <linux/list.h> | ||
| 26 | #include "kfd_device_queue_manager.h" | ||
| 27 | #include "kfd_priv.h" | ||
| 28 | #include "kfd_kernel_queue.h" | ||
| 29 | |||
| 30 | static inline struct process_queue_node *get_queue_by_qid( | ||
| 31 | struct process_queue_manager *pqm, unsigned int qid) | ||
| 32 | { | ||
| 33 | struct process_queue_node *pqn; | ||
| 34 | |||
| 35 | BUG_ON(!pqm); | ||
| 36 | |||
| 37 | list_for_each_entry(pqn, &pqm->queues, process_queue_list) { | ||
| 38 | if (pqn->q && pqn->q->properties.queue_id == qid) | ||
| 39 | return pqn; | ||
| 40 | if (pqn->kq && pqn->kq->queue->properties.queue_id == qid) | ||
| 41 | return pqn; | ||
| 42 | } | ||
| 43 | |||
| 44 | return NULL; | ||
| 45 | } | ||
| 46 | |||
| 47 | static int find_available_queue_slot(struct process_queue_manager *pqm, | ||
| 48 | unsigned int *qid) | ||
| 49 | { | ||
| 50 | unsigned long found; | ||
| 51 | |||
| 52 | BUG_ON(!pqm || !qid); | ||
| 53 | |||
| 54 | pr_debug("kfd: in %s\n", __func__); | ||
| 55 | |||
| 56 | found = find_first_zero_bit(pqm->queue_slot_bitmap, | ||
| 57 | max_num_of_queues_per_process); | ||
| 58 | |||
| 59 | pr_debug("kfd: the new slot id %lu\n", found); | ||
| 60 | |||
| 61 | if (found >= max_num_of_queues_per_process) { | ||
| 62 | pr_info("amdkfd: Can not open more queues for process with pasid %d\n", | ||
| 63 | pqm->process->pasid); | ||
| 64 | return -ENOMEM; | ||
| 65 | } | ||
| 66 | |||
| 67 | set_bit(found, pqm->queue_slot_bitmap); | ||
| 68 | *qid = found; | ||
| 69 | |||
| 70 | return 0; | ||
| 71 | } | ||
| 72 | |||
| 73 | int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p) | ||
| 74 | { | ||
| 75 | BUG_ON(!pqm); | ||
| 76 | |||
| 77 | INIT_LIST_HEAD(&pqm->queues); | ||
| 78 | pqm->queue_slot_bitmap = | ||
| 79 | kzalloc(DIV_ROUND_UP(max_num_of_queues_per_process, | ||
| 80 | BITS_PER_BYTE), GFP_KERNEL); | ||
| 81 | if (pqm->queue_slot_bitmap == NULL) | ||
| 82 | return -ENOMEM; | ||
| 83 | pqm->process = p; | ||
| 84 | |||
| 85 | return 0; | ||
| 86 | } | ||
| 87 | |||
| 88 | void pqm_uninit(struct process_queue_manager *pqm) | ||
| 89 | { | ||
| 90 | int retval; | ||
| 91 | struct process_queue_node *pqn, *next; | ||
| 92 | |||
| 93 | BUG_ON(!pqm); | ||
| 94 | |||
| 95 | pr_debug("In func %s\n", __func__); | ||
| 96 | |||
| 97 | list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) { | ||
| 98 | retval = pqm_destroy_queue( | ||
| 99 | pqm, | ||
| 100 | (pqn->q != NULL) ? | ||
| 101 | pqn->q->properties.queue_id : | ||
| 102 | pqn->kq->queue->properties.queue_id); | ||
| 103 | |||
| 104 | if (retval != 0) { | ||
| 105 | pr_err("kfd: failed to destroy queue\n"); | ||
| 106 | return; | ||
| 107 | } | ||
| 108 | } | ||
| 109 | kfree(pqm->queue_slot_bitmap); | ||
| 110 | pqm->queue_slot_bitmap = NULL; | ||
| 111 | } | ||
| 112 | |||
| 113 | static int create_cp_queue(struct process_queue_manager *pqm, | ||
| 114 | struct kfd_dev *dev, struct queue **q, | ||
| 115 | struct queue_properties *q_properties, | ||
| 116 | struct file *f, unsigned int qid) | ||
| 117 | { | ||
| 118 | int retval; | ||
| 119 | |||
| 120 | retval = 0; | ||
| 121 | |||
| 122 | /* Doorbell initialized in user space*/ | ||
| 123 | q_properties->doorbell_ptr = NULL; | ||
| 124 | |||
| 125 | q_properties->doorbell_off = | ||
| 126 | kfd_queue_id_to_doorbell(dev, pqm->process, qid); | ||
| 127 | |||
| 128 | /* let DQM handle it*/ | ||
| 129 | q_properties->vmid = 0; | ||
| 130 | q_properties->queue_id = qid; | ||
| 131 | q_properties->type = KFD_QUEUE_TYPE_COMPUTE; | ||
| 132 | |||
| 133 | retval = init_queue(q, *q_properties); | ||
| 134 | if (retval != 0) | ||
| 135 | goto err_init_queue; | ||
| 136 | |||
| 137 | (*q)->device = dev; | ||
| 138 | (*q)->process = pqm->process; | ||
| 139 | |||
| 140 | pr_debug("kfd: PQM After init queue"); | ||
| 141 | |||
| 142 | return retval; | ||
| 143 | |||
| 144 | err_init_queue: | ||
| 145 | return retval; | ||
| 146 | } | ||
| 147 | |||
| 148 | int pqm_create_queue(struct process_queue_manager *pqm, | ||
| 149 | struct kfd_dev *dev, | ||
| 150 | struct file *f, | ||
| 151 | struct queue_properties *properties, | ||
| 152 | unsigned int flags, | ||
| 153 | enum kfd_queue_type type, | ||
| 154 | unsigned int *qid) | ||
| 155 | { | ||
| 156 | int retval; | ||
| 157 | struct kfd_process_device *pdd; | ||
| 158 | struct queue_properties q_properties; | ||
| 159 | struct queue *q; | ||
| 160 | struct process_queue_node *pqn; | ||
| 161 | struct kernel_queue *kq; | ||
| 162 | |||
| 163 | BUG_ON(!pqm || !dev || !properties || !qid); | ||
| 164 | |||
| 165 | memset(&q_properties, 0, sizeof(struct queue_properties)); | ||
| 166 | memcpy(&q_properties, properties, sizeof(struct queue_properties)); | ||
| 167 | q = NULL; | ||
| 168 | kq = NULL; | ||
| 169 | |||
| 170 | pdd = kfd_get_process_device_data(dev, pqm->process, 1); | ||
| 171 | BUG_ON(!pdd); | ||
| 172 | |||
| 173 | retval = find_available_queue_slot(pqm, qid); | ||
| 174 | if (retval != 0) | ||
| 175 | return retval; | ||
| 176 | |||
| 177 | if (list_empty(&pqm->queues)) { | ||
| 178 | pdd->qpd.pqm = pqm; | ||
| 179 | dev->dqm->register_process(dev->dqm, &pdd->qpd); | ||
| 180 | } | ||
| 181 | |||
| 182 | pqn = kzalloc(sizeof(struct process_queue_node), GFP_KERNEL); | ||
| 183 | if (!pqn) { | ||
| 184 | retval = -ENOMEM; | ||
| 185 | goto err_allocate_pqn; | ||
| 186 | } | ||
| 187 | |||
| 188 | switch (type) { | ||
| 189 | case KFD_QUEUE_TYPE_COMPUTE: | ||
| 190 | /* check if there is over subscription */ | ||
| 191 | if ((sched_policy == KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) && | ||
| 192 | ((dev->dqm->processes_count >= VMID_PER_DEVICE) || | ||
| 193 | (dev->dqm->queue_count >= PIPE_PER_ME_CP_SCHEDULING * QUEUES_PER_PIPE))) { | ||
| 194 | pr_err("kfd: over-subscription is not allowed in radeon_kfd.sched_policy == 1\n"); | ||
| 195 | retval = -EPERM; | ||
| 196 | goto err_create_queue; | ||
| 197 | } | ||
| 198 | |||
| 199 | retval = create_cp_queue(pqm, dev, &q, &q_properties, f, *qid); | ||
| 200 | if (retval != 0) | ||
| 201 | goto err_create_queue; | ||
| 202 | pqn->q = q; | ||
| 203 | pqn->kq = NULL; | ||
| 204 | retval = dev->dqm->create_queue(dev->dqm, q, &pdd->qpd, | ||
| 205 | &q->properties.vmid); | ||
| 206 | print_queue(q); | ||
| 207 | break; | ||
| 208 | case KFD_QUEUE_TYPE_DIQ: | ||
| 209 | kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ); | ||
| 210 | if (kq == NULL) { | ||
| 211 | kernel_queue_uninit(kq); | ||
| 212 | goto err_create_queue; | ||
| 213 | } | ||
| 214 | kq->queue->properties.queue_id = *qid; | ||
| 215 | pqn->kq = kq; | ||
| 216 | pqn->q = NULL; | ||
| 217 | retval = dev->dqm->create_kernel_queue(dev->dqm, kq, &pdd->qpd); | ||
| 218 | break; | ||
| 219 | default: | ||
| 220 | BUG(); | ||
| 221 | break; | ||
| 222 | } | ||
| 223 | |||
| 224 | if (retval != 0) { | ||
| 225 | pr_err("kfd: error dqm create queue\n"); | ||
| 226 | goto err_create_queue; | ||
| 227 | } | ||
| 228 | |||
| 229 | pr_debug("kfd: PQM After DQM create queue\n"); | ||
| 230 | |||
| 231 | list_add(&pqn->process_queue_list, &pqm->queues); | ||
| 232 | |||
| 233 | if (q) { | ||
| 234 | *properties = q->properties; | ||
| 235 | pr_debug("kfd: PQM done creating queue\n"); | ||
| 236 | print_queue_properties(properties); | ||
| 237 | } | ||
| 238 | |||
| 239 | return retval; | ||
| 240 | |||
| 241 | err_create_queue: | ||
| 242 | kfree(pqn); | ||
| 243 | err_allocate_pqn: | ||
| 244 | clear_bit(*qid, pqm->queue_slot_bitmap); | ||
| 245 | return retval; | ||
| 246 | } | ||
| 247 | |||
| 248 | int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid) | ||
| 249 | { | ||
| 250 | struct process_queue_node *pqn; | ||
| 251 | struct kfd_process_device *pdd; | ||
| 252 | struct device_queue_manager *dqm; | ||
| 253 | struct kfd_dev *dev; | ||
| 254 | int retval; | ||
| 255 | |||
| 256 | dqm = NULL; | ||
| 257 | |||
| 258 | BUG_ON(!pqm); | ||
| 259 | retval = 0; | ||
| 260 | |||
| 261 | pr_debug("kfd: In Func %s\n", __func__); | ||
| 262 | |||
| 263 | pqn = get_queue_by_qid(pqm, qid); | ||
| 264 | if (pqn == NULL) { | ||
| 265 | pr_err("kfd: queue id does not match any known queue\n"); | ||
| 266 | return -EINVAL; | ||
| 267 | } | ||
| 268 | |||
| 269 | dev = NULL; | ||
| 270 | if (pqn->kq) | ||
| 271 | dev = pqn->kq->dev; | ||
| 272 | if (pqn->q) | ||
| 273 | dev = pqn->q->device; | ||
| 274 | BUG_ON(!dev); | ||
| 275 | |||
| 276 | pdd = kfd_get_process_device_data(dev, pqm->process, 1); | ||
| 277 | BUG_ON(!pdd); | ||
| 278 | |||
| 279 | if (pqn->kq) { | ||
| 280 | /* destroy kernel queue (DIQ) */ | ||
| 281 | dqm = pqn->kq->dev->dqm; | ||
| 282 | dqm->destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd); | ||
| 283 | kernel_queue_uninit(pqn->kq); | ||
| 284 | } | ||
| 285 | |||
| 286 | if (pqn->q) { | ||
| 287 | dqm = pqn->q->device->dqm; | ||
| 288 | retval = dqm->destroy_queue(dqm, &pdd->qpd, pqn->q); | ||
| 289 | if (retval != 0) | ||
| 290 | return retval; | ||
| 291 | |||
| 292 | uninit_queue(pqn->q); | ||
| 293 | } | ||
| 294 | |||
| 295 | list_del(&pqn->process_queue_list); | ||
| 296 | kfree(pqn); | ||
| 297 | clear_bit(qid, pqm->queue_slot_bitmap); | ||
| 298 | |||
| 299 | if (list_empty(&pqm->queues)) | ||
| 300 | dqm->unregister_process(dqm, &pdd->qpd); | ||
| 301 | |||
| 302 | return retval; | ||
| 303 | } | ||
| 304 | |||
| 305 | int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid, | ||
| 306 | struct queue_properties *p) | ||
| 307 | { | ||
| 308 | int retval; | ||
| 309 | struct process_queue_node *pqn; | ||
| 310 | |||
| 311 | BUG_ON(!pqm); | ||
| 312 | |||
| 313 | pqn = get_queue_by_qid(pqm, qid); | ||
| 314 | BUG_ON(!pqn); | ||
| 315 | |||
| 316 | pqn->q->properties.queue_address = p->queue_address; | ||
| 317 | pqn->q->properties.queue_size = p->queue_size; | ||
| 318 | pqn->q->properties.queue_percent = p->queue_percent; | ||
| 319 | pqn->q->properties.priority = p->priority; | ||
| 320 | |||
| 321 | retval = pqn->q->device->dqm->update_queue(pqn->q->device->dqm, pqn->q); | ||
| 322 | if (retval != 0) | ||
| 323 | return retval; | ||
| 324 | |||
| 325 | return 0; | ||
| 326 | } | ||
| 327 | |||
| 328 | struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm, | ||
| 329 | unsigned int qid) | ||
| 330 | { | ||
| 331 | struct process_queue_node *pqn; | ||
| 332 | |||
| 333 | BUG_ON(!pqm); | ||
| 334 | |||
| 335 | pqn = get_queue_by_qid(pqm, qid); | ||
| 336 | if (pqn && pqn->kq) | ||
| 337 | return pqn->kq; | ||
| 338 | |||
| 339 | return NULL; | ||
| 340 | } | ||
| 341 | |||
| 342 | |||
| @@ -0,0 +1,85 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | #include <linux/slab.h> | ||
| 25 | #include "kfd_priv.h" | ||
| 26 | |||
| 27 | void print_queue_properties(struct queue_properties *q) | ||
| 28 | { | ||
| 29 | if (!q) | ||
| 30 | return; | ||
| 31 | |||
| 32 | pr_debug("Printing queue properties:\n"); | ||
| 33 | pr_debug("Queue Type: %u\n", q->type); | ||
| 34 | pr_debug("Queue Size: %llu\n", q->queue_size); | ||
| 35 | pr_debug("Queue percent: %u\n", q->queue_percent); | ||
| 36 | pr_debug("Queue Address: 0x%llX\n", q->queue_address); | ||
| 37 | pr_debug("Queue Id: %u\n", q->queue_id); | ||
| 38 | pr_debug("Queue Process Vmid: %u\n", q->vmid); | ||
| 39 | pr_debug("Queue Read Pointer: 0x%p\n", q->read_ptr); | ||
| 40 | pr_debug("Queue Write Pointer: 0x%p\n", q->write_ptr); | ||
| 41 | pr_debug("Queue Doorbell Pointer: 0x%p\n", q->doorbell_ptr); | ||
| 42 | pr_debug("Queue Doorbell Offset: %u\n", q->doorbell_off); | ||
| 43 | } | ||
| 44 | |||
| 45 | void print_queue(struct queue *q) | ||
| 46 | { | ||
| 47 | if (!q) | ||
| 48 | return; | ||
| 49 | pr_debug("Printing queue:\n"); | ||
| 50 | pr_debug("Queue Type: %u\n", q->properties.type); | ||
| 51 | pr_debug("Queue Size: %llu\n", q->properties.queue_size); | ||
| 52 | pr_debug("Queue percent: %u\n", q->properties.queue_percent); | ||
| 53 | pr_debug("Queue Address: 0x%llX\n", q->properties.queue_address); | ||
| 54 | pr_debug("Queue Id: %u\n", q->properties.queue_id); | ||
| 55 | pr_debug("Queue Process Vmid: %u\n", q->properties.vmid); | ||
| 56 | pr_debug("Queue Read Pointer: 0x%p\n", q->properties.read_ptr); | ||
| 57 | pr_debug("Queue Write Pointer: 0x%p\n", q->properties.write_ptr); | ||
| 58 | pr_debug("Queue Doorbell Pointer: 0x%p\n", q->properties.doorbell_ptr); | ||
| 59 | pr_debug("Queue Doorbell Offset: %u\n", q->properties.doorbell_off); | ||
| 60 | pr_debug("Queue MQD Address: 0x%p\n", q->mqd); | ||
| 61 | pr_debug("Queue MQD Gart: 0x%llX\n", q->gart_mqd_addr); | ||
| 62 | pr_debug("Queue Process Address: 0x%p\n", q->process); | ||
| 63 | pr_debug("Queue Device Address: 0x%p\n", q->device); | ||
| 64 | } | ||
| 65 | |||
| 66 | int init_queue(struct queue **q, struct queue_properties properties) | ||
| 67 | { | ||
| 68 | struct queue *tmp; | ||
| 69 | |||
| 70 | BUG_ON(!q); | ||
| 71 | |||
| 72 | tmp = kzalloc(sizeof(struct queue), GFP_KERNEL); | ||
| 73 | if (!tmp) | ||
| 74 | return -ENOMEM; | ||
| 75 | |||
| 76 | memcpy(&tmp->properties, &properties, sizeof(struct queue_properties)); | ||
| 77 | |||
| 78 | *q = tmp; | ||
| 79 | return 0; | ||
| 80 | } | ||
| 81 | |||
| 82 | void uninit_queue(struct queue *q) | ||
| 83 | { | ||
| 84 | kfree(q); | ||
| 85 | } | ||
| @@ -0,0 +1,1235 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | #include <linux/types.h> | ||
| 24 | #include <linux/kernel.h> | ||
| 25 | #include <linux/pci.h> | ||
| 26 | #include <linux/errno.h> | ||
| 27 | #include <linux/acpi.h> | ||
| 28 | #include <linux/hash.h> | ||
| 29 | #include <linux/cpufreq.h> | ||
| 30 | |||
| 31 | #include "kfd_priv.h" | ||
| 32 | #include "kfd_crat.h" | ||
| 33 | #include "kfd_topology.h" | ||
| 34 | |||
| 35 | static struct list_head topology_device_list; | ||
| 36 | static int topology_crat_parsed; | ||
| 37 | static struct kfd_system_properties sys_props; | ||
| 38 | |||
| 39 | static DECLARE_RWSEM(topology_lock); | ||
| 40 | |||
| 41 | struct kfd_dev *kfd_device_by_id(uint32_t gpu_id) | ||
| 42 | { | ||
| 43 | struct kfd_topology_device *top_dev; | ||
| 44 | struct kfd_dev *device = NULL; | ||
| 45 | |||
| 46 | down_read(&topology_lock); | ||
| 47 | |||
| 48 | list_for_each_entry(top_dev, &topology_device_list, list) | ||
| 49 | if (top_dev->gpu_id == gpu_id) { | ||
| 50 | device = top_dev->gpu; | ||
| 51 | break; | ||
| 52 | } | ||
| 53 | |||
| 54 | up_read(&topology_lock); | ||
| 55 | |||
| 56 | return device; | ||
| 57 | } | ||
| 58 | |||
| 59 | struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev) | ||
| 60 | { | ||
| 61 | struct kfd_topology_device *top_dev; | ||
| 62 | struct kfd_dev *device = NULL; | ||
| 63 | |||
| 64 | down_read(&topology_lock); | ||
| 65 | |||
| 66 | list_for_each_entry(top_dev, &topology_device_list, list) | ||
| 67 | if (top_dev->gpu->pdev == pdev) { | ||
| 68 | device = top_dev->gpu; | ||
| 69 | break; | ||
| 70 | } | ||
| 71 | |||
| 72 | up_read(&topology_lock); | ||
| 73 | |||
| 74 | return device; | ||
| 75 | } | ||
| 76 | |||
| 77 | static int kfd_topology_get_crat_acpi(void *crat_image, size_t *size) | ||
| 78 | { | ||
| 79 | struct acpi_table_header *crat_table; | ||
| 80 | acpi_status status; | ||
| 81 | |||
| 82 | if (!size) | ||
| 83 | return -EINVAL; | ||
| 84 | |||
| 85 | /* | ||
| 86 | * Fetch the CRAT table from ACPI | ||
| 87 | */ | ||
| 88 | status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table); | ||
| 89 | if (status == AE_NOT_FOUND) { | ||
| 90 | pr_warn("CRAT table not found\n"); | ||
| 91 | return -ENODATA; | ||
| 92 | } else if (ACPI_FAILURE(status)) { | ||
| 93 | const char *err = acpi_format_exception(status); | ||
| 94 | |||
| 95 | pr_err("CRAT table error: %s\n", err); | ||
| 96 | return -EINVAL; | ||
| 97 | } | ||
| 98 | |||
| 99 | if (*size >= crat_table->length && crat_image != 0) | ||
| 100 | memcpy(crat_image, crat_table, crat_table->length); | ||
| 101 | |||
| 102 | *size = crat_table->length; | ||
| 103 | |||
| 104 | return 0; | ||
| 105 | } | ||
| 106 | |||
| 107 | static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev, | ||
| 108 | struct crat_subtype_computeunit *cu) | ||
| 109 | { | ||
| 110 | BUG_ON(!dev); | ||
| 111 | BUG_ON(!cu); | ||
| 112 | |||
| 113 | dev->node_props.cpu_cores_count = cu->num_cpu_cores; | ||
| 114 | dev->node_props.cpu_core_id_base = cu->processor_id_low; | ||
| 115 | if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT) | ||
| 116 | dev->node_props.capability |= HSA_CAP_ATS_PRESENT; | ||
| 117 | |||
| 118 | pr_info("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores, | ||
| 119 | cu->processor_id_low); | ||
| 120 | } | ||
| 121 | |||
| 122 | static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev, | ||
| 123 | struct crat_subtype_computeunit *cu) | ||
| 124 | { | ||
| 125 | BUG_ON(!dev); | ||
| 126 | BUG_ON(!cu); | ||
| 127 | |||
| 128 | dev->node_props.simd_id_base = cu->processor_id_low; | ||
| 129 | dev->node_props.simd_count = cu->num_simd_cores; | ||
| 130 | dev->node_props.lds_size_in_kb = cu->lds_size_in_kb; | ||
| 131 | dev->node_props.max_waves_per_simd = cu->max_waves_simd; | ||
| 132 | dev->node_props.wave_front_size = cu->wave_front_size; | ||
| 133 | dev->node_props.mem_banks_count = cu->num_banks; | ||
| 134 | dev->node_props.array_count = cu->num_arrays; | ||
| 135 | dev->node_props.cu_per_simd_array = cu->num_cu_per_array; | ||
| 136 | dev->node_props.simd_per_cu = cu->num_simd_per_cu; | ||
| 137 | dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu; | ||
| 138 | if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE) | ||
| 139 | dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE; | ||
| 140 | pr_info("CU GPU: simds=%d id_base=%d\n", cu->num_simd_cores, | ||
| 141 | cu->processor_id_low); | ||
| 142 | } | ||
| 143 | |||
| 144 | /* kfd_parse_subtype_cu is called when the topology mutex is already acquired */ | ||
| 145 | static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu) | ||
| 146 | { | ||
| 147 | struct kfd_topology_device *dev; | ||
| 148 | int i = 0; | ||
| 149 | |||
| 150 | BUG_ON(!cu); | ||
| 151 | |||
| 152 | pr_info("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n", | ||
| 153 | cu->proximity_domain, cu->hsa_capability); | ||
| 154 | list_for_each_entry(dev, &topology_device_list, list) { | ||
| 155 | if (cu->proximity_domain == i) { | ||
| 156 | if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT) | ||
| 157 | kfd_populated_cu_info_cpu(dev, cu); | ||
| 158 | |||
| 159 | if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT) | ||
| 160 | kfd_populated_cu_info_gpu(dev, cu); | ||
| 161 | break; | ||
| 162 | } | ||
| 163 | i++; | ||
| 164 | } | ||
| 165 | |||
| 166 | return 0; | ||
| 167 | } | ||
| 168 | |||
| 169 | /* | ||
| 170 | * kfd_parse_subtype_mem is called when the topology mutex is | ||
| 171 | * already acquired | ||
| 172 | */ | ||
| 173 | static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem) | ||
| 174 | { | ||
| 175 | struct kfd_mem_properties *props; | ||
| 176 | struct kfd_topology_device *dev; | ||
| 177 | int i = 0; | ||
| 178 | |||
| 179 | BUG_ON(!mem); | ||
| 180 | |||
| 181 | pr_info("Found memory entry in CRAT table with proximity_domain=%d\n", | ||
| 182 | mem->promixity_domain); | ||
| 183 | list_for_each_entry(dev, &topology_device_list, list) { | ||
| 184 | if (mem->promixity_domain == i) { | ||
| 185 | props = kfd_alloc_struct(props); | ||
| 186 | if (props == 0) | ||
| 187 | return -ENOMEM; | ||
| 188 | |||
| 189 | if (dev->node_props.cpu_cores_count == 0) | ||
| 190 | props->heap_type = HSA_MEM_HEAP_TYPE_FB_PRIVATE; | ||
| 191 | else | ||
| 192 | props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM; | ||
| 193 | |||
| 194 | if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE) | ||
| 195 | props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE; | ||
| 196 | if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE) | ||
| 197 | props->flags |= HSA_MEM_FLAGS_NON_VOLATILE; | ||
| 198 | |||
| 199 | props->size_in_bytes = | ||
| 200 | ((uint64_t)mem->length_high << 32) + | ||
| 201 | mem->length_low; | ||
| 202 | props->width = mem->width; | ||
| 203 | |||
| 204 | dev->mem_bank_count++; | ||
| 205 | list_add_tail(&props->list, &dev->mem_props); | ||
| 206 | |||
| 207 | break; | ||
| 208 | } | ||
| 209 | i++; | ||
| 210 | } | ||
| 211 | |||
| 212 | return 0; | ||
| 213 | } | ||
| 214 | |||
| 215 | /* | ||
| 216 | * kfd_parse_subtype_cache is called when the topology mutex | ||
| 217 | * is already acquired | ||
| 218 | */ | ||
| 219 | static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache) | ||
| 220 | { | ||
| 221 | struct kfd_cache_properties *props; | ||
| 222 | struct kfd_topology_device *dev; | ||
| 223 | uint32_t id; | ||
| 224 | |||
| 225 | BUG_ON(!cache); | ||
| 226 | |||
| 227 | id = cache->processor_id_low; | ||
| 228 | |||
| 229 | pr_info("Found cache entry in CRAT table with processor_id=%d\n", id); | ||
| 230 | list_for_each_entry(dev, &topology_device_list, list) | ||
| 231 | if (id == dev->node_props.cpu_core_id_base || | ||
| 232 | id == dev->node_props.simd_id_base) { | ||
| 233 | props = kfd_alloc_struct(props); | ||
| 234 | if (props == 0) | ||
| 235 | return -ENOMEM; | ||
| 236 | |||
| 237 | props->processor_id_low = id; | ||
| 238 | props->cache_level = cache->cache_level; | ||
| 239 | props->cache_size = cache->cache_size; | ||
| 240 | props->cacheline_size = cache->cache_line_size; | ||
| 241 | props->cachelines_per_tag = cache->lines_per_tag; | ||
| 242 | props->cache_assoc = cache->associativity; | ||
| 243 | props->cache_latency = cache->cache_latency; | ||
| 244 | |||
| 245 | if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE) | ||
| 246 | props->cache_type |= HSA_CACHE_TYPE_DATA; | ||
| 247 | if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE) | ||
| 248 | props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION; | ||
| 249 | if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE) | ||
| 250 | props->cache_type |= HSA_CACHE_TYPE_CPU; | ||
| 251 | if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE) | ||
| 252 | props->cache_type |= HSA_CACHE_TYPE_HSACU; | ||
| 253 | |||
| 254 | dev->cache_count++; | ||
| 255 | dev->node_props.caches_count++; | ||
| 256 | list_add_tail(&props->list, &dev->cache_props); | ||
| 257 | |||
| 258 | break; | ||
| 259 | } | ||
| 260 | |||
| 261 | return 0; | ||
| 262 | } | ||
| 263 | |||
| 264 | /* | ||
| 265 | * kfd_parse_subtype_iolink is called when the topology mutex | ||
| 266 | * is already acquired | ||
| 267 | */ | ||
| 268 | static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink) | ||
| 269 | { | ||
| 270 | struct kfd_iolink_properties *props; | ||
| 271 | struct kfd_topology_device *dev; | ||
| 272 | uint32_t i = 0; | ||
| 273 | uint32_t id_from; | ||
| 274 | uint32_t id_to; | ||
| 275 | |||
| 276 | BUG_ON(!iolink); | ||
| 277 | |||
| 278 | id_from = iolink->proximity_domain_from; | ||
| 279 | id_to = iolink->proximity_domain_to; | ||
| 280 | |||
| 281 | pr_info("Found IO link entry in CRAT table with id_from=%d\n", id_from); | ||
| 282 | list_for_each_entry(dev, &topology_device_list, list) { | ||
| 283 | if (id_from == i) { | ||
| 284 | props = kfd_alloc_struct(props); | ||
| 285 | if (props == 0) | ||
| 286 | return -ENOMEM; | ||
| 287 | |||
| 288 | props->node_from = id_from; | ||
| 289 | props->node_to = id_to; | ||
| 290 | props->ver_maj = iolink->version_major; | ||
| 291 | props->ver_min = iolink->version_minor; | ||
| 292 | |||
| 293 | /* | ||
| 294 | * weight factor (derived from CDIR), currently always 1 | ||
| 295 | */ | ||
| 296 | props->weight = 1; | ||
| 297 | |||
| 298 | props->min_latency = iolink->minimum_latency; | ||
| 299 | props->max_latency = iolink->maximum_latency; | ||
| 300 | props->min_bandwidth = iolink->minimum_bandwidth_mbs; | ||
| 301 | props->max_bandwidth = iolink->maximum_bandwidth_mbs; | ||
| 302 | props->rec_transfer_size = | ||
| 303 | iolink->recommended_transfer_size; | ||
| 304 | |||
| 305 | dev->io_link_count++; | ||
| 306 | dev->node_props.io_links_count++; | ||
| 307 | list_add_tail(&props->list, &dev->io_link_props); | ||
| 308 | |||
| 309 | break; | ||
| 310 | } | ||
| 311 | i++; | ||
| 312 | } | ||
| 313 | |||
| 314 | return 0; | ||
| 315 | } | ||
| 316 | |||
| 317 | static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr) | ||
| 318 | { | ||
| 319 | struct crat_subtype_computeunit *cu; | ||
| 320 | struct crat_subtype_memory *mem; | ||
| 321 | struct crat_subtype_cache *cache; | ||
| 322 | struct crat_subtype_iolink *iolink; | ||
| 323 | int ret = 0; | ||
| 324 | |||
| 325 | BUG_ON(!sub_type_hdr); | ||
| 326 | |||
| 327 | switch (sub_type_hdr->type) { | ||
| 328 | case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY: | ||
| 329 | cu = (struct crat_subtype_computeunit *)sub_type_hdr; | ||
| 330 | ret = kfd_parse_subtype_cu(cu); | ||
| 331 | break; | ||
| 332 | case CRAT_SUBTYPE_MEMORY_AFFINITY: | ||
| 333 | mem = (struct crat_subtype_memory *)sub_type_hdr; | ||
| 334 | ret = kfd_parse_subtype_mem(mem); | ||
| 335 | break; | ||
| 336 | case CRAT_SUBTYPE_CACHE_AFFINITY: | ||
| 337 | cache = (struct crat_subtype_cache *)sub_type_hdr; | ||
| 338 | ret = kfd_parse_subtype_cache(cache); | ||
| 339 | break; | ||
| 340 | case CRAT_SUBTYPE_TLB_AFFINITY: | ||
| 341 | /* | ||
| 342 | * For now, nothing to do here | ||
| 343 | */ | ||
| 344 | pr_info("Found TLB entry in CRAT table (not processing)\n"); | ||
| 345 | break; | ||
| 346 | case CRAT_SUBTYPE_CCOMPUTE_AFFINITY: | ||
| 347 | /* | ||
| 348 | * For now, nothing to do here | ||
| 349 | */ | ||
| 350 | pr_info("Found CCOMPUTE entry in CRAT table (not processing)\n"); | ||
| 351 | break; | ||
| 352 | case CRAT_SUBTYPE_IOLINK_AFFINITY: | ||
| 353 | iolink = (struct crat_subtype_iolink *)sub_type_hdr; | ||
| 354 | ret = kfd_parse_subtype_iolink(iolink); | ||
| 355 | break; | ||
| 356 | default: | ||
| 357 | pr_warn("Unknown subtype (%d) in CRAT\n", | ||
| 358 | sub_type_hdr->type); | ||
| 359 | } | ||
| 360 | |||
| 361 | return ret; | ||
| 362 | } | ||
| 363 | |||
| 364 | static void kfd_release_topology_device(struct kfd_topology_device *dev) | ||
| 365 | { | ||
| 366 | struct kfd_mem_properties *mem; | ||
| 367 | struct kfd_cache_properties *cache; | ||
| 368 | struct kfd_iolink_properties *iolink; | ||
| 369 | |||
| 370 | BUG_ON(!dev); | ||
| 371 | |||
| 372 | list_del(&dev->list); | ||
| 373 | |||
| 374 | while (dev->mem_props.next != &dev->mem_props) { | ||
| 375 | mem = container_of(dev->mem_props.next, | ||
| 376 | struct kfd_mem_properties, list); | ||
| 377 | list_del(&mem->list); | ||
| 378 | kfree(mem); | ||
| 379 | } | ||
| 380 | |||
| 381 | while (dev->cache_props.next != &dev->cache_props) { | ||
| 382 | cache = container_of(dev->cache_props.next, | ||
| 383 | struct kfd_cache_properties, list); | ||
| 384 | list_del(&cache->list); | ||
| 385 | kfree(cache); | ||
| 386 | } | ||
| 387 | |||
| 388 | while (dev->io_link_props.next != &dev->io_link_props) { | ||
| 389 | iolink = container_of(dev->io_link_props.next, | ||
| 390 | struct kfd_iolink_properties, list); | ||
| 391 | list_del(&iolink->list); | ||
| 392 | kfree(iolink); | ||
| 393 | } | ||
| 394 | |||
| 395 | kfree(dev); | ||
| 396 | |||
| 397 | sys_props.num_devices--; | ||
| 398 | } | ||
| 399 | |||
| 400 | static void kfd_release_live_view(void) | ||
| 401 | { | ||
| 402 | struct kfd_topology_device *dev; | ||
| 403 | |||
| 404 | while (topology_device_list.next != &topology_device_list) { | ||
| 405 | dev = container_of(topology_device_list.next, | ||
| 406 | struct kfd_topology_device, list); | ||
| 407 | kfd_release_topology_device(dev); | ||
| 408 | } | ||
| 409 | |||
| 410 | memset(&sys_props, 0, sizeof(sys_props)); | ||
| 411 | } | ||
| 412 | |||
| 413 | static struct kfd_topology_device *kfd_create_topology_device(void) | ||
| 414 | { | ||
| 415 | struct kfd_topology_device *dev; | ||
| 416 | |||
| 417 | dev = kfd_alloc_struct(dev); | ||
| 418 | if (dev == 0) { | ||
| 419 | pr_err("No memory to allocate a topology device"); | ||
| 420 | return 0; | ||
| 421 | } | ||
| 422 | |||
| 423 | INIT_LIST_HEAD(&dev->mem_props); | ||
| 424 | INIT_LIST_HEAD(&dev->cache_props); | ||
| 425 | INIT_LIST_HEAD(&dev->io_link_props); | ||
| 426 | |||
| 427 | list_add_tail(&dev->list, &topology_device_list); | ||
| 428 | sys_props.num_devices++; | ||
| 429 | |||
| 430 | return dev; | ||
| 431 | } | ||
| 432 | |||
| 433 | static int kfd_parse_crat_table(void *crat_image) | ||
| 434 | { | ||
| 435 | struct kfd_topology_device *top_dev; | ||
| 436 | struct crat_subtype_generic *sub_type_hdr; | ||
| 437 | uint16_t node_id; | ||
| 438 | int ret; | ||
| 439 | struct crat_header *crat_table = (struct crat_header *)crat_image; | ||
| 440 | uint16_t num_nodes; | ||
| 441 | uint32_t image_len; | ||
| 442 | |||
| 443 | if (!crat_image) | ||
| 444 | return -EINVAL; | ||
| 445 | |||
| 446 | num_nodes = crat_table->num_domains; | ||
| 447 | image_len = crat_table->length; | ||
| 448 | |||
| 449 | pr_info("Parsing CRAT table with %d nodes\n", num_nodes); | ||
| 450 | |||
| 451 | for (node_id = 0; node_id < num_nodes; node_id++) { | ||
| 452 | top_dev = kfd_create_topology_device(); | ||
| 453 | if (!top_dev) { | ||
| 454 | kfd_release_live_view(); | ||
| 455 | return -ENOMEM; | ||
| 456 | } | ||
| 457 | } | ||
| 458 | |||
| 459 | sys_props.platform_id = | ||
| 460 | (*((uint64_t *)crat_table->oem_id)) & CRAT_OEMID_64BIT_MASK; | ||
| 461 | sys_props.platform_oem = *((uint64_t *)crat_table->oem_table_id); | ||
| 462 | sys_props.platform_rev = crat_table->revision; | ||
| 463 | |||
| 464 | sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1); | ||
| 465 | while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) < | ||
| 466 | ((char *)crat_image) + image_len) { | ||
| 467 | if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) { | ||
| 468 | ret = kfd_parse_subtype(sub_type_hdr); | ||
| 469 | if (ret != 0) { | ||
| 470 | kfd_release_live_view(); | ||
| 471 | return ret; | ||
| 472 | } | ||
| 473 | } | ||
| 474 | |||
| 475 | sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr + | ||
| 476 | sub_type_hdr->length); | ||
| 477 | } | ||
| 478 | |||
| 479 | sys_props.generation_count++; | ||
| 480 | topology_crat_parsed = 1; | ||
| 481 | |||
| 482 | return 0; | ||
| 483 | } | ||
| 484 | |||
| 485 | |||
| 486 | #define sysfs_show_gen_prop(buffer, fmt, ...) \ | ||
| 487 | snprintf(buffer, PAGE_SIZE, "%s"fmt, buffer, __VA_ARGS__) | ||
| 488 | #define sysfs_show_32bit_prop(buffer, name, value) \ | ||
| 489 | sysfs_show_gen_prop(buffer, "%s %u\n", name, value) | ||
| 490 | #define sysfs_show_64bit_prop(buffer, name, value) \ | ||
| 491 | sysfs_show_gen_prop(buffer, "%s %llu\n", name, value) | ||
| 492 | #define sysfs_show_32bit_val(buffer, value) \ | ||
| 493 | sysfs_show_gen_prop(buffer, "%u\n", value) | ||
| 494 | #define sysfs_show_str_val(buffer, value) \ | ||
| 495 | sysfs_show_gen_prop(buffer, "%s\n", value) | ||
| 496 | |||
| 497 | static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr, | ||
| 498 | char *buffer) | ||
| 499 | { | ||
| 500 | ssize_t ret; | ||
| 501 | |||
| 502 | /* Making sure that the buffer is an empty string */ | ||
| 503 | buffer[0] = 0; | ||
| 504 | |||
| 505 | if (attr == &sys_props.attr_genid) { | ||
| 506 | ret = sysfs_show_32bit_val(buffer, sys_props.generation_count); | ||
| 507 | } else if (attr == &sys_props.attr_props) { | ||
| 508 | sysfs_show_64bit_prop(buffer, "platform_oem", | ||
| 509 | sys_props.platform_oem); | ||
| 510 | sysfs_show_64bit_prop(buffer, "platform_id", | ||
| 511 | sys_props.platform_id); | ||
| 512 | ret = sysfs_show_64bit_prop(buffer, "platform_rev", | ||
| 513 | sys_props.platform_rev); | ||
| 514 | } else { | ||
| 515 | ret = -EINVAL; | ||
| 516 | } | ||
| 517 | |||
| 518 | return ret; | ||
| 519 | } | ||
| 520 | |||
| 521 | static const struct sysfs_ops sysprops_ops = { | ||
| 522 | .show = sysprops_show, | ||
| 523 | }; | ||
| 524 | |||
| 525 | static struct kobj_type sysprops_type = { | ||
| 526 | .sysfs_ops = &sysprops_ops, | ||
| 527 | }; | ||
| 528 | |||
| 529 | static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr, | ||
| 530 | char *buffer) | ||
| 531 | { | ||
| 532 | ssize_t ret; | ||
| 533 | struct kfd_iolink_properties *iolink; | ||
| 534 | |||
| 535 | /* Making sure that the buffer is an empty string */ | ||
| 536 | buffer[0] = 0; | ||
| 537 | |||
| 538 | iolink = container_of(attr, struct kfd_iolink_properties, attr); | ||
| 539 | sysfs_show_32bit_prop(buffer, "type", iolink->iolink_type); | ||
| 540 | sysfs_show_32bit_prop(buffer, "version_major", iolink->ver_maj); | ||
| 541 | sysfs_show_32bit_prop(buffer, "version_minor", iolink->ver_min); | ||
| 542 | sysfs_show_32bit_prop(buffer, "node_from", iolink->node_from); | ||
| 543 | sysfs_show_32bit_prop(buffer, "node_to", iolink->node_to); | ||
| 544 | sysfs_show_32bit_prop(buffer, "weight", iolink->weight); | ||
| 545 | sysfs_show_32bit_prop(buffer, "min_latency", iolink->min_latency); | ||
| 546 | sysfs_show_32bit_prop(buffer, "max_latency", iolink->max_latency); | ||
| 547 | sysfs_show_32bit_prop(buffer, "min_bandwidth", iolink->min_bandwidth); | ||
| 548 | sysfs_show_32bit_prop(buffer, "max_bandwidth", iolink->max_bandwidth); | ||
| 549 | sysfs_show_32bit_prop(buffer, "recommended_transfer_size", | ||
| 550 | iolink->rec_transfer_size); | ||
| 551 | ret = sysfs_show_32bit_prop(buffer, "flags", iolink->flags); | ||
| 552 | |||
| 553 | return ret; | ||
| 554 | } | ||
| 555 | |||
| 556 | static const struct sysfs_ops iolink_ops = { | ||
| 557 | .show = iolink_show, | ||
| 558 | }; | ||
| 559 | |||
| 560 | static struct kobj_type iolink_type = { | ||
| 561 | .sysfs_ops = &iolink_ops, | ||
| 562 | }; | ||
| 563 | |||
| 564 | static ssize_t mem_show(struct kobject *kobj, struct attribute *attr, | ||
| 565 | char *buffer) | ||
| 566 | { | ||
| 567 | ssize_t ret; | ||
| 568 | struct kfd_mem_properties *mem; | ||
| 569 | |||
| 570 | /* Making sure that the buffer is an empty string */ | ||
| 571 | buffer[0] = 0; | ||
| 572 | |||
| 573 | mem = container_of(attr, struct kfd_mem_properties, attr); | ||
| 574 | sysfs_show_32bit_prop(buffer, "heap_type", mem->heap_type); | ||
| 575 | sysfs_show_64bit_prop(buffer, "size_in_bytes", mem->size_in_bytes); | ||
| 576 | sysfs_show_32bit_prop(buffer, "flags", mem->flags); | ||
| 577 | sysfs_show_32bit_prop(buffer, "width", mem->width); | ||
| 578 | ret = sysfs_show_32bit_prop(buffer, "mem_clk_max", mem->mem_clk_max); | ||
| 579 | |||
| 580 | return ret; | ||
| 581 | } | ||
| 582 | |||
| 583 | static const struct sysfs_ops mem_ops = { | ||
| 584 | .show = mem_show, | ||
| 585 | }; | ||
| 586 | |||
| 587 | static struct kobj_type mem_type = { | ||
| 588 | .sysfs_ops = &mem_ops, | ||
| 589 | }; | ||
| 590 | |||
| 591 | static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr, | ||
| 592 | char *buffer) | ||
| 593 | { | ||
| 594 | ssize_t ret; | ||
| 595 | uint32_t i; | ||
| 596 | struct kfd_cache_properties *cache; | ||
| 597 | |||
| 598 | /* Making sure that the buffer is an empty string */ | ||
| 599 | buffer[0] = 0; | ||
| 600 | |||
| 601 | cache = container_of(attr, struct kfd_cache_properties, attr); | ||
| 602 | sysfs_show_32bit_prop(buffer, "processor_id_low", | ||
| 603 | cache->processor_id_low); | ||
| 604 | sysfs_show_32bit_prop(buffer, "level", cache->cache_level); | ||
| 605 | sysfs_show_32bit_prop(buffer, "size", cache->cache_size); | ||
| 606 | sysfs_show_32bit_prop(buffer, "cache_line_size", cache->cacheline_size); | ||
| 607 | sysfs_show_32bit_prop(buffer, "cache_lines_per_tag", | ||
| 608 | cache->cachelines_per_tag); | ||
| 609 | sysfs_show_32bit_prop(buffer, "association", cache->cache_assoc); | ||
| 610 | sysfs_show_32bit_prop(buffer, "latency", cache->cache_latency); | ||
| 611 | sysfs_show_32bit_prop(buffer, "type", cache->cache_type); | ||
| 612 | snprintf(buffer, PAGE_SIZE, "%ssibling_map ", buffer); | ||
| 613 | for (i = 0; i < KFD_TOPOLOGY_CPU_SIBLINGS; i++) | ||
| 614 | ret = snprintf(buffer, PAGE_SIZE, "%s%d%s", | ||
| 615 | buffer, cache->sibling_map[i], | ||
| 616 | (i == KFD_TOPOLOGY_CPU_SIBLINGS-1) ? | ||
| 617 | "\n" : ","); | ||
| 618 | |||
| 619 | return ret; | ||
| 620 | } | ||
| 621 | |||
| 622 | static const struct sysfs_ops cache_ops = { | ||
| 623 | .show = kfd_cache_show, | ||
| 624 | }; | ||
| 625 | |||
| 626 | static struct kobj_type cache_type = { | ||
| 627 | .sysfs_ops = &cache_ops, | ||
| 628 | }; | ||
| 629 | |||
| 630 | static ssize_t node_show(struct kobject *kobj, struct attribute *attr, | ||
| 631 | char *buffer) | ||
| 632 | { | ||
| 633 | ssize_t ret; | ||
| 634 | struct kfd_topology_device *dev; | ||
| 635 | char public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE]; | ||
| 636 | uint32_t i; | ||
| 637 | |||
| 638 | /* Making sure that the buffer is an empty string */ | ||
| 639 | buffer[0] = 0; | ||
| 640 | |||
| 641 | if (strcmp(attr->name, "gpu_id") == 0) { | ||
| 642 | dev = container_of(attr, struct kfd_topology_device, | ||
| 643 | attr_gpuid); | ||
| 644 | ret = sysfs_show_32bit_val(buffer, dev->gpu_id); | ||
| 645 | } else if (strcmp(attr->name, "name") == 0) { | ||
| 646 | dev = container_of(attr, struct kfd_topology_device, | ||
| 647 | attr_name); | ||
| 648 | for (i = 0; i < KFD_TOPOLOGY_PUBLIC_NAME_SIZE; i++) { | ||
| 649 | public_name[i] = | ||
| 650 | (char)dev->node_props.marketing_name[i]; | ||
| 651 | if (dev->node_props.marketing_name[i] == 0) | ||
| 652 | break; | ||
| 653 | } | ||
| 654 | public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE-1] = 0x0; | ||
| 655 | ret = sysfs_show_str_val(buffer, public_name); | ||
| 656 | } else { | ||
| 657 | dev = container_of(attr, struct kfd_topology_device, | ||
| 658 | attr_props); | ||
| 659 | sysfs_show_32bit_prop(buffer, "cpu_cores_count", | ||
| 660 | dev->node_props.cpu_cores_count); | ||
| 661 | sysfs_show_32bit_prop(buffer, "simd_count", | ||
| 662 | dev->node_props.simd_count); | ||
| 663 | |||
| 664 | if (dev->mem_bank_count < dev->node_props.mem_banks_count) { | ||
| 665 | pr_warn("kfd: mem_banks_count truncated from %d to %d\n", | ||
| 666 | dev->node_props.mem_banks_count, | ||
| 667 | dev->mem_bank_count); | ||
| 668 | sysfs_show_32bit_prop(buffer, "mem_banks_count", | ||
| 669 | dev->mem_bank_count); | ||
| 670 | } else { | ||
| 671 | sysfs_show_32bit_prop(buffer, "mem_banks_count", | ||
| 672 | dev->node_props.mem_banks_count); | ||
| 673 | } | ||
| 674 | |||
| 675 | sysfs_show_32bit_prop(buffer, "caches_count", | ||
| 676 | dev->node_props.caches_count); | ||
| 677 | sysfs_show_32bit_prop(buffer, "io_links_count", | ||
| 678 | dev->node_props.io_links_count); | ||
| 679 | sysfs_show_32bit_prop(buffer, "cpu_core_id_base", | ||
| 680 | dev->node_props.cpu_core_id_base); | ||
| 681 | sysfs_show_32bit_prop(buffer, "simd_id_base", | ||
| 682 | dev->node_props.simd_id_base); | ||
| 683 | sysfs_show_32bit_prop(buffer, "capability", | ||
| 684 | dev->node_props.capability); | ||
| 685 | sysfs_show_32bit_prop(buffer, "max_waves_per_simd", | ||
| 686 | dev->node_props.max_waves_per_simd); | ||
| 687 | sysfs_show_32bit_prop(buffer, "lds_size_in_kb", | ||
| 688 | dev->node_props.lds_size_in_kb); | ||
| 689 | sysfs_show_32bit_prop(buffer, "gds_size_in_kb", | ||
| 690 | dev->node_props.gds_size_in_kb); | ||
| 691 | sysfs_show_32bit_prop(buffer, "wave_front_size", | ||
| 692 | dev->node_props.wave_front_size); | ||
| 693 | sysfs_show_32bit_prop(buffer, "array_count", | ||
| 694 | dev->node_props.array_count); | ||
| 695 | sysfs_show_32bit_prop(buffer, "simd_arrays_per_engine", | ||
| 696 | dev->node_props.simd_arrays_per_engine); | ||
| 697 | sysfs_show_32bit_prop(buffer, "cu_per_simd_array", | ||
| 698 | dev->node_props.cu_per_simd_array); | ||
| 699 | sysfs_show_32bit_prop(buffer, "simd_per_cu", | ||
| 700 | dev->node_props.simd_per_cu); | ||
| 701 | sysfs_show_32bit_prop(buffer, "max_slots_scratch_cu", | ||
| 702 | dev->node_props.max_slots_scratch_cu); | ||
| 703 | sysfs_show_32bit_prop(buffer, "engine_id", | ||
| 704 | dev->node_props.engine_id); | ||
| 705 | sysfs_show_32bit_prop(buffer, "vendor_id", | ||
| 706 | dev->node_props.vendor_id); | ||
| 707 | sysfs_show_32bit_prop(buffer, "device_id", | ||
| 708 | dev->node_props.device_id); | ||
| 709 | sysfs_show_32bit_prop(buffer, "location_id", | ||
| 710 | dev->node_props.location_id); | ||
| 711 | |||
| 712 | if (dev->gpu) { | ||
| 713 | sysfs_show_32bit_prop(buffer, "max_engine_clk_fcompute", | ||
| 714 | kfd2kgd->get_max_engine_clock_in_mhz( | ||
| 715 | dev->gpu->kgd)); | ||
| 716 | sysfs_show_64bit_prop(buffer, "local_mem_size", | ||
| 717 | kfd2kgd->get_vmem_size(dev->gpu->kgd)); | ||
| 718 | } | ||
| 719 | |||
| 720 | ret = sysfs_show_32bit_prop(buffer, "max_engine_clk_ccompute", | ||
| 721 | cpufreq_quick_get_max(0)/1000); | ||
| 722 | } | ||
| 723 | |||
| 724 | return ret; | ||
| 725 | } | ||
| 726 | |||
| 727 | static const struct sysfs_ops node_ops = { | ||
| 728 | .show = node_show, | ||
| 729 | }; | ||
| 730 | |||
| 731 | static struct kobj_type node_type = { | ||
| 732 | .sysfs_ops = &node_ops, | ||
| 733 | }; | ||
| 734 | |||
| 735 | static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr) | ||
| 736 | { | ||
| 737 | sysfs_remove_file(kobj, attr); | ||
| 738 | kobject_del(kobj); | ||
| 739 | kobject_put(kobj); | ||
| 740 | } | ||
| 741 | |||
| 742 | static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev) | ||
| 743 | { | ||
| 744 | struct kfd_iolink_properties *iolink; | ||
| 745 | struct kfd_cache_properties *cache; | ||
| 746 | struct kfd_mem_properties *mem; | ||
| 747 | |||
| 748 | BUG_ON(!dev); | ||
| 749 | |||
| 750 | if (dev->kobj_iolink) { | ||
| 751 | list_for_each_entry(iolink, &dev->io_link_props, list) | ||
| 752 | if (iolink->kobj) { | ||
| 753 | kfd_remove_sysfs_file(iolink->kobj, | ||
| 754 | &iolink->attr); | ||
| 755 | iolink->kobj = 0; | ||
| 756 | } | ||
| 757 | kobject_del(dev->kobj_iolink); | ||
| 758 | kobject_put(dev->kobj_iolink); | ||
| 759 | dev->kobj_iolink = 0; | ||
| 760 | } | ||
| 761 | |||
| 762 | if (dev->kobj_cache) { | ||
| 763 | list_for_each_entry(cache, &dev->cache_props, list) | ||
| 764 | if (cache->kobj) { | ||
| 765 | kfd_remove_sysfs_file(cache->kobj, | ||
| 766 | &cache->attr); | ||
| 767 | cache->kobj = 0; | ||
| 768 | } | ||
| 769 | kobject_del(dev->kobj_cache); | ||
| 770 | kobject_put(dev->kobj_cache); | ||
| 771 | dev->kobj_cache = 0; | ||
| 772 | } | ||
| 773 | |||
| 774 | if (dev->kobj_mem) { | ||
| 775 | list_for_each_entry(mem, &dev->mem_props, list) | ||
| 776 | if (mem->kobj) { | ||
| 777 | kfd_remove_sysfs_file(mem->kobj, &mem->attr); | ||
| 778 | mem->kobj = 0; | ||
| 779 | } | ||
| 780 | kobject_del(dev->kobj_mem); | ||
| 781 | kobject_put(dev->kobj_mem); | ||
| 782 | dev->kobj_mem = 0; | ||
| 783 | } | ||
| 784 | |||
| 785 | if (dev->kobj_node) { | ||
| 786 | sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid); | ||
| 787 | sysfs_remove_file(dev->kobj_node, &dev->attr_name); | ||
| 788 | sysfs_remove_file(dev->kobj_node, &dev->attr_props); | ||
| 789 | kobject_del(dev->kobj_node); | ||
| 790 | kobject_put(dev->kobj_node); | ||
| 791 | dev->kobj_node = 0; | ||
| 792 | } | ||
| 793 | } | ||
| 794 | |||
| 795 | static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev, | ||
| 796 | uint32_t id) | ||
| 797 | { | ||
| 798 | struct kfd_iolink_properties *iolink; | ||
| 799 | struct kfd_cache_properties *cache; | ||
| 800 | struct kfd_mem_properties *mem; | ||
| 801 | int ret; | ||
| 802 | uint32_t i; | ||
| 803 | |||
| 804 | BUG_ON(!dev); | ||
| 805 | |||
| 806 | /* | ||
| 807 | * Creating the sysfs folders | ||
| 808 | */ | ||
| 809 | BUG_ON(dev->kobj_node); | ||
| 810 | dev->kobj_node = kfd_alloc_struct(dev->kobj_node); | ||
| 811 | if (!dev->kobj_node) | ||
| 812 | return -ENOMEM; | ||
| 813 | |||
| 814 | ret = kobject_init_and_add(dev->kobj_node, &node_type, | ||
| 815 | sys_props.kobj_nodes, "%d", id); | ||
| 816 | if (ret < 0) | ||
| 817 | return ret; | ||
| 818 | |||
| 819 | dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node); | ||
| 820 | if (!dev->kobj_mem) | ||
| 821 | return -ENOMEM; | ||
| 822 | |||
| 823 | dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node); | ||
| 824 | if (!dev->kobj_cache) | ||
| 825 | return -ENOMEM; | ||
| 826 | |||
| 827 | dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node); | ||
| 828 | if (!dev->kobj_iolink) | ||
| 829 | return -ENOMEM; | ||
| 830 | |||
| 831 | /* | ||
| 832 | * Creating sysfs files for node properties | ||
| 833 | */ | ||
| 834 | dev->attr_gpuid.name = "gpu_id"; | ||
| 835 | dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE; | ||
| 836 | sysfs_attr_init(&dev->attr_gpuid); | ||
| 837 | dev->attr_name.name = "name"; | ||
| 838 | dev->attr_name.mode = KFD_SYSFS_FILE_MODE; | ||
| 839 | sysfs_attr_init(&dev->attr_name); | ||
| 840 | dev->attr_props.name = "properties"; | ||
| 841 | dev->attr_props.mode = KFD_SYSFS_FILE_MODE; | ||
| 842 | sysfs_attr_init(&dev->attr_props); | ||
| 843 | ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid); | ||
| 844 | if (ret < 0) | ||
| 845 | return ret; | ||
| 846 | ret = sysfs_create_file(dev->kobj_node, &dev->attr_name); | ||
| 847 | if (ret < 0) | ||
| 848 | return ret; | ||
| 849 | ret = sysfs_create_file(dev->kobj_node, &dev->attr_props); | ||
| 850 | if (ret < 0) | ||
| 851 | return ret; | ||
| 852 | |||
| 853 | i = 0; | ||
| 854 | list_for_each_entry(mem, &dev->mem_props, list) { | ||
| 855 | mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); | ||
| 856 | if (!mem->kobj) | ||
| 857 | return -ENOMEM; | ||
| 858 | ret = kobject_init_and_add(mem->kobj, &mem_type, | ||
| 859 | dev->kobj_mem, "%d", i); | ||
| 860 | if (ret < 0) | ||
| 861 | return ret; | ||
| 862 | |||
| 863 | mem->attr.name = "properties"; | ||
| 864 | mem->attr.mode = KFD_SYSFS_FILE_MODE; | ||
| 865 | sysfs_attr_init(&mem->attr); | ||
| 866 | ret = sysfs_create_file(mem->kobj, &mem->attr); | ||
| 867 | if (ret < 0) | ||
| 868 | return ret; | ||
| 869 | i++; | ||
| 870 | } | ||
| 871 | |||
| 872 | i = 0; | ||
| 873 | list_for_each_entry(cache, &dev->cache_props, list) { | ||
| 874 | cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); | ||
| 875 | if (!cache->kobj) | ||
| 876 | return -ENOMEM; | ||
| 877 | ret = kobject_init_and_add(cache->kobj, &cache_type, | ||
| 878 | dev->kobj_cache, "%d", i); | ||
| 879 | if (ret < 0) | ||
| 880 | return ret; | ||
| 881 | |||
| 882 | cache->attr.name = "properties"; | ||
| 883 | cache->attr.mode = KFD_SYSFS_FILE_MODE; | ||
| 884 | sysfs_attr_init(&cache->attr); | ||
| 885 | ret = sysfs_create_file(cache->kobj, &cache->attr); | ||
| 886 | if (ret < 0) | ||
| 887 | return ret; | ||
| 888 | i++; | ||
| 889 | } | ||
| 890 | |||
| 891 | i = 0; | ||
| 892 | list_for_each_entry(iolink, &dev->io_link_props, list) { | ||
| 893 | iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); | ||
| 894 | if (!iolink->kobj) | ||
| 895 | return -ENOMEM; | ||
| 896 | ret = kobject_init_and_add(iolink->kobj, &iolink_type, | ||
| 897 | dev->kobj_iolink, "%d", i); | ||
| 898 | if (ret < 0) | ||
| 899 | return ret; | ||
| 900 | |||
| 901 | iolink->attr.name = "properties"; | ||
| 902 | iolink->attr.mode = KFD_SYSFS_FILE_MODE; | ||
| 903 | sysfs_attr_init(&iolink->attr); | ||
| 904 | ret = sysfs_create_file(iolink->kobj, &iolink->attr); | ||
| 905 | if (ret < 0) | ||
| 906 | return ret; | ||
| 907 | i++; | ||
| 908 | } | ||
| 909 | |||
| 910 | return 0; | ||
| 911 | } | ||
| 912 | |||
| 913 | static int kfd_build_sysfs_node_tree(void) | ||
| 914 | { | ||
| 915 | struct kfd_topology_device *dev; | ||
| 916 | int ret; | ||
| 917 | uint32_t i = 0; | ||
| 918 | |||
| 919 | list_for_each_entry(dev, &topology_device_list, list) { | ||
| 920 | ret = kfd_build_sysfs_node_entry(dev, 0); | ||
| 921 | if (ret < 0) | ||
| 922 | return ret; | ||
| 923 | i++; | ||
| 924 | } | ||
| 925 | |||
| 926 | return 0; | ||
| 927 | } | ||
| 928 | |||
| 929 | static void kfd_remove_sysfs_node_tree(void) | ||
| 930 | { | ||
| 931 | struct kfd_topology_device *dev; | ||
| 932 | |||
| 933 | list_for_each_entry(dev, &topology_device_list, list) | ||
| 934 | kfd_remove_sysfs_node_entry(dev); | ||
| 935 | } | ||
| 936 | |||
| 937 | static int kfd_topology_update_sysfs(void) | ||
| 938 | { | ||
| 939 | int ret; | ||
| 940 | |||
| 941 | pr_info("Creating topology SYSFS entries\n"); | ||
| 942 | if (sys_props.kobj_topology == 0) { | ||
| 943 | sys_props.kobj_topology = | ||
| 944 | kfd_alloc_struct(sys_props.kobj_topology); | ||
| 945 | if (!sys_props.kobj_topology) | ||
| 946 | return -ENOMEM; | ||
| 947 | |||
| 948 | ret = kobject_init_and_add(sys_props.kobj_topology, | ||
| 949 | &sysprops_type, &kfd_device->kobj, | ||
| 950 | "topology"); | ||
| 951 | if (ret < 0) | ||
| 952 | return ret; | ||
| 953 | |||
| 954 | sys_props.kobj_nodes = kobject_create_and_add("nodes", | ||
| 955 | sys_props.kobj_topology); | ||
| 956 | if (!sys_props.kobj_nodes) | ||
| 957 | return -ENOMEM; | ||
| 958 | |||
| 959 | sys_props.attr_genid.name = "generation_id"; | ||
| 960 | sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE; | ||
| 961 | sysfs_attr_init(&sys_props.attr_genid); | ||
| 962 | ret = sysfs_create_file(sys_props.kobj_topology, | ||
| 963 | &sys_props.attr_genid); | ||
| 964 | if (ret < 0) | ||
| 965 | return ret; | ||
| 966 | |||
| 967 | sys_props.attr_props.name = "system_properties"; | ||
| 968 | sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE; | ||
| 969 | sysfs_attr_init(&sys_props.attr_props); | ||
| 970 | ret = sysfs_create_file(sys_props.kobj_topology, | ||
| 971 | &sys_props.attr_props); | ||
| 972 | if (ret < 0) | ||
| 973 | return ret; | ||
| 974 | } | ||
| 975 | |||
| 976 | kfd_remove_sysfs_node_tree(); | ||
| 977 | |||
| 978 | return kfd_build_sysfs_node_tree(); | ||
| 979 | } | ||
| 980 | |||
| 981 | static void kfd_topology_release_sysfs(void) | ||
| 982 | { | ||
| 983 | kfd_remove_sysfs_node_tree(); | ||
| 984 | if (sys_props.kobj_topology) { | ||
| 985 | sysfs_remove_file(sys_props.kobj_topology, | ||
| 986 | &sys_props.attr_genid); | ||
| 987 | sysfs_remove_file(sys_props.kobj_topology, | ||
| 988 | &sys_props.attr_props); | ||
| 989 | if (sys_props.kobj_nodes) { | ||
| 990 | kobject_del(sys_props.kobj_nodes); | ||
| 991 | kobject_put(sys_props.kobj_nodes); | ||
| 992 | sys_props.kobj_nodes = 0; | ||
| 993 | } | ||
| 994 | kobject_del(sys_props.kobj_topology); | ||
| 995 | kobject_put(sys_props.kobj_topology); | ||
| 996 | sys_props.kobj_topology = 0; | ||
| 997 | } | ||
| 998 | } | ||
| 999 | |||
| 1000 | int kfd_topology_init(void) | ||
| 1001 | { | ||
| 1002 | void *crat_image = 0; | ||
| 1003 | size_t image_size = 0; | ||
| 1004 | int ret; | ||
| 1005 | |||
| 1006 | /* | ||
| 1007 | * Initialize the head for the topology device list | ||
| 1008 | */ | ||
| 1009 | INIT_LIST_HEAD(&topology_device_list); | ||
| 1010 | init_rwsem(&topology_lock); | ||
| 1011 | topology_crat_parsed = 0; | ||
| 1012 | |||
| 1013 | memset(&sys_props, 0, sizeof(sys_props)); | ||
| 1014 | |||
| 1015 | /* | ||
| 1016 | * Get the CRAT image from the ACPI | ||
| 1017 | */ | ||
| 1018 | ret = kfd_topology_get_crat_acpi(crat_image, &image_size); | ||
| 1019 | if (ret == 0 && image_size > 0) { | ||
| 1020 | pr_info("Found CRAT image with size=%zd\n", image_size); | ||
| 1021 | crat_image = kmalloc(image_size, GFP_KERNEL); | ||
| 1022 | if (!crat_image) { | ||
| 1023 | ret = -ENOMEM; | ||
| 1024 | pr_err("No memory for allocating CRAT image\n"); | ||
| 1025 | goto err; | ||
| 1026 | } | ||
| 1027 | ret = kfd_topology_get_crat_acpi(crat_image, &image_size); | ||
| 1028 | |||
| 1029 | if (ret == 0) { | ||
| 1030 | down_write(&topology_lock); | ||
| 1031 | ret = kfd_parse_crat_table(crat_image); | ||
| 1032 | if (ret == 0) | ||
| 1033 | ret = kfd_topology_update_sysfs(); | ||
| 1034 | up_write(&topology_lock); | ||
| 1035 | } else { | ||
| 1036 | pr_err("Couldn't get CRAT table size from ACPI\n"); | ||
| 1037 | } | ||
| 1038 | kfree(crat_image); | ||
| 1039 | } else if (ret == -ENODATA) { | ||
| 1040 | ret = 0; | ||
| 1041 | } else { | ||
| 1042 | pr_err("Couldn't get CRAT table size from ACPI\n"); | ||
| 1043 | } | ||
| 1044 | |||
| 1045 | err: | ||
| 1046 | pr_info("Finished initializing topology ret=%d\n", ret); | ||
| 1047 | return ret; | ||
| 1048 | } | ||
| 1049 | |||
| 1050 | void kfd_topology_shutdown(void) | ||
| 1051 | { | ||
| 1052 | kfd_topology_release_sysfs(); | ||
| 1053 | kfd_release_live_view(); | ||
| 1054 | } | ||
| 1055 | |||
| 1056 | static void kfd_debug_print_topology(void) | ||
| 1057 | { | ||
| 1058 | struct kfd_topology_device *dev; | ||
| 1059 | uint32_t i = 0; | ||
| 1060 | |||
| 1061 | pr_info("DEBUG PRINT OF TOPOLOGY:"); | ||
| 1062 | list_for_each_entry(dev, &topology_device_list, list) { | ||
| 1063 | pr_info("Node: %d\n", i); | ||
| 1064 | pr_info("\tGPU assigned: %s\n", (dev->gpu ? "yes" : "no")); | ||
| 1065 | pr_info("\tCPU count: %d\n", dev->node_props.cpu_cores_count); | ||
| 1066 | pr_info("\tSIMD count: %d", dev->node_props.simd_count); | ||
| 1067 | i++; | ||
| 1068 | } | ||
| 1069 | } | ||
| 1070 | |||
| 1071 | static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu) | ||
| 1072 | { | ||
| 1073 | uint32_t hashout; | ||
| 1074 | uint32_t buf[7]; | ||
| 1075 | int i; | ||
| 1076 | |||
| 1077 | if (!gpu) | ||
| 1078 | return 0; | ||
| 1079 | |||
| 1080 | buf[0] = gpu->pdev->devfn; | ||
| 1081 | buf[1] = gpu->pdev->subsystem_vendor; | ||
| 1082 | buf[2] = gpu->pdev->subsystem_device; | ||
| 1083 | buf[3] = gpu->pdev->device; | ||
| 1084 | buf[4] = gpu->pdev->bus->number; | ||
| 1085 | buf[5] = (uint32_t)(kfd2kgd->get_vmem_size(gpu->kgd) & 0xffffffff); | ||
| 1086 | buf[6] = (uint32_t)(kfd2kgd->get_vmem_size(gpu->kgd) >> 32); | ||
| 1087 | |||
| 1088 | for (i = 0, hashout = 0; i < 7; i++) | ||
| 1089 | hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH); | ||
| 1090 | |||
| 1091 | return hashout; | ||
| 1092 | } | ||
| 1093 | |||
| 1094 | static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu) | ||
| 1095 | { | ||
| 1096 | struct kfd_topology_device *dev; | ||
| 1097 | struct kfd_topology_device *out_dev = 0; | ||
| 1098 | |||
| 1099 | BUG_ON(!gpu); | ||
| 1100 | |||
| 1101 | list_for_each_entry(dev, &topology_device_list, list) | ||
| 1102 | if (dev->gpu == 0 && dev->node_props.simd_count > 0) { | ||
| 1103 | dev->gpu = gpu; | ||
| 1104 | out_dev = dev; | ||
| 1105 | break; | ||
| 1106 | } | ||
| 1107 | |||
| 1108 | return out_dev; | ||
| 1109 | } | ||
| 1110 | |||
| 1111 | static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival) | ||
| 1112 | { | ||
| 1113 | /* | ||
| 1114 | * TODO: Generate an event for thunk about the arrival/removal | ||
| 1115 | * of the GPU | ||
| 1116 | */ | ||
| 1117 | } | ||
| 1118 | |||
| 1119 | int kfd_topology_add_device(struct kfd_dev *gpu) | ||
| 1120 | { | ||
| 1121 | uint32_t gpu_id; | ||
| 1122 | struct kfd_topology_device *dev; | ||
| 1123 | int res; | ||
| 1124 | |||
| 1125 | BUG_ON(!gpu); | ||
| 1126 | |||
| 1127 | gpu_id = kfd_generate_gpu_id(gpu); | ||
| 1128 | |||
| 1129 | pr_debug("kfd: Adding new GPU (ID: 0x%x) to topology\n", gpu_id); | ||
| 1130 | |||
| 1131 | down_write(&topology_lock); | ||
| 1132 | /* | ||
| 1133 | * Try to assign the GPU to existing topology device (generated from | ||
| 1134 | * CRAT table | ||
| 1135 | */ | ||
| 1136 | dev = kfd_assign_gpu(gpu); | ||
| 1137 | if (!dev) { | ||
| 1138 | pr_info("GPU was not found in the current topology. Extending.\n"); | ||
| 1139 | kfd_debug_print_topology(); | ||
| 1140 | dev = kfd_create_topology_device(); | ||
| 1141 | if (!dev) { | ||
| 1142 | res = -ENOMEM; | ||
| 1143 | goto err; | ||
| 1144 | } | ||
| 1145 | dev->gpu = gpu; | ||
| 1146 | |||
| 1147 | /* | ||
| 1148 | * TODO: Make a call to retrieve topology information from the | ||
| 1149 | * GPU vBIOS | ||
| 1150 | */ | ||
| 1151 | |||
| 1152 | /* | ||
| 1153 | * Update the SYSFS tree, since we added another topology device | ||
| 1154 | */ | ||
| 1155 | if (kfd_topology_update_sysfs() < 0) | ||
| 1156 | kfd_topology_release_sysfs(); | ||
| 1157 | |||
| 1158 | } | ||
| 1159 | |||
| 1160 | dev->gpu_id = gpu_id; | ||
| 1161 | gpu->id = gpu_id; | ||
| 1162 | dev->node_props.vendor_id = gpu->pdev->vendor; | ||
| 1163 | dev->node_props.device_id = gpu->pdev->device; | ||
| 1164 | dev->node_props.location_id = (gpu->pdev->bus->number << 24) + | ||
| 1165 | (gpu->pdev->devfn & 0xffffff); | ||
| 1166 | /* | ||
| 1167 | * TODO: Retrieve max engine clock values from KGD | ||
| 1168 | */ | ||
| 1169 | |||
| 1170 | res = 0; | ||
| 1171 | |||
| 1172 | err: | ||
| 1173 | up_write(&topology_lock); | ||
| 1174 | |||
| 1175 | if (res == 0) | ||
| 1176 | kfd_notify_gpu_change(gpu_id, 1); | ||
| 1177 | |||
| 1178 | return res; | ||
| 1179 | } | ||
| 1180 | |||
| 1181 | int kfd_topology_remove_device(struct kfd_dev *gpu) | ||
| 1182 | { | ||
| 1183 | struct kfd_topology_device *dev; | ||
| 1184 | uint32_t gpu_id; | ||
| 1185 | int res = -ENODEV; | ||
| 1186 | |||
| 1187 | BUG_ON(!gpu); | ||
| 1188 | |||
| 1189 | down_write(&topology_lock); | ||
| 1190 | |||
| 1191 | list_for_each_entry(dev, &topology_device_list, list) | ||
| 1192 | if (dev->gpu == gpu) { | ||
| 1193 | gpu_id = dev->gpu_id; | ||
| 1194 | kfd_remove_sysfs_node_entry(dev); | ||
| 1195 | kfd_release_topology_device(dev); | ||
| 1196 | res = 0; | ||
| 1197 | if (kfd_topology_update_sysfs() < 0) | ||
| 1198 | kfd_topology_release_sysfs(); | ||
| 1199 | break; | ||
| 1200 | } | ||
| 1201 | |||
| 1202 | up_write(&topology_lock); | ||
| 1203 | |||
| 1204 | if (res == 0) | ||
| 1205 | kfd_notify_gpu_change(gpu_id, 0); | ||
| 1206 | |||
| 1207 | return res; | ||
| 1208 | } | ||
| 1209 | |||
| 1210 | /* | ||
| 1211 | * When idx is out of bounds, the function will return NULL | ||
| 1212 | */ | ||
| 1213 | struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx) | ||
| 1214 | { | ||
| 1215 | |||
| 1216 | struct kfd_topology_device *top_dev; | ||
| 1217 | struct kfd_dev *device = NULL; | ||
| 1218 | uint8_t device_idx = 0; | ||
| 1219 | |||
| 1220 | down_read(&topology_lock); | ||
| 1221 | |||
| 1222 | list_for_each_entry(top_dev, &topology_device_list, list) { | ||
| 1223 | if (device_idx == idx) { | ||
| 1224 | device = top_dev->gpu; | ||
| 1225 | break; | ||
| 1226 | } | ||
| 1227 | |||
| 1228 | device_idx++; | ||
| 1229 | } | ||
| 1230 | |||
| 1231 | up_read(&topology_lock); | ||
| 1232 | |||
| 1233 | return device; | ||
| 1234 | |||
| 1235 | } | ||
| @@ -0,0 +1,168 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | #ifndef __KFD_TOPOLOGY_H__ | ||
| 24 | #define __KFD_TOPOLOGY_H__ | ||
| 25 | |||
| 26 | #include <linux/types.h> | ||
| 27 | #include <linux/list.h> | ||
| 28 | #include "kfd_priv.h" | ||
| 29 | |||
| 30 | #define KFD_TOPOLOGY_PUBLIC_NAME_SIZE 128 | ||
| 31 | |||
| 32 | #define HSA_CAP_HOT_PLUGGABLE 0x00000001 | ||
| 33 | #define HSA_CAP_ATS_PRESENT 0x00000002 | ||
| 34 | #define HSA_CAP_SHARED_WITH_GRAPHICS 0x00000004 | ||
| 35 | #define HSA_CAP_QUEUE_SIZE_POW2 0x00000008 | ||
| 36 | #define HSA_CAP_QUEUE_SIZE_32BIT 0x00000010 | ||
| 37 | #define HSA_CAP_QUEUE_IDLE_EVENT 0x00000020 | ||
| 38 | #define HSA_CAP_VA_LIMIT 0x00000040 | ||
| 39 | #define HSA_CAP_WATCH_POINTS_SUPPORTED 0x00000080 | ||
| 40 | #define HSA_CAP_WATCH_POINTS_TOTALBITS_MASK 0x00000f00 | ||
| 41 | #define HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT 8 | ||
| 42 | #define HSA_CAP_RESERVED 0xfffff000 | ||
| 43 | |||
| 44 | struct kfd_node_properties { | ||
| 45 | uint32_t cpu_cores_count; | ||
| 46 | uint32_t simd_count; | ||
| 47 | uint32_t mem_banks_count; | ||
| 48 | uint32_t caches_count; | ||
| 49 | uint32_t io_links_count; | ||
| 50 | uint32_t cpu_core_id_base; | ||
| 51 | uint32_t simd_id_base; | ||
| 52 | uint32_t capability; | ||
| 53 | uint32_t max_waves_per_simd; | ||
| 54 | uint32_t lds_size_in_kb; | ||
| 55 | uint32_t gds_size_in_kb; | ||
| 56 | uint32_t wave_front_size; | ||
| 57 | uint32_t array_count; | ||
| 58 | uint32_t simd_arrays_per_engine; | ||
| 59 | uint32_t cu_per_simd_array; | ||
| 60 | uint32_t simd_per_cu; | ||
| 61 | uint32_t max_slots_scratch_cu; | ||
| 62 | uint32_t engine_id; | ||
| 63 | uint32_t vendor_id; | ||
| 64 | uint32_t device_id; | ||
| 65 | uint32_t location_id; | ||
| 66 | uint32_t max_engine_clk_fcompute; | ||
| 67 | uint32_t max_engine_clk_ccompute; | ||
| 68 | uint16_t marketing_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE]; | ||
| 69 | }; | ||
| 70 | |||
| 71 | #define HSA_MEM_HEAP_TYPE_SYSTEM 0 | ||
| 72 | #define HSA_MEM_HEAP_TYPE_FB_PUBLIC 1 | ||
| 73 | #define HSA_MEM_HEAP_TYPE_FB_PRIVATE 2 | ||
| 74 | #define HSA_MEM_HEAP_TYPE_GPU_GDS 3 | ||
| 75 | #define HSA_MEM_HEAP_TYPE_GPU_LDS 4 | ||
| 76 | #define HSA_MEM_HEAP_TYPE_GPU_SCRATCH 5 | ||
| 77 | |||
| 78 | #define HSA_MEM_FLAGS_HOT_PLUGGABLE 0x00000001 | ||
| 79 | #define HSA_MEM_FLAGS_NON_VOLATILE 0x00000002 | ||
| 80 | #define HSA_MEM_FLAGS_RESERVED 0xfffffffc | ||
| 81 | |||
| 82 | struct kfd_mem_properties { | ||
| 83 | struct list_head list; | ||
| 84 | uint32_t heap_type; | ||
| 85 | uint64_t size_in_bytes; | ||
| 86 | uint32_t flags; | ||
| 87 | uint32_t width; | ||
| 88 | uint32_t mem_clk_max; | ||
| 89 | struct kobject *kobj; | ||
| 90 | struct attribute attr; | ||
| 91 | }; | ||
| 92 | |||
| 93 | #define KFD_TOPOLOGY_CPU_SIBLINGS 256 | ||
| 94 | |||
| 95 | #define HSA_CACHE_TYPE_DATA 0x00000001 | ||
| 96 | #define HSA_CACHE_TYPE_INSTRUCTION 0x00000002 | ||
| 97 | #define HSA_CACHE_TYPE_CPU 0x00000004 | ||
| 98 | #define HSA_CACHE_TYPE_HSACU 0x00000008 | ||
| 99 | #define HSA_CACHE_TYPE_RESERVED 0xfffffff0 | ||
| 100 | |||
| 101 | struct kfd_cache_properties { | ||
| 102 | struct list_head list; | ||
| 103 | uint32_t processor_id_low; | ||
| 104 | uint32_t cache_level; | ||
| 105 | uint32_t cache_size; | ||
| 106 | uint32_t cacheline_size; | ||
| 107 | uint32_t cachelines_per_tag; | ||
| 108 | uint32_t cache_assoc; | ||
| 109 | uint32_t cache_latency; | ||
| 110 | uint32_t cache_type; | ||
| 111 | uint8_t sibling_map[KFD_TOPOLOGY_CPU_SIBLINGS]; | ||
| 112 | struct kobject *kobj; | ||
| 113 | struct attribute attr; | ||
| 114 | }; | ||
| 115 | |||
| 116 | struct kfd_iolink_properties { | ||
| 117 | struct list_head list; | ||
| 118 | uint32_t iolink_type; | ||
| 119 | uint32_t ver_maj; | ||
| 120 | uint32_t ver_min; | ||
| 121 | uint32_t node_from; | ||
| 122 | uint32_t node_to; | ||
| 123 | uint32_t weight; | ||
| 124 | uint32_t min_latency; | ||
| 125 | uint32_t max_latency; | ||
| 126 | uint32_t min_bandwidth; | ||
| 127 | uint32_t max_bandwidth; | ||
| 128 | uint32_t rec_transfer_size; | ||
| 129 | uint32_t flags; | ||
| 130 | struct kobject *kobj; | ||
| 131 | struct attribute attr; | ||
| 132 | }; | ||
| 133 | |||
| 134 | struct kfd_topology_device { | ||
| 135 | struct list_head list; | ||
| 136 | uint32_t gpu_id; | ||
| 137 | struct kfd_node_properties node_props; | ||
| 138 | uint32_t mem_bank_count; | ||
| 139 | struct list_head mem_props; | ||
| 140 | uint32_t cache_count; | ||
| 141 | struct list_head cache_props; | ||
| 142 | uint32_t io_link_count; | ||
| 143 | struct list_head io_link_props; | ||
| 144 | struct kfd_dev *gpu; | ||
| 145 | struct kobject *kobj_node; | ||
| 146 | struct kobject *kobj_mem; | ||
| 147 | struct kobject *kobj_cache; | ||
| 148 | struct kobject *kobj_iolink; | ||
| 149 | struct attribute attr_gpuid; | ||
| 150 | struct attribute attr_name; | ||
| 151 | struct attribute attr_props; | ||
| 152 | }; | ||
| 153 | |||
| 154 | struct kfd_system_properties { | ||
| 155 | uint32_t num_devices; /* Number of H-NUMA nodes */ | ||
| 156 | uint32_t generation_count; | ||
| 157 | uint64_t platform_oem; | ||
| 158 | uint64_t platform_id; | ||
| 159 | uint64_t platform_rev; | ||
| 160 | struct kobject *kobj_topology; | ||
| 161 | struct kobject *kobj_nodes; | ||
| 162 | struct attribute attr_genid; | ||
| 163 | struct attribute attr_props; | ||
| 164 | }; | ||
| 165 | |||
| 166 | |||
| 167 | |||
| 168 | #endif /* __KFD_TOPOLOGY_H__ */ | ||
| @@ -0,0 +1,185 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | /* | ||
| 24 | * This file defines the private interface between the | ||
| 25 | * AMD kernel graphics drivers and the AMD KFD. | ||
| 26 | */ | ||
| 27 | |||
| 28 | #ifndef KGD_KFD_INTERFACE_H_INCLUDED | ||
| 29 | #define KGD_KFD_INTERFACE_H_INCLUDED | ||
| 30 | |||
| 31 | #include <linux/types.h> | ||
| 32 | |||
| 33 | struct pci_dev; | ||
| 34 | |||
| 35 | #define KFD_INTERFACE_VERSION 1 | ||
| 36 | |||
| 37 | struct kfd_dev; | ||
| 38 | struct kgd_dev; | ||
| 39 | |||
| 40 | struct kgd_mem; | ||
| 41 | |||
| 42 | enum kgd_memory_pool { | ||
| 43 | KGD_POOL_SYSTEM_CACHEABLE = 1, | ||
| 44 | KGD_POOL_SYSTEM_WRITECOMBINE = 2, | ||
| 45 | KGD_POOL_FRAMEBUFFER = 3, | ||
| 46 | }; | ||
| 47 | |||
| 48 | struct kgd2kfd_shared_resources { | ||
| 49 | /* Bit n == 1 means VMID n is available for KFD. */ | ||
| 50 | unsigned int compute_vmid_bitmap; | ||
| 51 | |||
| 52 | /* Compute pipes are counted starting from MEC0/pipe0 as 0. */ | ||
| 53 | unsigned int first_compute_pipe; | ||
| 54 | |||
| 55 | /* Number of MEC pipes available for KFD. */ | ||
| 56 | unsigned int compute_pipe_count; | ||
| 57 | |||
| 58 | /* Base address of doorbell aperture. */ | ||
| 59 | phys_addr_t doorbell_physical_address; | ||
| 60 | |||
| 61 | /* Size in bytes of doorbell aperture. */ | ||
| 62 | size_t doorbell_aperture_size; | ||
| 63 | |||
| 64 | /* Number of bytes at start of aperture reserved for KGD. */ | ||
| 65 | size_t doorbell_start_offset; | ||
| 66 | }; | ||
| 67 | |||
| 68 | /** | ||
| 69 | * struct kgd2kfd_calls | ||
| 70 | * | ||
| 71 | * @exit: Notifies amdkfd that kgd module is unloaded | ||
| 72 | * | ||
| 73 | * @probe: Notifies amdkfd about a probe done on a device in the kgd driver. | ||
| 74 | * | ||
| 75 | * @device_init: Initialize the newly probed device (if it is a device that | ||
| 76 | * amdkfd supports) | ||
| 77 | * | ||
| 78 | * @device_exit: Notifies amdkfd about a removal of a kgd device | ||
| 79 | * | ||
| 80 | * @suspend: Notifies amdkfd about a suspend action done to a kgd device | ||
| 81 | * | ||
| 82 | * @resume: Notifies amdkfd about a resume action done to a kgd device | ||
| 83 | * | ||
| 84 | * This structure contains function callback pointers so the kgd driver | ||
| 85 | * will notify to the amdkfd about certain status changes. | ||
| 86 | * | ||
| 87 | */ | ||
| 88 | struct kgd2kfd_calls { | ||
| 89 | void (*exit)(void); | ||
| 90 | struct kfd_dev* (*probe)(struct kgd_dev *kgd, struct pci_dev *pdev); | ||
| 91 | bool (*device_init)(struct kfd_dev *kfd, | ||
| 92 | const struct kgd2kfd_shared_resources *gpu_resources); | ||
| 93 | void (*device_exit)(struct kfd_dev *kfd); | ||
| 94 | void (*interrupt)(struct kfd_dev *kfd, const void *ih_ring_entry); | ||
| 95 | void (*suspend)(struct kfd_dev *kfd); | ||
| 96 | int (*resume)(struct kfd_dev *kfd); | ||
| 97 | }; | ||
| 98 | |||
| 99 | /** | ||
| 100 | * struct kfd2kgd_calls | ||
| 101 | * | ||
| 102 | * @init_sa_manager: Initialize an instance of the sa manager, used by | ||
| 103 | * amdkfd for all system memory allocations that are mapped to the GART | ||
| 104 | * address space | ||
| 105 | * | ||
| 106 | * @fini_sa_manager: Releases all memory allocations for amdkfd that are | ||
| 107 | * handled by kgd sa manager | ||
| 108 | * | ||
| 109 | * @allocate_mem: Allocate a buffer from amdkfd's sa manager. The buffer can | ||
| 110 | * be used for mqds, hpds, kernel queue, fence and runlists | ||
| 111 | * | ||
| 112 | * @free_mem: Frees a buffer that was allocated by amdkfd's sa manager | ||
| 113 | * | ||
| 114 | * @get_vmem_size: Retrieves (physical) size of VRAM | ||
| 115 | * | ||
| 116 | * @get_gpu_clock_counter: Retrieves GPU clock counter | ||
| 117 | * | ||
| 118 | * @get_max_engine_clock_in_mhz: Retrieves maximum GPU clock in MHz | ||
| 119 | * | ||
| 120 | * @program_sh_mem_settings: A function that should initiate the memory | ||
| 121 | * properties such as main aperture memory type (cache / non cached) and | ||
| 122 | * secondary aperture base address, size and memory type. | ||
| 123 | * This function is used only for no cp scheduling mode. | ||
| 124 | * | ||
| 125 | * @set_pasid_vmid_mapping: Exposes pasid/vmid pair to the H/W for no cp | ||
| 126 | * scheduling mode. Only used for no cp scheduling mode. | ||
| 127 | * | ||
| 128 | * @init_memory: Initializes memory apertures to fixed base/limit address | ||
| 129 | * and non cached memory types. | ||
| 130 | * | ||
| 131 | * @init_pipeline: Initialized the compute pipelines. | ||
| 132 | * | ||
| 133 | * @hqd_load: Loads the mqd structure to a H/W hqd slot. used only for no cp | ||
| 134 | * sceduling mode. | ||
| 135 | * | ||
| 136 | * @hqd_is_occupies: Checks if a hqd slot is occupied. | ||
| 137 | * | ||
| 138 | * @hqd_destroy: Destructs and preempts the queue assigned to that hqd slot. | ||
| 139 | * | ||
| 140 | * This structure contains function pointers to services that the kgd driver | ||
| 141 | * provides to amdkfd driver. | ||
| 142 | * | ||
| 143 | */ | ||
| 144 | struct kfd2kgd_calls { | ||
| 145 | /* Memory management. */ | ||
| 146 | int (*init_sa_manager)(struct kgd_dev *kgd, unsigned int size); | ||
| 147 | void (*fini_sa_manager)(struct kgd_dev *kgd); | ||
| 148 | int (*allocate_mem)(struct kgd_dev *kgd, size_t size, size_t alignment, | ||
| 149 | enum kgd_memory_pool pool, struct kgd_mem **mem); | ||
| 150 | |||
| 151 | void (*free_mem)(struct kgd_dev *kgd, struct kgd_mem *mem); | ||
| 152 | |||
| 153 | uint64_t (*get_vmem_size)(struct kgd_dev *kgd); | ||
| 154 | uint64_t (*get_gpu_clock_counter)(struct kgd_dev *kgd); | ||
| 155 | |||
| 156 | uint32_t (*get_max_engine_clock_in_mhz)(struct kgd_dev *kgd); | ||
| 157 | |||
| 158 | /* Register access functions */ | ||
| 159 | void (*program_sh_mem_settings)(struct kgd_dev *kgd, uint32_t vmid, | ||
| 160 | uint32_t sh_mem_config, uint32_t sh_mem_ape1_base, | ||
| 161 | uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases); | ||
| 162 | |||
| 163 | int (*set_pasid_vmid_mapping)(struct kgd_dev *kgd, unsigned int pasid, | ||
| 164 | unsigned int vmid); | ||
| 165 | |||
| 166 | int (*init_memory)(struct kgd_dev *kgd); | ||
| 167 | int (*init_pipeline)(struct kgd_dev *kgd, uint32_t pipe_id, | ||
| 168 | uint32_t hpd_size, uint64_t hpd_gpu_addr); | ||
| 169 | |||
| 170 | int (*hqd_load)(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, | ||
| 171 | uint32_t queue_id, uint32_t __user *wptr); | ||
| 172 | |||
| 173 | bool (*hqd_is_occupies)(struct kgd_dev *kgd, uint64_t queue_address, | ||
| 174 | uint32_t pipe_id, uint32_t queue_id); | ||
| 175 | |||
| 176 | int (*hqd_destroy)(struct kgd_dev *kgd, uint32_t reset_type, | ||
| 177 | unsigned int timeout, uint32_t pipe_id, | ||
| 178 | uint32_t queue_id); | ||
| 179 | }; | ||
| 180 | |||
| 181 | bool kgd2kfd_init(unsigned interface_version, | ||
| 182 | const struct kfd2kgd_calls *f2g, | ||
| 183 | const struct kgd2kfd_calls **g2f); | ||
| 184 | |||
| 185 | #endif /* KGD_KFD_INTERFACE_H_INCLUDED */ | ||
| @@ -104,6 +104,7 @@ radeon-y += \ · 103 unmodified lines | |||
| 104 | radeon_vce.o \ | 104 | radeon_vce.o \ |
| 105 | vce_v1_0.o \ | 105 | vce_v1_0.o \ |
| 106 | vce_v2_0.o \ | 106 | vce_v2_0.o \ |
| 107 | radeon_kfd.o | ||
| 107 | 108 | ||
| 108 | radeon-$(CONFIG_COMPAT) += radeon_ioc32.o | 109 | radeon-$(CONFIG_COMPAT) += radeon_ioc32.o |
| 109 | radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o | 110 | radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o |
| @@ -32,6 +32,7 @@ · 31 unmodified lines | |||
| 32 | #include "cik_blit_shaders.h" | 32 | #include "cik_blit_shaders.h" |
| 33 | #include "radeon_ucode.h" | 33 | #include "radeon_ucode.h" |
| 34 | #include "clearstate_ci.h" | 34 | #include "clearstate_ci.h" |
| 35 | #include "radeon_kfd.h" | ||
| 35 | 36 | ||
| 36 | MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin"); | 37 | MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin"); |
| 37 | MODULE_FIRMWARE("radeon/BONAIRE_me.bin"); | 38 | MODULE_FIRMWARE("radeon/BONAIRE_me.bin"); |
| @@ -1563,6 +1564,8 @@ static const u32 godavari_golden_registers[] = · 1525 unmodified lines | |||
| 1563 | 1564 | ||
| 1564 | static void cik_init_golden_registers(struct radeon_device *rdev) | 1565 | static void cik_init_golden_registers(struct radeon_device *rdev) |
| 1565 | { | 1566 | { |
| 1567 | /* Some of the registers might be dependent on GRBM_GFX_INDEX */ | ||
| 1568 | mutex_lock(&rdev->grbm_idx_mutex); | ||
| 1566 | switch (rdev->family) { | 1569 | switch (rdev->family) { |
| 1567 | case CHIP_BONAIRE: | 1570 | case CHIP_BONAIRE: |
| 1568 | radeon_program_register_sequence(rdev, | 1571 | radeon_program_register_sequence(rdev, |
| @@ -1637,6 +1640,7 @@ static void cik_init_golden_registers(struct radeon_device *rdev) · 68 unmodified lines | |||
| 1637 | default: | 1640 | default: |
| 1638 | break; | 1641 | break; |
| 1639 | } | 1642 | } |
| 1643 | mutex_unlock(&rdev->grbm_idx_mutex); | ||
| 1640 | } | 1644 | } |
| 1641 | 1645 | ||
| 1642 | /** | 1646 | /** |
| @@ -3428,6 +3432,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 1785 unmodified lines | |||
| 3428 | u32 disabled_rbs = 0; | 3432 | u32 disabled_rbs = 0; |
| 3429 | u32 enabled_rbs = 0; | 3433 | u32 enabled_rbs = 0; |
| 3430 | 3434 | ||
| 3435 | mutex_lock(&rdev->grbm_idx_mutex); | ||
| 3431 | for (i = 0; i < se_num; i++) { | 3436 | for (i = 0; i < se_num; i++) { |
| 3432 | for (j = 0; j < sh_per_se; j++) { | 3437 | for (j = 0; j < sh_per_se; j++) { |
| 3433 | cik_select_se_sh(rdev, i, j); | 3438 | cik_select_se_sh(rdev, i, j); |
| @@ -3439,6 +3444,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 5 unmodified lines | |||
| 3439 | } | 3444 | } |
| 3440 | } | 3445 | } |
| 3441 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); | 3446 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 3447 | mutex_unlock(&rdev->grbm_idx_mutex); | ||
| 3442 | 3448 | ||
| 3443 | mask = 1; | 3449 | mask = 1; |
| 3444 | for (i = 0; i < max_rb_num_per_se * se_num; i++) { | 3450 | for (i = 0; i < max_rb_num_per_se * se_num; i++) { |
| @@ -3449,6 +3455,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 4 unmodified lines | |||
| 3449 | 3455 | ||
| 3450 | rdev->config.cik.backend_enable_mask = enabled_rbs; | 3456 | rdev->config.cik.backend_enable_mask = enabled_rbs; |
| 3451 | 3457 | ||
| 3458 | mutex_lock(&rdev->grbm_idx_mutex); | ||
| 3452 | for (i = 0; i < se_num; i++) { | 3459 | for (i = 0; i < se_num; i++) { |
| 3453 | cik_select_se_sh(rdev, i, 0xffffffff); | 3460 | cik_select_se_sh(rdev, i, 0xffffffff); |
| 3454 | data = 0; | 3461 | data = 0; |
| @@ -3476,6 +3483,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 21 unmodified lines | |||
| 3476 | WREG32(PA_SC_RASTER_CONFIG, data); | 3483 | WREG32(PA_SC_RASTER_CONFIG, data); |
| 3477 | } | 3484 | } |
| 3478 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); | 3485 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 3486 | mutex_unlock(&rdev->grbm_idx_mutex); | ||
| 3479 | } | 3487 | } |
| 3480 | 3488 | ||
| 3481 | /** | 3489 | /** |
| @@ -3693,6 +3701,12 @@ static void cik_gpu_init(struct radeon_device *rdev) · 211 unmodified lines | |||
| 3693 | /* set HW defaults for 3D engine */ | 3701 | /* set HW defaults for 3D engine */ |
| 3694 | WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); | 3702 | WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); |
| 3695 | 3703 | ||
| 3704 | mutex_lock(&rdev->grbm_idx_mutex); | ||
| 3705 | /* | ||
| 3706 | * making sure that the following register writes will be broadcasted | ||
| 3707 | * to all the shaders | ||
| 3708 | */ | ||
| 3709 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); | ||
| 3696 | WREG32(SX_DEBUG_1, 0x20); | 3710 | WREG32(SX_DEBUG_1, 0x20); |
| 3697 | 3711 | ||
| 3698 | WREG32(TA_CNTL_AUX, 0x00010000); | 3712 | WREG32(TA_CNTL_AUX, 0x00010000); |
| @@ -3748,6 +3762,7 @@ static void cik_gpu_init(struct radeon_device *rdev) · 49 unmodified lines | |||
| 3748 | 3762 | ||
| 3749 | WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); | 3763 | WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); |
| 3750 | WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER); | 3764 | WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER); |
| 3765 | mutex_unlock(&rdev->grbm_idx_mutex); | ||
| 3751 | 3766 | ||
| 3752 | udelay(50); | 3767 | udelay(50); |
| 3753 | } | 3768 | } |
| @@ -4684,12 +4699,11 @@ static int cik_mec_init(struct radeon_device *rdev) · 930 unmodified lines | |||
| 4684 | /* | 4699 | /* |
| 4685 | * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total | 4700 | * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total |
| 4686 | * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total | 4701 | * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total |
| 4702 | * Nonetheless, we assign only 1 pipe because all other pipes will | ||
| 4703 | * be handled by KFD | ||
| 4687 | */ | 4704 | */ |
| 4688 | if (rdev->family == CHIP_KAVERI) | 4705 | rdev->mec.num_mec = 1; |
| 4689 | rdev->mec.num_mec = 2; | 4706 | rdev->mec.num_pipe = 1; |
| 4690 | else | ||
| 4691 | rdev->mec.num_mec = 1; | ||
| 4692 | rdev->mec.num_pipe = 4; | ||
| 4693 | rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8; | 4707 | rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8; |
| 4694 | 4708 | ||
| 4695 | if (rdev->mec.hpd_eop_obj == NULL) { | 4709 | if (rdev->mec.hpd_eop_obj == NULL) { |
| @@ -4831,28 +4845,24 @@ static int cik_cp_compute_resume(struct radeon_device *rdev) · 135 unmodified lines | |||
| 4831 | 4845 | ||
| 4832 | /* init the pipes */ | 4846 | /* init the pipes */ |
| 4833 | mutex_lock(&rdev->srbm_mutex); | 4847 | mutex_lock(&rdev->srbm_mutex); |
| 4834 | for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) { | ||
| 4835 | int me = (i < 4) ? 1 : 2; | ||
| 4836 | int pipe = (i < 4) ? i : (i - 4); | ||
| 4837 | 4848 | ||
| 4838 | eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2); | 4849 | eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr; |
| 4839 | 4850 | ||
| 4840 | cik_srbm_select(rdev, me, pipe, 0, 0); | 4851 | cik_srbm_select(rdev, 0, 0, 0, 0); |
| 4841 | 4852 | ||
| 4842 | /* write the EOP addr */ | 4853 | /* write the EOP addr */ |
| 4843 | WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8); | 4854 | WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8); |
| 4844 | WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8); | 4855 | WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8); |
| 4845 | 4856 | ||
| 4846 | /* set the VMID assigned */ | 4857 | /* set the VMID assigned */ |
| 4847 | WREG32(CP_HPD_EOP_VMID, 0); | 4858 | WREG32(CP_HPD_EOP_VMID, 0); |
| 4859 | |||
| 4860 | /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ | ||
| 4861 | tmp = RREG32(CP_HPD_EOP_CONTROL); | ||
| 4862 | tmp &= ~EOP_SIZE_MASK; | ||
| 4863 | tmp |= order_base_2(MEC_HPD_SIZE / 8); | ||
| 4864 | WREG32(CP_HPD_EOP_CONTROL, tmp); | ||
| 4848 | 4865 | ||
| 4849 | /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ | ||
| 4850 | tmp = RREG32(CP_HPD_EOP_CONTROL); | ||
| 4851 | tmp &= ~EOP_SIZE_MASK; | ||
| 4852 | tmp |= order_base_2(MEC_HPD_SIZE / 8); | ||
| 4853 | WREG32(CP_HPD_EOP_CONTROL, tmp); | ||
| 4854 | } | ||
| 4855 | cik_srbm_select(rdev, 0, 0, 0, 0); | ||
| 4856 | mutex_unlock(&rdev->srbm_mutex); | 4866 | mutex_unlock(&rdev->srbm_mutex); |
| 4857 | 4867 | ||
| 4858 | /* init the queues. Just two for now. */ | 4868 | /* init the queues. Just two for now. */ |
| @@ -5906,8 +5916,13 @@ int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) · 1047 unmodified lines | |||
| 5906 | */ | 5916 | */ |
| 5907 | int cik_vm_init(struct radeon_device *rdev) | 5917 | int cik_vm_init(struct radeon_device *rdev) |
| 5908 | { | 5918 | { |
| 5909 | /* number of VMs */ | 5919 | /* |
| 5910 | rdev->vm_manager.nvm = 16; | 5920 | * number of VMs |
| 5921 | * VMID 0 is reserved for System | ||
| 5922 | * radeon graphics/compute will use VMIDs 1-7 | ||
| 5923 | * amdkfd will use VMIDs 8-15 | ||
| 5924 | */ | ||
| 5925 | rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS; | ||
| 5911 | /* base offset of vram pages */ | 5926 | /* base offset of vram pages */ |
| 5912 | if (rdev->flags & RADEON_IS_IGP) { | 5927 | if (rdev->flags & RADEON_IS_IGP) { |
| 5913 | u64 tmp = RREG32(MC_VM_FB_OFFSET); | 5928 | u64 tmp = RREG32(MC_VM_FB_OFFSET); |
| @@ -6068,6 +6083,7 @@ static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) · 154 unmodified lines | |||
| 6068 | u32 i, j, k; | 6083 | u32 i, j, k; |
| 6069 | u32 mask; | 6084 | u32 mask; |
| 6070 | 6085 | ||
| 6086 | mutex_lock(&rdev->grbm_idx_mutex); | ||
| 6071 | for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { | 6087 | for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { |
| 6072 | for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { | 6088 | for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { |
| 6073 | cik_select_se_sh(rdev, i, j); | 6089 | cik_select_se_sh(rdev, i, j); |
| @@ -6079,6 +6095,7 @@ static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) · 5 unmodified lines | |||
| 6079 | } | 6095 | } |
| 6080 | } | 6096 | } |
| 6081 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); | 6097 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 6098 | mutex_unlock(&rdev->grbm_idx_mutex); | ||
| 6082 | 6099 | ||
| 6083 | mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY; | 6100 | mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY; |
| 6084 | for (k = 0; k < rdev->usec_timeout; k++) { | 6101 | for (k = 0; k < rdev->usec_timeout; k++) { |
| @@ -6213,10 +6230,12 @@ static int cik_rlc_resume(struct radeon_device *rdev) · 128 unmodified lines | |||
| 6213 | WREG32(RLC_LB_CNTR_INIT, 0); | 6230 | WREG32(RLC_LB_CNTR_INIT, 0); |
| 6214 | WREG32(RLC_LB_CNTR_MAX, 0x00008000); | 6231 | WREG32(RLC_LB_CNTR_MAX, 0x00008000); |
| 6215 | 6232 | ||
| 6233 | mutex_lock(&rdev->grbm_idx_mutex); | ||
| 6216 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); | 6234 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 6217 | WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); | 6235 | WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); |
| 6218 | WREG32(RLC_LB_PARAMS, 0x00600408); | 6236 | WREG32(RLC_LB_PARAMS, 0x00600408); |
| 6219 | WREG32(RLC_LB_CNTL, 0x80000004); | 6237 | WREG32(RLC_LB_CNTL, 0x80000004); |
| 6238 | mutex_unlock(&rdev->grbm_idx_mutex); | ||
| 6220 | 6239 | ||
| 6221 | WREG32(RLC_MC_CNTL, 0); | 6240 | WREG32(RLC_MC_CNTL, 0); |
| 6222 | WREG32(RLC_UCODE_CNTL, 0); | 6241 | WREG32(RLC_UCODE_CNTL, 0); |
| @@ -6283,11 +6302,13 @@ static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) · 60 unmodified lines | |||
| 6283 | 6302 | ||
| 6284 | tmp = cik_halt_rlc(rdev); | 6303 | tmp = cik_halt_rlc(rdev); |
| 6285 | 6304 | ||
| 6305 | mutex_lock(&rdev->grbm_idx_mutex); | ||
| 6286 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); | 6306 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 6287 | WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); | 6307 | WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); |
| 6288 | WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); | 6308 | WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); |
| 6289 | tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE; | 6309 | tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE; |
| 6290 | WREG32(RLC_SERDES_WR_CTRL, tmp2); | 6310 | WREG32(RLC_SERDES_WR_CTRL, tmp2); |
| 6311 | mutex_unlock(&rdev->grbm_idx_mutex); | ||
| 6291 | 6312 | ||
| 6292 | cik_update_rlc(rdev, tmp); | 6313 | cik_update_rlc(rdev, tmp); |
| 6293 | 6314 | ||
| @@ -6329,11 +6350,13 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) · 35 unmodified lines | |||
| 6329 | 6350 | ||
| 6330 | tmp = cik_halt_rlc(rdev); | 6351 | tmp = cik_halt_rlc(rdev); |
| 6331 | 6352 | ||
| 6353 | mutex_lock(&rdev->grbm_idx_mutex); | ||
| 6332 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); | 6354 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 6333 | WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); | 6355 | WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); |
| 6334 | WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); | 6356 | WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); |
| 6335 | data = BPM_ADDR_MASK | MGCG_OVERRIDE_0; | 6357 | data = BPM_ADDR_MASK | MGCG_OVERRIDE_0; |
| 6336 | WREG32(RLC_SERDES_WR_CTRL, data); | 6358 | WREG32(RLC_SERDES_WR_CTRL, data); |
| 6359 | mutex_unlock(&rdev->grbm_idx_mutex); | ||
| 6337 | 6360 | ||
| 6338 | cik_update_rlc(rdev, tmp); | 6361 | cik_update_rlc(rdev, tmp); |
| 6339 | 6362 | ||
| @@ -6377,11 +6400,13 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) · 37 unmodified lines | |||
| 6377 | 6400 | ||
| 6378 | tmp = cik_halt_rlc(rdev); | 6401 | tmp = cik_halt_rlc(rdev); |
| 6379 | 6402 | ||
| 6403 | mutex_lock(&rdev->grbm_idx_mutex); | ||
| 6380 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); | 6404 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 6381 | WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); | 6405 | WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); |
| 6382 | WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); | 6406 | WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); |
| 6383 | data = BPM_ADDR_MASK | MGCG_OVERRIDE_1; | 6407 | data = BPM_ADDR_MASK | MGCG_OVERRIDE_1; |
| 6384 | WREG32(RLC_SERDES_WR_CTRL, data); | 6408 | WREG32(RLC_SERDES_WR_CTRL, data); |
| 6409 | mutex_unlock(&rdev->grbm_idx_mutex); | ||
| 6385 | 6410 | ||
| 6386 | cik_update_rlc(rdev, tmp); | 6411 | cik_update_rlc(rdev, tmp); |
| 6387 | } | 6412 | } |
| @@ -6810,10 +6835,12 @@ static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) · 422 unmodified lines | |||
| 6810 | u32 mask = 0, tmp, tmp1; | 6835 | u32 mask = 0, tmp, tmp1; |
| 6811 | int i; | 6836 | int i; |
| 6812 | 6837 | ||
| 6838 | mutex_lock(&rdev->grbm_idx_mutex); | ||
| 6813 | cik_select_se_sh(rdev, se, sh); | 6839 | cik_select_se_sh(rdev, se, sh); |
| 6814 | tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); | 6840 | tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); |
| 6815 | tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); | 6841 | tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); |
| 6816 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); | 6842 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 6843 | mutex_unlock(&rdev->grbm_idx_mutex); | ||
| 6817 | 6844 | ||
| 6818 | tmp &= 0xffff0000; | 6845 | tmp &= 0xffff0000; |
| 6819 | 6846 | ||
| @@ -7297,8 +7324,7 @@ static int cik_irq_init(struct radeon_device *rdev) · 477 unmodified lines | |||
| 7297 | int cik_irq_set(struct radeon_device *rdev) | 7324 | int cik_irq_set(struct radeon_device *rdev) |
| 7298 | { | 7325 | { |
| 7299 | u32 cp_int_cntl; | 7326 | u32 cp_int_cntl; |
| 7300 | u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3; | 7327 | u32 cp_m1p0; |
| 7301 | u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3; | ||
| 7302 | u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; | 7328 | u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; |
| 7303 | u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; | 7329 | u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; |
| 7304 | u32 grbm_int_cntl = 0; | 7330 | u32 grbm_int_cntl = 0; |
| @@ -7332,13 +7358,6 @@ int cik_irq_set(struct radeon_device *rdev) · 27 unmodified lines | |||
| 7332 | dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; | 7358 | dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; |
| 7333 | 7359 | ||
| 7334 | cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | 7360 | cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; |
| 7335 | cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | ||
| 7336 | cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | ||
| 7337 | cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | ||
| 7338 | cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | ||
| 7339 | cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | ||
| 7340 | cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | ||
| 7341 | cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | ||
| 7342 | 7361 | ||
| 7343 | if (rdev->flags & RADEON_IS_IGP) | 7362 | if (rdev->flags & RADEON_IS_IGP) |
| 7344 | thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) & | 7363 | thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) & |
| @@ -7360,33 +7379,6 @@ int cik_irq_set(struct radeon_device *rdev) · 15 unmodified lines | |||
| 7360 | case 0: | 7379 | case 0: |
| 7361 | cp_m1p0 |= TIME_STAMP_INT_ENABLE; | 7380 | cp_m1p0 |= TIME_STAMP_INT_ENABLE; |
| 7362 | break; | 7381 | break; |
| 7363 | case 1: | ||
| 7364 | cp_m1p1 |= TIME_STAMP_INT_ENABLE; | ||
| 7365 | break; | ||
| 7366 | case 2: | ||
| 7367 | cp_m1p2 |= TIME_STAMP_INT_ENABLE; | ||
| 7368 | break; | ||
| 7369 | case 3: | ||
| 7370 | cp_m1p2 |= TIME_STAMP_INT_ENABLE; | ||
| 7371 | break; | ||
| 7372 | default: | ||
| 7373 | DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); | ||
| 7374 | break; | ||
| 7375 | } | ||
| 7376 | } else if (ring->me == 2) { | ||
| 7377 | switch (ring->pipe) { | ||
| 7378 | case 0: | ||
| 7379 | cp_m2p0 |= TIME_STAMP_INT_ENABLE; | ||
| 7380 | break; | ||
| 7381 | case 1: | ||
| 7382 | cp_m2p1 |= TIME_STAMP_INT_ENABLE; | ||
| 7383 | break; | ||
| 7384 | case 2: | ||
| 7385 | cp_m2p2 |= TIME_STAMP_INT_ENABLE; | ||
| 7386 | break; | ||
| 7387 | case 3: | ||
| 7388 | cp_m2p2 |= TIME_STAMP_INT_ENABLE; | ||
| 7389 | break; | ||
| 7390 | default: | 7382 | default: |
| 7391 | DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); | 7383 | DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); |
| 7392 | break; | 7384 | break; |
| @@ -7403,33 +7395,6 @@ int cik_irq_set(struct radeon_device *rdev) · 10 unmodified lines | |||
| 7403 | case 0: | 7395 | case 0: |
| 7404 | cp_m1p0 |= TIME_STAMP_INT_ENABLE; | 7396 | cp_m1p0 |= TIME_STAMP_INT_ENABLE; |
| 7405 | break; | 7397 | break; |
| 7406 | case 1: | ||
| 7407 | cp_m1p1 |= TIME_STAMP_INT_ENABLE; | ||
| 7408 | break; | ||
| 7409 | case 2: | ||
| 7410 | cp_m1p2 |= TIME_STAMP_INT_ENABLE; | ||
| 7411 | break; | ||
| 7412 | case 3: | ||
| 7413 | cp_m1p2 |= TIME_STAMP_INT_ENABLE; | ||
| 7414 | break; | ||
| 7415 | default: | ||
| 7416 | DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); | ||
| 7417 | break; | ||
| 7418 | } | ||
| 7419 | } else if (ring->me == 2) { | ||
| 7420 | switch (ring->pipe) { | ||
| 7421 | case 0: | ||
| 7422 | cp_m2p0 |= TIME_STAMP_INT_ENABLE; | ||
| 7423 | break; | ||
| 7424 | case 1: | ||
| 7425 | cp_m2p1 |= TIME_STAMP_INT_ENABLE; | ||
| 7426 | break; | ||
| 7427 | case 2: | ||
| 7428 | cp_m2p2 |= TIME_STAMP_INT_ENABLE; | ||
| 7429 | break; | ||
| 7430 | case 3: | ||
| 7431 | cp_m2p2 |= TIME_STAMP_INT_ENABLE; | ||
| 7432 | break; | ||
| 7433 | default: | 7398 | default: |
| 7434 | DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); | 7399 | DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); |
| 7435 | break; | 7400 | break; |
| @@ -7518,13 +7483,6 @@ int cik_irq_set(struct radeon_device *rdev) · 82 unmodified lines | |||
| 7518 | WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1); | 7483 | WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1); |
| 7519 | 7484 | ||
| 7520 | WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0); | 7485 | WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0); |
| 7521 | WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1); | ||
| 7522 | WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2); | ||
| 7523 | WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3); | ||
| 7524 | WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0); | ||
| 7525 | WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1); | ||
| 7526 | WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2); | ||
| 7527 | WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3); | ||
| 7528 | 7486 | ||
| 7529 | WREG32(GRBM_INT_CNTL, grbm_int_cntl); | 7487 | WREG32(GRBM_INT_CNTL, grbm_int_cntl); |
| 7530 | 7488 | ||
| @@ -7841,6 +7799,10 @@ restart_ih: · 310 unmodified lines | |||
| 7841 | while (rptr != wptr) { | 7799 | while (rptr != wptr) { |
| 7842 | /* wptr/rptr are in bytes! */ | 7800 | /* wptr/rptr are in bytes! */ |
| 7843 | ring_index = rptr / 4; | 7801 | ring_index = rptr / 4; |
| 7802 | |||
| 7803 | radeon_kfd_interrupt(rdev, | ||
| 7804 | (const void *) &rdev->ih.ring[ring_index]); | ||
| 7805 | |||
| 7844 | src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; | 7806 | src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; |
| 7845 | src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; | 7807 | src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; |
| 7846 | ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; | 7808 | ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; |
| @@ -8530,6 +8492,10 @@ static int cik_startup(struct radeon_device *rdev) · 683 unmodified lines | |||
| 8530 | if (r) | 8492 | if (r) |
| 8531 | return r; | 8493 | return r; |
| 8532 | 8494 | ||
| 8495 | r = radeon_kfd_resume(rdev); | ||
| 8496 | if (r) | ||
| 8497 | return r; | ||
| 8498 | |||
| 8533 | return 0; | 8499 | return 0; |
| 8534 | } | 8500 | } |
| 8535 | 8501 | ||
| @@ -8578,6 +8544,7 @@ int cik_resume(struct radeon_device *rdev) · 42 unmodified lines | |||
| 8578 | */ | 8544 | */ |
| 8579 | int cik_suspend(struct radeon_device *rdev) | 8545 | int cik_suspend(struct radeon_device *rdev) |
| 8580 | { | 8546 | { |
| 8547 | radeon_kfd_suspend(rdev); | ||
| 8581 | radeon_pm_suspend(rdev); | 8548 | radeon_pm_suspend(rdev); |
| 8582 | dce6_audio_fini(rdev); | 8549 | dce6_audio_fini(rdev); |
| 8583 | radeon_vm_manager_fini(rdev); | 8550 | radeon_vm_manager_fini(rdev); |
| @@ -147,4 +147,140 @@ · 146 unmodified lines | |||
| 147 | 147 | ||
| 148 | #define CIK_LB_DESKTOP_HEIGHT 0x6b0c | 148 | #define CIK_LB_DESKTOP_HEIGHT 0x6b0c |
| 149 | 149 | ||
| 150 | #define CP_HQD_IQ_RPTR 0xC970u | ||
| 151 | #define AQL_ENABLE (1U << 0) | ||
| 152 | |||
| 153 | #define IDLE (1 << 2) | ||
| 154 | |||
| 155 | struct cik_mqd { | ||
| 156 | uint32_t header; | ||
| 157 | uint32_t compute_dispatch_initiator; | ||
| 158 | uint32_t compute_dim_x; | ||
| 159 | uint32_t compute_dim_y; | ||
| 160 | uint32_t compute_dim_z; | ||
| 161 | uint32_t compute_start_x; | ||
| 162 | uint32_t compute_start_y; | ||
| 163 | uint32_t compute_start_z; | ||
| 164 | uint32_t compute_num_thread_x; | ||
| 165 | uint32_t compute_num_thread_y; | ||
| 166 | uint32_t compute_num_thread_z; | ||
| 167 | uint32_t compute_pipelinestat_enable; | ||
| 168 | uint32_t compute_perfcount_enable; | ||
| 169 | uint32_t compute_pgm_lo; | ||
| 170 | uint32_t compute_pgm_hi; | ||
| 171 | uint32_t compute_tba_lo; | ||
| 172 | uint32_t compute_tba_hi; | ||
| 173 | uint32_t compute_tma_lo; | ||
| 174 | uint32_t compute_tma_hi; | ||
| 175 | uint32_t compute_pgm_rsrc1; | ||
| 176 | uint32_t compute_pgm_rsrc2; | ||
| 177 | uint32_t compute_vmid; | ||
| 178 | uint32_t compute_resource_limits; | ||
| 179 | uint32_t compute_static_thread_mgmt_se0; | ||
| 180 | uint32_t compute_static_thread_mgmt_se1; | ||
| 181 | uint32_t compute_tmpring_size; | ||
| 182 | uint32_t compute_static_thread_mgmt_se2; | ||
| 183 | uint32_t compute_static_thread_mgmt_se3; | ||
| 184 | uint32_t compute_restart_x; | ||
| 185 | uint32_t compute_restart_y; | ||
| 186 | uint32_t compute_restart_z; | ||
| 187 | uint32_t compute_thread_trace_enable; | ||
| 188 | uint32_t compute_misc_reserved; | ||
| 189 | uint32_t compute_user_data_0; | ||
| 190 | uint32_t compute_user_data_1; | ||
| 191 | uint32_t compute_user_data_2; | ||
| 192 | uint32_t compute_user_data_3; | ||
| 193 | uint32_t compute_user_data_4; | ||
| 194 | uint32_t compute_user_data_5; | ||
| 195 | uint32_t compute_user_data_6; | ||
| 196 | uint32_t compute_user_data_7; | ||
| 197 | uint32_t compute_user_data_8; | ||
| 198 | uint32_t compute_user_data_9; | ||
| 199 | uint32_t compute_user_data_10; | ||
| 200 | uint32_t compute_user_data_11; | ||
| 201 | uint32_t compute_user_data_12; | ||
| 202 | uint32_t compute_user_data_13; | ||
| 203 | uint32_t compute_user_data_14; | ||
| 204 | uint32_t compute_user_data_15; | ||
| 205 | uint32_t cp_compute_csinvoc_count_lo; | ||
| 206 | uint32_t cp_compute_csinvoc_count_hi; | ||
| 207 | uint32_t cp_mqd_base_addr_lo; | ||
| 208 | uint32_t cp_mqd_base_addr_hi; | ||
| 209 | uint32_t cp_hqd_active; | ||
| 210 | uint32_t cp_hqd_vmid; | ||
| 211 | uint32_t cp_hqd_persistent_state; | ||
| 212 | uint32_t cp_hqd_pipe_priority; | ||
| 213 | uint32_t cp_hqd_queue_priority; | ||
| 214 | uint32_t cp_hqd_quantum; | ||
| 215 | uint32_t cp_hqd_pq_base_lo; | ||
| 216 | uint32_t cp_hqd_pq_base_hi; | ||
| 217 | uint32_t cp_hqd_pq_rptr; | ||
| 218 | uint32_t cp_hqd_pq_rptr_report_addr_lo; | ||
| 219 | uint32_t cp_hqd_pq_rptr_report_addr_hi; | ||
| 220 | uint32_t cp_hqd_pq_wptr_poll_addr_lo; | ||
| 221 | uint32_t cp_hqd_pq_wptr_poll_addr_hi; | ||
| 222 | uint32_t cp_hqd_pq_doorbell_control; | ||
| 223 | uint32_t cp_hqd_pq_wptr; | ||
| 224 | uint32_t cp_hqd_pq_control; | ||
| 225 | uint32_t cp_hqd_ib_base_addr_lo; | ||
| 226 | uint32_t cp_hqd_ib_base_addr_hi; | ||
| 227 | uint32_t cp_hqd_ib_rptr; | ||
| 228 | uint32_t cp_hqd_ib_control; | ||
| 229 | uint32_t cp_hqd_iq_timer; | ||
| 230 | uint32_t cp_hqd_iq_rptr; | ||
| 231 | uint32_t cp_hqd_dequeue_request; | ||
| 232 | uint32_t cp_hqd_dma_offload; | ||
| 233 | uint32_t cp_hqd_sema_cmd; | ||
| 234 | uint32_t cp_hqd_msg_type; | ||
| 235 | uint32_t cp_hqd_atomic0_preop_lo; | ||
| 236 | uint32_t cp_hqd_atomic0_preop_hi; | ||
| 237 | uint32_t cp_hqd_atomic1_preop_lo; | ||
| 238 | uint32_t cp_hqd_atomic1_preop_hi; | ||
| 239 | uint32_t cp_hqd_hq_status0; | ||
| 240 | uint32_t cp_hqd_hq_control0; | ||
| 241 | uint32_t cp_mqd_control; | ||
| 242 | uint32_t cp_mqd_query_time_lo; | ||
| 243 | uint32_t cp_mqd_query_time_hi; | ||
| 244 | uint32_t cp_mqd_connect_start_time_lo; | ||
| 245 | uint32_t cp_mqd_connect_start_time_hi; | ||
| 246 | uint32_t cp_mqd_connect_end_time_lo; | ||
| 247 | uint32_t cp_mqd_connect_end_time_hi; | ||
| 248 | uint32_t cp_mqd_connect_end_wf_count; | ||
| 249 | uint32_t cp_mqd_connect_end_pq_rptr; | ||
| 250 | uint32_t cp_mqd_connect_end_pq_wptr; | ||
| 251 | uint32_t cp_mqd_connect_end_ib_rptr; | ||
| 252 | uint32_t reserved_96; | ||
| 253 | uint32_t reserved_97; | ||
| 254 | uint32_t reserved_98; | ||
| 255 | uint32_t reserved_99; | ||
| 256 | uint32_t iqtimer_pkt_header; | ||
| 257 | uint32_t iqtimer_pkt_dw0; | ||
| 258 | uint32_t iqtimer_pkt_dw1; | ||
| 259 | uint32_t iqtimer_pkt_dw2; | ||
| 260 | uint32_t iqtimer_pkt_dw3; | ||
| 261 | uint32_t iqtimer_pkt_dw4; | ||
| 262 | uint32_t iqtimer_pkt_dw5; | ||
| 263 | uint32_t iqtimer_pkt_dw6; | ||
| 264 | uint32_t reserved_108; | ||
| 265 | uint32_t reserved_109; | ||
| 266 | uint32_t reserved_110; | ||
| 267 | uint32_t reserved_111; | ||
| 268 | uint32_t queue_doorbell_id0; | ||
| 269 | uint32_t queue_doorbell_id1; | ||
| 270 | uint32_t queue_doorbell_id2; | ||
| 271 | uint32_t queue_doorbell_id3; | ||
| 272 | uint32_t queue_doorbell_id4; | ||
| 273 | uint32_t queue_doorbell_id5; | ||
| 274 | uint32_t queue_doorbell_id6; | ||
| 275 | uint32_t queue_doorbell_id7; | ||
| 276 | uint32_t queue_doorbell_id8; | ||
| 277 | uint32_t queue_doorbell_id9; | ||
| 278 | uint32_t queue_doorbell_id10; | ||
| 279 | uint32_t queue_doorbell_id11; | ||
| 280 | uint32_t queue_doorbell_id12; | ||
| 281 | uint32_t queue_doorbell_id13; | ||
| 282 | uint32_t queue_doorbell_id14; | ||
| 283 | uint32_t queue_doorbell_id15; | ||
| 284 | }; | ||
| 285 | |||
| 150 | #endif | 286 | #endif |
| @@ -30,6 +30,8 @@ · 29 unmodified lines | |||
| 30 | #define CIK_RB_BITMAP_WIDTH_PER_SH 2 | 30 | #define CIK_RB_BITMAP_WIDTH_PER_SH 2 |
| 31 | #define HAWAII_RB_BITMAP_WIDTH_PER_SH 4 | 31 | #define HAWAII_RB_BITMAP_WIDTH_PER_SH 4 |
| 32 | 32 | ||
| 33 | #define RADEON_NUM_OF_VMIDS 8 | ||
| 34 | |||
| 33 | /* DIDT IND registers */ | 35 | /* DIDT IND registers */ |
| 34 | #define DIDT_SQ_CTRL0 0x0 | 36 | #define DIDT_SQ_CTRL0 0x0 |
| 35 | # define DIDT_CTRL_EN (1 << 0) | 37 | # define DIDT_CTRL_EN (1 << 0) |
| @@ -1137,6 +1139,9 @@ · 1101 unmodified lines | |||
| 1137 | #define SH_MEM_ALIGNMENT_MODE_UNALIGNED 3 | 1139 | #define SH_MEM_ALIGNMENT_MODE_UNALIGNED 3 |
| 1138 | #define DEFAULT_MTYPE(x) ((x) << 4) | 1140 | #define DEFAULT_MTYPE(x) ((x) << 4) |
| 1139 | #define APE1_MTYPE(x) ((x) << 7) | 1141 | #define APE1_MTYPE(x) ((x) << 7) |
| 1142 | /* valid for both DEFAULT_MTYPE and APE1_MTYPE */ | ||
| 1143 | #define MTYPE_CACHED 0 | ||
| 1144 | #define MTYPE_NONCACHED 3 | ||
| 1140 | 1145 | ||
| 1141 | #define SX_DEBUG_1 0x9060 | 1146 | #define SX_DEBUG_1 0x9060 |
| 1142 | 1147 | ||
| @@ -1447,6 +1452,16 @@ · 304 unmodified lines | |||
| 1447 | #define CP_HQD_ACTIVE 0xC91C | 1452 | #define CP_HQD_ACTIVE 0xC91C |
| 1448 | #define CP_HQD_VMID 0xC920 | 1453 | #define CP_HQD_VMID 0xC920 |
| 1449 | 1454 | ||
| 1455 | #define CP_HQD_PERSISTENT_STATE 0xC924u | ||
| 1456 | #define DEFAULT_CP_HQD_PERSISTENT_STATE (0x33U << 8) | ||
| 1457 | |||
| 1458 | #define CP_HQD_PIPE_PRIORITY 0xC928u | ||
| 1459 | #define CP_HQD_QUEUE_PRIORITY 0xC92Cu | ||
| 1460 | #define CP_HQD_QUANTUM 0xC930u | ||
| 1461 | #define QUANTUM_EN 1U | ||
| 1462 | #define QUANTUM_SCALE_1MS (1U << 4) | ||
| 1463 | #define QUANTUM_DURATION(x) ((x) << 8) | ||
| 1464 | |||
| 1450 | #define CP_HQD_PQ_BASE 0xC934 | 1465 | #define CP_HQD_PQ_BASE 0xC934 |
| 1451 | #define CP_HQD_PQ_BASE_HI 0xC938 | 1466 | #define CP_HQD_PQ_BASE_HI 0xC938 |
| 1452 | #define CP_HQD_PQ_RPTR 0xC93C | 1467 | #define CP_HQD_PQ_RPTR 0xC93C |
| @@ -1474,12 +1489,32 @@ · 21 unmodified lines | |||
| 1474 | #define PRIV_STATE (1 << 30) | 1489 | #define PRIV_STATE (1 << 30) |
| 1475 | #define KMD_QUEUE (1 << 31) | 1490 | #define KMD_QUEUE (1 << 31) |
| 1476 | 1491 | ||
| 1477 | #define CP_HQD_DEQUEUE_REQUEST 0xC974 | 1492 | #define CP_HQD_IB_BASE_ADDR 0xC95Cu |
| 1493 | #define CP_HQD_IB_BASE_ADDR_HI 0xC960u | ||
| 1494 | #define CP_HQD_IB_RPTR 0xC964u | ||
| 1495 | #define CP_HQD_IB_CONTROL 0xC968u | ||
| 1496 | #define IB_ATC_EN (1U << 23) | ||
| 1497 | #define DEFAULT_MIN_IB_AVAIL_SIZE (3U << 20) | ||
| 1498 | |||
| 1499 | #define CP_HQD_DEQUEUE_REQUEST 0xC974 | ||
| 1500 | #define DEQUEUE_REQUEST_DRAIN 1 | ||
| 1501 | #define DEQUEUE_REQUEST_RESET 2 | ||
| 1478 | 1502 | ||
| 1479 | #define CP_MQD_CONTROL 0xC99C | 1503 | #define CP_MQD_CONTROL 0xC99C |
| 1480 | #define MQD_VMID(x) ((x) << 0) | 1504 | #define MQD_VMID(x) ((x) << 0) |
| 1481 | #define MQD_VMID_MASK (0xf << 0) | 1505 | #define MQD_VMID_MASK (0xf << 0) |
| 1482 | 1506 | ||
| 1507 | #define CP_HQD_SEMA_CMD 0xC97Cu | ||
| 1508 | #define CP_HQD_MSG_TYPE 0xC980u | ||
| 1509 | #define CP_HQD_ATOMIC0_PREOP_LO 0xC984u | ||
| 1510 | #define CP_HQD_ATOMIC0_PREOP_HI 0xC988u | ||
| 1511 | #define CP_HQD_ATOMIC1_PREOP_LO 0xC98Cu | ||
| 1512 | #define CP_HQD_ATOMIC1_PREOP_HI 0xC990u | ||
| 1513 | #define CP_HQD_HQ_SCHEDULER0 0xC994u | ||
| 1514 | #define CP_HQD_HQ_SCHEDULER1 0xC998u | ||
| 1515 | |||
| 1516 | #define SH_STATIC_MEM_CONFIG 0x9604u | ||
| 1517 | |||
| 1483 | #define DB_RENDER_CONTROL 0x28000 | 1518 | #define DB_RENDER_CONTROL 0x28000 |
| 1484 | 1519 | ||
| 1485 | #define PA_SC_RASTER_CONFIG 0x28350 | 1520 | #define PA_SC_RASTER_CONFIG 0x28350 |
| @@ -2069,4 +2104,20 @@ · 583 unmodified lines | |||
| 2069 | #define VCE_CMD_IB_AUTO 0x00000005 | 2104 | #define VCE_CMD_IB_AUTO 0x00000005 |
| 2070 | #define VCE_CMD_SEMAPHORE 0x00000006 | 2105 | #define VCE_CMD_SEMAPHORE 0x00000006 |
| 2071 | 2106 | ||
| 2107 | #define ATC_VMID0_PASID_MAPPING 0x339Cu | ||
| 2108 | #define ATC_VMID_PASID_MAPPING_UPDATE_STATUS 0x3398u | ||
| 2109 | #define ATC_VMID_PASID_MAPPING_VALID (1U << 31) | ||
| 2110 | |||
| 2111 | #define ATC_VM_APERTURE0_CNTL 0x3310u | ||
| 2112 | #define ATS_ACCESS_MODE_NEVER 0 | ||
| 2113 | #define ATS_ACCESS_MODE_ALWAYS 1 | ||
| 2114 | |||
| 2115 | #define ATC_VM_APERTURE0_CNTL2 0x3318u | ||
| 2116 | #define ATC_VM_APERTURE0_HIGH_ADDR 0x3308u | ||
| 2117 | #define ATC_VM_APERTURE0_LOW_ADDR 0x3300u | ||
| 2118 | #define ATC_VM_APERTURE1_CNTL 0x3314u | ||
| 2119 | #define ATC_VM_APERTURE1_CNTL2 0x331Cu | ||
| 2120 | #define ATC_VM_APERTURE1_HIGH_ADDR 0x330Cu | ||
| 2121 | #define ATC_VM_APERTURE1_LOW_ADDR 0x3304u | ||
| 2122 | |||
| 2072 | #endif | 2123 | #endif |
| @@ -701,6 +701,10 @@ struct radeon_doorbell { · 700 unmodified lines | |||
| 701 | 701 | ||
| 702 | int radeon_doorbell_get(struct radeon_device *rdev, u32 *page); | 702 | int radeon_doorbell_get(struct radeon_device *rdev, u32 *page); |
| 703 | void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell); | 703 | void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell); |
| 704 | void radeon_doorbell_get_kfd_info(struct radeon_device *rdev, | ||
| 705 | phys_addr_t *aperture_base, | ||
| 706 | size_t *aperture_size, | ||
| 707 | size_t *start_offset); | ||
| 704 | 708 | ||
| 705 | /* | 709 | /* |
| 706 | * IRQS. | 710 | * IRQS. |
| @@ -2393,6 +2397,8 @@ struct radeon_device { · 1686 unmodified lines | |||
| 2393 | struct radeon_atcs atcs; | 2397 | struct radeon_atcs atcs; |
| 2394 | /* srbm instance registers */ | 2398 | /* srbm instance registers */ |
| 2395 | struct mutex srbm_mutex; | 2399 | struct mutex srbm_mutex; |
| 2400 | /* GRBM index mutex. Protects concurrents access to GRBM index */ | ||
| 2401 | struct mutex grbm_idx_mutex; | ||
| 2396 | /* clock, powergating flags */ | 2402 | /* clock, powergating flags */ |
| 2397 | u32 cg_flags; | 2403 | u32 cg_flags; |
| 2398 | u32 pg_flags; | 2404 | u32 pg_flags; |
| @@ -2405,6 +2411,10 @@ struct radeon_device { · 6 unmodified lines | |||
| 2405 | u64 vram_pin_size; | 2411 | u64 vram_pin_size; |
| 2406 | u64 gart_pin_size; | 2412 | u64 gart_pin_size; |
| 2407 | 2413 | ||
| 2414 | /* amdkfd interface */ | ||
| 2415 | struct kfd_dev *kfd; | ||
| 2416 | struct radeon_sa_manager kfd_bo; | ||
| 2417 | |||
| 2408 | struct mutex mn_lock; | 2418 | struct mutex mn_lock; |
| 2409 | DECLARE_HASHTABLE(mn_hash, 7); | 2419 | DECLARE_HASHTABLE(mn_hash, 7); |
| 2410 | }; | 2420 | }; |
| @@ -377,6 +377,37 @@ void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell) · 376 unmodified lines | |||
| 377 | __clear_bit(doorbell, rdev->doorbell.used); | 377 | __clear_bit(doorbell, rdev->doorbell.used); |
| 378 | } | 378 | } |
| 379 | 379 | ||
| 380 | /** | ||
| 381 | * radeon_doorbell_get_kfd_info - Report doorbell configuration required to | ||
| 382 | * setup KFD | ||
| 383 | * | ||
| 384 | * @rdev: radeon_device pointer | ||
| 385 | * @aperture_base: output returning doorbell aperture base physical address | ||
| 386 | * @aperture_size: output returning doorbell aperture size in bytes | ||
| 387 | * @start_offset: output returning # of doorbell bytes reserved for radeon. | ||
| 388 | * | ||
| 389 | * Radeon and the KFD share the doorbell aperture. Radeon sets it up, | ||
| 390 | * takes doorbells required for its own rings and reports the setup to KFD. | ||
| 391 | * Radeon reserved doorbells are at the start of the doorbell aperture. | ||
| 392 | */ | ||
| 393 | void radeon_doorbell_get_kfd_info(struct radeon_device *rdev, | ||
| 394 | phys_addr_t *aperture_base, | ||
| 395 | size_t *aperture_size, | ||
| 396 | size_t *start_offset) | ||
| 397 | { | ||
| 398 | /* The first num_doorbells are used by radeon. | ||
| 399 | * KFD takes whatever's left in the aperture. */ | ||
| 400 | if (rdev->doorbell.size > rdev->doorbell.num_doorbells * sizeof(u32)) { | ||
| 401 | *aperture_base = rdev->doorbell.base; | ||
| 402 | *aperture_size = rdev->doorbell.size; | ||
| 403 | *start_offset = rdev->doorbell.num_doorbells * sizeof(u32); | ||
| 404 | } else { | ||
| 405 | *aperture_base = 0; | ||
| 406 | *aperture_size = 0; | ||
| 407 | *start_offset = 0; | ||
| 408 | } | ||
| 409 | } | ||
| 410 | |||
| 380 | /* | 411 | /* |
| 381 | * radeon_wb_*() | 412 | * radeon_wb_*() |
| 382 | * Writeback is the the method by which the the GPU updates special pages | 413 | * Writeback is the the method by which the the GPU updates special pages |
| @@ -1272,6 +1303,7 @@ int radeon_device_init(struct radeon_device *rdev, · 889 unmodified lines | |||
| 1272 | mutex_init(&rdev->pm.mutex); | 1303 | mutex_init(&rdev->pm.mutex); |
| 1273 | mutex_init(&rdev->gpu_clock_mutex); | 1304 | mutex_init(&rdev->gpu_clock_mutex); |
| 1274 | mutex_init(&rdev->srbm_mutex); | 1305 | mutex_init(&rdev->srbm_mutex); |
| 1306 | mutex_init(&rdev->grbm_idx_mutex); | ||
| 1275 | init_rwsem(&rdev->pm.mclk_lock); | 1307 | init_rwsem(&rdev->pm.mclk_lock); |
| 1276 | init_rwsem(&rdev->exclusive_lock); | 1308 | init_rwsem(&rdev->exclusive_lock); |
| 1277 | init_waitqueue_head(&rdev->irq.vblank_queue); | 1309 | init_waitqueue_head(&rdev->irq.vblank_queue); |
| @@ -41,6 +41,8 @@ · 40 unmodified lines | |||
| 41 | #include <drm/drm_gem.h> | 41 | #include <drm/drm_gem.h> |
| 42 | 42 | ||
| 43 | #include "drm_crtc_helper.h" | 43 | #include "drm_crtc_helper.h" |
| 44 | #include "radeon_kfd.h" | ||
| 45 | |||
| 44 | /* | 46 | /* |
| 45 | * KMS wrapper. | 47 | * KMS wrapper. |
| 46 | * - 2.0.0 - initial interface | 48 | * - 2.0.0 - initial interface |
| @@ -654,12 +656,15 @@ static int __init radeon_init(void) · 607 unmodified lines | |||
| 654 | #endif | 656 | #endif |
| 655 | } | 657 | } |
| 656 | 658 | ||
| 659 | radeon_kfd_init(); | ||
| 660 | |||
| 657 | /* let modprobe override vga console setting */ | 661 | /* let modprobe override vga console setting */ |
| 658 | return drm_pci_init(driver, pdriver); | 662 | return drm_pci_init(driver, pdriver); |
| 659 | } | 663 | } |
| 660 | 664 | ||
| 661 | static void __exit radeon_exit(void) | 665 | static void __exit radeon_exit(void) |
| 662 | { | 666 | { |
| 667 | radeon_kfd_fini(); | ||
| 663 | drm_pci_exit(driver, pdriver); | 668 | drm_pci_exit(driver, pdriver); |
| 664 | radeon_unregister_atpx_handler(); | 669 | radeon_unregister_atpx_handler(); |
| 665 | } | 670 | } |
| @@ -0,0 +1,563 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | #include <linux/module.h> | ||
| 24 | #include <linux/fdtable.h> | ||
| 25 | #include <linux/uaccess.h> | ||
| 26 | #include <drm/drmP.h> | ||
| 27 | #include "radeon.h" | ||
| 28 | #include "cikd.h" | ||
| 29 | #include "cik_reg.h" | ||
| 30 | #include "radeon_kfd.h" | ||
| 31 | |||
| 32 | #define CIK_PIPE_PER_MEC (4) | ||
| 33 | |||
| 34 | struct kgd_mem { | ||
| 35 | struct radeon_sa_bo *sa_bo; | ||
| 36 | uint64_t gpu_addr; | ||
| 37 | void *ptr; | ||
| 38 | }; | ||
| 39 | |||
| 40 | static int init_sa_manager(struct kgd_dev *kgd, unsigned int size); | ||
| 41 | static void fini_sa_manager(struct kgd_dev *kgd); | ||
| 42 | |||
| 43 | static int allocate_mem(struct kgd_dev *kgd, size_t size, size_t alignment, | ||
| 44 | enum kgd_memory_pool pool, struct kgd_mem **mem); | ||
| 45 | |||
| 46 | static void free_mem(struct kgd_dev *kgd, struct kgd_mem *mem); | ||
| 47 | |||
| 48 | static uint64_t get_vmem_size(struct kgd_dev *kgd); | ||
| 49 | static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd); | ||
| 50 | |||
| 51 | static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd); | ||
| 52 | |||
| 53 | /* | ||
| 54 | * Register access functions | ||
| 55 | */ | ||
| 56 | |||
| 57 | static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, | ||
| 58 | uint32_t sh_mem_config, uint32_t sh_mem_ape1_base, | ||
| 59 | uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases); | ||
| 60 | |||
| 61 | static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, | ||
| 62 | unsigned int vmid); | ||
| 63 | |||
| 64 | static int kgd_init_memory(struct kgd_dev *kgd); | ||
| 65 | |||
| 66 | static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id, | ||
| 67 | uint32_t hpd_size, uint64_t hpd_gpu_addr); | ||
| 68 | |||
| 69 | static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, | ||
| 70 | uint32_t queue_id, uint32_t __user *wptr); | ||
| 71 | |||
| 72 | static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address, | ||
| 73 | uint32_t pipe_id, uint32_t queue_id); | ||
| 74 | |||
| 75 | static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, | ||
| 76 | unsigned int timeout, uint32_t pipe_id, | ||
| 77 | uint32_t queue_id); | ||
| 78 | |||
| 79 | static const struct kfd2kgd_calls kfd2kgd = { | ||
| 80 | .init_sa_manager = init_sa_manager, | ||
| 81 | .fini_sa_manager = fini_sa_manager, | ||
| 82 | .allocate_mem = allocate_mem, | ||
| 83 | .free_mem = free_mem, | ||
| 84 | .get_vmem_size = get_vmem_size, | ||
| 85 | .get_gpu_clock_counter = get_gpu_clock_counter, | ||
| 86 | .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz, | ||
| 87 | .program_sh_mem_settings = kgd_program_sh_mem_settings, | ||
| 88 | .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, | ||
| 89 | .init_memory = kgd_init_memory, | ||
| 90 | .init_pipeline = kgd_init_pipeline, | ||
| 91 | .hqd_load = kgd_hqd_load, | ||
| 92 | .hqd_is_occupies = kgd_hqd_is_occupies, | ||
| 93 | .hqd_destroy = kgd_hqd_destroy, | ||
| 94 | }; | ||
| 95 | |||
| 96 | static const struct kgd2kfd_calls *kgd2kfd; | ||
| 97 | |||
| 98 | bool radeon_kfd_init(void) | ||
| 99 | { | ||
| 100 | bool (*kgd2kfd_init_p)(unsigned, const struct kfd2kgd_calls*, | ||
| 101 | const struct kgd2kfd_calls**); | ||
| 102 | |||
| 103 | kgd2kfd_init_p = symbol_request(kgd2kfd_init); | ||
| 104 | |||
| 105 | if (kgd2kfd_init_p == NULL) | ||
| 106 | return false; | ||
| 107 | |||
| 108 | if (!kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kfd2kgd, &kgd2kfd)) { | ||
| 109 | symbol_put(kgd2kfd_init); | ||
| 110 | kgd2kfd = NULL; | ||
| 111 | |||
| 112 | return false; | ||
| 113 | } | ||
| 114 | |||
| 115 | return true; | ||
| 116 | } | ||
| 117 | |||
| 118 | void radeon_kfd_fini(void) | ||
| 119 | { | ||
| 120 | if (kgd2kfd) { | ||
| 121 | kgd2kfd->exit(); | ||
| 122 | symbol_put(kgd2kfd_init); | ||
| 123 | } | ||
| 124 | } | ||
| 125 | |||
| 126 | void radeon_kfd_device_probe(struct radeon_device *rdev) | ||
| 127 | { | ||
| 128 | if (kgd2kfd) | ||
| 129 | rdev->kfd = kgd2kfd->probe((struct kgd_dev *)rdev, rdev->pdev); | ||
| 130 | } | ||
| 131 | |||
| 132 | void radeon_kfd_device_init(struct radeon_device *rdev) | ||
| 133 | { | ||
| 134 | if (rdev->kfd) { | ||
| 135 | struct kgd2kfd_shared_resources gpu_resources = { | ||
| 136 | .compute_vmid_bitmap = 0xFF00, | ||
| 137 | |||
| 138 | .first_compute_pipe = 1, | ||
| 139 | .compute_pipe_count = 8 - 1, | ||
| 140 | }; | ||
| 141 | |||
| 142 | radeon_doorbell_get_kfd_info(rdev, | ||
| 143 | &gpu_resources.doorbell_physical_address, | ||
| 144 | &gpu_resources.doorbell_aperture_size, | ||
| 145 | &gpu_resources.doorbell_start_offset); | ||
| 146 | |||
| 147 | kgd2kfd->device_init(rdev->kfd, &gpu_resources); | ||
| 148 | } | ||
| 149 | } | ||
| 150 | |||
| 151 | void radeon_kfd_device_fini(struct radeon_device *rdev) | ||
| 152 | { | ||
| 153 | if (rdev->kfd) { | ||
| 154 | kgd2kfd->device_exit(rdev->kfd); | ||
| 155 | rdev->kfd = NULL; | ||
| 156 | } | ||
| 157 | } | ||
| 158 | |||
| 159 | void radeon_kfd_interrupt(struct radeon_device *rdev, const void *ih_ring_entry) | ||
| 160 | { | ||
| 161 | if (rdev->kfd) | ||
| 162 | kgd2kfd->interrupt(rdev->kfd, ih_ring_entry); | ||
| 163 | } | ||
| 164 | |||
| 165 | void radeon_kfd_suspend(struct radeon_device *rdev) | ||
| 166 | { | ||
| 167 | if (rdev->kfd) | ||
| 168 | kgd2kfd->suspend(rdev->kfd); | ||
| 169 | } | ||
| 170 | |||
| 171 | int radeon_kfd_resume(struct radeon_device *rdev) | ||
| 172 | { | ||
| 173 | int r = 0; | ||
| 174 | |||
| 175 | if (rdev->kfd) | ||
| 176 | r = kgd2kfd->resume(rdev->kfd); | ||
| 177 | |||
| 178 | return r; | ||
| 179 | } | ||
| 180 | |||
| 181 | static u32 pool_to_domain(enum kgd_memory_pool p) | ||
| 182 | { | ||
| 183 | switch (p) { | ||
| 184 | case KGD_POOL_FRAMEBUFFER: return RADEON_GEM_DOMAIN_VRAM; | ||
| 185 | default: return RADEON_GEM_DOMAIN_GTT; | ||
| 186 | } | ||
| 187 | } | ||
| 188 | |||
| 189 | static int init_sa_manager(struct kgd_dev *kgd, unsigned int size) | ||
| 190 | { | ||
| 191 | struct radeon_device *rdev = (struct radeon_device *)kgd; | ||
| 192 | int r; | ||
| 193 | |||
| 194 | BUG_ON(kgd == NULL); | ||
| 195 | |||
| 196 | r = radeon_sa_bo_manager_init(rdev, &rdev->kfd_bo, | ||
| 197 | size, | ||
| 198 | RADEON_GPU_PAGE_SIZE, | ||
| 199 | RADEON_GEM_DOMAIN_GTT, | ||
| 200 | RADEON_GEM_GTT_WC); | ||
| 201 | |||
| 202 | if (r) | ||
| 203 | return r; | ||
| 204 | |||
| 205 | r = radeon_sa_bo_manager_start(rdev, &rdev->kfd_bo); | ||
| 206 | if (r) | ||
| 207 | radeon_sa_bo_manager_fini(rdev, &rdev->kfd_bo); | ||
| 208 | |||
| 209 | return r; | ||
| 210 | } | ||
| 211 | |||
| 212 | static void fini_sa_manager(struct kgd_dev *kgd) | ||
| 213 | { | ||
| 214 | struct radeon_device *rdev = (struct radeon_device *)kgd; | ||
| 215 | |||
| 216 | BUG_ON(kgd == NULL); | ||
| 217 | |||
| 218 | radeon_sa_bo_manager_suspend(rdev, &rdev->kfd_bo); | ||
| 219 | radeon_sa_bo_manager_fini(rdev, &rdev->kfd_bo); | ||
| 220 | } | ||
| 221 | |||
| 222 | static int allocate_mem(struct kgd_dev *kgd, size_t size, size_t alignment, | ||
| 223 | enum kgd_memory_pool pool, struct kgd_mem **mem) | ||
| 224 | { | ||
| 225 | struct radeon_device *rdev = (struct radeon_device *)kgd; | ||
| 226 | u32 domain; | ||
| 227 | int r; | ||
| 228 | |||
| 229 | BUG_ON(kgd == NULL); | ||
| 230 | |||
| 231 | domain = pool_to_domain(pool); | ||
| 232 | if (domain != RADEON_GEM_DOMAIN_GTT) { | ||
| 233 | dev_err(rdev->dev, | ||
| 234 | "Only allowed to allocate gart memory for kfd\n"); | ||
| 235 | return -EINVAL; | ||
| 236 | } | ||
| 237 | |||
| 238 | *mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL); | ||
| 239 | if ((*mem) == NULL) | ||
| 240 | return -ENOMEM; | ||
| 241 | |||
| 242 | r = radeon_sa_bo_new(rdev, &rdev->kfd_bo, &(*mem)->sa_bo, size, | ||
| 243 | alignment); | ||
| 244 | if (r) { | ||
| 245 | dev_err(rdev->dev, "failed to get memory for kfd (%d)\n", r); | ||
| 246 | return r; | ||
| 247 | } | ||
| 248 | |||
| 249 | (*mem)->ptr = radeon_sa_bo_cpu_addr((*mem)->sa_bo); | ||
| 250 | (*mem)->gpu_addr = radeon_sa_bo_gpu_addr((*mem)->sa_bo); | ||
| 251 | |||
| 252 | return 0; | ||
| 253 | } | ||
| 254 | |||
| 255 | static void free_mem(struct kgd_dev *kgd, struct kgd_mem *mem) | ||
| 256 | { | ||
| 257 | struct radeon_device *rdev = (struct radeon_device *)kgd; | ||
| 258 | |||
| 259 | BUG_ON(kgd == NULL); | ||
| 260 | |||
| 261 | radeon_sa_bo_free(rdev, &mem->sa_bo, NULL); | ||
| 262 | kfree(mem); | ||
| 263 | } | ||
| 264 | |||
| 265 | static uint64_t get_vmem_size(struct kgd_dev *kgd) | ||
| 266 | { | ||
| 267 | struct radeon_device *rdev = (struct radeon_device *)kgd; | ||
| 268 | |||
| 269 | BUG_ON(kgd == NULL); | ||
| 270 | |||
| 271 | return rdev->mc.real_vram_size; | ||
| 272 | } | ||
| 273 | |||
| 274 | static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd) | ||
| 275 | { | ||
| 276 | struct radeon_device *rdev = (struct radeon_device *)kgd; | ||
| 277 | |||
| 278 | return rdev->asic->get_gpu_clock_counter(rdev); | ||
| 279 | } | ||
| 280 | |||
| 281 | static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd) | ||
| 282 | { | ||
| 283 | struct radeon_device *rdev = (struct radeon_device *)kgd; | ||
| 284 | |||
| 285 | /* The sclk is in quantas of 10kHz */ | ||
| 286 | return rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100; | ||
| 287 | } | ||
| 288 | |||
| 289 | static inline struct radeon_device *get_radeon_device(struct kgd_dev *kgd) | ||
| 290 | { | ||
| 291 | return (struct radeon_device *)kgd; | ||
| 292 | } | ||
| 293 | |||
| 294 | static void write_register(struct kgd_dev *kgd, uint32_t offset, uint32_t value) | ||
| 295 | { | ||
| 296 | struct radeon_device *rdev = get_radeon_device(kgd); | ||
| 297 | |||
| 298 | writel(value, (void __iomem *)(rdev->rmmio + offset)); | ||
| 299 | } | ||
| 300 | |||
| 301 | static uint32_t read_register(struct kgd_dev *kgd, uint32_t offset) | ||
| 302 | { | ||
| 303 | struct radeon_device *rdev = get_radeon_device(kgd); | ||
| 304 | |||
| 305 | return readl((void __iomem *)(rdev->rmmio + offset)); | ||
| 306 | } | ||
| 307 | |||
| 308 | static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe, | ||
| 309 | uint32_t queue, uint32_t vmid) | ||
| 310 | { | ||
| 311 | struct radeon_device *rdev = get_radeon_device(kgd); | ||
| 312 | uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue); | ||
| 313 | |||
| 314 | mutex_lock(&rdev->srbm_mutex); | ||
| 315 | write_register(kgd, SRBM_GFX_CNTL, value); | ||
| 316 | } | ||
| 317 | |||
| 318 | static void unlock_srbm(struct kgd_dev *kgd) | ||
| 319 | { | ||
| 320 | struct radeon_device *rdev = get_radeon_device(kgd); | ||
| 321 | |||
| 322 | write_register(kgd, SRBM_GFX_CNTL, 0); | ||
| 323 | mutex_unlock(&rdev->srbm_mutex); | ||
| 324 | } | ||
| 325 | |||
| 326 | static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id, | ||
| 327 | uint32_t queue_id) | ||
| 328 | { | ||
| 329 | uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1; | ||
| 330 | uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC); | ||
| 331 | |||
| 332 | lock_srbm(kgd, mec, pipe, queue_id, 0); | ||
| 333 | } | ||
| 334 | |||
| 335 | static void release_queue(struct kgd_dev *kgd) | ||
| 336 | { | ||
| 337 | unlock_srbm(kgd); | ||
| 338 | } | ||
| 339 | |||
| 340 | static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, | ||
| 341 | uint32_t sh_mem_config, | ||
| 342 | uint32_t sh_mem_ape1_base, | ||
| 343 | uint32_t sh_mem_ape1_limit, | ||
| 344 | uint32_t sh_mem_bases) | ||
| 345 | { | ||
| 346 | lock_srbm(kgd, 0, 0, 0, vmid); | ||
| 347 | |||
| 348 | write_register(kgd, SH_MEM_CONFIG, sh_mem_config); | ||
| 349 | write_register(kgd, SH_MEM_APE1_BASE, sh_mem_ape1_base); | ||
| 350 | write_register(kgd, SH_MEM_APE1_LIMIT, sh_mem_ape1_limit); | ||
| 351 | write_register(kgd, SH_MEM_BASES, sh_mem_bases); | ||
| 352 | |||
| 353 | unlock_srbm(kgd); | ||
| 354 | } | ||
| 355 | |||
| 356 | static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, | ||
| 357 | unsigned int vmid) | ||
| 358 | { | ||
| 359 | /* | ||
| 360 | * We have to assume that there is no outstanding mapping. | ||
| 361 | * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 | ||
| 362 | * because a mapping is in progress or because a mapping finished and | ||
| 363 | * the SW cleared it. | ||
| 364 | * So the protocol is to always wait & clear. | ||
| 365 | */ | ||
| 366 | uint32_t pasid_mapping = (pasid == 0) ? 0 : | ||
| 367 | (uint32_t)pasid | ATC_VMID_PASID_MAPPING_VALID; | ||
| 368 | |||
| 369 | write_register(kgd, ATC_VMID0_PASID_MAPPING + vmid*sizeof(uint32_t), | ||
| 370 | pasid_mapping); | ||
| 371 | |||
| 372 | while (!(read_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS) & | ||
| 373 | (1U << vmid))) | ||
| 374 | cpu_relax(); | ||
| 375 | write_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid); | ||
| 376 | |||
| 377 | return 0; | ||
| 378 | } | ||
| 379 | |||
| 380 | static int kgd_init_memory(struct kgd_dev *kgd) | ||
| 381 | { | ||
| 382 | /* | ||
| 383 | * Configure apertures: | ||
| 384 | * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) | ||
| 385 | * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) | ||
| 386 | * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) | ||
| 387 | */ | ||
| 388 | int i; | ||
| 389 | uint32_t sh_mem_bases = PRIVATE_BASE(0x6000) | SHARED_BASE(0x6000); | ||
| 390 | |||
| 391 | for (i = 8; i < 16; i++) { | ||
| 392 | uint32_t sh_mem_config; | ||
| 393 | |||
| 394 | lock_srbm(kgd, 0, 0, 0, i); | ||
| 395 | |||
| 396 | sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED); | ||
| 397 | sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED); | ||
| 398 | |||
| 399 | write_register(kgd, SH_MEM_CONFIG, sh_mem_config); | ||
| 400 | |||
| 401 | write_register(kgd, SH_MEM_BASES, sh_mem_bases); | ||
| 402 | |||
| 403 | /* Scratch aperture is not supported for now. */ | ||
| 404 | write_register(kgd, SH_STATIC_MEM_CONFIG, 0); | ||
| 405 | |||
| 406 | /* APE1 disabled for now. */ | ||
| 407 | write_register(kgd, SH_MEM_APE1_BASE, 1); | ||
| 408 | write_register(kgd, SH_MEM_APE1_LIMIT, 0); | ||
| 409 | |||
| 410 | unlock_srbm(kgd); | ||
| 411 | } | ||
| 412 | |||
| 413 | return 0; | ||
| 414 | } | ||
| 415 | |||
| 416 | static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id, | ||
| 417 | uint32_t hpd_size, uint64_t hpd_gpu_addr) | ||
| 418 | { | ||
| 419 | uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1; | ||
| 420 | uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC); | ||
| 421 | |||
| 422 | lock_srbm(kgd, mec, pipe, 0, 0); | ||
| 423 | write_register(kgd, CP_HPD_EOP_BASE_ADDR, | ||
| 424 | lower_32_bits(hpd_gpu_addr >> 8)); | ||
| 425 | write_register(kgd, CP_HPD_EOP_BASE_ADDR_HI, | ||
| 426 | upper_32_bits(hpd_gpu_addr >> 8)); | ||
| 427 | write_register(kgd, CP_HPD_EOP_VMID, 0); | ||
| 428 | write_register(kgd, CP_HPD_EOP_CONTROL, hpd_size); | ||
| 429 | unlock_srbm(kgd); | ||
| 430 | |||
| 431 | return 0; | ||
| 432 | } | ||
| 433 | |||
| 434 | static inline struct cik_mqd *get_mqd(void *mqd) | ||
| 435 | { | ||
| 436 | return (struct cik_mqd *)mqd; | ||
| 437 | } | ||
| 438 | |||
| 439 | static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, | ||
| 440 | uint32_t queue_id, uint32_t __user *wptr) | ||
| 441 | { | ||
| 442 | uint32_t wptr_shadow, is_wptr_shadow_valid; | ||
| 443 | struct cik_mqd *m; | ||
| 444 | |||
| 445 | m = get_mqd(mqd); | ||
| 446 | |||
| 447 | is_wptr_shadow_valid = !get_user(wptr_shadow, wptr); | ||
| 448 | |||
| 449 | acquire_queue(kgd, pipe_id, queue_id); | ||
| 450 | write_register(kgd, CP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo); | ||
| 451 | write_register(kgd, CP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi); | ||
| 452 | write_register(kgd, CP_MQD_CONTROL, m->cp_mqd_control); | ||
| 453 | |||
| 454 | write_register(kgd, CP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo); | ||
| 455 | write_register(kgd, CP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi); | ||
| 456 | write_register(kgd, CP_HQD_PQ_CONTROL, m->cp_hqd_pq_control); | ||
| 457 | |||
| 458 | write_register(kgd, CP_HQD_IB_CONTROL, m->cp_hqd_ib_control); | ||
| 459 | write_register(kgd, CP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo); | ||
| 460 | write_register(kgd, CP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi); | ||
| 461 | |||
| 462 | write_register(kgd, CP_HQD_IB_RPTR, m->cp_hqd_ib_rptr); | ||
| 463 | |||
| 464 | write_register(kgd, CP_HQD_PERSISTENT_STATE, | ||
| 465 | m->cp_hqd_persistent_state); | ||
| 466 | write_register(kgd, CP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd); | ||
| 467 | write_register(kgd, CP_HQD_MSG_TYPE, m->cp_hqd_msg_type); | ||
| 468 | |||
| 469 | write_register(kgd, CP_HQD_ATOMIC0_PREOP_LO, | ||
| 470 | m->cp_hqd_atomic0_preop_lo); | ||
| 471 | |||
| 472 | write_register(kgd, CP_HQD_ATOMIC0_PREOP_HI, | ||
| 473 | m->cp_hqd_atomic0_preop_hi); | ||
| 474 | |||
| 475 | write_register(kgd, CP_HQD_ATOMIC1_PREOP_LO, | ||
| 476 | m->cp_hqd_atomic1_preop_lo); | ||
| 477 | |||
| 478 | write_register(kgd, CP_HQD_ATOMIC1_PREOP_HI, | ||
| 479 | m->cp_hqd_atomic1_preop_hi); | ||
| 480 | |||
| 481 | write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR, | ||
| 482 | m->cp_hqd_pq_rptr_report_addr_lo); | ||
| 483 | |||
| 484 | write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR_HI, | ||
| 485 | m->cp_hqd_pq_rptr_report_addr_hi); | ||
| 486 | |||
| 487 | write_register(kgd, CP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr); | ||
| 488 | |||
| 489 | write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR, | ||
| 490 | m->cp_hqd_pq_wptr_poll_addr_lo); | ||
| 491 | |||
| 492 | write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR_HI, | ||
| 493 | m->cp_hqd_pq_wptr_poll_addr_hi); | ||
| 494 | |||
| 495 | write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, | ||
| 496 | m->cp_hqd_pq_doorbell_control); | ||
| 497 | |||
| 498 | write_register(kgd, CP_HQD_VMID, m->cp_hqd_vmid); | ||
| 499 | |||
| 500 | write_register(kgd, CP_HQD_QUANTUM, m->cp_hqd_quantum); | ||
| 501 | |||
| 502 | write_register(kgd, CP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority); | ||
| 503 | write_register(kgd, CP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority); | ||
| 504 | |||
| 505 | write_register(kgd, CP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr); | ||
| 506 | |||
| 507 | if (is_wptr_shadow_valid) | ||
| 508 | write_register(kgd, CP_HQD_PQ_WPTR, wptr_shadow); | ||
| 509 | |||
| 510 | write_register(kgd, CP_HQD_ACTIVE, m->cp_hqd_active); | ||
| 511 | release_queue(kgd); | ||
| 512 | |||
| 513 | return 0; | ||
| 514 | } | ||
| 515 | |||
| 516 | static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address, | ||
| 517 | uint32_t pipe_id, uint32_t queue_id) | ||
| 518 | { | ||
| 519 | uint32_t act; | ||
| 520 | bool retval = false; | ||
| 521 | uint32_t low, high; | ||
| 522 | |||
| 523 | acquire_queue(kgd, pipe_id, queue_id); | ||
| 524 | act = read_register(kgd, CP_HQD_ACTIVE); | ||
| 525 | if (act) { | ||
| 526 | low = lower_32_bits(queue_address >> 8); | ||
| 527 | high = upper_32_bits(queue_address >> 8); | ||
| 528 | |||
| 529 | if (low == read_register(kgd, CP_HQD_PQ_BASE) && | ||
| 530 | high == read_register(kgd, CP_HQD_PQ_BASE_HI)) | ||
| 531 | retval = true; | ||
| 532 | } | ||
| 533 | release_queue(kgd); | ||
| 534 | return retval; | ||
| 535 | } | ||
| 536 | |||
| 537 | static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, | ||
| 538 | unsigned int timeout, uint32_t pipe_id, | ||
| 539 | uint32_t queue_id) | ||
| 540 | { | ||
| 541 | uint32_t temp; | ||
| 542 | |||
| 543 | acquire_queue(kgd, pipe_id, queue_id); | ||
| 544 | write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, 0); | ||
| 545 | |||
| 546 | write_register(kgd, CP_HQD_DEQUEUE_REQUEST, reset_type); | ||
| 547 | |||
| 548 | while (true) { | ||
| 549 | temp = read_register(kgd, CP_HQD_ACTIVE); | ||
| 550 | if (temp & 0x1) | ||
| 551 | break; | ||
| 552 | if (timeout == 0) { | ||
| 553 | pr_err("kfd: cp queue preemption time out (%dms)\n", | ||
| 554 | temp); | ||
| 555 | return -ETIME; | ||
| 556 | } | ||
| 557 | msleep(20); | ||
| 558 | timeout -= 20; | ||
| 559 | } | ||
| 560 | |||
| 561 | release_queue(kgd); | ||
| 562 | return 0; | ||
| 563 | } | ||
| @@ -0,0 +1,47 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | /* | ||
| 24 | * radeon_kfd.h defines the private interface between the | ||
| 25 | * AMD kernel graphics drivers and the AMD KFD. | ||
| 26 | */ | ||
| 27 | |||
| 28 | #ifndef RADEON_KFD_H_INCLUDED | ||
| 29 | #define RADEON_KFD_H_INCLUDED | ||
| 30 | |||
| 31 | #include <linux/types.h> | ||
| 32 | #include "../amd/include/kgd_kfd_interface.h" | ||
| 33 | |||
| 34 | struct radeon_device; | ||
| 35 | |||
| 36 | bool radeon_kfd_init(void); | ||
| 37 | void radeon_kfd_fini(void); | ||
| 38 | |||
| 39 | void radeon_kfd_suspend(struct radeon_device *rdev); | ||
| 40 | int radeon_kfd_resume(struct radeon_device *rdev); | ||
| 41 | void radeon_kfd_interrupt(struct radeon_device *rdev, | ||
| 42 | const void *ih_ring_entry); | ||
| 43 | void radeon_kfd_device_probe(struct radeon_device *rdev); | ||
| 44 | void radeon_kfd_device_init(struct radeon_device *rdev); | ||
| 45 | void radeon_kfd_device_fini(struct radeon_device *rdev); | ||
| 46 | |||
| 47 | #endif /* RADEON_KFD_H_INCLUDED */ | ||
| @@ -34,6 +34,8 @@ · 33 unmodified lines | |||
| 34 | #include <linux/slab.h> | 34 | #include <linux/slab.h> |
| 35 | #include <linux/pm_runtime.h> | 35 | #include <linux/pm_runtime.h> |
| 36 | 36 | ||
| 37 | #include "radeon_kfd.h" | ||
| 38 | |||
| 37 | #if defined(CONFIG_VGA_SWITCHEROO) | 39 | #if defined(CONFIG_VGA_SWITCHEROO) |
| 38 | bool radeon_has_atpx(void); | 40 | bool radeon_has_atpx(void); |
| 39 | #else | 41 | #else |
| @@ -63,6 +65,8 @@ int radeon_driver_unload_kms(struct drm_device *dev) · 23 unmodified lines | |||
| 63 | 65 | ||
| 64 | pm_runtime_get_sync(dev->dev); | 66 | pm_runtime_get_sync(dev->dev); |
| 65 | 67 | ||
| 68 | radeon_kfd_device_fini(rdev); | ||
| 69 | |||
| 66 | radeon_acpi_fini(rdev); | 70 | radeon_acpi_fini(rdev); |
| 67 | 71 | ||
| 68 | radeon_modeset_fini(rdev); | 72 | radeon_modeset_fini(rdev); |
| @@ -142,6 +146,9 @@ int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags) · 73 unmodified lines | |||
| 142 | "Error during ACPI methods call\n"); | 146 | "Error during ACPI methods call\n"); |
| 143 | } | 147 | } |
| 144 | 148 | ||
| 149 | radeon_kfd_device_probe(rdev); | ||
| 150 | radeon_kfd_device_init(rdev); | ||
| 151 | |||
| 145 | if (radeon_is_px(dev)) { | 152 | if (radeon_is_px(dev)) { |
| 146 | pm_runtime_use_autosuspend(dev->dev); | 153 | pm_runtime_use_autosuspend(dev->dev); |
| 147 | pm_runtime_set_autosuspend_delay(dev->dev, 5000); | 154 | pm_runtime_set_autosuspend_delay(dev->dev, 5000); |
| @@ -92,13 +92,6 @@ static spinlock_t state_lock; · 91 unmodified lines | |||
| 92 | 92 | ||
| 93 | static struct workqueue_struct *iommu_wq; | 93 | static struct workqueue_struct *iommu_wq; |
| 94 | 94 | ||
| 95 | /* | ||
| 96 | * Empty page table - Used between | ||
| 97 | * mmu_notifier_invalidate_range_start and | ||
| 98 | * mmu_notifier_invalidate_range_end | ||
| 99 | */ | ||
| 100 | static u64 *empty_page_table; | ||
| 101 | |||
| 102 | static void free_pasid_states(struct device_state *dev_state); | 95 | static void free_pasid_states(struct device_state *dev_state); |
| 103 | 96 | ||
| 104 | static u16 device_id(struct pci_dev *pdev) | 97 | static u16 device_id(struct pci_dev *pdev) |
| @@ -279,10 +272,8 @@ static void free_pasid_state(struct pasid_state *pasid_state) · 174 unmodified lines | |||
| 279 | 272 | ||
| 280 | static void put_pasid_state(struct pasid_state *pasid_state) | 273 | static void put_pasid_state(struct pasid_state *pasid_state) |
| 281 | { | 274 | { |
| 282 | if (atomic_dec_and_test(&pasid_state->count)) { | 275 | if (atomic_dec_and_test(&pasid_state->count)) |
| 283 | put_device_state(pasid_state->device_state); | ||
| 284 | wake_up(&pasid_state->wq); | 276 | wake_up(&pasid_state->wq); |
| 285 | } | ||
| 286 | } | 277 | } |
| 287 | 278 | ||
| 288 | static void put_pasid_state_wait(struct pasid_state *pasid_state) | 279 | static void put_pasid_state_wait(struct pasid_state *pasid_state) |
| @@ -291,9 +282,7 @@ static void put_pasid_state_wait(struct pasid_state *pasid_state) · 2 unmodified lines | |||
| 291 | 282 | ||
| 292 | prepare_to_wait(&pasid_state->wq, &wait, TASK_UNINTERRUPTIBLE); | 283 | prepare_to_wait(&pasid_state->wq, &wait, TASK_UNINTERRUPTIBLE); |
| 293 | 284 | ||
| 294 | if (atomic_dec_and_test(&pasid_state->count)) | 285 | if (!atomic_dec_and_test(&pasid_state->count)) |
| 295 | put_device_state(pasid_state->device_state); | ||
| 296 | else | ||
| 297 | schedule(); | 286 | schedule(); |
| 298 | 287 | ||
| 299 | finish_wait(&pasid_state->wq, &wait); | 288 | finish_wait(&pasid_state->wq, &wait); |
| @@ -418,46 +407,21 @@ static void mn_invalidate_page(struct mmu_notifier *mn, · 118 unmodified lines | |||
| 418 | __mn_flush_page(mn, address); | 407 | __mn_flush_page(mn, address); |
| 419 | } | 408 | } |
| 420 | 409 | ||
| 421 | static void mn_invalidate_range_start(struct mmu_notifier *mn, | 410 | static void mn_invalidate_range(struct mmu_notifier *mn, |
| 422 | struct mm_struct *mm, | 411 | struct mm_struct *mm, |
| 423 | unsigned long start, unsigned long end) | 412 | unsigned long start, unsigned long end) |
| 424 | { | ||
| 425 | struct pasid_state *pasid_state; | ||
| 426 | struct device_state *dev_state; | ||
| 427 | unsigned long flags; | ||
| 428 | |||
| 429 | pasid_state = mn_to_state(mn); | ||
| 430 | dev_state = pasid_state->device_state; | ||
| 431 | |||
| 432 | spin_lock_irqsave(&pasid_state->lock, flags); | ||
| 433 | if (pasid_state->mmu_notifier_count == 0) { | ||
| 434 | amd_iommu_domain_set_gcr3(dev_state->domain, | ||
| 435 | pasid_state->pasid, | ||
| 436 | __pa(empty_page_table)); | ||
| 437 | } | ||
| 438 | pasid_state->mmu_notifier_count += 1; | ||
| 439 | spin_unlock_irqrestore(&pasid_state->lock, flags); | ||
| 440 | } | ||
| 441 | |||
| 442 | static void mn_invalidate_range_end(struct mmu_notifier *mn, | ||
| 443 | struct mm_struct *mm, | ||
| 444 | unsigned long start, unsigned long end) | ||
| 445 | { | 413 | { |
| 446 | struct pasid_state *pasid_state; | 414 | struct pasid_state *pasid_state; |
| 447 | struct device_state *dev_state; | 415 | struct device_state *dev_state; |
| 448 | unsigned long flags; | ||
| 449 | 416 | ||
| 450 | pasid_state = mn_to_state(mn); | 417 | pasid_state = mn_to_state(mn); |
| 451 | dev_state = pasid_state->device_state; | 418 | dev_state = pasid_state->device_state; |
| 452 | 419 | ||
| 453 | spin_lock_irqsave(&pasid_state->lock, flags); | 420 | if ((start ^ (end - 1)) < PAGE_SIZE) |
| 454 | pasid_state->mmu_notifier_count -= 1; | 421 | amd_iommu_flush_page(dev_state->domain, pasid_state->pasid, |
| 455 | if (pasid_state->mmu_notifier_count == 0) { | 422 | start); |
| 456 | amd_iommu_domain_set_gcr3(dev_state->domain, | 423 | else |
| 457 | pasid_state->pasid, | 424 | amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid); |
| 458 | __pa(pasid_state->mm->pgd)); | ||
| 459 | } | ||
| 460 | spin_unlock_irqrestore(&pasid_state->lock, flags); | ||
| 461 | } | 425 | } |
| 462 | 426 | ||
| 463 | static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm) | 427 | static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm) |
| @@ -482,8 +446,7 @@ static struct mmu_notifier_ops iommu_mn = { · 18 unmodified lines | |||
| 482 | .release = mn_release, | 446 | .release = mn_release, |
| 483 | .clear_flush_young = mn_clear_flush_young, | 447 | .clear_flush_young = mn_clear_flush_young, |
| 484 | .invalidate_page = mn_invalidate_page, | 448 | .invalidate_page = mn_invalidate_page, |
| 485 | .invalidate_range_start = mn_invalidate_range_start, | 449 | .invalidate_range = mn_invalidate_range, |
| 486 | .invalidate_range_end = mn_invalidate_range_end, | ||
| 487 | }; | 450 | }; |
| 488 | 451 | ||
| 489 | static void set_pri_tag_status(struct pasid_state *pasid_state, | 452 | static void set_pri_tag_status(struct pasid_state *pasid_state, |
| @@ -954,18 +917,10 @@ static int __init amd_iommu_v2_init(void) · 464 unmodified lines | |||
| 954 | if (iommu_wq == NULL) | 917 | if (iommu_wq == NULL) |
| 955 | goto out; | 918 | goto out; |
| 956 | 919 | ||
| 957 | ret = -ENOMEM; | ||
| 958 | empty_page_table = (u64 *)get_zeroed_page(GFP_KERNEL); | ||
| 959 | if (empty_page_table == NULL) | ||
| 960 | goto out_destroy_wq; | ||
| 961 | |||
| 962 | amd_iommu_register_ppr_notifier(&ppr_nb); | 920 | amd_iommu_register_ppr_notifier(&ppr_nb); |
| 963 | 921 | ||
| 964 | return 0; | 922 | return 0; |
| 965 | 923 | ||
| 966 | out_destroy_wq: | ||
| 967 | destroy_workqueue(iommu_wq); | ||
| 968 | |||
| 969 | out: | 924 | out: |
| 970 | return ret; | 925 | return ret; |
| 971 | } | 926 | } |
| @@ -999,8 +954,6 @@ static void __exit amd_iommu_v2_exit(void) · 27 unmodified lines | |||
| 999 | } | 954 | } |
| 1000 | 955 | ||
| 1001 | destroy_workqueue(iommu_wq); | 956 | destroy_workqueue(iommu_wq); |
| 1002 | |||
| 1003 | free_page((unsigned long)empty_page_table); | ||
| 1004 | } | 957 | } |
| 1005 | 958 | ||
| 1006 | module_init(amd_iommu_v2_init); | 959 | module_init(amd_iommu_v2_init); |
| @@ -98,11 +98,11 @@ struct mmu_notifier_ops { · 97 unmodified lines | |||
| 98 | /* | 98 | /* |
| 99 | * invalidate_range_start() and invalidate_range_end() must be | 99 | * invalidate_range_start() and invalidate_range_end() must be |
| 100 | * paired and are called only when the mmap_sem and/or the | 100 | * paired and are called only when the mmap_sem and/or the |
| 101 | * locks protecting the reverse maps are held. The subsystem | 101 | * locks protecting the reverse maps are held. If the subsystem |
| 102 | * must guarantee that no additional references are taken to | 102 | * can't guarantee that no additional references are taken to |
| 103 | * the pages in the range established between the call to | 103 | * the pages in the range, it has to implement the |
| 104 | * invalidate_range_start() and the matching call to | 104 | * invalidate_range() notifier to remove any references taken |
| 105 | * invalidate_range_end(). | 105 | * after invalidate_range_start(). |
| 106 | * | 106 | * |
| 107 | * Invalidation of multiple concurrent ranges may be | 107 | * Invalidation of multiple concurrent ranges may be |
| 108 | * optionally permitted by the driver. Either way the | 108 | * optionally permitted by the driver. Either way the |
| @@ -144,6 +144,29 @@ struct mmu_notifier_ops { · 35 unmodified lines | |||
| 144 | void (*invalidate_range_end)(struct mmu_notifier *mn, | 144 | void (*invalidate_range_end)(struct mmu_notifier *mn, |
| 145 | struct mm_struct *mm, | 145 | struct mm_struct *mm, |
| 146 | unsigned long start, unsigned long end); | 146 | unsigned long start, unsigned long end); |
| 147 | |||
| 148 | /* | ||
| 149 | * invalidate_range() is either called between | ||
| 150 | * invalidate_range_start() and invalidate_range_end() when the | ||
| 151 | * VM has to free pages that where unmapped, but before the | ||
| 152 | * pages are actually freed, or outside of _start()/_end() when | ||
| 153 | * a (remote) TLB is necessary. | ||
| 154 | * | ||
| 155 | * If invalidate_range() is used to manage a non-CPU TLB with | ||
| 156 | * shared page-tables, it not necessary to implement the | ||
| 157 | * invalidate_range_start()/end() notifiers, as | ||
| 158 | * invalidate_range() alread catches the points in time when an | ||
| 159 | * external TLB range needs to be flushed. | ||
| 160 | * | ||
| 161 | * The invalidate_range() function is called under the ptl | ||
| 162 | * spin-lock and not allowed to sleep. | ||
| 163 | * | ||
| 164 | * Note that this function might be called with just a sub-range | ||
| 165 | * of what was passed to invalidate_range_start()/end(), if | ||
| 166 | * called between those functions. | ||
| 167 | */ | ||
| 168 | void (*invalidate_range)(struct mmu_notifier *mn, struct mm_struct *mm, | ||
| 169 | unsigned long start, unsigned long end); | ||
| 147 | }; | 170 | }; |
| 148 | 171 | ||
| 149 | /* | 172 | /* |
| @@ -190,6 +213,8 @@ extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm, · 40 unmodified lines | |||
| 190 | unsigned long start, unsigned long end); | 213 | unsigned long start, unsigned long end); |
| 191 | extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm, | 214 | extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm, |
| 192 | unsigned long start, unsigned long end); | 215 | unsigned long start, unsigned long end); |
| 216 | extern void __mmu_notifier_invalidate_range(struct mm_struct *mm, | ||
| 217 | unsigned long start, unsigned long end); | ||
| 193 | 218 | ||
| 194 | static inline void mmu_notifier_release(struct mm_struct *mm) | 219 | static inline void mmu_notifier_release(struct mm_struct *mm) |
| 195 | { | 220 | { |
| @@ -242,6 +267,13 @@ static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 46 unmodified lines | |||
| 242 | __mmu_notifier_invalidate_range_end(mm, start, end); | 267 | __mmu_notifier_invalidate_range_end(mm, start, end); |
| 243 | } | 268 | } |
| 244 | 269 | ||
| 270 | static inline void mmu_notifier_invalidate_range(struct mm_struct *mm, | ||
| 271 | unsigned long start, unsigned long end) | ||
| 272 | { | ||
| 273 | if (mm_has_notifiers(mm)) | ||
| 274 | __mmu_notifier_invalidate_range(mm, start, end); | ||
| 275 | } | ||
| 276 | |||
| 245 | static inline void mmu_notifier_mm_init(struct mm_struct *mm) | 277 | static inline void mmu_notifier_mm_init(struct mm_struct *mm) |
| 246 | { | 278 | { |
| 247 | mm->mmu_notifier_mm = NULL; | 279 | mm->mmu_notifier_mm = NULL; |
| @@ -279,6 +311,44 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) · 31 unmodified lines | |||
| 279 | __young; \ | 311 | __young; \ |
| 280 | }) | 312 | }) |
| 281 | 313 | ||
| 314 | #define ptep_clear_flush_notify(__vma, __address, __ptep) \ | ||
| 315 | ({ \ | ||
| 316 | unsigned long ___addr = __address & PAGE_MASK; \ | ||
| 317 | struct mm_struct *___mm = (__vma)->vm_mm; \ | ||
| 318 | pte_t ___pte; \ | ||
| 319 | \ | ||
| 320 | ___pte = ptep_clear_flush(__vma, __address, __ptep); \ | ||
| 321 | mmu_notifier_invalidate_range(___mm, ___addr, \ | ||
| 322 | ___addr + PAGE_SIZE); \ | ||
| 323 | \ | ||
| 324 | ___pte; \ | ||
| 325 | }) | ||
| 326 | |||
| 327 | #define pmdp_clear_flush_notify(__vma, __haddr, __pmd) \ | ||
| 328 | ({ \ | ||
| 329 | unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \ | ||
| 330 | struct mm_struct *___mm = (__vma)->vm_mm; \ | ||
| 331 | pmd_t ___pmd; \ | ||
| 332 | \ | ||
| 333 | ___pmd = pmdp_clear_flush(__vma, __haddr, __pmd); \ | ||
| 334 | mmu_notifier_invalidate_range(___mm, ___haddr, \ | ||
| 335 | ___haddr + HPAGE_PMD_SIZE); \ | ||
| 336 | \ | ||
| 337 | ___pmd; \ | ||
| 338 | }) | ||
| 339 | |||
| 340 | #define pmdp_get_and_clear_notify(__mm, __haddr, __pmd) \ | ||
| 341 | ({ \ | ||
| 342 | unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \ | ||
| 343 | pmd_t ___pmd; \ | ||
| 344 | \ | ||
| 345 | ___pmd = pmdp_get_and_clear(__mm, __haddr, __pmd); \ | ||
| 346 | mmu_notifier_invalidate_range(__mm, ___haddr, \ | ||
| 347 | ___haddr + HPAGE_PMD_SIZE); \ | ||
| 348 | \ | ||
| 349 | ___pmd; \ | ||
| 350 | }) | ||
| 351 | |||
| 282 | /* | 352 | /* |
| 283 | * set_pte_at_notify() sets the pte _after_ running the notifier. | 353 | * set_pte_at_notify() sets the pte _after_ running the notifier. |
| 284 | * This is safe to start by updating the secondary MMUs, because the primary MMU | 354 | * This is safe to start by updating the secondary MMUs, because the primary MMU |
| @@ -342,6 +412,11 @@ static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 57 unmodified lines | |||
| 342 | { | 412 | { |
| 343 | } | 413 | } |
| 344 | 414 | ||
| 415 | static inline void mmu_notifier_invalidate_range(struct mm_struct *mm, | ||
| 416 | unsigned long start, unsigned long end) | ||
| 417 | { | ||
| 418 | } | ||
| 419 | |||
| 345 | static inline void mmu_notifier_mm_init(struct mm_struct *mm) | 420 | static inline void mmu_notifier_mm_init(struct mm_struct *mm) |
| 346 | { | 421 | { |
| 347 | } | 422 | } |
| @@ -352,6 +427,9 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) · 4 unmodified lines | |||
| 352 | 427 | ||
| 353 | #define ptep_clear_flush_young_notify ptep_clear_flush_young | 428 | #define ptep_clear_flush_young_notify ptep_clear_flush_young |
| 354 | #define pmdp_clear_flush_young_notify pmdp_clear_flush_young | 429 | #define pmdp_clear_flush_young_notify pmdp_clear_flush_young |
| 430 | #define ptep_clear_flush_notify ptep_clear_flush | ||
| 431 | #define pmdp_clear_flush_notify pmdp_clear_flush | ||
| 432 | #define pmdp_get_and_clear_notify pmdp_get_and_clear | ||
| 355 | #define set_pte_at_notify set_pte_at | 433 | #define set_pte_at_notify set_pte_at |
| 356 | 434 | ||
| 357 | #endif /* CONFIG_MMU_NOTIFIER */ | 435 | #endif /* CONFIG_MMU_NOTIFIER */ |
| @@ -0,0 +1,154 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | #ifndef KFD_IOCTL_H_INCLUDED | ||
| 24 | #define KFD_IOCTL_H_INCLUDED | ||
| 25 | |||
| 26 | #include <linux/types.h> | ||
| 27 | #include <linux/ioctl.h> | ||
| 28 | |||
| 29 | #define KFD_IOCTL_MAJOR_VERSION 1 | ||
| 30 | #define KFD_IOCTL_MINOR_VERSION 0 | ||
| 31 | |||
| 32 | struct kfd_ioctl_get_version_args { | ||
| 33 | uint32_t major_version; /* from KFD */ | ||
| 34 | uint32_t minor_version; /* from KFD */ | ||
| 35 | }; | ||
| 36 | |||
| 37 | /* For kfd_ioctl_create_queue_args.queue_type. */ | ||
| 38 | #define KFD_IOC_QUEUE_TYPE_COMPUTE 0 | ||
| 39 | #define KFD_IOC_QUEUE_TYPE_SDMA 1 | ||
| 40 | #define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 2 | ||
| 41 | |||
| 42 | #define KFD_MAX_QUEUE_PERCENTAGE 100 | ||
| 43 | #define KFD_MAX_QUEUE_PRIORITY 15 | ||
| 44 | |||
| 45 | struct kfd_ioctl_create_queue_args { | ||
| 46 | uint64_t ring_base_address; /* to KFD */ | ||
| 47 | uint64_t write_pointer_address; /* from KFD */ | ||
| 48 | uint64_t read_pointer_address; /* from KFD */ | ||
| 49 | uint64_t doorbell_offset; /* from KFD */ | ||
| 50 | |||
| 51 | uint32_t ring_size; /* to KFD */ | ||
| 52 | uint32_t gpu_id; /* to KFD */ | ||
| 53 | uint32_t queue_type; /* to KFD */ | ||
| 54 | uint32_t queue_percentage; /* to KFD */ | ||
| 55 | uint32_t queue_priority; /* to KFD */ | ||
| 56 | uint32_t queue_id; /* from KFD */ | ||
| 57 | |||
| 58 | uint64_t eop_buffer_address; /* to KFD */ | ||
| 59 | uint64_t eop_buffer_size; /* to KFD */ | ||
| 60 | uint64_t ctx_save_restore_address; /* to KFD */ | ||
| 61 | uint64_t ctx_save_restore_size; /* to KFD */ | ||
| 62 | }; | ||
| 63 | |||
| 64 | struct kfd_ioctl_destroy_queue_args { | ||
| 65 | uint32_t queue_id; /* to KFD */ | ||
| 66 | uint32_t pad; | ||
| 67 | }; | ||
| 68 | |||
| 69 | struct kfd_ioctl_update_queue_args { | ||
| 70 | uint64_t ring_base_address; /* to KFD */ | ||
| 71 | |||
| 72 | uint32_t queue_id; /* to KFD */ | ||
| 73 | uint32_t ring_size; /* to KFD */ | ||
| 74 | uint32_t queue_percentage; /* to KFD */ | ||
| 75 | uint32_t queue_priority; /* to KFD */ | ||
| 76 | }; | ||
| 77 | |||
| 78 | /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */ | ||
| 79 | #define KFD_IOC_CACHE_POLICY_COHERENT 0 | ||
| 80 | #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1 | ||
| 81 | |||
| 82 | struct kfd_ioctl_set_memory_policy_args { | ||
| 83 | uint64_t alternate_aperture_base; /* to KFD */ | ||
| 84 | uint64_t alternate_aperture_size; /* to KFD */ | ||
| 85 | |||
| 86 | uint32_t gpu_id; /* to KFD */ | ||
| 87 | uint32_t default_policy; /* to KFD */ | ||
| 88 | uint32_t alternate_policy; /* to KFD */ | ||
| 89 | uint32_t pad; | ||
| 90 | }; | ||
| 91 | |||
| 92 | /* | ||
| 93 | * All counters are monotonic. They are used for profiling of compute jobs. | ||
| 94 | * The profiling is done by userspace. | ||
| 95 | * | ||
| 96 | * In case of GPU reset, the counter should not be affected. | ||
| 97 | */ | ||
| 98 | |||
| 99 | struct kfd_ioctl_get_clock_counters_args { | ||
| 100 | uint64_t gpu_clock_counter; /* from KFD */ | ||
| 101 | uint64_t cpu_clock_counter; /* from KFD */ | ||
| 102 | uint64_t system_clock_counter; /* from KFD */ | ||
| 103 | uint64_t system_clock_freq; /* from KFD */ | ||
| 104 | |||
| 105 | uint32_t gpu_id; /* to KFD */ | ||
| 106 | uint32_t pad; | ||
| 107 | }; | ||
| 108 | |||
| 109 | #define NUM_OF_SUPPORTED_GPUS 7 | ||
| 110 | |||
| 111 | struct kfd_process_device_apertures { | ||
| 112 | uint64_t lds_base; /* from KFD */ | ||
| 113 | uint64_t lds_limit; /* from KFD */ | ||
| 114 | uint64_t scratch_base; /* from KFD */ | ||
| 115 | uint64_t scratch_limit; /* from KFD */ | ||
| 116 | uint64_t gpuvm_base; /* from KFD */ | ||
| 117 | uint64_t gpuvm_limit; /* from KFD */ | ||
| 118 | uint32_t gpu_id; /* from KFD */ | ||
| 119 | uint32_t pad; | ||
| 120 | }; | ||
| 121 | |||
| 122 | struct kfd_ioctl_get_process_apertures_args { | ||
| 123 | struct kfd_process_device_apertures | ||
| 124 | process_apertures[NUM_OF_SUPPORTED_GPUS];/* from KFD */ | ||
| 125 | |||
| 126 | /* from KFD, should be in the range [1 - NUM_OF_SUPPORTED_GPUS] */ | ||
| 127 | uint32_t num_of_nodes; | ||
| 128 | uint32_t pad; | ||
| 129 | }; | ||
| 130 | |||
| 131 | #define KFD_IOC_MAGIC 'K' | ||
| 132 | |||
| 133 | #define KFD_IOC_GET_VERSION \ | ||
| 134 | _IOR(KFD_IOC_MAGIC, 1, struct kfd_ioctl_get_version_args) | ||
| 135 | |||
| 136 | #define KFD_IOC_CREATE_QUEUE \ | ||
| 137 | _IOWR(KFD_IOC_MAGIC, 2, struct kfd_ioctl_create_queue_args) | ||
| 138 | |||
| 139 | #define KFD_IOC_DESTROY_QUEUE \ | ||
| 140 | _IOWR(KFD_IOC_MAGIC, 3, struct kfd_ioctl_destroy_queue_args) | ||
| 141 | |||
| 142 | #define KFD_IOC_SET_MEMORY_POLICY \ | ||
| 143 | _IOW(KFD_IOC_MAGIC, 4, struct kfd_ioctl_set_memory_policy_args) | ||
| 144 | |||
| 145 | #define KFD_IOC_GET_CLOCK_COUNTERS \ | ||
| 146 | _IOWR(KFD_IOC_MAGIC, 5, struct kfd_ioctl_get_clock_counters_args) | ||
| 147 | |||
| 148 | #define KFD_IOC_GET_PROCESS_APERTURES \ | ||
| 149 | _IOR(KFD_IOC_MAGIC, 6, struct kfd_ioctl_get_process_apertures_args) | ||
| 150 | |||
| 151 | #define KFD_IOC_UPDATE_QUEUE \ | ||
| 152 | _IOW(KFD_IOC_MAGIC, 7, struct kfd_ioctl_update_queue_args) | ||
| 153 | |||
| 154 | #endif | ||
| @@ -193,7 +193,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, · 192 unmodified lines | |||
| 193 | } | 193 | } |
| 194 | 194 | ||
| 195 | flush_cache_page(vma, addr, pte_pfn(*ptep)); | 195 | flush_cache_page(vma, addr, pte_pfn(*ptep)); |
| 196 | ptep_clear_flush(vma, addr, ptep); | 196 | ptep_clear_flush_notify(vma, addr, ptep); |
| 197 | set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot)); | 197 | set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot)); |
| 198 | 198 | ||
| 199 | page_remove_rmap(page); | 199 | page_remove_rmap(page); |
| @@ -37,7 +37,7 @@ static void zap_pte(struct mm_struct *mm, struct vm_area_struct *vma, · 36 unmodified lines | |||
| 37 | 37 | ||
| 38 | if (pte_present(pte)) { | 38 | if (pte_present(pte)) { |
| 39 | flush_cache_page(vma, addr, pte_pfn(pte)); | 39 | flush_cache_page(vma, addr, pte_pfn(pte)); |
| 40 | pte = ptep_clear_flush(vma, addr, ptep); | 40 | pte = ptep_clear_flush_notify(vma, addr, ptep); |
| 41 | page = vm_normal_page(vma, addr, pte); | 41 | page = vm_normal_page(vma, addr, pte); |
| 42 | if (page) { | 42 | if (page) { |
| 43 | if (pte_dirty(pte)) | 43 | if (pte_dirty(pte)) |
| @@ -1036,7 +1036,7 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, · 1035 unmodified lines | |||
| 1036 | goto out_free_pages; | 1036 | goto out_free_pages; |
| 1037 | VM_BUG_ON_PAGE(!PageHead(page), page); | 1037 | VM_BUG_ON_PAGE(!PageHead(page), page); |
| 1038 | 1038 | ||
| 1039 | pmdp_clear_flush(vma, haddr, pmd); | 1039 | pmdp_clear_flush_notify(vma, haddr, pmd); |
| 1040 | /* leave pmd empty until pte is filled */ | 1040 | /* leave pmd empty until pte is filled */ |
| 1041 | 1041 | ||
| 1042 | pgtable = pgtable_trans_huge_withdraw(mm, pmd); | 1042 | pgtable = pgtable_trans_huge_withdraw(mm, pmd); |
| @@ -1179,7 +1179,7 @@ alloc: · 136 unmodified lines | |||
| 1179 | pmd_t entry; | 1179 | pmd_t entry; |
| 1180 | entry = mk_huge_pmd(new_page, vma->vm_page_prot); | 1180 | entry = mk_huge_pmd(new_page, vma->vm_page_prot); |
| 1181 | entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); | 1181 | entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); |
| 1182 | pmdp_clear_flush(vma, haddr, pmd); | 1182 | pmdp_clear_flush_notify(vma, haddr, pmd); |
| 1183 | page_add_new_anon_rmap(new_page, vma, haddr); | 1183 | page_add_new_anon_rmap(new_page, vma, haddr); |
| 1184 | mem_cgroup_commit_charge(new_page, memcg, false); | 1184 | mem_cgroup_commit_charge(new_page, memcg, false); |
| 1185 | lru_cache_add_active_or_unevictable(new_page, vma); | 1185 | lru_cache_add_active_or_unevictable(new_page, vma); |
| @@ -1512,7 +1512,7 @@ int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, · 326 unmodified lines | |||
| 1512 | pmd_t entry; | 1512 | pmd_t entry; |
| 1513 | ret = 1; | 1513 | ret = 1; |
| 1514 | if (!prot_numa) { | 1514 | if (!prot_numa) { |
| 1515 | entry = pmdp_get_and_clear(mm, addr, pmd); | 1515 | entry = pmdp_get_and_clear_notify(mm, addr, pmd); |
| 1516 | if (pmd_numa(entry)) | 1516 | if (pmd_numa(entry)) |
| 1517 | entry = pmd_mknonnuma(entry); | 1517 | entry = pmd_mknonnuma(entry); |
| 1518 | entry = pmd_modify(entry, newprot); | 1518 | entry = pmd_modify(entry, newprot); |
| @@ -1644,6 +1644,7 @@ static int __split_huge_page_splitting(struct page *page, · 125 unmodified lines | |||
| 1644 | * serialize against split_huge_page*. | 1644 | * serialize against split_huge_page*. |
| 1645 | */ | 1645 | */ |
| 1646 | pmdp_splitting_flush(vma, address, pmd); | 1646 | pmdp_splitting_flush(vma, address, pmd); |
| 1647 | |||
| 1647 | ret = 1; | 1648 | ret = 1; |
| 1648 | spin_unlock(ptl); | 1649 | spin_unlock(ptl); |
| 1649 | } | 1650 | } |
| @@ -2834,7 +2835,7 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, · 1184 unmodified lines | |||
| 2834 | pmd_t _pmd; | 2835 | pmd_t _pmd; |
| 2835 | int i; | 2836 | int i; |
| 2836 | 2837 | ||
| 2837 | pmdp_clear_flush(vma, haddr, pmd); | 2838 | pmdp_clear_flush_notify(vma, haddr, pmd); |
| 2838 | /* leave pmd empty until pte is filled */ | 2839 | /* leave pmd empty until pte is filled */ |
| 2839 | 2840 | ||
| 2840 | pgtable = pgtable_trans_huge_withdraw(mm, pmd); | 2841 | pgtable = pgtable_trans_huge_withdraw(mm, pmd); |
| @@ -2598,8 +2598,11 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, · 2597 unmodified lines | |||
| 2598 | } | 2598 | } |
| 2599 | set_huge_pte_at(dst, addr, dst_pte, entry); | 2599 | set_huge_pte_at(dst, addr, dst_pte, entry); |
| 2600 | } else { | 2600 | } else { |
| 2601 | if (cow) | 2601 | if (cow) { |
| 2602 | huge_ptep_set_wrprotect(src, addr, src_pte); | 2602 | huge_ptep_set_wrprotect(src, addr, src_pte); |
| 2603 | mmu_notifier_invalidate_range(src, mmun_start, | ||
| 2604 | mmun_end); | ||
| 2605 | } | ||
| 2603 | entry = huge_ptep_get(src_pte); | 2606 | entry = huge_ptep_get(src_pte); |
| 2604 | ptepage = pte_page(entry); | 2607 | ptepage = pte_page(entry); |
| 2605 | get_page(ptepage); | 2608 | get_page(ptepage); |
| @@ -2899,6 +2902,7 @@ retry_avoidcopy: · 293 unmodified lines | |||
| 2899 | 2902 | ||
| 2900 | /* Break COW */ | 2903 | /* Break COW */ |
| 2901 | huge_ptep_clear_flush(vma, address, ptep); | 2904 | huge_ptep_clear_flush(vma, address, ptep); |
| 2905 | mmu_notifier_invalidate_range(mm, mmun_start, mmun_end); | ||
| 2902 | set_huge_pte_at(mm, address, ptep, | 2906 | set_huge_pte_at(mm, address, ptep, |
| 2903 | make_huge_pte(vma, new_page, 1)); | 2907 | make_huge_pte(vma, new_page, 1)); |
| 2904 | page_remove_rmap(old_page); | 2908 | page_remove_rmap(old_page); |
| @@ -3374,6 +3378,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, · 469 unmodified lines | |||
| 3374 | * and that page table be reused and filled with junk. | 3378 | * and that page table be reused and filled with junk. |
| 3375 | */ | 3379 | */ |
| 3376 | flush_tlb_range(vma, start, end); | 3380 | flush_tlb_range(vma, start, end); |
| 3381 | mmu_notifier_invalidate_range(mm, start, end); | ||
| 3377 | mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex); | 3382 | mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex); |
| 3378 | mmu_notifier_invalidate_range_end(mm, start, end); | 3383 | mmu_notifier_invalidate_range_end(mm, start, end); |
| 3379 | 3384 | ||
| @@ -892,7 +892,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page, · 891 unmodified lines | |||
| 892 | * this assure us that no O_DIRECT can happen after the check | 892 | * this assure us that no O_DIRECT can happen after the check |
| 893 | * or in the middle of the check. | 893 | * or in the middle of the check. |
| 894 | */ | 894 | */ |
| 895 | entry = ptep_clear_flush(vma, addr, ptep); | 895 | entry = ptep_clear_flush_notify(vma, addr, ptep); |
| 896 | /* | 896 | /* |
| 897 | * Check that no O_DIRECT or similar I/O is in progress on the | 897 | * Check that no O_DIRECT or similar I/O is in progress on the |
| 898 | * page | 898 | * page |
| @@ -960,7 +960,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, · 61 unmodified lines | |||
| 960 | page_add_anon_rmap(kpage, vma, addr); | 960 | page_add_anon_rmap(kpage, vma, addr); |
| 961 | 961 | ||
| 962 | flush_cache_page(vma, addr, pte_pfn(*ptep)); | 962 | flush_cache_page(vma, addr, pte_pfn(*ptep)); |
| 963 | ptep_clear_flush(vma, addr, ptep); | 963 | ptep_clear_flush_notify(vma, addr, ptep); |
| 964 | set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot)); | 964 | set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot)); |
| 965 | 965 | ||
| 966 | page_remove_rmap(page); | 966 | page_remove_rmap(page); |
| @@ -238,6 +238,7 @@ static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) · 237 unmodified lines | |||
| 238 | { | 238 | { |
| 239 | tlb->need_flush = 0; | 239 | tlb->need_flush = 0; |
| 240 | tlb_flush(tlb); | 240 | tlb_flush(tlb); |
| 241 | mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end); | ||
| 241 | #ifdef CONFIG_HAVE_RCU_TABLE_FREE | 242 | #ifdef CONFIG_HAVE_RCU_TABLE_FREE |
| 242 | tlb_table_flush(tlb); | 243 | tlb_table_flush(tlb); |
| 243 | #endif | 244 | #endif |
| @@ -2234,7 +2235,7 @@ gotten: · 1990 unmodified lines | |||
| 2234 | * seen in the presence of one thread doing SMC and another | 2235 | * seen in the presence of one thread doing SMC and another |
| 2235 | * thread doing COW. | 2236 | * thread doing COW. |
| 2236 | */ | 2237 | */ |
| 2237 | ptep_clear_flush(vma, address, page_table); | 2238 | ptep_clear_flush_notify(vma, address, page_table); |
| 2238 | page_add_new_anon_rmap(new_page, vma, address); | 2239 | page_add_new_anon_rmap(new_page, vma, address); |
| 2239 | mem_cgroup_commit_charge(new_page, memcg, false); | 2240 | mem_cgroup_commit_charge(new_page, memcg, false); |
| 2240 | lru_cache_add_active_or_unevictable(new_page, vma); | 2241 | lru_cache_add_active_or_unevictable(new_page, vma); |
| @@ -1854,7 +1854,7 @@ fail_putback: · 1853 unmodified lines | |||
| 1854 | */ | 1854 | */ |
| 1855 | flush_cache_range(vma, mmun_start, mmun_end); | 1855 | flush_cache_range(vma, mmun_start, mmun_end); |
| 1856 | page_add_anon_rmap(new_page, vma, mmun_start); | 1856 | page_add_anon_rmap(new_page, vma, mmun_start); |
| 1857 | pmdp_clear_flush(vma, mmun_start, pmd); | 1857 | pmdp_clear_flush_notify(vma, mmun_start, pmd); |
| 1858 | set_pmd_at(mm, mmun_start, pmd, entry); | 1858 | set_pmd_at(mm, mmun_start, pmd, entry); |
| 1859 | flush_tlb_range(vma, mmun_start, mmun_end); | 1859 | flush_tlb_range(vma, mmun_start, mmun_end); |
| 1860 | update_mmu_cache_pmd(vma, address, &entry); | 1860 | update_mmu_cache_pmd(vma, address, &entry); |
| @@ -1862,6 +1862,7 @@ fail_putback: · 1 unmodified line | |||
| 1862 | if (page_count(page) != 2) { | 1862 | if (page_count(page) != 2) { |
| 1863 | set_pmd_at(mm, mmun_start, pmd, orig_entry); | 1863 | set_pmd_at(mm, mmun_start, pmd, orig_entry); |
| 1864 | flush_tlb_range(vma, mmun_start, mmun_end); | 1864 | flush_tlb_range(vma, mmun_start, mmun_end); |
| 1865 | mmu_notifier_invalidate_range(mm, mmun_start, mmun_end); | ||
| 1865 | update_mmu_cache_pmd(vma, address, &entry); | 1866 | update_mmu_cache_pmd(vma, address, &entry); |
| 1866 | page_remove_rmap(new_page); | 1867 | page_remove_rmap(new_page); |
| 1867 | goto fail_putback; | 1868 | goto fail_putback; |
| @@ -193,6 +193,16 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 192 unmodified lines | |||
| 193 | 193 | ||
| 194 | id = srcu_read_lock(&srcu); | 194 | id = srcu_read_lock(&srcu); |
| 195 | hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) { | 195 | hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) { |
| 196 | /* | ||
| 197 | * Call invalidate_range here too to avoid the need for the | ||
| 198 | * subsystem of having to register an invalidate_range_end | ||
| 199 | * call-back when there is invalidate_range already. Usually a | ||
| 200 | * subsystem registers either invalidate_range_start()/end() or | ||
| 201 | * invalidate_range(), so this will be no additional overhead | ||
| 202 | * (besides the pointer check). | ||
| 203 | */ | ||
| 204 | if (mn->ops->invalidate_range) | ||
| 205 | mn->ops->invalidate_range(mn, mm, start, end); | ||
| 196 | if (mn->ops->invalidate_range_end) | 206 | if (mn->ops->invalidate_range_end) |
| 197 | mn->ops->invalidate_range_end(mn, mm, start, end); | 207 | mn->ops->invalidate_range_end(mn, mm, start, end); |
| 198 | } | 208 | } |
| @@ -200,6 +210,21 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 1 unmodified line | |||
| 200 | } | 210 | } |
| 201 | EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_end); | 211 | EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_end); |
| 202 | 212 | ||
| 213 | void __mmu_notifier_invalidate_range(struct mm_struct *mm, | ||
| 214 | unsigned long start, unsigned long end) | ||
| 215 | { | ||
| 216 | struct mmu_notifier *mn; | ||
| 217 | int id; | ||
| 218 | |||
| 219 | id = srcu_read_lock(&srcu); | ||
| 220 | hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) { | ||
| 221 | if (mn->ops->invalidate_range) | ||
| 222 | mn->ops->invalidate_range(mn, mm, start, end); | ||
| 223 | } | ||
| 224 | srcu_read_unlock(&srcu, id); | ||
| 225 | } | ||
| 226 | EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range); | ||
| 227 | |||
| 203 | static int do_mmu_notifier_register(struct mmu_notifier *mn, | 228 | static int do_mmu_notifier_register(struct mmu_notifier *mn, |
| 204 | struct mm_struct *mm, | 229 | struct mm_struct *mm, |
| 205 | int take_mmap_sem) | 230 | int take_mmap_sem) |
| @@ -1378,7 +1378,7 @@ static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount, · 1377 unmodified lines | |||
| 1378 | 1378 | ||
| 1379 | /* Nuke the page table entry. */ | 1379 | /* Nuke the page table entry. */ |
| 1380 | flush_cache_page(vma, address, pte_pfn(*pte)); | 1380 | flush_cache_page(vma, address, pte_pfn(*pte)); |
| 1381 | pteval = ptep_clear_flush(vma, address, pte); | 1381 | pteval = ptep_clear_flush_notify(vma, address, pte); |
| 1382 | 1382 | ||
| 1383 | /* If nonlinear, store the file page offset in the pte. */ | 1383 | /* If nonlinear, store the file page offset in the pte. */ |
| 1384 | if (page->index != linear_page_index(vma, address)) { | 1384 | if (page->index != linear_page_index(vma, address)) { |