Merge AMDKFD it seems clean enough. * 'amdkfd-v6' of git://people.freedesktop.org/~gabbayo/linux: (29 commits) amdkfd: Implement the Get Version IOCTL amdkfd: Implement the Get Process Aperture IOCTL amdkfd: Implement the Get Clock Counters IOCTL amdkfd: Implement the Set Memory Policy IOCTL amdkfd: Implement the create/destroy/update queue IOCTLs amdkfd: Add interrupt handling module amdkfd: Add device queue manager module amdkfd: Add process queue manager module amdkfd: Add packet manager module amdkfd: Add module parameter of scheduling policy amdkfd: Add kernel queue module amdkfd: Add mqd_manager module amdkfd: Add queue module amdkfd: Add binding/unbinding calls to amd_iommu driver amdkfd: Add basic modules to amdkfd amdkfd: Add topology module to amdkfd amdkfd: Add amdkfd skeleton driver amdkfd: Add IOCTL set definitions of amdkfd Update MAINTAINERS and CREDITS files with amdkfd info drm/radeon: Add radeon <--> amdkfd interface ...
Unified · Split
| @@ -1197,6 +1197,13 @@ S: R. Tocantins, 89 - Cristo Rei · 1196 unmodified lines | ||
| 1197 | 1197 | S: 80050-430 - Curitiba - Paraná |
| 1198 | 1198 | S: Brazil |
| 1199 | 1199 | |
| 1200 | +N: Oded Gabbay | |
| 1201 | +E: oded.gabbay@gmail.com | |
| 1202 | +D: AMD KFD maintainer | |
| 1203 | +S: 12 Shraga Raphaeli | |
| 1204 | +S: Petah-Tikva, 4906418 | |
| 1205 | +S: Israel | |
| 1206 | + | |
| 1200 | 1207 | N: Kumar Gala |
| 1201 | 1208 | E: galak@kernel.crashing.org |
| 1202 | 1209 | D: Embedded PowerPC 6xx/7xx/74xx/82xx/83xx/85xx support |
| @@ -618,6 +618,16 @@ S: Maintained · 617 unmodified lines | ||
| 618 | 618 | F: drivers/iommu/amd_iommu*.[ch] |
| 619 | 619 | F: include/linux/amd-iommu.h |
| 620 | 620 | |
| 621 | +AMD KFD | |
| 622 | +M: Oded Gabbay <oded.gabbay@amd.com> | |
| 623 | +L: dri-devel@lists.freedesktop.org | |
| 624 | +T: git git://people.freedesktop.org/~gabbayo/linux.git | |
| 625 | +S: Supported | |
| 626 | +F: drivers/gpu/drm/amd/amdkfd/ | |
| 627 | +F: drivers/gpu/drm/radeon/radeon_kfd.c | |
| 628 | +F: drivers/gpu/drm/radeon/radeon_kfd.h | |
| 629 | +F: include/uapi/linux/kfd_ioctl.h | |
| 630 | + | |
| 621 | 631 | AMD MICROCODE UPDATE SUPPORT |
| 622 | 632 | M: Andreas Herrmann <herrmann.der.user@googlemail.com> |
| 623 | 633 | L: amd64-microcode@amd64.org |
| @@ -200,3 +200,5 @@ source "drivers/gpu/drm/tegra/Kconfig" · 199 unmodified lines | ||
| 200 | 200 | source "drivers/gpu/drm/panel/Kconfig" |
| 201 | 201 | |
| 202 | 202 | source "drivers/gpu/drm/sti/Kconfig" |
| 203 | + | |
| 204 | +source "drivers/gpu/drm/amd/amdkfd/Kconfig" | |
| @@ -65,3 +65,4 @@ obj-$(CONFIG_DRM_STI) += sti/ · 64 unmodified lines | ||
| 65 | 65 | obj-y += i2c/ |
| 66 | 66 | obj-y += panel/ |
| 67 | 67 | obj-y += bridge/ |
| 68 | +obj-$(CONFIG_HSA_AMD) += amd/amdkfd/ | |
| @@ -0,0 +1,9 @@ | ||
| 1 | +# | |
| 2 | +# Heterogenous system architecture configuration | |
| 3 | +# | |
| 4 | + | |
| 5 | +config HSA_AMD | |
| 6 | + tristate "HSA kernel driver for AMD GPU devices" | |
| 7 | + depends on (DRM_RADEON || DRM_AMDGPU) && AMD_IOMMU_V2 && X86_64 | |
| 8 | + help | |
| 9 | + Enable this if you want to use HSA features on AMD GPU devices. | |
| @@ -0,0 +1,14 @@ | ||
| 1 | +# | |
| 2 | +# Makefile for Heterogenous System Architecture support for AMD GPU devices | |
| 3 | +# | |
| 4 | + | |
| 5 | +ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/amd/include/ | |
| 6 | + | |
| 7 | +amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \ | |
| 8 | + kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \ | |
| 9 | + kfd_process.o kfd_queue.o kfd_mqd_manager.o \ | |
| 10 | + kfd_kernel_queue.o kfd_packet_manager.o \ | |
| 11 | + kfd_process_queue_manager.o kfd_device_queue_manager.o \ | |
| 12 | + kfd_interrupt.o | |
| 13 | + | |
| 14 | +obj-$(CONFIG_HSA_AMD) += amdkfd.o | |
| @@ -0,0 +1,221 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +#ifndef CIK_REGS_H | |
| 24 | +#define CIK_REGS_H | |
| 25 | + | |
| 26 | +#define IH_VMID_0_LUT 0x3D40u | |
| 27 | + | |
| 28 | +#define BIF_DOORBELL_CNTL 0x530Cu | |
| 29 | + | |
| 30 | +#define SRBM_GFX_CNTL 0xE44 | |
| 31 | +#define PIPEID(x) ((x) << 0) | |
| 32 | +#define MEID(x) ((x) << 2) | |
| 33 | +#define VMID(x) ((x) << 4) | |
| 34 | +#define QUEUEID(x) ((x) << 8) | |
| 35 | + | |
| 36 | +#define SQ_CONFIG 0x8C00 | |
| 37 | + | |
| 38 | +#define SH_MEM_BASES 0x8C28 | |
| 39 | +/* if PTR32, these are the bases for scratch and lds */ | |
| 40 | +#define PRIVATE_BASE(x) ((x) << 0) /* scratch */ | |
| 41 | +#define SHARED_BASE(x) ((x) << 16) /* LDS */ | |
| 42 | +#define SH_MEM_APE1_BASE 0x8C2C | |
| 43 | +/* if PTR32, this is the base location of GPUVM */ | |
| 44 | +#define SH_MEM_APE1_LIMIT 0x8C30 | |
| 45 | +/* if PTR32, this is the upper limit of GPUVM */ | |
| 46 | +#define SH_MEM_CONFIG 0x8C34 | |
| 47 | +#define PTR32 (1 << 0) | |
| 48 | +#define PRIVATE_ATC (1 << 1) | |
| 49 | +#define ALIGNMENT_MODE(x) ((x) << 2) | |
| 50 | +#define SH_MEM_ALIGNMENT_MODE_DWORD 0 | |
| 51 | +#define SH_MEM_ALIGNMENT_MODE_DWORD_STRICT 1 | |
| 52 | +#define SH_MEM_ALIGNMENT_MODE_STRICT 2 | |
| 53 | +#define SH_MEM_ALIGNMENT_MODE_UNALIGNED 3 | |
| 54 | +#define DEFAULT_MTYPE(x) ((x) << 4) | |
| 55 | +#define APE1_MTYPE(x) ((x) << 7) | |
| 56 | + | |
| 57 | +/* valid for both DEFAULT_MTYPE and APE1_MTYPE */ | |
| 58 | +#define MTYPE_CACHED 0 | |
| 59 | +#define MTYPE_NONCACHED 3 | |
| 60 | + | |
| 61 | + | |
| 62 | +#define SH_STATIC_MEM_CONFIG 0x9604u | |
| 63 | + | |
| 64 | +#define TC_CFG_L1_LOAD_POLICY0 0xAC68 | |
| 65 | +#define TC_CFG_L1_LOAD_POLICY1 0xAC6C | |
| 66 | +#define TC_CFG_L1_STORE_POLICY 0xAC70 | |
| 67 | +#define TC_CFG_L2_LOAD_POLICY0 0xAC74 | |
| 68 | +#define TC_CFG_L2_LOAD_POLICY1 0xAC78 | |
| 69 | +#define TC_CFG_L2_STORE_POLICY0 0xAC7C | |
| 70 | +#define TC_CFG_L2_STORE_POLICY1 0xAC80 | |
| 71 | +#define TC_CFG_L2_ATOMIC_POLICY 0xAC84 | |
| 72 | +#define TC_CFG_L1_VOLATILE 0xAC88 | |
| 73 | +#define TC_CFG_L2_VOLATILE 0xAC8C | |
| 74 | + | |
| 75 | +#define CP_PQ_WPTR_POLL_CNTL 0xC20C | |
| 76 | +#define WPTR_POLL_EN (1 << 31) | |
| 77 | + | |
| 78 | +#define CPC_INT_CNTL 0xC2D0 | |
| 79 | +#define CP_ME1_PIPE0_INT_CNTL 0xC214 | |
| 80 | +#define CP_ME1_PIPE1_INT_CNTL 0xC218 | |
| 81 | +#define CP_ME1_PIPE2_INT_CNTL 0xC21C | |
| 82 | +#define CP_ME1_PIPE3_INT_CNTL 0xC220 | |
| 83 | +#define CP_ME2_PIPE0_INT_CNTL 0xC224 | |
| 84 | +#define CP_ME2_PIPE1_INT_CNTL 0xC228 | |
| 85 | +#define CP_ME2_PIPE2_INT_CNTL 0xC22C | |
| 86 | +#define CP_ME2_PIPE3_INT_CNTL 0xC230 | |
| 87 | +#define DEQUEUE_REQUEST_INT_ENABLE (1 << 13) | |
| 88 | +#define WRM_POLL_TIMEOUT_INT_ENABLE (1 << 17) | |
| 89 | +#define PRIV_REG_INT_ENABLE (1 << 23) | |
| 90 | +#define TIME_STAMP_INT_ENABLE (1 << 26) | |
| 91 | +#define GENERIC2_INT_ENABLE (1 << 29) | |
| 92 | +#define GENERIC1_INT_ENABLE (1 << 30) | |
| 93 | +#define GENERIC0_INT_ENABLE (1 << 31) | |
| 94 | +#define CP_ME1_PIPE0_INT_STATUS 0xC214 | |
| 95 | +#define CP_ME1_PIPE1_INT_STATUS 0xC218 | |
| 96 | +#define CP_ME1_PIPE2_INT_STATUS 0xC21C | |
| 97 | +#define CP_ME1_PIPE3_INT_STATUS 0xC220 | |
| 98 | +#define CP_ME2_PIPE0_INT_STATUS 0xC224 | |
| 99 | +#define CP_ME2_PIPE1_INT_STATUS 0xC228 | |
| 100 | +#define CP_ME2_PIPE2_INT_STATUS 0xC22C | |
| 101 | +#define CP_ME2_PIPE3_INT_STATUS 0xC230 | |
| 102 | +#define DEQUEUE_REQUEST_INT_STATUS (1 << 13) | |
| 103 | +#define WRM_POLL_TIMEOUT_INT_STATUS (1 << 17) | |
| 104 | +#define PRIV_REG_INT_STATUS (1 << 23) | |
| 105 | +#define TIME_STAMP_INT_STATUS (1 << 26) | |
| 106 | +#define GENERIC2_INT_STATUS (1 << 29) | |
| 107 | +#define GENERIC1_INT_STATUS (1 << 30) | |
| 108 | +#define GENERIC0_INT_STATUS (1 << 31) | |
| 109 | + | |
| 110 | +#define CP_HPD_EOP_BASE_ADDR 0xC904 | |
| 111 | +#define CP_HPD_EOP_BASE_ADDR_HI 0xC908 | |
| 112 | +#define CP_HPD_EOP_VMID 0xC90C | |
| 113 | +#define CP_HPD_EOP_CONTROL 0xC910 | |
| 114 | +#define EOP_SIZE(x) ((x) << 0) | |
| 115 | +#define EOP_SIZE_MASK (0x3f << 0) | |
| 116 | +#define CP_MQD_BASE_ADDR 0xC914 | |
| 117 | +#define CP_MQD_BASE_ADDR_HI 0xC918 | |
| 118 | +#define CP_HQD_ACTIVE 0xC91C | |
| 119 | +#define CP_HQD_VMID 0xC920 | |
| 120 | + | |
| 121 | +#define CP_HQD_PERSISTENT_STATE 0xC924u | |
| 122 | +#define DEFAULT_CP_HQD_PERSISTENT_STATE (0x33U << 8) | |
| 123 | +#define PRELOAD_REQ (1 << 0) | |
| 124 | + | |
| 125 | +#define CP_HQD_PIPE_PRIORITY 0xC928u | |
| 126 | +#define CP_HQD_QUEUE_PRIORITY 0xC92Cu | |
| 127 | +#define CP_HQD_QUANTUM 0xC930u | |
| 128 | +#define QUANTUM_EN 1U | |
| 129 | +#define QUANTUM_SCALE_1MS (1U << 4) | |
| 130 | +#define QUANTUM_DURATION(x) ((x) << 8) | |
| 131 | + | |
| 132 | +#define CP_HQD_PQ_BASE 0xC934 | |
| 133 | +#define CP_HQD_PQ_BASE_HI 0xC938 | |
| 134 | +#define CP_HQD_PQ_RPTR 0xC93C | |
| 135 | +#define CP_HQD_PQ_RPTR_REPORT_ADDR 0xC940 | |
| 136 | +#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI 0xC944 | |
| 137 | +#define CP_HQD_PQ_WPTR_POLL_ADDR 0xC948 | |
| 138 | +#define CP_HQD_PQ_WPTR_POLL_ADDR_HI 0xC94C | |
| 139 | +#define CP_HQD_PQ_DOORBELL_CONTROL 0xC950 | |
| 140 | +#define DOORBELL_OFFSET(x) ((x) << 2) | |
| 141 | +#define DOORBELL_OFFSET_MASK (0x1fffff << 2) | |
| 142 | +#define DOORBELL_SOURCE (1 << 28) | |
| 143 | +#define DOORBELL_SCHD_HIT (1 << 29) | |
| 144 | +#define DOORBELL_EN (1 << 30) | |
| 145 | +#define DOORBELL_HIT (1 << 31) | |
| 146 | +#define CP_HQD_PQ_WPTR 0xC954 | |
| 147 | +#define CP_HQD_PQ_CONTROL 0xC958 | |
| 148 | +#define QUEUE_SIZE(x) ((x) << 0) | |
| 149 | +#define QUEUE_SIZE_MASK (0x3f << 0) | |
| 150 | +#define RPTR_BLOCK_SIZE(x) ((x) << 8) | |
| 151 | +#define RPTR_BLOCK_SIZE_MASK (0x3f << 8) | |
| 152 | +#define MIN_AVAIL_SIZE(x) ((x) << 20) | |
| 153 | +#define PQ_ATC_EN (1 << 23) | |
| 154 | +#define PQ_VOLATILE (1 << 26) | |
| 155 | +#define NO_UPDATE_RPTR (1 << 27) | |
| 156 | +#define UNORD_DISPATCH (1 << 28) | |
| 157 | +#define ROQ_PQ_IB_FLIP (1 << 29) | |
| 158 | +#define PRIV_STATE (1 << 30) | |
| 159 | +#define KMD_QUEUE (1 << 31) | |
| 160 | + | |
| 161 | +#define DEFAULT_RPTR_BLOCK_SIZE RPTR_BLOCK_SIZE(5) | |
| 162 | +#define DEFAULT_MIN_AVAIL_SIZE MIN_AVAIL_SIZE(3) | |
| 163 | + | |
| 164 | +#define CP_HQD_IB_BASE_ADDR 0xC95Cu | |
| 165 | +#define CP_HQD_IB_BASE_ADDR_HI 0xC960u | |
| 166 | +#define CP_HQD_IB_RPTR 0xC964u | |
| 167 | +#define CP_HQD_IB_CONTROL 0xC968u | |
| 168 | +#define IB_ATC_EN (1U << 23) | |
| 169 | +#define DEFAULT_MIN_IB_AVAIL_SIZE (3U << 20) | |
| 170 | + | |
| 171 | +#define CP_HQD_DEQUEUE_REQUEST 0xC974 | |
| 172 | +#define DEQUEUE_REQUEST_DRAIN 1 | |
| 173 | +#define DEQUEUE_REQUEST_RESET 2 | |
| 174 | +#define DEQUEUE_INT (1U << 8) | |
| 175 | + | |
| 176 | +#define CP_HQD_SEMA_CMD 0xC97Cu | |
| 177 | +#define CP_HQD_MSG_TYPE 0xC980u | |
| 178 | +#define CP_HQD_ATOMIC0_PREOP_LO 0xC984u | |
| 179 | +#define CP_HQD_ATOMIC0_PREOP_HI 0xC988u | |
| 180 | +#define CP_HQD_ATOMIC1_PREOP_LO 0xC98Cu | |
| 181 | +#define CP_HQD_ATOMIC1_PREOP_HI 0xC990u | |
| 182 | +#define CP_HQD_HQ_SCHEDULER0 0xC994u | |
| 183 | +#define CP_HQD_HQ_SCHEDULER1 0xC998u | |
| 184 | + | |
| 185 | + | |
| 186 | +#define CP_MQD_CONTROL 0xC99C | |
| 187 | +#define MQD_VMID(x) ((x) << 0) | |
| 188 | +#define MQD_VMID_MASK (0xf << 0) | |
| 189 | +#define MQD_CONTROL_PRIV_STATE_EN (1U << 8) | |
| 190 | + | |
| 191 | +#define GRBM_GFX_INDEX 0x30800 | |
| 192 | +#define INSTANCE_INDEX(x) ((x) << 0) | |
| 193 | +#define SH_INDEX(x) ((x) << 8) | |
| 194 | +#define SE_INDEX(x) ((x) << 16) | |
| 195 | +#define SH_BROADCAST_WRITES (1 << 29) | |
| 196 | +#define INSTANCE_BROADCAST_WRITES (1 << 30) | |
| 197 | +#define SE_BROADCAST_WRITES (1 << 31) | |
| 198 | + | |
| 199 | +#define SQC_CACHES 0x30d20 | |
| 200 | +#define SQC_POLICY 0x8C38u | |
| 201 | +#define SQC_VOLATILE 0x8C3Cu | |
| 202 | + | |
| 203 | +#define CP_PERFMON_CNTL 0x36020 | |
| 204 | + | |
| 205 | +#define ATC_VMID0_PASID_MAPPING 0x339Cu | |
| 206 | +#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS 0x3398u | |
| 207 | +#define ATC_VMID_PASID_MAPPING_VALID (1U << 31) | |
| 208 | + | |
| 209 | +#define ATC_VM_APERTURE0_CNTL 0x3310u | |
| 210 | +#define ATS_ACCESS_MODE_NEVER 0 | |
| 211 | +#define ATS_ACCESS_MODE_ALWAYS 1 | |
| 212 | + | |
| 213 | +#define ATC_VM_APERTURE0_CNTL2 0x3318u | |
| 214 | +#define ATC_VM_APERTURE0_HIGH_ADDR 0x3308u | |
| 215 | +#define ATC_VM_APERTURE0_LOW_ADDR 0x3300u | |
| 216 | +#define ATC_VM_APERTURE1_CNTL 0x3314u | |
| 217 | +#define ATC_VM_APERTURE1_CNTL2 0x331Cu | |
| 218 | +#define ATC_VM_APERTURE1_HIGH_ADDR 0x330Cu | |
| 219 | +#define ATC_VM_APERTURE1_LOW_ADDR 0x3304u | |
| 220 | + | |
| 221 | +#endif | |
| @@ -0,0 +1,576 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +#include <linux/device.h> | |
| 24 | +#include <linux/export.h> | |
| 25 | +#include <linux/err.h> | |
| 26 | +#include <linux/fs.h> | |
| 27 | +#include <linux/sched.h> | |
| 28 | +#include <linux/slab.h> | |
| 29 | +#include <linux/uaccess.h> | |
| 30 | +#include <linux/compat.h> | |
| 31 | +#include <uapi/linux/kfd_ioctl.h> | |
| 32 | +#include <linux/time.h> | |
| 33 | +#include <linux/mm.h> | |
| 34 | +#include <linux/uaccess.h> | |
| 35 | +#include <uapi/asm-generic/mman-common.h> | |
| 36 | +#include <asm/processor.h> | |
| 37 | +#include "kfd_priv.h" | |
| 38 | +#include "kfd_device_queue_manager.h" | |
| 39 | + | |
| 40 | +static long kfd_ioctl(struct file *, unsigned int, unsigned long); | |
| 41 | +static int kfd_open(struct inode *, struct file *); | |
| 42 | +static int kfd_mmap(struct file *, struct vm_area_struct *); | |
| 43 | + | |
| 44 | +static const char kfd_dev_name[] = "kfd"; | |
| 45 | + | |
| 46 | +static const struct file_operations kfd_fops = { | |
| 47 | + .owner = THIS_MODULE, | |
| 48 | + .unlocked_ioctl = kfd_ioctl, | |
| 49 | + .compat_ioctl = kfd_ioctl, | |
| 50 | + .open = kfd_open, | |
| 51 | + .mmap = kfd_mmap, | |
| 52 | +}; | |
| 53 | + | |
| 54 | +static int kfd_char_dev_major = -1; | |
| 55 | +static struct class *kfd_class; | |
| 56 | +struct device *kfd_device; | |
| 57 | + | |
| 58 | +int kfd_chardev_init(void) | |
| 59 | +{ | |
| 60 | + int err = 0; | |
| 61 | + | |
| 62 | + kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops); | |
| 63 | + err = kfd_char_dev_major; | |
| 64 | + if (err < 0) | |
| 65 | + goto err_register_chrdev; | |
| 66 | + | |
| 67 | + kfd_class = class_create(THIS_MODULE, kfd_dev_name); | |
| 68 | + err = PTR_ERR(kfd_class); | |
| 69 | + if (IS_ERR(kfd_class)) | |
| 70 | + goto err_class_create; | |
| 71 | + | |
| 72 | + kfd_device = device_create(kfd_class, NULL, | |
| 73 | + MKDEV(kfd_char_dev_major, 0), | |
| 74 | + NULL, kfd_dev_name); | |
| 75 | + err = PTR_ERR(kfd_device); | |
| 76 | + if (IS_ERR(kfd_device)) | |
| 77 | + goto err_device_create; | |
| 78 | + | |
| 79 | + return 0; | |
| 80 | + | |
| 81 | +err_device_create: | |
| 82 | + class_destroy(kfd_class); | |
| 83 | +err_class_create: | |
| 84 | + unregister_chrdev(kfd_char_dev_major, kfd_dev_name); | |
| 85 | +err_register_chrdev: | |
| 86 | + return err; | |
| 87 | +} | |
| 88 | + | |
| 89 | +void kfd_chardev_exit(void) | |
| 90 | +{ | |
| 91 | + device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0)); | |
| 92 | + class_destroy(kfd_class); | |
| 93 | + unregister_chrdev(kfd_char_dev_major, kfd_dev_name); | |
| 94 | +} | |
| 95 | + | |
| 96 | +struct device *kfd_chardev(void) | |
| 97 | +{ | |
| 98 | + return kfd_device; | |
| 99 | +} | |
| 100 | + | |
| 101 | + | |
| 102 | +static int kfd_open(struct inode *inode, struct file *filep) | |
| 103 | +{ | |
| 104 | + struct kfd_process *process; | |
| 105 | + | |
| 106 | + if (iminor(inode) != 0) | |
| 107 | + return -ENODEV; | |
| 108 | + | |
| 109 | + process = kfd_create_process(current); | |
| 110 | + if (IS_ERR(process)) | |
| 111 | + return PTR_ERR(process); | |
| 112 | + | |
| 113 | + process->is_32bit_user_mode = is_compat_task(); | |
| 114 | + | |
| 115 | + dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n", | |
| 116 | + process->pasid, process->is_32bit_user_mode); | |
| 117 | + | |
| 118 | + kfd_init_apertures(process); | |
| 119 | + | |
| 120 | + return 0; | |
| 121 | +} | |
| 122 | + | |
| 123 | +static long kfd_ioctl_get_version(struct file *filep, struct kfd_process *p, | |
| 124 | + void __user *arg) | |
| 125 | +{ | |
| 126 | + struct kfd_ioctl_get_version_args args; | |
| 127 | + int err = 0; | |
| 128 | + | |
| 129 | + args.major_version = KFD_IOCTL_MAJOR_VERSION; | |
| 130 | + args.minor_version = KFD_IOCTL_MINOR_VERSION; | |
| 131 | + | |
| 132 | + if (copy_to_user(arg, &args, sizeof(args))) | |
| 133 | + err = -EFAULT; | |
| 134 | + | |
| 135 | + return err; | |
| 136 | +} | |
| 137 | + | |
| 138 | +static int set_queue_properties_from_user(struct queue_properties *q_properties, | |
| 139 | + struct kfd_ioctl_create_queue_args *args) | |
| 140 | +{ | |
| 141 | + if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { | |
| 142 | + pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); | |
| 143 | + return -EINVAL; | |
| 144 | + } | |
| 145 | + | |
| 146 | + if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { | |
| 147 | + pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); | |
| 148 | + return -EINVAL; | |
| 149 | + } | |
| 150 | + | |
| 151 | + if ((args->ring_base_address) && | |
| 152 | + (!access_ok(VERIFY_WRITE, args->ring_base_address, sizeof(uint64_t)))) { | |
| 153 | + pr_err("kfd: can't access ring base address\n"); | |
| 154 | + return -EFAULT; | |
| 155 | + } | |
| 156 | + | |
| 157 | + if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { | |
| 158 | + pr_err("kfd: ring size must be a power of 2 or 0\n"); | |
| 159 | + return -EINVAL; | |
| 160 | + } | |
| 161 | + | |
| 162 | + if (!access_ok(VERIFY_WRITE, args->read_pointer_address, sizeof(uint32_t))) { | |
| 163 | + pr_err("kfd: can't access read pointer\n"); | |
| 164 | + return -EFAULT; | |
| 165 | + } | |
| 166 | + | |
| 167 | + if (!access_ok(VERIFY_WRITE, args->write_pointer_address, sizeof(uint32_t))) { | |
| 168 | + pr_err("kfd: can't access write pointer\n"); | |
| 169 | + return -EFAULT; | |
| 170 | + } | |
| 171 | + | |
| 172 | + q_properties->is_interop = false; | |
| 173 | + q_properties->queue_percent = args->queue_percentage; | |
| 174 | + q_properties->priority = args->queue_priority; | |
| 175 | + q_properties->queue_address = args->ring_base_address; | |
| 176 | + q_properties->queue_size = args->ring_size; | |
| 177 | + q_properties->read_ptr = (uint32_t *) args->read_pointer_address; | |
| 178 | + q_properties->write_ptr = (uint32_t *) args->write_pointer_address; | |
| 179 | + if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE || | |
| 180 | + args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) | |
| 181 | + q_properties->type = KFD_QUEUE_TYPE_COMPUTE; | |
| 182 | + else | |
| 183 | + return -ENOTSUPP; | |
| 184 | + | |
| 185 | + if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) | |
| 186 | + q_properties->format = KFD_QUEUE_FORMAT_AQL; | |
| 187 | + else | |
| 188 | + q_properties->format = KFD_QUEUE_FORMAT_PM4; | |
| 189 | + | |
| 190 | + pr_debug("Queue Percentage (%d, %d)\n", | |
| 191 | + q_properties->queue_percent, args->queue_percentage); | |
| 192 | + | |
| 193 | + pr_debug("Queue Priority (%d, %d)\n", | |
| 194 | + q_properties->priority, args->queue_priority); | |
| 195 | + | |
| 196 | + pr_debug("Queue Address (0x%llX, 0x%llX)\n", | |
| 197 | + q_properties->queue_address, args->ring_base_address); | |
| 198 | + | |
| 199 | + pr_debug("Queue Size (0x%llX, %u)\n", | |
| 200 | + q_properties->queue_size, args->ring_size); | |
| 201 | + | |
| 202 | + pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n", | |
| 203 | + (uint64_t) q_properties->read_ptr, | |
| 204 | + (uint64_t) q_properties->write_ptr); | |
| 205 | + | |
| 206 | + pr_debug("Queue Format (%d)\n", q_properties->format); | |
| 207 | + | |
| 208 | + return 0; | |
| 209 | +} | |
| 210 | + | |
| 211 | +static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, | |
| 212 | + void __user *arg) | |
| 213 | +{ | |
| 214 | + struct kfd_ioctl_create_queue_args args; | |
| 215 | + struct kfd_dev *dev; | |
| 216 | + int err = 0; | |
| 217 | + unsigned int queue_id; | |
| 218 | + struct kfd_process_device *pdd; | |
| 219 | + struct queue_properties q_properties; | |
| 220 | + | |
| 221 | + memset(&q_properties, 0, sizeof(struct queue_properties)); | |
| 222 | + | |
| 223 | + if (copy_from_user(&args, arg, sizeof(args))) | |
| 224 | + return -EFAULT; | |
| 225 | + | |
| 226 | + pr_debug("kfd: creating queue ioctl\n"); | |
| 227 | + | |
| 228 | + err = set_queue_properties_from_user(&q_properties, &args); | |
| 229 | + if (err) | |
| 230 | + return err; | |
| 231 | + | |
| 232 | + dev = kfd_device_by_id(args.gpu_id); | |
| 233 | + if (dev == NULL) | |
| 234 | + return -EINVAL; | |
| 235 | + | |
| 236 | + mutex_lock(&p->mutex); | |
| 237 | + | |
| 238 | + pdd = kfd_bind_process_to_device(dev, p); | |
| 239 | + if (IS_ERR(pdd) < 0) { | |
| 240 | + err = PTR_ERR(pdd); | |
| 241 | + goto err_bind_process; | |
| 242 | + } | |
| 243 | + | |
| 244 | + pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n", | |
| 245 | + p->pasid, | |
| 246 | + dev->id); | |
| 247 | + | |
| 248 | + err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, 0, | |
| 249 | + KFD_QUEUE_TYPE_COMPUTE, &queue_id); | |
| 250 | + if (err != 0) | |
| 251 | + goto err_create_queue; | |
| 252 | + | |
| 253 | + args.queue_id = queue_id; | |
| 254 | + | |
| 255 | + /* Return gpu_id as doorbell offset for mmap usage */ | |
| 256 | + args.doorbell_offset = args.gpu_id << PAGE_SHIFT; | |
| 257 | + | |
| 258 | + if (copy_to_user(arg, &args, sizeof(args))) { | |
| 259 | + err = -EFAULT; | |
| 260 | + goto err_copy_args_out; | |
| 261 | + } | |
| 262 | + | |
| 263 | + mutex_unlock(&p->mutex); | |
| 264 | + | |
| 265 | + pr_debug("kfd: queue id %d was created successfully\n", args.queue_id); | |
| 266 | + | |
| 267 | + pr_debug("ring buffer address == 0x%016llX\n", | |
| 268 | + args.ring_base_address); | |
| 269 | + | |
| 270 | + pr_debug("read ptr address == 0x%016llX\n", | |
| 271 | + args.read_pointer_address); | |
| 272 | + | |
| 273 | + pr_debug("write ptr address == 0x%016llX\n", | |
| 274 | + args.write_pointer_address); | |
| 275 | + | |
| 276 | + return 0; | |
| 277 | + | |
| 278 | +err_copy_args_out: | |
| 279 | + pqm_destroy_queue(&p->pqm, queue_id); | |
| 280 | +err_create_queue: | |
| 281 | +err_bind_process: | |
| 282 | + mutex_unlock(&p->mutex); | |
| 283 | + return err; | |
| 284 | +} | |
| 285 | + | |
| 286 | +static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, | |
| 287 | + void __user *arg) | |
| 288 | +{ | |
| 289 | + int retval; | |
| 290 | + struct kfd_ioctl_destroy_queue_args args; | |
| 291 | + | |
| 292 | + if (copy_from_user(&args, arg, sizeof(args))) | |
| 293 | + return -EFAULT; | |
| 294 | + | |
| 295 | + pr_debug("kfd: destroying queue id %d for PASID %d\n", | |
| 296 | + args.queue_id, | |
| 297 | + p->pasid); | |
| 298 | + | |
| 299 | + mutex_lock(&p->mutex); | |
| 300 | + | |
| 301 | + retval = pqm_destroy_queue(&p->pqm, args.queue_id); | |
| 302 | + | |
| 303 | + mutex_unlock(&p->mutex); | |
| 304 | + return retval; | |
| 305 | +} | |
| 306 | + | |
| 307 | +static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, | |
| 308 | + void __user *arg) | |
| 309 | +{ | |
| 310 | + int retval; | |
| 311 | + struct kfd_ioctl_update_queue_args args; | |
| 312 | + struct queue_properties properties; | |
| 313 | + | |
| 314 | + if (copy_from_user(&args, arg, sizeof(args))) | |
| 315 | + return -EFAULT; | |
| 316 | + | |
| 317 | + if (args.queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { | |
| 318 | + pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); | |
| 319 | + return -EINVAL; | |
| 320 | + } | |
| 321 | + | |
| 322 | + if (args.queue_priority > KFD_MAX_QUEUE_PRIORITY) { | |
| 323 | + pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); | |
| 324 | + return -EINVAL; | |
| 325 | + } | |
| 326 | + | |
| 327 | + if ((args.ring_base_address) && | |
| 328 | + (!access_ok(VERIFY_WRITE, args.ring_base_address, sizeof(uint64_t)))) { | |
| 329 | + pr_err("kfd: can't access ring base address\n"); | |
| 330 | + return -EFAULT; | |
| 331 | + } | |
| 332 | + | |
| 333 | + if (!is_power_of_2(args.ring_size) && (args.ring_size != 0)) { | |
| 334 | + pr_err("kfd: ring size must be a power of 2 or 0\n"); | |
| 335 | + return -EINVAL; | |
| 336 | + } | |
| 337 | + | |
| 338 | + properties.queue_address = args.ring_base_address; | |
| 339 | + properties.queue_size = args.ring_size; | |
| 340 | + properties.queue_percent = args.queue_percentage; | |
| 341 | + properties.priority = args.queue_priority; | |
| 342 | + | |
| 343 | + pr_debug("kfd: updating queue id %d for PASID %d\n", | |
| 344 | + args.queue_id, p->pasid); | |
| 345 | + | |
| 346 | + mutex_lock(&p->mutex); | |
| 347 | + | |
| 348 | + retval = pqm_update_queue(&p->pqm, args.queue_id, &properties); | |
| 349 | + | |
| 350 | + mutex_unlock(&p->mutex); | |
| 351 | + | |
| 352 | + return retval; | |
| 353 | +} | |
| 354 | + | |
| 355 | +static long kfd_ioctl_set_memory_policy(struct file *filep, | |
| 356 | + struct kfd_process *p, void __user *arg) | |
| 357 | +{ | |
| 358 | + struct kfd_ioctl_set_memory_policy_args args; | |
| 359 | + struct kfd_dev *dev; | |
| 360 | + int err = 0; | |
| 361 | + struct kfd_process_device *pdd; | |
| 362 | + enum cache_policy default_policy, alternate_policy; | |
| 363 | + | |
| 364 | + if (copy_from_user(&args, arg, sizeof(args))) | |
| 365 | + return -EFAULT; | |
| 366 | + | |
| 367 | + if (args.default_policy != KFD_IOC_CACHE_POLICY_COHERENT | |
| 368 | + && args.default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { | |
| 369 | + return -EINVAL; | |
| 370 | + } | |
| 371 | + | |
| 372 | + if (args.alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT | |
| 373 | + && args.alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { | |
| 374 | + return -EINVAL; | |
| 375 | + } | |
| 376 | + | |
| 377 | + dev = kfd_device_by_id(args.gpu_id); | |
| 378 | + if (dev == NULL) | |
| 379 | + return -EINVAL; | |
| 380 | + | |
| 381 | + mutex_lock(&p->mutex); | |
| 382 | + | |
| 383 | + pdd = kfd_bind_process_to_device(dev, p); | |
| 384 | + if (IS_ERR(pdd) < 0) { | |
| 385 | + err = PTR_ERR(pdd); | |
| 386 | + goto out; | |
| 387 | + } | |
| 388 | + | |
| 389 | + default_policy = (args.default_policy == KFD_IOC_CACHE_POLICY_COHERENT) | |
| 390 | + ? cache_policy_coherent : cache_policy_noncoherent; | |
| 391 | + | |
| 392 | + alternate_policy = | |
| 393 | + (args.alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT) | |
| 394 | + ? cache_policy_coherent : cache_policy_noncoherent; | |
| 395 | + | |
| 396 | + if (!dev->dqm->set_cache_memory_policy(dev->dqm, | |
| 397 | + &pdd->qpd, | |
| 398 | + default_policy, | |
| 399 | + alternate_policy, | |
| 400 | + (void __user *)args.alternate_aperture_base, | |
| 401 | + args.alternate_aperture_size)) | |
| 402 | + err = -EINVAL; | |
| 403 | + | |
| 404 | +out: | |
| 405 | + mutex_unlock(&p->mutex); | |
| 406 | + | |
| 407 | + return err; | |
| 408 | +} | |
| 409 | + | |
| 410 | +static long kfd_ioctl_get_clock_counters(struct file *filep, | |
| 411 | + struct kfd_process *p, void __user *arg) | |
| 412 | +{ | |
| 413 | + struct kfd_ioctl_get_clock_counters_args args; | |
| 414 | + struct kfd_dev *dev; | |
| 415 | + struct timespec time; | |
| 416 | + | |
| 417 | + if (copy_from_user(&args, arg, sizeof(args))) | |
| 418 | + return -EFAULT; | |
| 419 | + | |
| 420 | + dev = kfd_device_by_id(args.gpu_id); | |
| 421 | + if (dev == NULL) | |
| 422 | + return -EINVAL; | |
| 423 | + | |
| 424 | + /* Reading GPU clock counter from KGD */ | |
| 425 | + args.gpu_clock_counter = kfd2kgd->get_gpu_clock_counter(dev->kgd); | |
| 426 | + | |
| 427 | + /* No access to rdtsc. Using raw monotonic time */ | |
| 428 | + getrawmonotonic(&time); | |
| 429 | + args.cpu_clock_counter = (uint64_t)timespec_to_ns(&time); | |
| 430 | + | |
| 431 | + get_monotonic_boottime(&time); | |
| 432 | + args.system_clock_counter = (uint64_t)timespec_to_ns(&time); | |
| 433 | + | |
| 434 | + /* Since the counter is in nano-seconds we use 1GHz frequency */ | |
| 435 | + args.system_clock_freq = 1000000000; | |
| 436 | + | |
| 437 | + if (copy_to_user(arg, &args, sizeof(args))) | |
| 438 | + return -EFAULT; | |
| 439 | + | |
| 440 | + return 0; | |
| 441 | +} | |
| 442 | + | |
| 443 | + | |
| 444 | +static int kfd_ioctl_get_process_apertures(struct file *filp, | |
| 445 | + struct kfd_process *p, void __user *arg) | |
| 446 | +{ | |
| 447 | + struct kfd_ioctl_get_process_apertures_args args; | |
| 448 | + struct kfd_process_device_apertures *pAperture; | |
| 449 | + struct kfd_process_device *pdd; | |
| 450 | + | |
| 451 | + dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid); | |
| 452 | + | |
| 453 | + if (copy_from_user(&args, arg, sizeof(args))) | |
| 454 | + return -EFAULT; | |
| 455 | + | |
| 456 | + args.num_of_nodes = 0; | |
| 457 | + | |
| 458 | + mutex_lock(&p->mutex); | |
| 459 | + | |
| 460 | + /*if the process-device list isn't empty*/ | |
| 461 | + if (kfd_has_process_device_data(p)) { | |
| 462 | + /* Run over all pdd of the process */ | |
| 463 | + pdd = kfd_get_first_process_device_data(p); | |
| 464 | + do { | |
| 465 | + pAperture = &args.process_apertures[args.num_of_nodes]; | |
| 466 | + pAperture->gpu_id = pdd->dev->id; | |
| 467 | + pAperture->lds_base = pdd->lds_base; | |
| 468 | + pAperture->lds_limit = pdd->lds_limit; | |
| 469 | + pAperture->gpuvm_base = pdd->gpuvm_base; | |
| 470 | + pAperture->gpuvm_limit = pdd->gpuvm_limit; | |
| 471 | + pAperture->scratch_base = pdd->scratch_base; | |
| 472 | + pAperture->scratch_limit = pdd->scratch_limit; | |
| 473 | + | |
| 474 | + dev_dbg(kfd_device, | |
| 475 | + "node id %u\n", args.num_of_nodes); | |
| 476 | + dev_dbg(kfd_device, | |
| 477 | + "gpu id %u\n", pdd->dev->id); | |
| 478 | + dev_dbg(kfd_device, | |
| 479 | + "lds_base %llX\n", pdd->lds_base); | |
| 480 | + dev_dbg(kfd_device, | |
| 481 | + "lds_limit %llX\n", pdd->lds_limit); | |
| 482 | + dev_dbg(kfd_device, | |
| 483 | + "gpuvm_base %llX\n", pdd->gpuvm_base); | |
| 484 | + dev_dbg(kfd_device, | |
| 485 | + "gpuvm_limit %llX\n", pdd->gpuvm_limit); | |
| 486 | + dev_dbg(kfd_device, | |
| 487 | + "scratch_base %llX\n", pdd->scratch_base); | |
| 488 | + dev_dbg(kfd_device, | |
| 489 | + "scratch_limit %llX\n", pdd->scratch_limit); | |
| 490 | + | |
| 491 | + args.num_of_nodes++; | |
| 492 | + } while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL && | |
| 493 | + (args.num_of_nodes < NUM_OF_SUPPORTED_GPUS)); | |
| 494 | + } | |
| 495 | + | |
| 496 | + mutex_unlock(&p->mutex); | |
| 497 | + | |
| 498 | + if (copy_to_user(arg, &args, sizeof(args))) | |
| 499 | + return -EFAULT; | |
| 500 | + | |
| 501 | + return 0; | |
| 502 | +} | |
| 503 | + | |
| 504 | +static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) | |
| 505 | +{ | |
| 506 | + struct kfd_process *process; | |
| 507 | + long err = -EINVAL; | |
| 508 | + | |
| 509 | + dev_dbg(kfd_device, | |
| 510 | + "ioctl cmd 0x%x (#%d), arg 0x%lx\n", | |
| 511 | + cmd, _IOC_NR(cmd), arg); | |
| 512 | + | |
| 513 | + process = kfd_get_process(current); | |
| 514 | + if (IS_ERR(process)) | |
| 515 | + return PTR_ERR(process); | |
| 516 | + | |
| 517 | + switch (cmd) { | |
| 518 | + case KFD_IOC_GET_VERSION: | |
| 519 | + err = kfd_ioctl_get_version(filep, process, (void __user *)arg); | |
| 520 | + break; | |
| 521 | + case KFD_IOC_CREATE_QUEUE: | |
| 522 | + err = kfd_ioctl_create_queue(filep, process, | |
| 523 | + (void __user *)arg); | |
| 524 | + break; | |
| 525 | + | |
| 526 | + case KFD_IOC_DESTROY_QUEUE: | |
| 527 | + err = kfd_ioctl_destroy_queue(filep, process, | |
| 528 | + (void __user *)arg); | |
| 529 | + break; | |
| 530 | + | |
| 531 | + case KFD_IOC_SET_MEMORY_POLICY: | |
| 532 | + err = kfd_ioctl_set_memory_policy(filep, process, | |
| 533 | + (void __user *)arg); | |
| 534 | + break; | |
| 535 | + | |
| 536 | + case KFD_IOC_GET_CLOCK_COUNTERS: | |
| 537 | + err = kfd_ioctl_get_clock_counters(filep, process, | |
| 538 | + (void __user *)arg); | |
| 539 | + break; | |
| 540 | + | |
| 541 | + case KFD_IOC_GET_PROCESS_APERTURES: | |
| 542 | + err = kfd_ioctl_get_process_apertures(filep, process, | |
| 543 | + (void __user *)arg); | |
| 544 | + break; | |
| 545 | + | |
| 546 | + case KFD_IOC_UPDATE_QUEUE: | |
| 547 | + err = kfd_ioctl_update_queue(filep, process, | |
| 548 | + (void __user *)arg); | |
| 549 | + break; | |
| 550 | + | |
| 551 | + default: | |
| 552 | + dev_err(kfd_device, | |
| 553 | + "unknown ioctl cmd 0x%x, arg 0x%lx)\n", | |
| 554 | + cmd, arg); | |
| 555 | + err = -EINVAL; | |
| 556 | + break; | |
| 557 | + } | |
| 558 | + | |
| 559 | + if (err < 0) | |
| 560 | + dev_err(kfd_device, | |
| 561 | + "ioctl error %ld for ioctl cmd 0x%x (#%d)\n", | |
| 562 | + err, cmd, _IOC_NR(cmd)); | |
| 563 | + | |
| 564 | + return err; | |
| 565 | +} | |
| 566 | + | |
| 567 | +static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) | |
| 568 | +{ | |
| 569 | + struct kfd_process *process; | |
| 570 | + | |
| 571 | + process = kfd_get_process(current); | |
| 572 | + if (IS_ERR(process)) | |
| 573 | + return PTR_ERR(process); | |
| 574 | + | |
| 575 | + return kfd_doorbell_mmap(process, vma); | |
| 576 | +} | |
| @@ -0,0 +1,294 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +#ifndef KFD_CRAT_H_INCLUDED | |
| 24 | +#define KFD_CRAT_H_INCLUDED | |
| 25 | + | |
| 26 | +#include <linux/types.h> | |
| 27 | + | |
| 28 | +#pragma pack(1) | |
| 29 | + | |
| 30 | +/* | |
| 31 | + * 4CC signature values for the CRAT and CDIT ACPI tables | |
| 32 | + */ | |
| 33 | + | |
| 34 | +#define CRAT_SIGNATURE "CRAT" | |
| 35 | +#define CDIT_SIGNATURE "CDIT" | |
| 36 | + | |
| 37 | +/* | |
| 38 | + * Component Resource Association Table (CRAT) | |
| 39 | + */ | |
| 40 | + | |
| 41 | +#define CRAT_OEMID_LENGTH 6 | |
| 42 | +#define CRAT_OEMTABLEID_LENGTH 8 | |
| 43 | +#define CRAT_RESERVED_LENGTH 6 | |
| 44 | + | |
| 45 | +#define CRAT_OEMID_64BIT_MASK ((1ULL << (CRAT_OEMID_LENGTH * 8)) - 1) | |
| 46 | + | |
| 47 | +struct crat_header { | |
| 48 | + uint32_t signature; | |
| 49 | + uint32_t length; | |
| 50 | + uint8_t revision; | |
| 51 | + uint8_t checksum; | |
| 52 | + uint8_t oem_id[CRAT_OEMID_LENGTH]; | |
| 53 | + uint8_t oem_table_id[CRAT_OEMTABLEID_LENGTH]; | |
| 54 | + uint32_t oem_revision; | |
| 55 | + uint32_t creator_id; | |
| 56 | + uint32_t creator_revision; | |
| 57 | + uint32_t total_entries; | |
| 58 | + uint16_t num_domains; | |
| 59 | + uint8_t reserved[CRAT_RESERVED_LENGTH]; | |
| 60 | +}; | |
| 61 | + | |
| 62 | +/* | |
| 63 | + * The header structure is immediately followed by total_entries of the | |
| 64 | + * data definitions | |
| 65 | + */ | |
| 66 | + | |
| 67 | +/* | |
| 68 | + * The currently defined subtype entries in the CRAT | |
| 69 | + */ | |
| 70 | +#define CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY 0 | |
| 71 | +#define CRAT_SUBTYPE_MEMORY_AFFINITY 1 | |
| 72 | +#define CRAT_SUBTYPE_CACHE_AFFINITY 2 | |
| 73 | +#define CRAT_SUBTYPE_TLB_AFFINITY 3 | |
| 74 | +#define CRAT_SUBTYPE_CCOMPUTE_AFFINITY 4 | |
| 75 | +#define CRAT_SUBTYPE_IOLINK_AFFINITY 5 | |
| 76 | +#define CRAT_SUBTYPE_MAX 6 | |
| 77 | + | |
| 78 | +#define CRAT_SIBLINGMAP_SIZE 32 | |
| 79 | + | |
| 80 | +/* | |
| 81 | + * ComputeUnit Affinity structure and definitions | |
| 82 | + */ | |
| 83 | +#define CRAT_CU_FLAGS_ENABLED 0x00000001 | |
| 84 | +#define CRAT_CU_FLAGS_HOT_PLUGGABLE 0x00000002 | |
| 85 | +#define CRAT_CU_FLAGS_CPU_PRESENT 0x00000004 | |
| 86 | +#define CRAT_CU_FLAGS_GPU_PRESENT 0x00000008 | |
| 87 | +#define CRAT_CU_FLAGS_IOMMU_PRESENT 0x00000010 | |
| 88 | +#define CRAT_CU_FLAGS_RESERVED 0xffffffe0 | |
| 89 | + | |
| 90 | +#define CRAT_COMPUTEUNIT_RESERVED_LENGTH 4 | |
| 91 | + | |
| 92 | +struct crat_subtype_computeunit { | |
| 93 | + uint8_t type; | |
| 94 | + uint8_t length; | |
| 95 | + uint16_t reserved; | |
| 96 | + uint32_t flags; | |
| 97 | + uint32_t proximity_domain; | |
| 98 | + uint32_t processor_id_low; | |
| 99 | + uint16_t num_cpu_cores; | |
| 100 | + uint16_t num_simd_cores; | |
| 101 | + uint16_t max_waves_simd; | |
| 102 | + uint16_t io_count; | |
| 103 | + uint16_t hsa_capability; | |
| 104 | + uint16_t lds_size_in_kb; | |
| 105 | + uint8_t wave_front_size; | |
| 106 | + uint8_t num_banks; | |
| 107 | + uint16_t micro_engine_id; | |
| 108 | + uint8_t num_arrays; | |
| 109 | + uint8_t num_cu_per_array; | |
| 110 | + uint8_t num_simd_per_cu; | |
| 111 | + uint8_t max_slots_scatch_cu; | |
| 112 | + uint8_t reserved2[CRAT_COMPUTEUNIT_RESERVED_LENGTH]; | |
| 113 | +}; | |
| 114 | + | |
| 115 | +/* | |
| 116 | + * HSA Memory Affinity structure and definitions | |
| 117 | + */ | |
| 118 | +#define CRAT_MEM_FLAGS_ENABLED 0x00000001 | |
| 119 | +#define CRAT_MEM_FLAGS_HOT_PLUGGABLE 0x00000002 | |
| 120 | +#define CRAT_MEM_FLAGS_NON_VOLATILE 0x00000004 | |
| 121 | +#define CRAT_MEM_FLAGS_RESERVED 0xfffffff8 | |
| 122 | + | |
| 123 | +#define CRAT_MEMORY_RESERVED_LENGTH 8 | |
| 124 | + | |
| 125 | +struct crat_subtype_memory { | |
| 126 | + uint8_t type; | |
| 127 | + uint8_t length; | |
| 128 | + uint16_t reserved; | |
| 129 | + uint32_t flags; | |
| 130 | + uint32_t promixity_domain; | |
| 131 | + uint32_t base_addr_low; | |
| 132 | + uint32_t base_addr_high; | |
| 133 | + uint32_t length_low; | |
| 134 | + uint32_t length_high; | |
| 135 | + uint32_t width; | |
| 136 | + uint8_t reserved2[CRAT_MEMORY_RESERVED_LENGTH]; | |
| 137 | +}; | |
| 138 | + | |
| 139 | +/* | |
| 140 | + * HSA Cache Affinity structure and definitions | |
| 141 | + */ | |
| 142 | +#define CRAT_CACHE_FLAGS_ENABLED 0x00000001 | |
| 143 | +#define CRAT_CACHE_FLAGS_DATA_CACHE 0x00000002 | |
| 144 | +#define CRAT_CACHE_FLAGS_INST_CACHE 0x00000004 | |
| 145 | +#define CRAT_CACHE_FLAGS_CPU_CACHE 0x00000008 | |
| 146 | +#define CRAT_CACHE_FLAGS_SIMD_CACHE 0x00000010 | |
| 147 | +#define CRAT_CACHE_FLAGS_RESERVED 0xffffffe0 | |
| 148 | + | |
| 149 | +#define CRAT_CACHE_RESERVED_LENGTH 8 | |
| 150 | + | |
| 151 | +struct crat_subtype_cache { | |
| 152 | + uint8_t type; | |
| 153 | + uint8_t length; | |
| 154 | + uint16_t reserved; | |
| 155 | + uint32_t flags; | |
| 156 | + uint32_t processor_id_low; | |
| 157 | + uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE]; | |
| 158 | + uint32_t cache_size; | |
| 159 | + uint8_t cache_level; | |
| 160 | + uint8_t lines_per_tag; | |
| 161 | + uint16_t cache_line_size; | |
| 162 | + uint8_t associativity; | |
| 163 | + uint8_t cache_properties; | |
| 164 | + uint16_t cache_latency; | |
| 165 | + uint8_t reserved2[CRAT_CACHE_RESERVED_LENGTH]; | |
| 166 | +}; | |
| 167 | + | |
| 168 | +/* | |
| 169 | + * HSA TLB Affinity structure and definitions | |
| 170 | + */ | |
| 171 | +#define CRAT_TLB_FLAGS_ENABLED 0x00000001 | |
| 172 | +#define CRAT_TLB_FLAGS_DATA_TLB 0x00000002 | |
| 173 | +#define CRAT_TLB_FLAGS_INST_TLB 0x00000004 | |
| 174 | +#define CRAT_TLB_FLAGS_CPU_TLB 0x00000008 | |
| 175 | +#define CRAT_TLB_FLAGS_SIMD_TLB 0x00000010 | |
| 176 | +#define CRAT_TLB_FLAGS_RESERVED 0xffffffe0 | |
| 177 | + | |
| 178 | +#define CRAT_TLB_RESERVED_LENGTH 4 | |
| 179 | + | |
| 180 | +struct crat_subtype_tlb { | |
| 181 | + uint8_t type; | |
| 182 | + uint8_t length; | |
| 183 | + uint16_t reserved; | |
| 184 | + uint32_t flags; | |
| 185 | + uint32_t processor_id_low; | |
| 186 | + uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE]; | |
| 187 | + uint32_t tlb_level; | |
| 188 | + uint8_t data_tlb_associativity_2mb; | |
| 189 | + uint8_t data_tlb_size_2mb; | |
| 190 | + uint8_t instruction_tlb_associativity_2mb; | |
| 191 | + uint8_t instruction_tlb_size_2mb; | |
| 192 | + uint8_t data_tlb_associativity_4k; | |
| 193 | + uint8_t data_tlb_size_4k; | |
| 194 | + uint8_t instruction_tlb_associativity_4k; | |
| 195 | + uint8_t instruction_tlb_size_4k; | |
| 196 | + uint8_t data_tlb_associativity_1gb; | |
| 197 | + uint8_t data_tlb_size_1gb; | |
| 198 | + uint8_t instruction_tlb_associativity_1gb; | |
| 199 | + uint8_t instruction_tlb_size_1gb; | |
| 200 | + uint8_t reserved2[CRAT_TLB_RESERVED_LENGTH]; | |
| 201 | +}; | |
| 202 | + | |
| 203 | +/* | |
| 204 | + * HSA CCompute/APU Affinity structure and definitions | |
| 205 | + */ | |
| 206 | +#define CRAT_CCOMPUTE_FLAGS_ENABLED 0x00000001 | |
| 207 | +#define CRAT_CCOMPUTE_FLAGS_RESERVED 0xfffffffe | |
| 208 | + | |
| 209 | +#define CRAT_CCOMPUTE_RESERVED_LENGTH 16 | |
| 210 | + | |
| 211 | +struct crat_subtype_ccompute { | |
| 212 | + uint8_t type; | |
| 213 | + uint8_t length; | |
| 214 | + uint16_t reserved; | |
| 215 | + uint32_t flags; | |
| 216 | + uint32_t processor_id_low; | |
| 217 | + uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE]; | |
| 218 | + uint32_t apu_size; | |
| 219 | + uint8_t reserved2[CRAT_CCOMPUTE_RESERVED_LENGTH]; | |
| 220 | +}; | |
| 221 | + | |
| 222 | +/* | |
| 223 | + * HSA IO Link Affinity structure and definitions | |
| 224 | + */ | |
| 225 | +#define CRAT_IOLINK_FLAGS_ENABLED 0x00000001 | |
| 226 | +#define CRAT_IOLINK_FLAGS_COHERENCY 0x00000002 | |
| 227 | +#define CRAT_IOLINK_FLAGS_RESERVED 0xfffffffc | |
| 228 | + | |
| 229 | +/* | |
| 230 | + * IO interface types | |
| 231 | + */ | |
| 232 | +#define CRAT_IOLINK_TYPE_UNDEFINED 0 | |
| 233 | +#define CRAT_IOLINK_TYPE_HYPERTRANSPORT 1 | |
| 234 | +#define CRAT_IOLINK_TYPE_PCIEXPRESS 2 | |
| 235 | +#define CRAT_IOLINK_TYPE_OTHER 3 | |
| 236 | +#define CRAT_IOLINK_TYPE_MAX 255 | |
| 237 | + | |
| 238 | +#define CRAT_IOLINK_RESERVED_LENGTH 24 | |
| 239 | + | |
| 240 | +struct crat_subtype_iolink { | |
| 241 | + uint8_t type; | |
| 242 | + uint8_t length; | |
| 243 | + uint16_t reserved; | |
| 244 | + uint32_t flags; | |
| 245 | + uint32_t proximity_domain_from; | |
| 246 | + uint32_t proximity_domain_to; | |
| 247 | + uint8_t io_interface_type; | |
| 248 | + uint8_t version_major; | |
| 249 | + uint16_t version_minor; | |
| 250 | + uint32_t minimum_latency; | |
| 251 | + uint32_t maximum_latency; | |
| 252 | + uint32_t minimum_bandwidth_mbs; | |
| 253 | + uint32_t maximum_bandwidth_mbs; | |
| 254 | + uint32_t recommended_transfer_size; | |
| 255 | + uint8_t reserved2[CRAT_IOLINK_RESERVED_LENGTH]; | |
| 256 | +}; | |
| 257 | + | |
| 258 | +/* | |
| 259 | + * HSA generic sub-type header | |
| 260 | + */ | |
| 261 | + | |
| 262 | +#define CRAT_SUBTYPE_FLAGS_ENABLED 0x00000001 | |
| 263 | + | |
| 264 | +struct crat_subtype_generic { | |
| 265 | + uint8_t type; | |
| 266 | + uint8_t length; | |
| 267 | + uint16_t reserved; | |
| 268 | + uint32_t flags; | |
| 269 | +}; | |
| 270 | + | |
| 271 | +/* | |
| 272 | + * Component Locality Distance Information Table (CDIT) | |
| 273 | + */ | |
| 274 | +#define CDIT_OEMID_LENGTH 6 | |
| 275 | +#define CDIT_OEMTABLEID_LENGTH 8 | |
| 276 | + | |
| 277 | +struct cdit_header { | |
| 278 | + uint32_t signature; | |
| 279 | + uint32_t length; | |
| 280 | + uint8_t revision; | |
| 281 | + uint8_t checksum; | |
| 282 | + uint8_t oem_id[CDIT_OEMID_LENGTH]; | |
| 283 | + uint8_t oem_table_id[CDIT_OEMTABLEID_LENGTH]; | |
| 284 | + uint32_t oem_revision; | |
| 285 | + uint32_t creator_id; | |
| 286 | + uint32_t creator_revision; | |
| 287 | + uint32_t total_entries; | |
| 288 | + uint16_t num_domains; | |
| 289 | + uint8_t entry[1]; | |
| 290 | +}; | |
| 291 | + | |
| 292 | +#pragma pack() | |
| 293 | + | |
| 294 | +#endif /* KFD_CRAT_H_INCLUDED */ | |
| @@ -0,0 +1,307 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +#include <linux/amd-iommu.h> | |
| 24 | +#include <linux/bsearch.h> | |
| 25 | +#include <linux/pci.h> | |
| 26 | +#include <linux/slab.h> | |
| 27 | +#include "kfd_priv.h" | |
| 28 | +#include "kfd_device_queue_manager.h" | |
| 29 | + | |
| 30 | +#define MQD_SIZE_ALIGNED 768 | |
| 31 | + | |
| 32 | +static const struct kfd_device_info kaveri_device_info = { | |
| 33 | + .max_pasid_bits = 16, | |
| 34 | + .ih_ring_entry_size = 4 * sizeof(uint32_t), | |
| 35 | + .mqd_size_aligned = MQD_SIZE_ALIGNED | |
| 36 | +}; | |
| 37 | + | |
| 38 | +struct kfd_deviceid { | |
| 39 | + unsigned short did; | |
| 40 | + const struct kfd_device_info *device_info; | |
| 41 | +}; | |
| 42 | + | |
| 43 | +/* Please keep this sorted by increasing device id. */ | |
| 44 | +static const struct kfd_deviceid supported_devices[] = { | |
| 45 | + { 0x1304, &kaveri_device_info }, /* Kaveri */ | |
| 46 | + { 0x1305, &kaveri_device_info }, /* Kaveri */ | |
| 47 | + { 0x1306, &kaveri_device_info }, /* Kaveri */ | |
| 48 | + { 0x1307, &kaveri_device_info }, /* Kaveri */ | |
| 49 | + { 0x1309, &kaveri_device_info }, /* Kaveri */ | |
| 50 | + { 0x130A, &kaveri_device_info }, /* Kaveri */ | |
| 51 | + { 0x130B, &kaveri_device_info }, /* Kaveri */ | |
| 52 | + { 0x130C, &kaveri_device_info }, /* Kaveri */ | |
| 53 | + { 0x130D, &kaveri_device_info }, /* Kaveri */ | |
| 54 | + { 0x130E, &kaveri_device_info }, /* Kaveri */ | |
| 55 | + { 0x130F, &kaveri_device_info }, /* Kaveri */ | |
| 56 | + { 0x1310, &kaveri_device_info }, /* Kaveri */ | |
| 57 | + { 0x1311, &kaveri_device_info }, /* Kaveri */ | |
| 58 | + { 0x1312, &kaveri_device_info }, /* Kaveri */ | |
| 59 | + { 0x1313, &kaveri_device_info }, /* Kaveri */ | |
| 60 | + { 0x1315, &kaveri_device_info }, /* Kaveri */ | |
| 61 | + { 0x1316, &kaveri_device_info }, /* Kaveri */ | |
| 62 | + { 0x1317, &kaveri_device_info }, /* Kaveri */ | |
| 63 | + { 0x1318, &kaveri_device_info }, /* Kaveri */ | |
| 64 | + { 0x131B, &kaveri_device_info }, /* Kaveri */ | |
| 65 | + { 0x131C, &kaveri_device_info }, /* Kaveri */ | |
| 66 | + { 0x131D, &kaveri_device_info }, /* Kaveri */ | |
| 67 | +}; | |
| 68 | + | |
| 69 | +static const struct kfd_device_info *lookup_device_info(unsigned short did) | |
| 70 | +{ | |
| 71 | + size_t i; | |
| 72 | + | |
| 73 | + for (i = 0; i < ARRAY_SIZE(supported_devices); i++) { | |
| 74 | + if (supported_devices[i].did == did) { | |
| 75 | + BUG_ON(supported_devices[i].device_info == NULL); | |
| 76 | + return supported_devices[i].device_info; | |
| 77 | + } | |
| 78 | + } | |
| 79 | + | |
| 80 | + return NULL; | |
| 81 | +} | |
| 82 | + | |
| 83 | +struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev) | |
| 84 | +{ | |
| 85 | + struct kfd_dev *kfd; | |
| 86 | + | |
| 87 | + const struct kfd_device_info *device_info = | |
| 88 | + lookup_device_info(pdev->device); | |
| 89 | + | |
| 90 | + if (!device_info) | |
| 91 | + return NULL; | |
| 92 | + | |
| 93 | + kfd = kzalloc(sizeof(*kfd), GFP_KERNEL); | |
| 94 | + if (!kfd) | |
| 95 | + return NULL; | |
| 96 | + | |
| 97 | + kfd->kgd = kgd; | |
| 98 | + kfd->device_info = device_info; | |
| 99 | + kfd->pdev = pdev; | |
| 100 | + kfd->init_complete = false; | |
| 101 | + | |
| 102 | + return kfd; | |
| 103 | +} | |
| 104 | + | |
| 105 | +static bool device_iommu_pasid_init(struct kfd_dev *kfd) | |
| 106 | +{ | |
| 107 | + const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP | | |
| 108 | + AMD_IOMMU_DEVICE_FLAG_PRI_SUP | | |
| 109 | + AMD_IOMMU_DEVICE_FLAG_PASID_SUP; | |
| 110 | + | |
| 111 | + struct amd_iommu_device_info iommu_info; | |
| 112 | + unsigned int pasid_limit; | |
| 113 | + int err; | |
| 114 | + | |
| 115 | + err = amd_iommu_device_info(kfd->pdev, &iommu_info); | |
| 116 | + if (err < 0) { | |
| 117 | + dev_err(kfd_device, | |
| 118 | + "error getting iommu info. is the iommu enabled?\n"); | |
| 119 | + return false; | |
| 120 | + } | |
| 121 | + | |
| 122 | + if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) { | |
| 123 | + dev_err(kfd_device, "error required iommu flags ats(%i), pri(%i), pasid(%i)\n", | |
| 124 | + (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0, | |
| 125 | + (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0, | |
| 126 | + (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) != 0); | |
| 127 | + return false; | |
| 128 | + } | |
| 129 | + | |
| 130 | + pasid_limit = min_t(unsigned int, | |
| 131 | + (unsigned int)1 << kfd->device_info->max_pasid_bits, | |
| 132 | + iommu_info.max_pasids); | |
| 133 | + /* | |
| 134 | + * last pasid is used for kernel queues doorbells | |
| 135 | + * in the future the last pasid might be used for a kernel thread. | |
| 136 | + */ | |
| 137 | + pasid_limit = min_t(unsigned int, | |
| 138 | + pasid_limit, | |
| 139 | + kfd->doorbell_process_limit - 1); | |
| 140 | + | |
| 141 | + err = amd_iommu_init_device(kfd->pdev, pasid_limit); | |
| 142 | + if (err < 0) { | |
| 143 | + dev_err(kfd_device, "error initializing iommu device\n"); | |
| 144 | + return false; | |
| 145 | + } | |
| 146 | + | |
| 147 | + if (!kfd_set_pasid_limit(pasid_limit)) { | |
| 148 | + dev_err(kfd_device, "error setting pasid limit\n"); | |
| 149 | + amd_iommu_free_device(kfd->pdev); | |
| 150 | + return false; | |
| 151 | + } | |
| 152 | + | |
| 153 | + return true; | |
| 154 | +} | |
| 155 | + | |
| 156 | +static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int pasid) | |
| 157 | +{ | |
| 158 | + struct kfd_dev *dev = kfd_device_by_pci_dev(pdev); | |
| 159 | + | |
| 160 | + if (dev) | |
| 161 | + kfd_unbind_process_from_device(dev, pasid); | |
| 162 | +} | |
| 163 | + | |
| 164 | +bool kgd2kfd_device_init(struct kfd_dev *kfd, | |
| 165 | + const struct kgd2kfd_shared_resources *gpu_resources) | |
| 166 | +{ | |
| 167 | + unsigned int size; | |
| 168 | + | |
| 169 | + kfd->shared_resources = *gpu_resources; | |
| 170 | + | |
| 171 | + /* calculate max size of mqds needed for queues */ | |
| 172 | + size = max_num_of_processes * | |
| 173 | + max_num_of_queues_per_process * | |
| 174 | + kfd->device_info->mqd_size_aligned; | |
| 175 | + | |
| 176 | + /* add another 512KB for all other allocations on gart */ | |
| 177 | + size += 512 * 1024; | |
| 178 | + | |
| 179 | + if (kfd2kgd->init_sa_manager(kfd->kgd, size)) { | |
| 180 | + dev_err(kfd_device, | |
| 181 | + "Error initializing sa manager for device (%x:%x)\n", | |
| 182 | + kfd->pdev->vendor, kfd->pdev->device); | |
| 183 | + goto out; | |
| 184 | + } | |
| 185 | + | |
| 186 | + kfd_doorbell_init(kfd); | |
| 187 | + | |
| 188 | + if (kfd_topology_add_device(kfd) != 0) { | |
| 189 | + dev_err(kfd_device, | |
| 190 | + "Error adding device (%x:%x) to topology\n", | |
| 191 | + kfd->pdev->vendor, kfd->pdev->device); | |
| 192 | + goto kfd_topology_add_device_error; | |
| 193 | + } | |
| 194 | + | |
| 195 | + if (kfd_interrupt_init(kfd)) { | |
| 196 | + dev_err(kfd_device, | |
| 197 | + "Error initializing interrupts for device (%x:%x)\n", | |
| 198 | + kfd->pdev->vendor, kfd->pdev->device); | |
| 199 | + goto kfd_interrupt_error; | |
| 200 | + } | |
| 201 | + | |
| 202 | + if (!device_iommu_pasid_init(kfd)) { | |
| 203 | + dev_err(kfd_device, | |
| 204 | + "Error initializing iommuv2 for device (%x:%x)\n", | |
| 205 | + kfd->pdev->vendor, kfd->pdev->device); | |
| 206 | + goto device_iommu_pasid_error; | |
| 207 | + } | |
| 208 | + amd_iommu_set_invalidate_ctx_cb(kfd->pdev, | |
| 209 | + iommu_pasid_shutdown_callback); | |
| 210 | + | |
| 211 | + kfd->dqm = device_queue_manager_init(kfd); | |
| 212 | + if (!kfd->dqm) { | |
| 213 | + dev_err(kfd_device, | |
| 214 | + "Error initializing queue manager for device (%x:%x)\n", | |
| 215 | + kfd->pdev->vendor, kfd->pdev->device); | |
| 216 | + goto device_queue_manager_error; | |
| 217 | + } | |
| 218 | + | |
| 219 | + if (kfd->dqm->start(kfd->dqm) != 0) { | |
| 220 | + dev_err(kfd_device, | |
| 221 | + "Error starting queuen manager for device (%x:%x)\n", | |
| 222 | + kfd->pdev->vendor, kfd->pdev->device); | |
| 223 | + goto dqm_start_error; | |
| 224 | + } | |
| 225 | + | |
| 226 | + kfd->init_complete = true; | |
| 227 | + dev_info(kfd_device, "added device (%x:%x)\n", kfd->pdev->vendor, | |
| 228 | + kfd->pdev->device); | |
| 229 | + | |
| 230 | + pr_debug("kfd: Starting kfd with the following scheduling policy %d\n", | |
| 231 | + sched_policy); | |
| 232 | + | |
| 233 | + goto out; | |
| 234 | + | |
| 235 | +dqm_start_error: | |
| 236 | + device_queue_manager_uninit(kfd->dqm); | |
| 237 | +device_queue_manager_error: | |
| 238 | + amd_iommu_free_device(kfd->pdev); | |
| 239 | +device_iommu_pasid_error: | |
| 240 | + kfd_interrupt_exit(kfd); | |
| 241 | +kfd_interrupt_error: | |
| 242 | + kfd_topology_remove_device(kfd); | |
| 243 | +kfd_topology_add_device_error: | |
| 244 | + kfd2kgd->fini_sa_manager(kfd->kgd); | |
| 245 | + dev_err(kfd_device, | |
| 246 | + "device (%x:%x) NOT added due to errors\n", | |
| 247 | + kfd->pdev->vendor, kfd->pdev->device); | |
| 248 | +out: | |
| 249 | + return kfd->init_complete; | |
| 250 | +} | |
| 251 | + | |
| 252 | +void kgd2kfd_device_exit(struct kfd_dev *kfd) | |
| 253 | +{ | |
| 254 | + if (kfd->init_complete) { | |
| 255 | + device_queue_manager_uninit(kfd->dqm); | |
| 256 | + amd_iommu_free_device(kfd->pdev); | |
| 257 | + kfd_interrupt_exit(kfd); | |
| 258 | + kfd_topology_remove_device(kfd); | |
| 259 | + } | |
| 260 | + | |
| 261 | + kfree(kfd); | |
| 262 | +} | |
| 263 | + | |
| 264 | +void kgd2kfd_suspend(struct kfd_dev *kfd) | |
| 265 | +{ | |
| 266 | + BUG_ON(kfd == NULL); | |
| 267 | + | |
| 268 | + if (kfd->init_complete) { | |
| 269 | + kfd->dqm->stop(kfd->dqm); | |
| 270 | + amd_iommu_free_device(kfd->pdev); | |
| 271 | + } | |
| 272 | +} | |
| 273 | + | |
| 274 | +int kgd2kfd_resume(struct kfd_dev *kfd) | |
| 275 | +{ | |
| 276 | + unsigned int pasid_limit; | |
| 277 | + int err; | |
| 278 | + | |
| 279 | + BUG_ON(kfd == NULL); | |
| 280 | + | |
| 281 | + pasid_limit = kfd_get_pasid_limit(); | |
| 282 | + | |
| 283 | + if (kfd->init_complete) { | |
| 284 | + err = amd_iommu_init_device(kfd->pdev, pasid_limit); | |
| 285 | + if (err < 0) | |
| 286 | + return -ENXIO; | |
| 287 | + amd_iommu_set_invalidate_ctx_cb(kfd->pdev, | |
| 288 | + iommu_pasid_shutdown_callback); | |
| 289 | + kfd->dqm->start(kfd->dqm); | |
| 290 | + } | |
| 291 | + | |
| 292 | + return 0; | |
| 293 | +} | |
| 294 | + | |
| 295 | +/* This is called directly from KGD at ISR. */ | |
| 296 | +void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) | |
| 297 | +{ | |
| 298 | + if (kfd->init_complete) { | |
| 299 | + spin_lock(&kfd->interrupt_lock); | |
| 300 | + | |
| 301 | + if (kfd->interrupts_active | |
| 302 | + && enqueue_ih_ring_entry(kfd, ih_ring_entry)) | |
| 303 | + schedule_work(&kfd->interrupt_work); | |
| 304 | + | |
| 305 | + spin_unlock(&kfd->interrupt_lock); | |
| 306 | + } | |
| 307 | +} | |
| @@ -0,0 +1,1059 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + * | |
| 22 | + */ | |
| 23 | + | |
| 24 | +#include <linux/slab.h> | |
| 25 | +#include <linux/list.h> | |
| 26 | +#include <linux/types.h> | |
| 27 | +#include <linux/printk.h> | |
| 28 | +#include <linux/bitops.h> | |
| 29 | +#include "kfd_priv.h" | |
| 30 | +#include "kfd_device_queue_manager.h" | |
| 31 | +#include "kfd_mqd_manager.h" | |
| 32 | +#include "cik_regs.h" | |
| 33 | +#include "kfd_kernel_queue.h" | |
| 34 | +#include "../../radeon/cik_reg.h" | |
| 35 | + | |
| 36 | +/* Size of the per-pipe EOP queue */ | |
| 37 | +#define CIK_HPD_EOP_BYTES_LOG2 11 | |
| 38 | +#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) | |
| 39 | + | |
| 40 | +static bool is_mem_initialized; | |
| 41 | + | |
| 42 | +static int init_memory(struct device_queue_manager *dqm); | |
| 43 | +static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, | |
| 44 | + unsigned int pasid, unsigned int vmid); | |
| 45 | + | |
| 46 | +static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, | |
| 47 | + struct queue *q, | |
| 48 | + struct qcm_process_device *qpd); | |
| 49 | +static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock); | |
| 50 | +static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock); | |
| 51 | + | |
| 52 | + | |
| 53 | +static inline unsigned int get_pipes_num(struct device_queue_manager *dqm) | |
| 54 | +{ | |
| 55 | + BUG_ON(!dqm || !dqm->dev); | |
| 56 | + return dqm->dev->shared_resources.compute_pipe_count; | |
| 57 | +} | |
| 58 | + | |
| 59 | +static inline unsigned int get_first_pipe(struct device_queue_manager *dqm) | |
| 60 | +{ | |
| 61 | + BUG_ON(!dqm); | |
| 62 | + return dqm->dev->shared_resources.first_compute_pipe; | |
| 63 | +} | |
| 64 | + | |
| 65 | +static inline unsigned int get_pipes_num_cpsch(void) | |
| 66 | +{ | |
| 67 | + return PIPE_PER_ME_CP_SCHEDULING; | |
| 68 | +} | |
| 69 | + | |
| 70 | +static unsigned int get_sh_mem_bases_nybble_64(struct kfd_process *process, | |
| 71 | + struct kfd_dev *dev) | |
| 72 | +{ | |
| 73 | + struct kfd_process_device *pdd; | |
| 74 | + uint32_t nybble; | |
| 75 | + | |
| 76 | + pdd = kfd_get_process_device_data(dev, process, 1); | |
| 77 | + nybble = (pdd->lds_base >> 60) & 0x0E; | |
| 78 | + | |
| 79 | + return nybble; | |
| 80 | + | |
| 81 | +} | |
| 82 | + | |
| 83 | +static unsigned int get_sh_mem_bases_32(struct kfd_process *process, | |
| 84 | + struct kfd_dev *dev) | |
| 85 | +{ | |
| 86 | + struct kfd_process_device *pdd; | |
| 87 | + unsigned int shared_base; | |
| 88 | + | |
| 89 | + pdd = kfd_get_process_device_data(dev, process, 1); | |
| 90 | + shared_base = (pdd->lds_base >> 16) & 0xFF; | |
| 91 | + | |
| 92 | + return shared_base; | |
| 93 | +} | |
| 94 | + | |
| 95 | +static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble); | |
| 96 | +static void init_process_memory(struct device_queue_manager *dqm, | |
| 97 | + struct qcm_process_device *qpd) | |
| 98 | +{ | |
| 99 | + unsigned int temp; | |
| 100 | + | |
| 101 | + BUG_ON(!dqm || !qpd); | |
| 102 | + | |
| 103 | + /* check if sh_mem_config register already configured */ | |
| 104 | + if (qpd->sh_mem_config == 0) { | |
| 105 | + qpd->sh_mem_config = | |
| 106 | + ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) | | |
| 107 | + DEFAULT_MTYPE(MTYPE_NONCACHED) | | |
| 108 | + APE1_MTYPE(MTYPE_NONCACHED); | |
| 109 | + qpd->sh_mem_ape1_limit = 0; | |
| 110 | + qpd->sh_mem_ape1_base = 0; | |
| 111 | + } | |
| 112 | + | |
| 113 | + if (qpd->pqm->process->is_32bit_user_mode) { | |
| 114 | + temp = get_sh_mem_bases_32(qpd->pqm->process, dqm->dev); | |
| 115 | + qpd->sh_mem_bases = SHARED_BASE(temp); | |
| 116 | + qpd->sh_mem_config |= PTR32; | |
| 117 | + } else { | |
| 118 | + temp = get_sh_mem_bases_nybble_64(qpd->pqm->process, dqm->dev); | |
| 119 | + qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp); | |
| 120 | + } | |
| 121 | + | |
| 122 | + pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n", | |
| 123 | + qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases); | |
| 124 | +} | |
| 125 | + | |
| 126 | +static void program_sh_mem_settings(struct device_queue_manager *dqm, | |
| 127 | + struct qcm_process_device *qpd) | |
| 128 | +{ | |
| 129 | + return kfd2kgd->program_sh_mem_settings(dqm->dev->kgd, qpd->vmid, | |
| 130 | + qpd->sh_mem_config, | |
| 131 | + qpd->sh_mem_ape1_base, | |
| 132 | + qpd->sh_mem_ape1_limit, | |
| 133 | + qpd->sh_mem_bases); | |
| 134 | +} | |
| 135 | + | |
| 136 | +static int allocate_vmid(struct device_queue_manager *dqm, | |
| 137 | + struct qcm_process_device *qpd, | |
| 138 | + struct queue *q) | |
| 139 | +{ | |
| 140 | + int bit, allocated_vmid; | |
| 141 | + | |
| 142 | + if (dqm->vmid_bitmap == 0) | |
| 143 | + return -ENOMEM; | |
| 144 | + | |
| 145 | + bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM); | |
| 146 | + clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap); | |
| 147 | + | |
| 148 | + /* Kaveri kfd vmid's starts from vmid 8 */ | |
| 149 | + allocated_vmid = bit + KFD_VMID_START_OFFSET; | |
| 150 | + pr_debug("kfd: vmid allocation %d\n", allocated_vmid); | |
| 151 | + qpd->vmid = allocated_vmid; | |
| 152 | + q->properties.vmid = allocated_vmid; | |
| 153 | + | |
| 154 | + set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid); | |
| 155 | + program_sh_mem_settings(dqm, qpd); | |
| 156 | + | |
| 157 | + return 0; | |
| 158 | +} | |
| 159 | + | |
| 160 | +static void deallocate_vmid(struct device_queue_manager *dqm, | |
| 161 | + struct qcm_process_device *qpd, | |
| 162 | + struct queue *q) | |
| 163 | +{ | |
| 164 | + int bit = qpd->vmid - KFD_VMID_START_OFFSET; | |
| 165 | + | |
| 166 | + set_bit(bit, (unsigned long *)&dqm->vmid_bitmap); | |
| 167 | + qpd->vmid = 0; | |
| 168 | + q->properties.vmid = 0; | |
| 169 | +} | |
| 170 | + | |
| 171 | +static int create_queue_nocpsch(struct device_queue_manager *dqm, | |
| 172 | + struct queue *q, | |
| 173 | + struct qcm_process_device *qpd, | |
| 174 | + int *allocated_vmid) | |
| 175 | +{ | |
| 176 | + int retval; | |
| 177 | + | |
| 178 | + BUG_ON(!dqm || !q || !qpd || !allocated_vmid); | |
| 179 | + | |
| 180 | + pr_debug("kfd: In func %s\n", __func__); | |
| 181 | + print_queue(q); | |
| 182 | + | |
| 183 | + mutex_lock(&dqm->lock); | |
| 184 | + | |
| 185 | + if (list_empty(&qpd->queues_list)) { | |
| 186 | + retval = allocate_vmid(dqm, qpd, q); | |
| 187 | + if (retval != 0) { | |
| 188 | + mutex_unlock(&dqm->lock); | |
| 189 | + return retval; | |
| 190 | + } | |
| 191 | + } | |
| 192 | + *allocated_vmid = qpd->vmid; | |
| 193 | + q->properties.vmid = qpd->vmid; | |
| 194 | + | |
| 195 | + retval = create_compute_queue_nocpsch(dqm, q, qpd); | |
| 196 | + | |
| 197 | + if (retval != 0) { | |
| 198 | + if (list_empty(&qpd->queues_list)) { | |
| 199 | + deallocate_vmid(dqm, qpd, q); | |
| 200 | + *allocated_vmid = 0; | |
| 201 | + } | |
| 202 | + mutex_unlock(&dqm->lock); | |
| 203 | + return retval; | |
| 204 | + } | |
| 205 | + | |
| 206 | + list_add(&q->list, &qpd->queues_list); | |
| 207 | + dqm->queue_count++; | |
| 208 | + | |
| 209 | + mutex_unlock(&dqm->lock); | |
| 210 | + return 0; | |
| 211 | +} | |
| 212 | + | |
| 213 | +static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) | |
| 214 | +{ | |
| 215 | + bool set; | |
| 216 | + int pipe, bit; | |
| 217 | + | |
| 218 | + set = false; | |
| 219 | + | |
| 220 | + for (pipe = dqm->next_pipe_to_allocate; pipe < get_pipes_num(dqm); | |
| 221 | + pipe = (pipe + 1) % get_pipes_num(dqm)) { | |
| 222 | + if (dqm->allocated_queues[pipe] != 0) { | |
| 223 | + bit = find_first_bit( | |
| 224 | + (unsigned long *)&dqm->allocated_queues[pipe], | |
| 225 | + QUEUES_PER_PIPE); | |
| 226 | + | |
| 227 | + clear_bit(bit, | |
| 228 | + (unsigned long *)&dqm->allocated_queues[pipe]); | |
| 229 | + q->pipe = pipe; | |
| 230 | + q->queue = bit; | |
| 231 | + set = true; | |
| 232 | + break; | |
| 233 | + } | |
| 234 | + } | |
| 235 | + | |
| 236 | + if (set == false) | |
| 237 | + return -EBUSY; | |
| 238 | + | |
| 239 | + pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n", | |
| 240 | + __func__, q->pipe, q->queue); | |
| 241 | + /* horizontal hqd allocation */ | |
| 242 | + dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm); | |
| 243 | + | |
| 244 | + return 0; | |
| 245 | +} | |
| 246 | + | |
| 247 | +static inline void deallocate_hqd(struct device_queue_manager *dqm, | |
| 248 | + struct queue *q) | |
| 249 | +{ | |
| 250 | + set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]); | |
| 251 | +} | |
| 252 | + | |
| 253 | +static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, | |
| 254 | + struct queue *q, | |
| 255 | + struct qcm_process_device *qpd) | |
| 256 | +{ | |
| 257 | + int retval; | |
| 258 | + struct mqd_manager *mqd; | |
| 259 | + | |
| 260 | + BUG_ON(!dqm || !q || !qpd); | |
| 261 | + | |
| 262 | + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); | |
| 263 | + if (mqd == NULL) | |
| 264 | + return -ENOMEM; | |
| 265 | + | |
| 266 | + retval = allocate_hqd(dqm, q); | |
| 267 | + if (retval != 0) | |
| 268 | + return retval; | |
| 269 | + | |
| 270 | + retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, | |
| 271 | + &q->gart_mqd_addr, &q->properties); | |
| 272 | + if (retval != 0) { | |
| 273 | + deallocate_hqd(dqm, q); | |
| 274 | + return retval; | |
| 275 | + } | |
| 276 | + | |
| 277 | + return 0; | |
| 278 | +} | |
| 279 | + | |
| 280 | +static int destroy_queue_nocpsch(struct device_queue_manager *dqm, | |
| 281 | + struct qcm_process_device *qpd, | |
| 282 | + struct queue *q) | |
| 283 | +{ | |
| 284 | + int retval; | |
| 285 | + struct mqd_manager *mqd; | |
| 286 | + | |
| 287 | + BUG_ON(!dqm || !q || !q->mqd || !qpd); | |
| 288 | + | |
| 289 | + retval = 0; | |
| 290 | + | |
| 291 | + pr_debug("kfd: In Func %s\n", __func__); | |
| 292 | + | |
| 293 | + mutex_lock(&dqm->lock); | |
| 294 | + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); | |
| 295 | + if (mqd == NULL) { | |
| 296 | + retval = -ENOMEM; | |
| 297 | + goto out; | |
| 298 | + } | |
| 299 | + | |
| 300 | + retval = mqd->destroy_mqd(mqd, q->mqd, | |
| 301 | + KFD_PREEMPT_TYPE_WAVEFRONT, | |
| 302 | + QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS, | |
| 303 | + q->pipe, q->queue); | |
| 304 | + | |
| 305 | + if (retval != 0) | |
| 306 | + goto out; | |
| 307 | + | |
| 308 | + deallocate_hqd(dqm, q); | |
| 309 | + | |
| 310 | + mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); | |
| 311 | + | |
| 312 | + list_del(&q->list); | |
| 313 | + if (list_empty(&qpd->queues_list)) | |
| 314 | + deallocate_vmid(dqm, qpd, q); | |
| 315 | + dqm->queue_count--; | |
| 316 | +out: | |
| 317 | + mutex_unlock(&dqm->lock); | |
| 318 | + return retval; | |
| 319 | +} | |
| 320 | + | |
| 321 | +static int update_queue(struct device_queue_manager *dqm, struct queue *q) | |
| 322 | +{ | |
| 323 | + int retval; | |
| 324 | + struct mqd_manager *mqd; | |
| 325 | + | |
| 326 | + BUG_ON(!dqm || !q || !q->mqd); | |
| 327 | + | |
| 328 | + mutex_lock(&dqm->lock); | |
| 329 | + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); | |
| 330 | + if (mqd == NULL) { | |
| 331 | + mutex_unlock(&dqm->lock); | |
| 332 | + return -ENOMEM; | |
| 333 | + } | |
| 334 | + | |
| 335 | + retval = mqd->update_mqd(mqd, q->mqd, &q->properties); | |
| 336 | + if (q->properties.is_active == true) | |
| 337 | + dqm->queue_count++; | |
| 338 | + else | |
| 339 | + dqm->queue_count--; | |
| 340 | + | |
| 341 | + if (sched_policy != KFD_SCHED_POLICY_NO_HWS) | |
| 342 | + retval = execute_queues_cpsch(dqm, false); | |
| 343 | + | |
| 344 | + mutex_unlock(&dqm->lock); | |
| 345 | + return retval; | |
| 346 | +} | |
| 347 | + | |
| 348 | +static struct mqd_manager *get_mqd_manager_nocpsch( | |
| 349 | + struct device_queue_manager *dqm, enum KFD_MQD_TYPE type) | |
| 350 | +{ | |
| 351 | + struct mqd_manager *mqd; | |
| 352 | + | |
| 353 | + BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX); | |
| 354 | + | |
| 355 | + pr_debug("kfd: In func %s mqd type %d\n", __func__, type); | |
| 356 | + | |
| 357 | + mqd = dqm->mqds[type]; | |
| 358 | + if (!mqd) { | |
| 359 | + mqd = mqd_manager_init(type, dqm->dev); | |
| 360 | + if (mqd == NULL) | |
| 361 | + pr_err("kfd: mqd manager is NULL"); | |
| 362 | + dqm->mqds[type] = mqd; | |
| 363 | + } | |
| 364 | + | |
| 365 | + return mqd; | |
| 366 | +} | |
| 367 | + | |
| 368 | +static int register_process_nocpsch(struct device_queue_manager *dqm, | |
| 369 | + struct qcm_process_device *qpd) | |
| 370 | +{ | |
| 371 | + struct device_process_node *n; | |
| 372 | + | |
| 373 | + BUG_ON(!dqm || !qpd); | |
| 374 | + | |
| 375 | + pr_debug("kfd: In func %s\n", __func__); | |
| 376 | + | |
| 377 | + n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL); | |
| 378 | + if (!n) | |
| 379 | + return -ENOMEM; | |
| 380 | + | |
| 381 | + n->qpd = qpd; | |
| 382 | + | |
| 383 | + mutex_lock(&dqm->lock); | |
| 384 | + list_add(&n->list, &dqm->queues); | |
| 385 | + | |
| 386 | + init_process_memory(dqm, qpd); | |
| 387 | + dqm->processes_count++; | |
| 388 | + | |
| 389 | + mutex_unlock(&dqm->lock); | |
| 390 | + | |
| 391 | + return 0; | |
| 392 | +} | |
| 393 | + | |
| 394 | +static int unregister_process_nocpsch(struct device_queue_manager *dqm, | |
| 395 | + struct qcm_process_device *qpd) | |
| 396 | +{ | |
| 397 | + int retval; | |
| 398 | + struct device_process_node *cur, *next; | |
| 399 | + | |
| 400 | + BUG_ON(!dqm || !qpd); | |
| 401 | + | |
| 402 | + BUG_ON(!list_empty(&qpd->queues_list)); | |
| 403 | + | |
| 404 | + pr_debug("kfd: In func %s\n", __func__); | |
| 405 | + | |
| 406 | + retval = 0; | |
| 407 | + mutex_lock(&dqm->lock); | |
| 408 | + | |
| 409 | + list_for_each_entry_safe(cur, next, &dqm->queues, list) { | |
| 410 | + if (qpd == cur->qpd) { | |
| 411 | + list_del(&cur->list); | |
| 412 | + dqm->processes_count--; | |
| 413 | + goto out; | |
| 414 | + } | |
| 415 | + } | |
| 416 | + /* qpd not found in dqm list */ | |
| 417 | + retval = 1; | |
| 418 | +out: | |
| 419 | + mutex_unlock(&dqm->lock); | |
| 420 | + return retval; | |
| 421 | +} | |
| 422 | + | |
| 423 | +static int | |
| 424 | +set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid, | |
| 425 | + unsigned int vmid) | |
| 426 | +{ | |
| 427 | + uint32_t pasid_mapping; | |
| 428 | + | |
| 429 | + pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | | |
| 430 | + ATC_VMID_PASID_MAPPING_VALID; | |
| 431 | + return kfd2kgd->set_pasid_vmid_mapping(dqm->dev->kgd, pasid_mapping, | |
| 432 | + vmid); | |
| 433 | +} | |
| 434 | + | |
| 435 | +static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble) | |
| 436 | +{ | |
| 437 | + /* In 64-bit mode, we can only control the top 3 bits of the LDS, | |
| 438 | + * scratch and GPUVM apertures. | |
| 439 | + * The hardware fills in the remaining 59 bits according to the | |
| 440 | + * following pattern: | |
| 441 | + * LDS: X0000000'00000000 - X0000001'00000000 (4GB) | |
| 442 | + * Scratch: X0000001'00000000 - X0000002'00000000 (4GB) | |
| 443 | + * GPUVM: Y0010000'00000000 - Y0020000'00000000 (1TB) | |
| 444 | + * | |
| 445 | + * (where X/Y is the configurable nybble with the low-bit 0) | |
| 446 | + * | |
| 447 | + * LDS and scratch will have the same top nybble programmed in the | |
| 448 | + * top 3 bits of SH_MEM_BASES.PRIVATE_BASE. | |
| 449 | + * GPUVM can have a different top nybble programmed in the | |
| 450 | + * top 3 bits of SH_MEM_BASES.SHARED_BASE. | |
| 451 | + * We don't bother to support different top nybbles | |
| 452 | + * for LDS/Scratch and GPUVM. | |
| 453 | + */ | |
| 454 | + | |
| 455 | + BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE || | |
| 456 | + top_address_nybble == 0); | |
| 457 | + | |
| 458 | + return PRIVATE_BASE(top_address_nybble << 12) | | |
| 459 | + SHARED_BASE(top_address_nybble << 12); | |
| 460 | +} | |
| 461 | + | |
| 462 | +static int init_memory(struct device_queue_manager *dqm) | |
| 463 | +{ | |
| 464 | + int i, retval; | |
| 465 | + | |
| 466 | + for (i = 8; i < 16; i++) | |
| 467 | + set_pasid_vmid_mapping(dqm, 0, i); | |
| 468 | + | |
| 469 | + retval = kfd2kgd->init_memory(dqm->dev->kgd); | |
| 470 | + if (retval == 0) | |
| 471 | + is_mem_initialized = true; | |
| 472 | + return retval; | |
| 473 | +} | |
| 474 | + | |
| 475 | + | |
| 476 | +static int init_pipelines(struct device_queue_manager *dqm, | |
| 477 | + unsigned int pipes_num, unsigned int first_pipe) | |
| 478 | +{ | |
| 479 | + void *hpdptr; | |
| 480 | + struct mqd_manager *mqd; | |
| 481 | + unsigned int i, err, inx; | |
| 482 | + uint64_t pipe_hpd_addr; | |
| 483 | + | |
| 484 | + BUG_ON(!dqm || !dqm->dev); | |
| 485 | + | |
| 486 | + pr_debug("kfd: In func %s\n", __func__); | |
| 487 | + | |
| 488 | + /* | |
| 489 | + * Allocate memory for the HPDs. This is hardware-owned per-pipe data. | |
| 490 | + * The driver never accesses this memory after zeroing it. | |
| 491 | + * It doesn't even have to be saved/restored on suspend/resume | |
| 492 | + * because it contains no data when there are no active queues. | |
| 493 | + */ | |
| 494 | + | |
| 495 | + err = kfd2kgd->allocate_mem(dqm->dev->kgd, | |
| 496 | + CIK_HPD_EOP_BYTES * pipes_num, | |
| 497 | + PAGE_SIZE, | |
| 498 | + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | |
| 499 | + (struct kgd_mem **) &dqm->pipeline_mem); | |
| 500 | + | |
| 501 | + if (err) { | |
| 502 | + pr_err("kfd: error allocate vidmem num pipes: %d\n", | |
| 503 | + pipes_num); | |
| 504 | + return -ENOMEM; | |
| 505 | + } | |
| 506 | + | |
| 507 | + hpdptr = dqm->pipeline_mem->cpu_ptr; | |
| 508 | + dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr; | |
| 509 | + | |
| 510 | + memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num); | |
| 511 | + | |
| 512 | + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); | |
| 513 | + if (mqd == NULL) { | |
| 514 | + kfd2kgd->free_mem(dqm->dev->kgd, | |
| 515 | + (struct kgd_mem *) dqm->pipeline_mem); | |
| 516 | + return -ENOMEM; | |
| 517 | + } | |
| 518 | + | |
| 519 | + for (i = 0; i < pipes_num; i++) { | |
| 520 | + inx = i + first_pipe; | |
| 521 | + pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES; | |
| 522 | + pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr); | |
| 523 | + /* = log2(bytes/4)-1 */ | |
| 524 | + kfd2kgd->init_pipeline(dqm->dev->kgd, i, | |
| 525 | + CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr); | |
| 526 | + } | |
| 527 | + | |
| 528 | + return 0; | |
| 529 | +} | |
| 530 | + | |
| 531 | + | |
| 532 | +static int init_scheduler(struct device_queue_manager *dqm) | |
| 533 | +{ | |
| 534 | + int retval; | |
| 535 | + | |
| 536 | + BUG_ON(!dqm); | |
| 537 | + | |
| 538 | + pr_debug("kfd: In %s\n", __func__); | |
| 539 | + | |
| 540 | + retval = init_pipelines(dqm, get_pipes_num(dqm), KFD_DQM_FIRST_PIPE); | |
| 541 | + if (retval != 0) | |
| 542 | + return retval; | |
| 543 | + | |
| 544 | + retval = init_memory(dqm); | |
| 545 | + | |
| 546 | + return retval; | |
| 547 | +} | |
| 548 | + | |
| 549 | +static int initialize_nocpsch(struct device_queue_manager *dqm) | |
| 550 | +{ | |
| 551 | + int i; | |
| 552 | + | |
| 553 | + BUG_ON(!dqm); | |
| 554 | + | |
| 555 | + pr_debug("kfd: In func %s num of pipes: %d\n", | |
| 556 | + __func__, get_pipes_num(dqm)); | |
| 557 | + | |
| 558 | + mutex_init(&dqm->lock); | |
| 559 | + INIT_LIST_HEAD(&dqm->queues); | |
| 560 | + dqm->queue_count = dqm->next_pipe_to_allocate = 0; | |
| 561 | + dqm->allocated_queues = kcalloc(get_pipes_num(dqm), | |
| 562 | + sizeof(unsigned int), GFP_KERNEL); | |
| 563 | + if (!dqm->allocated_queues) { | |
| 564 | + mutex_destroy(&dqm->lock); | |
| 565 | + return -ENOMEM; | |
| 566 | + } | |
| 567 | + | |
| 568 | + for (i = 0; i < get_pipes_num(dqm); i++) | |
| 569 | + dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1; | |
| 570 | + | |
| 571 | + dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1; | |
| 572 | + | |
| 573 | + init_scheduler(dqm); | |
| 574 | + return 0; | |
| 575 | +} | |
| 576 | + | |
| 577 | +static void uninitialize_nocpsch(struct device_queue_manager *dqm) | |
| 578 | +{ | |
| 579 | + BUG_ON(!dqm); | |
| 580 | + | |
| 581 | + BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0); | |
| 582 | + | |
| 583 | + kfree(dqm->allocated_queues); | |
| 584 | + mutex_destroy(&dqm->lock); | |
| 585 | + kfd2kgd->free_mem(dqm->dev->kgd, | |
| 586 | + (struct kgd_mem *) dqm->pipeline_mem); | |
| 587 | +} | |
| 588 | + | |
| 589 | +static int start_nocpsch(struct device_queue_manager *dqm) | |
| 590 | +{ | |
| 591 | + return 0; | |
| 592 | +} | |
| 593 | + | |
| 594 | +static int stop_nocpsch(struct device_queue_manager *dqm) | |
| 595 | +{ | |
| 596 | + return 0; | |
| 597 | +} | |
| 598 | + | |
| 599 | +/* | |
| 600 | + * Device Queue Manager implementation for cp scheduler | |
| 601 | + */ | |
| 602 | + | |
| 603 | +static int set_sched_resources(struct device_queue_manager *dqm) | |
| 604 | +{ | |
| 605 | + struct scheduling_resources res; | |
| 606 | + unsigned int queue_num, queue_mask; | |
| 607 | + | |
| 608 | + BUG_ON(!dqm); | |
| 609 | + | |
| 610 | + pr_debug("kfd: In func %s\n", __func__); | |
| 611 | + | |
| 612 | + queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE; | |
| 613 | + queue_mask = (1 << queue_num) - 1; | |
| 614 | + res.vmid_mask = (1 << VMID_PER_DEVICE) - 1; | |
| 615 | + res.vmid_mask <<= KFD_VMID_START_OFFSET; | |
| 616 | + res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE); | |
| 617 | + res.gws_mask = res.oac_mask = res.gds_heap_base = | |
| 618 | + res.gds_heap_size = 0; | |
| 619 | + | |
| 620 | + pr_debug("kfd: scheduling resources:\n" | |
| 621 | + " vmid mask: 0x%8X\n" | |
| 622 | + " queue mask: 0x%8llX\n", | |
| 623 | + res.vmid_mask, res.queue_mask); | |
| 624 | + | |
| 625 | + return pm_send_set_resources(&dqm->packets, &res); | |
| 626 | +} | |
| 627 | + | |
| 628 | +static int initialize_cpsch(struct device_queue_manager *dqm) | |
| 629 | +{ | |
| 630 | + int retval; | |
| 631 | + | |
| 632 | + BUG_ON(!dqm); | |
| 633 | + | |
| 634 | + pr_debug("kfd: In func %s num of pipes: %d\n", | |
| 635 | + __func__, get_pipes_num_cpsch()); | |
| 636 | + | |
| 637 | + mutex_init(&dqm->lock); | |
| 638 | + INIT_LIST_HEAD(&dqm->queues); | |
| 639 | + dqm->queue_count = dqm->processes_count = 0; | |
| 640 | + dqm->active_runlist = false; | |
| 641 | + retval = init_pipelines(dqm, get_pipes_num(dqm), 0); | |
| 642 | + if (retval != 0) | |
| 643 | + goto fail_init_pipelines; | |
| 644 | + | |
| 645 | + return 0; | |
| 646 | + | |
| 647 | +fail_init_pipelines: | |
| 648 | + mutex_destroy(&dqm->lock); | |
| 649 | + return retval; | |
| 650 | +} | |
| 651 | + | |
| 652 | +static int start_cpsch(struct device_queue_manager *dqm) | |
| 653 | +{ | |
| 654 | + struct device_process_node *node; | |
| 655 | + int retval; | |
| 656 | + | |
| 657 | + BUG_ON(!dqm); | |
| 658 | + | |
| 659 | + retval = 0; | |
| 660 | + | |
| 661 | + retval = pm_init(&dqm->packets, dqm); | |
| 662 | + if (retval != 0) | |
| 663 | + goto fail_packet_manager_init; | |
| 664 | + | |
| 665 | + retval = set_sched_resources(dqm); | |
| 666 | + if (retval != 0) | |
| 667 | + goto fail_set_sched_resources; | |
| 668 | + | |
| 669 | + pr_debug("kfd: allocating fence memory\n"); | |
| 670 | + | |
| 671 | + /* allocate fence memory on the gart */ | |
| 672 | + retval = kfd2kgd->allocate_mem(dqm->dev->kgd, | |
| 673 | + sizeof(*dqm->fence_addr), | |
| 674 | + 32, | |
| 675 | + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | |
| 676 | + (struct kgd_mem **) &dqm->fence_mem); | |
| 677 | + | |
| 678 | + if (retval != 0) | |
| 679 | + goto fail_allocate_vidmem; | |
| 680 | + | |
| 681 | + dqm->fence_addr = dqm->fence_mem->cpu_ptr; | |
| 682 | + dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; | |
| 683 | + | |
| 684 | + list_for_each_entry(node, &dqm->queues, list) | |
| 685 | + if (node->qpd->pqm->process && dqm->dev) | |
| 686 | + kfd_bind_process_to_device(dqm->dev, | |
| 687 | + node->qpd->pqm->process); | |
| 688 | + | |
| 689 | + execute_queues_cpsch(dqm, true); | |
| 690 | + | |
| 691 | + return 0; | |
| 692 | +fail_allocate_vidmem: | |
| 693 | +fail_set_sched_resources: | |
| 694 | + pm_uninit(&dqm->packets); | |
| 695 | +fail_packet_manager_init: | |
| 696 | + return retval; | |
| 697 | +} | |
| 698 | + | |
| 699 | +static int stop_cpsch(struct device_queue_manager *dqm) | |
| 700 | +{ | |
| 701 | + struct device_process_node *node; | |
| 702 | + struct kfd_process_device *pdd; | |
| 703 | + | |
| 704 | + BUG_ON(!dqm); | |
| 705 | + | |
| 706 | + destroy_queues_cpsch(dqm, true); | |
| 707 | + | |
| 708 | + list_for_each_entry(node, &dqm->queues, list) { | |
| 709 | + pdd = kfd_get_process_device_data(dqm->dev, | |
| 710 | + node->qpd->pqm->process, 1); | |
| 711 | + pdd->bound = false; | |
| 712 | + } | |
| 713 | + kfd2kgd->free_mem(dqm->dev->kgd, | |
| 714 | + (struct kgd_mem *) dqm->fence_mem); | |
| 715 | + pm_uninit(&dqm->packets); | |
| 716 | + | |
| 717 | + return 0; | |
| 718 | +} | |
| 719 | + | |
| 720 | +static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, | |
| 721 | + struct kernel_queue *kq, | |
| 722 | + struct qcm_process_device *qpd) | |
| 723 | +{ | |
| 724 | + BUG_ON(!dqm || !kq || !qpd); | |
| 725 | + | |
| 726 | + pr_debug("kfd: In func %s\n", __func__); | |
| 727 | + | |
| 728 | + mutex_lock(&dqm->lock); | |
| 729 | + list_add(&kq->list, &qpd->priv_queue_list); | |
| 730 | + dqm->queue_count++; | |
| 731 | + qpd->is_debug = true; | |
| 732 | + execute_queues_cpsch(dqm, false); | |
| 733 | + mutex_unlock(&dqm->lock); | |
| 734 | + | |
| 735 | + return 0; | |
| 736 | +} | |
| 737 | + | |
| 738 | +static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, | |
| 739 | + struct kernel_queue *kq, | |
| 740 | + struct qcm_process_device *qpd) | |
| 741 | +{ | |
| 742 | + BUG_ON(!dqm || !kq); | |
| 743 | + | |
| 744 | + pr_debug("kfd: In %s\n", __func__); | |
| 745 | + | |
| 746 | + mutex_lock(&dqm->lock); | |
| 747 | + destroy_queues_cpsch(dqm, false); | |
| 748 | + list_del(&kq->list); | |
| 749 | + dqm->queue_count--; | |
| 750 | + qpd->is_debug = false; | |
| 751 | + execute_queues_cpsch(dqm, false); | |
| 752 | + mutex_unlock(&dqm->lock); | |
| 753 | +} | |
| 754 | + | |
| 755 | +static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, | |
| 756 | + struct qcm_process_device *qpd, int *allocate_vmid) | |
| 757 | +{ | |
| 758 | + int retval; | |
| 759 | + struct mqd_manager *mqd; | |
| 760 | + | |
| 761 | + BUG_ON(!dqm || !q || !qpd); | |
| 762 | + | |
| 763 | + retval = 0; | |
| 764 | + | |
| 765 | + if (allocate_vmid) | |
| 766 | + *allocate_vmid = 0; | |
| 767 | + | |
| 768 | + mutex_lock(&dqm->lock); | |
| 769 | + | |
| 770 | + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_CP); | |
| 771 | + if (mqd == NULL) { | |
| 772 | + mutex_unlock(&dqm->lock); | |
| 773 | + return -ENOMEM; | |
| 774 | + } | |
| 775 | + | |
| 776 | + retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, | |
| 777 | + &q->gart_mqd_addr, &q->properties); | |
| 778 | + if (retval != 0) | |
| 779 | + goto out; | |
| 780 | + | |
| 781 | + list_add(&q->list, &qpd->queues_list); | |
| 782 | + if (q->properties.is_active) { | |
| 783 | + dqm->queue_count++; | |
| 784 | + retval = execute_queues_cpsch(dqm, false); | |
| 785 | + } | |
| 786 | + | |
| 787 | +out: | |
| 788 | + mutex_unlock(&dqm->lock); | |
| 789 | + return retval; | |
| 790 | +} | |
| 791 | + | |
| 792 | +int fence_wait_timeout(unsigned int *fence_addr, unsigned int fence_value, | |
| 793 | + unsigned long timeout) | |
| 794 | +{ | |
| 795 | + BUG_ON(!fence_addr); | |
| 796 | + timeout += jiffies; | |
| 797 | + | |
| 798 | + while (*fence_addr != fence_value) { | |
| 799 | + if (time_after(jiffies, timeout)) { | |
| 800 | + pr_err("kfd: qcm fence wait loop timeout expired\n"); | |
| 801 | + return -ETIME; | |
| 802 | + } | |
| 803 | + cpu_relax(); | |
| 804 | + } | |
| 805 | + | |
| 806 | + return 0; | |
| 807 | +} | |
| 808 | + | |
| 809 | +static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock) | |
| 810 | +{ | |
| 811 | + int retval; | |
| 812 | + | |
| 813 | + BUG_ON(!dqm); | |
| 814 | + | |
| 815 | + retval = 0; | |
| 816 | + | |
| 817 | + if (lock) | |
| 818 | + mutex_lock(&dqm->lock); | |
| 819 | + if (dqm->active_runlist == false) | |
| 820 | + goto out; | |
| 821 | + retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE, | |
| 822 | + KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0); | |
| 823 | + if (retval != 0) | |
| 824 | + goto out; | |
| 825 | + | |
| 826 | + *dqm->fence_addr = KFD_FENCE_INIT; | |
| 827 | + pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr, | |
| 828 | + KFD_FENCE_COMPLETED); | |
| 829 | + /* should be timed out */ | |
| 830 | + fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED, | |
| 831 | + QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS); | |
| 832 | + pm_release_ib(&dqm->packets); | |
| 833 | + dqm->active_runlist = false; | |
| 834 | + | |
| 835 | +out: | |
| 836 | + if (lock) | |
| 837 | + mutex_unlock(&dqm->lock); | |
| 838 | + return retval; | |
| 839 | +} | |
| 840 | + | |
| 841 | +static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock) | |
| 842 | +{ | |
| 843 | + int retval; | |
| 844 | + | |
| 845 | + BUG_ON(!dqm); | |
| 846 | + | |
| 847 | + if (lock) | |
| 848 | + mutex_lock(&dqm->lock); | |
| 849 | + | |
| 850 | + retval = destroy_queues_cpsch(dqm, false); | |
| 851 | + if (retval != 0) { | |
| 852 | + pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption"); | |
| 853 | + goto out; | |
| 854 | + } | |
| 855 | + | |
| 856 | + if (dqm->queue_count <= 0 || dqm->processes_count <= 0) { | |
| 857 | + retval = 0; | |
| 858 | + goto out; | |
| 859 | + } | |
| 860 | + | |
| 861 | + if (dqm->active_runlist) { | |
| 862 | + retval = 0; | |
| 863 | + goto out; | |
| 864 | + } | |
| 865 | + | |
| 866 | + retval = pm_send_runlist(&dqm->packets, &dqm->queues); | |
| 867 | + if (retval != 0) { | |
| 868 | + pr_err("kfd: failed to execute runlist"); | |
| 869 | + goto out; | |
| 870 | + } | |
| 871 | + dqm->active_runlist = true; | |
| 872 | + | |
| 873 | +out: | |
| 874 | + if (lock) | |
| 875 | + mutex_unlock(&dqm->lock); | |
| 876 | + return retval; | |
| 877 | +} | |
| 878 | + | |
| 879 | +static int destroy_queue_cpsch(struct device_queue_manager *dqm, | |
| 880 | + struct qcm_process_device *qpd, | |
| 881 | + struct queue *q) | |
| 882 | +{ | |
| 883 | + int retval; | |
| 884 | + struct mqd_manager *mqd; | |
| 885 | + | |
| 886 | + BUG_ON(!dqm || !qpd || !q); | |
| 887 | + | |
| 888 | + retval = 0; | |
| 889 | + | |
| 890 | + /* remove queue from list to prevent rescheduling after preemption */ | |
| 891 | + mutex_lock(&dqm->lock); | |
| 892 | + | |
| 893 | + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_CP); | |
| 894 | + if (!mqd) { | |
| 895 | + retval = -ENOMEM; | |
| 896 | + goto failed; | |
| 897 | + } | |
| 898 | + | |
| 899 | + list_del(&q->list); | |
| 900 | + dqm->queue_count--; | |
| 901 | + | |
| 902 | + execute_queues_cpsch(dqm, false); | |
| 903 | + | |
| 904 | + mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); | |
| 905 | + | |
| 906 | + mutex_unlock(&dqm->lock); | |
| 907 | + | |
| 908 | + return 0; | |
| 909 | + | |
| 910 | +failed: | |
| 911 | + mutex_unlock(&dqm->lock); | |
| 912 | + return retval; | |
| 913 | +} | |
| 914 | + | |
| 915 | +/* | |
| 916 | + * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to | |
| 917 | + * stay in user mode. | |
| 918 | + */ | |
| 919 | +#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL | |
| 920 | +/* APE1 limit is inclusive and 64K aligned. */ | |
| 921 | +#define APE1_LIMIT_ALIGNMENT 0xFFFF | |
| 922 | + | |
| 923 | +static bool set_cache_memory_policy(struct device_queue_manager *dqm, | |
| 924 | + struct qcm_process_device *qpd, | |
| 925 | + enum cache_policy default_policy, | |
| 926 | + enum cache_policy alternate_policy, | |
| 927 | + void __user *alternate_aperture_base, | |
| 928 | + uint64_t alternate_aperture_size) | |
| 929 | +{ | |
| 930 | + uint32_t default_mtype; | |
| 931 | + uint32_t ape1_mtype; | |
| 932 | + | |
| 933 | + pr_debug("kfd: In func %s\n", __func__); | |
| 934 | + | |
| 935 | + mutex_lock(&dqm->lock); | |
| 936 | + | |
| 937 | + if (alternate_aperture_size == 0) { | |
| 938 | + /* base > limit disables APE1 */ | |
| 939 | + qpd->sh_mem_ape1_base = 1; | |
| 940 | + qpd->sh_mem_ape1_limit = 0; | |
| 941 | + } else { | |
| 942 | + /* | |
| 943 | + * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]}, | |
| 944 | + * SH_MEM_APE1_BASE[31:0], 0x0000 } | |
| 945 | + * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]}, | |
| 946 | + * SH_MEM_APE1_LIMIT[31:0], 0xFFFF } | |
| 947 | + * Verify that the base and size parameters can be | |
| 948 | + * represented in this format and convert them. | |
| 949 | + * Additionally restrict APE1 to user-mode addresses. | |
| 950 | + */ | |
| 951 | + | |
| 952 | + uint64_t base = (uintptr_t)alternate_aperture_base; | |
| 953 | + uint64_t limit = base + alternate_aperture_size - 1; | |
| 954 | + | |
| 955 | + if (limit <= base) | |
| 956 | + goto out; | |
| 957 | + | |
| 958 | + if ((base & APE1_FIXED_BITS_MASK) != 0) | |
| 959 | + goto out; | |
| 960 | + | |
| 961 | + if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) | |
| 962 | + goto out; | |
| 963 | + | |
| 964 | + qpd->sh_mem_ape1_base = base >> 16; | |
| 965 | + qpd->sh_mem_ape1_limit = limit >> 16; | |
| 966 | + } | |
| 967 | + | |
| 968 | + default_mtype = (default_policy == cache_policy_coherent) ? | |
| 969 | + MTYPE_NONCACHED : | |
| 970 | + MTYPE_CACHED; | |
| 971 | + | |
| 972 | + ape1_mtype = (alternate_policy == cache_policy_coherent) ? | |
| 973 | + MTYPE_NONCACHED : | |
| 974 | + MTYPE_CACHED; | |
| 975 | + | |
| 976 | + qpd->sh_mem_config = (qpd->sh_mem_config & PTR32) | |
| 977 | + | ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) | |
| 978 | + | DEFAULT_MTYPE(default_mtype) | |
| 979 | + | APE1_MTYPE(ape1_mtype); | |
| 980 | + | |
| 981 | + if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) | |
| 982 | + program_sh_mem_settings(dqm, qpd); | |
| 983 | + | |
| 984 | + pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", | |
| 985 | + qpd->sh_mem_config, qpd->sh_mem_ape1_base, | |
| 986 | + qpd->sh_mem_ape1_limit); | |
| 987 | + | |
| 988 | + mutex_unlock(&dqm->lock); | |
| 989 | + return true; | |
| 990 | + | |
| 991 | +out: | |
| 992 | + mutex_unlock(&dqm->lock); | |
| 993 | + return false; | |
| 994 | +} | |
| 995 | + | |
| 996 | +struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev) | |
| 997 | +{ | |
| 998 | + struct device_queue_manager *dqm; | |
| 999 | + | |
| 1000 | + BUG_ON(!dev); | |
| 1001 | + | |
| 1002 | + dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL); | |
| 1003 | + if (!dqm) | |
| 1004 | + return NULL; | |
| 1005 | + | |
| 1006 | + dqm->dev = dev; | |
| 1007 | + switch (sched_policy) { | |
| 1008 | + case KFD_SCHED_POLICY_HWS: | |
| 1009 | + case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: | |
| 1010 | + /* initialize dqm for cp scheduling */ | |
| 1011 | + dqm->create_queue = create_queue_cpsch; | |
| 1012 | + dqm->initialize = initialize_cpsch; | |
| 1013 | + dqm->start = start_cpsch; | |
| 1014 | + dqm->stop = stop_cpsch; | |
| 1015 | + dqm->destroy_queue = destroy_queue_cpsch; | |
| 1016 | + dqm->update_queue = update_queue; | |
| 1017 | + dqm->get_mqd_manager = get_mqd_manager_nocpsch; | |
| 1018 | + dqm->register_process = register_process_nocpsch; | |
| 1019 | + dqm->unregister_process = unregister_process_nocpsch; | |
| 1020 | + dqm->uninitialize = uninitialize_nocpsch; | |
| 1021 | + dqm->create_kernel_queue = create_kernel_queue_cpsch; | |
| 1022 | + dqm->destroy_kernel_queue = destroy_kernel_queue_cpsch; | |
| 1023 | + dqm->set_cache_memory_policy = set_cache_memory_policy; | |
| 1024 | + break; | |
| 1025 | + case KFD_SCHED_POLICY_NO_HWS: | |
| 1026 | + /* initialize dqm for no cp scheduling */ | |
| 1027 | + dqm->start = start_nocpsch; | |
| 1028 | + dqm->stop = stop_nocpsch; | |
| 1029 | + dqm->create_queue = create_queue_nocpsch; | |
| 1030 | + dqm->destroy_queue = destroy_queue_nocpsch; | |
| 1031 | + dqm->update_queue = update_queue; | |
| 1032 | + dqm->get_mqd_manager = get_mqd_manager_nocpsch; | |
| 1033 | + dqm->register_process = register_process_nocpsch; | |
| 1034 | + dqm->unregister_process = unregister_process_nocpsch; | |
| 1035 | + dqm->initialize = initialize_nocpsch; | |
| 1036 | + dqm->uninitialize = uninitialize_nocpsch; | |
| 1037 | + dqm->set_cache_memory_policy = set_cache_memory_policy; | |
| 1038 | + break; | |
| 1039 | + default: | |
| 1040 | + BUG(); | |
| 1041 | + break; | |
| 1042 | + } | |
| 1043 | + | |
| 1044 | + if (dqm->initialize(dqm) != 0) { | |
| 1045 | + kfree(dqm); | |
| 1046 | + return NULL; | |
| 1047 | + } | |
| 1048 | + | |
| 1049 | + return dqm; | |
| 1050 | +} | |
| 1051 | + | |
| 1052 | +void device_queue_manager_uninit(struct device_queue_manager *dqm) | |
| 1053 | +{ | |
| 1054 | + BUG_ON(!dqm); | |
| 1055 | + | |
| 1056 | + dqm->uninitialize(dqm); | |
| 1057 | + kfree(dqm); | |
| 1058 | +} | |
| 1059 | + | |
| @@ -0,0 +1,146 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + * | |
| 22 | + */ | |
| 23 | + | |
| 24 | +#ifndef KFD_DEVICE_QUEUE_MANAGER_H_ | |
| 25 | +#define KFD_DEVICE_QUEUE_MANAGER_H_ | |
| 26 | + | |
| 27 | +#include <linux/rwsem.h> | |
| 28 | +#include <linux/list.h> | |
| 29 | +#include "kfd_priv.h" | |
| 30 | +#include "kfd_mqd_manager.h" | |
| 31 | + | |
| 32 | +#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (500) | |
| 33 | +#define QUEUES_PER_PIPE (8) | |
| 34 | +#define PIPE_PER_ME_CP_SCHEDULING (3) | |
| 35 | +#define CIK_VMID_NUM (8) | |
| 36 | +#define KFD_VMID_START_OFFSET (8) | |
| 37 | +#define VMID_PER_DEVICE CIK_VMID_NUM | |
| 38 | +#define KFD_DQM_FIRST_PIPE (0) | |
| 39 | + | |
| 40 | +struct device_process_node { | |
| 41 | + struct qcm_process_device *qpd; | |
| 42 | + struct list_head list; | |
| 43 | +}; | |
| 44 | + | |
| 45 | +/** | |
| 46 | + * struct device_queue_manager | |
| 47 | + * | |
| 48 | + * @create_queue: Queue creation routine. | |
| 49 | + * | |
| 50 | + * @destroy_queue: Queue destruction routine. | |
| 51 | + * | |
| 52 | + * @update_queue: Queue update routine. | |
| 53 | + * | |
| 54 | + * @get_mqd_manager: Returns the mqd manager according to the mqd type. | |
| 55 | + * | |
| 56 | + * @exeute_queues: Dispatches the queues list to the H/W. | |
| 57 | + * | |
| 58 | + * @register_process: This routine associates a specific process with device. | |
| 59 | + * | |
| 60 | + * @unregister_process: destroys the associations between process to device. | |
| 61 | + * | |
| 62 | + * @initialize: Initializes the pipelines and memory module for that device. | |
| 63 | + * | |
| 64 | + * @start: Initializes the resources/modules the the device needs for queues | |
| 65 | + * execution. This function is called on device initialization and after the | |
| 66 | + * system woke up after suspension. | |
| 67 | + * | |
| 68 | + * @stop: This routine stops execution of all the active queue running on the | |
| 69 | + * H/W and basically this function called on system suspend. | |
| 70 | + * | |
| 71 | + * @uninitialize: Destroys all the device queue manager resources allocated in | |
| 72 | + * initialize routine. | |
| 73 | + * | |
| 74 | + * @create_kernel_queue: Creates kernel queue. Used for debug queue. | |
| 75 | + * | |
| 76 | + * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue. | |
| 77 | + * | |
| 78 | + * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the | |
| 79 | + * memory apertures. | |
| 80 | + * | |
| 81 | + * This struct is a base class for the kfd queues scheduler in the | |
| 82 | + * device level. The device base class should expose the basic operations | |
| 83 | + * for queue creation and queue destruction. This base class hides the | |
| 84 | + * scheduling mode of the driver and the specific implementation of the | |
| 85 | + * concrete device. This class is the only class in the queues scheduler | |
| 86 | + * that configures the H/W. | |
| 87 | + */ | |
| 88 | + | |
| 89 | +struct device_queue_manager { | |
| 90 | + int (*create_queue)(struct device_queue_manager *dqm, | |
| 91 | + struct queue *q, | |
| 92 | + struct qcm_process_device *qpd, | |
| 93 | + int *allocate_vmid); | |
| 94 | + int (*destroy_queue)(struct device_queue_manager *dqm, | |
| 95 | + struct qcm_process_device *qpd, | |
| 96 | + struct queue *q); | |
| 97 | + int (*update_queue)(struct device_queue_manager *dqm, | |
| 98 | + struct queue *q); | |
| 99 | + | |
| 100 | + struct mqd_manager * (*get_mqd_manager) | |
| 101 | + (struct device_queue_manager *dqm, | |
| 102 | + enum KFD_MQD_TYPE type); | |
| 103 | + | |
| 104 | + int (*register_process)(struct device_queue_manager *dqm, | |
| 105 | + struct qcm_process_device *qpd); | |
| 106 | + int (*unregister_process)(struct device_queue_manager *dqm, | |
| 107 | + struct qcm_process_device *qpd); | |
| 108 | + int (*initialize)(struct device_queue_manager *dqm); | |
| 109 | + int (*start)(struct device_queue_manager *dqm); | |
| 110 | + int (*stop)(struct device_queue_manager *dqm); | |
| 111 | + void (*uninitialize)(struct device_queue_manager *dqm); | |
| 112 | + int (*create_kernel_queue)(struct device_queue_manager *dqm, | |
| 113 | + struct kernel_queue *kq, | |
| 114 | + struct qcm_process_device *qpd); | |
| 115 | + void (*destroy_kernel_queue)(struct device_queue_manager *dqm, | |
| 116 | + struct kernel_queue *kq, | |
| 117 | + struct qcm_process_device *qpd); | |
| 118 | + bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, | |
| 119 | + struct qcm_process_device *qpd, | |
| 120 | + enum cache_policy default_policy, | |
| 121 | + enum cache_policy alternate_policy, | |
| 122 | + void __user *alternate_aperture_base, | |
| 123 | + uint64_t alternate_aperture_size); | |
| 124 | + | |
| 125 | + | |
| 126 | + struct mqd_manager *mqds[KFD_MQD_TYPE_MAX]; | |
| 127 | + struct packet_manager packets; | |
| 128 | + struct kfd_dev *dev; | |
| 129 | + struct mutex lock; | |
| 130 | + struct list_head queues; | |
| 131 | + unsigned int processes_count; | |
| 132 | + unsigned int queue_count; | |
| 133 | + unsigned int next_pipe_to_allocate; | |
| 134 | + unsigned int *allocated_queues; | |
| 135 | + unsigned int vmid_bitmap; | |
| 136 | + uint64_t pipelines_addr; | |
| 137 | + struct kfd_mem_obj *pipeline_mem; | |
| 138 | + uint64_t fence_gpu_addr; | |
| 139 | + unsigned int *fence_addr; | |
| 140 | + struct kfd_mem_obj *fence_mem; | |
| 141 | + bool active_runlist; | |
| 142 | +}; | |
| 143 | + | |
| 144 | + | |
| 145 | + | |
| 146 | +#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */ | |
| @@ -0,0 +1,255 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | +#include "kfd_priv.h" | |
| 23 | +#include <linux/mm.h> | |
| 24 | +#include <linux/mman.h> | |
| 25 | +#include <linux/slab.h> | |
| 26 | + | |
| 27 | +/* | |
| 28 | + * This extension supports a kernel level doorbells management for | |
| 29 | + * the kernel queues. | |
| 30 | + * Basically the last doorbells page is devoted to kernel queues | |
| 31 | + * and that's assures that any user process won't get access to the | |
| 32 | + * kernel doorbells page | |
| 33 | + */ | |
| 34 | +static DEFINE_MUTEX(doorbell_mutex); | |
| 35 | +static unsigned long doorbell_available_index[ | |
| 36 | + DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)] = { 0 }; | |
| 37 | + | |
| 38 | +#define KERNEL_DOORBELL_PASID 1 | |
| 39 | +#define KFD_SIZE_OF_DOORBELL_IN_BYTES 4 | |
| 40 | + | |
| 41 | +/* | |
| 42 | + * Each device exposes a doorbell aperture, a PCI MMIO aperture that | |
| 43 | + * receives 32-bit writes that are passed to queues as wptr values. | |
| 44 | + * The doorbells are intended to be written by applications as part | |
| 45 | + * of queueing work on user-mode queues. | |
| 46 | + * We assign doorbells to applications in PAGE_SIZE-sized and aligned chunks. | |
| 47 | + * We map the doorbell address space into user-mode when a process creates | |
| 48 | + * its first queue on each device. | |
| 49 | + * Although the mapping is done by KFD, it is equivalent to an mmap of | |
| 50 | + * the /dev/kfd with the particular device encoded in the mmap offset. | |
| 51 | + * There will be other uses for mmap of /dev/kfd, so only a range of | |
| 52 | + * offsets (KFD_MMAP_DOORBELL_START-END) is used for doorbells. | |
| 53 | + */ | |
| 54 | + | |
| 55 | +/* # of doorbell bytes allocated for each process. */ | |
| 56 | +static inline size_t doorbell_process_allocation(void) | |
| 57 | +{ | |
| 58 | + return roundup(KFD_SIZE_OF_DOORBELL_IN_BYTES * | |
| 59 | + KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, | |
| 60 | + PAGE_SIZE); | |
| 61 | +} | |
| 62 | + | |
| 63 | +/* Doorbell calculations for device init. */ | |
| 64 | +void kfd_doorbell_init(struct kfd_dev *kfd) | |
| 65 | +{ | |
| 66 | + size_t doorbell_start_offset; | |
| 67 | + size_t doorbell_aperture_size; | |
| 68 | + size_t doorbell_process_limit; | |
| 69 | + | |
| 70 | + /* | |
| 71 | + * We start with calculations in bytes because the input data might | |
| 72 | + * only be byte-aligned. | |
| 73 | + * Only after we have done the rounding can we assume any alignment. | |
| 74 | + */ | |
| 75 | + | |
| 76 | + doorbell_start_offset = | |
| 77 | + roundup(kfd->shared_resources.doorbell_start_offset, | |
| 78 | + doorbell_process_allocation()); | |
| 79 | + | |
| 80 | + doorbell_aperture_size = | |
| 81 | + rounddown(kfd->shared_resources.doorbell_aperture_size, | |
| 82 | + doorbell_process_allocation()); | |
| 83 | + | |
| 84 | + if (doorbell_aperture_size > doorbell_start_offset) | |
| 85 | + doorbell_process_limit = | |
| 86 | + (doorbell_aperture_size - doorbell_start_offset) / | |
| 87 | + doorbell_process_allocation(); | |
| 88 | + else | |
| 89 | + doorbell_process_limit = 0; | |
| 90 | + | |
| 91 | + kfd->doorbell_base = kfd->shared_resources.doorbell_physical_address + | |
| 92 | + doorbell_start_offset; | |
| 93 | + | |
| 94 | + kfd->doorbell_id_offset = doorbell_start_offset / sizeof(u32); | |
| 95 | + kfd->doorbell_process_limit = doorbell_process_limit - 1; | |
| 96 | + | |
| 97 | + kfd->doorbell_kernel_ptr = ioremap(kfd->doorbell_base, | |
| 98 | + doorbell_process_allocation()); | |
| 99 | + | |
| 100 | + BUG_ON(!kfd->doorbell_kernel_ptr); | |
| 101 | + | |
| 102 | + pr_debug("kfd: doorbell initialization:\n"); | |
| 103 | + pr_debug("kfd: doorbell base == 0x%08lX\n", | |
| 104 | + (uintptr_t)kfd->doorbell_base); | |
| 105 | + | |
| 106 | + pr_debug("kfd: doorbell_id_offset == 0x%08lX\n", | |
| 107 | + kfd->doorbell_id_offset); | |
| 108 | + | |
| 109 | + pr_debug("kfd: doorbell_process_limit == 0x%08lX\n", | |
| 110 | + doorbell_process_limit); | |
| 111 | + | |
| 112 | + pr_debug("kfd: doorbell_kernel_offset == 0x%08lX\n", | |
| 113 | + (uintptr_t)kfd->doorbell_base); | |
| 114 | + | |
| 115 | + pr_debug("kfd: doorbell aperture size == 0x%08lX\n", | |
| 116 | + kfd->shared_resources.doorbell_aperture_size); | |
| 117 | + | |
| 118 | + pr_debug("kfd: doorbell kernel address == 0x%08lX\n", | |
| 119 | + (uintptr_t)kfd->doorbell_kernel_ptr); | |
| 120 | +} | |
| 121 | + | |
| 122 | +int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma) | |
| 123 | +{ | |
| 124 | + phys_addr_t address; | |
| 125 | + struct kfd_dev *dev; | |
| 126 | + | |
| 127 | + /* | |
| 128 | + * For simplicitly we only allow mapping of the entire doorbell | |
| 129 | + * allocation of a single device & process. | |
| 130 | + */ | |
| 131 | + if (vma->vm_end - vma->vm_start != doorbell_process_allocation()) | |
| 132 | + return -EINVAL; | |
| 133 | + | |
| 134 | + /* Find kfd device according to gpu id */ | |
| 135 | + dev = kfd_device_by_id(vma->vm_pgoff); | |
| 136 | + if (dev == NULL) | |
| 137 | + return -EINVAL; | |
| 138 | + | |
| 139 | + /* Find if pdd exists for combination of process and gpu id */ | |
| 140 | + if (!kfd_get_process_device_data(dev, process, 0)) | |
| 141 | + return -EINVAL; | |
| 142 | + | |
| 143 | + /* Calculate physical address of doorbell */ | |
| 144 | + address = kfd_get_process_doorbells(dev, process); | |
| 145 | + | |
| 146 | + vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE | | |
| 147 | + VM_DONTDUMP | VM_PFNMAP; | |
| 148 | + | |
| 149 | + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); | |
| 150 | + | |
| 151 | + pr_debug("kfd: mapping doorbell page in kfd_doorbell_mmap\n" | |
| 152 | + " target user address == 0x%08llX\n" | |
| 153 | + " physical address == 0x%08llX\n" | |
| 154 | + " vm_flags == 0x%04lX\n" | |
| 155 | + " size == 0x%04lX\n", | |
| 156 | + (unsigned long long) vma->vm_start, address, vma->vm_flags, | |
| 157 | + doorbell_process_allocation()); | |
| 158 | + | |
| 159 | + | |
| 160 | + return io_remap_pfn_range(vma, | |
| 161 | + vma->vm_start, | |
| 162 | + address >> PAGE_SHIFT, | |
| 163 | + doorbell_process_allocation(), | |
| 164 | + vma->vm_page_prot); | |
| 165 | +} | |
| 166 | + | |
| 167 | + | |
| 168 | +/* get kernel iomem pointer for a doorbell */ | |
| 169 | +u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, | |
| 170 | + unsigned int *doorbell_off) | |
| 171 | +{ | |
| 172 | + u32 inx; | |
| 173 | + | |
| 174 | + BUG_ON(!kfd || !doorbell_off); | |
| 175 | + | |
| 176 | + mutex_lock(&doorbell_mutex); | |
| 177 | + inx = find_first_zero_bit(doorbell_available_index, | |
| 178 | + KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); | |
| 179 | + | |
| 180 | + __set_bit(inx, doorbell_available_index); | |
| 181 | + mutex_unlock(&doorbell_mutex); | |
| 182 | + | |
| 183 | + if (inx >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) | |
| 184 | + return NULL; | |
| 185 | + | |
| 186 | + /* | |
| 187 | + * Calculating the kernel doorbell offset using "faked" kernel | |
| 188 | + * pasid that allocated for kernel queues only | |
| 189 | + */ | |
| 190 | + *doorbell_off = KERNEL_DOORBELL_PASID * (doorbell_process_allocation() / | |
| 191 | + sizeof(u32)) + inx; | |
| 192 | + | |
| 193 | + pr_debug("kfd: get kernel queue doorbell\n" | |
| 194 | + " doorbell offset == 0x%08d\n" | |
| 195 | + " kernel address == 0x%08lX\n", | |
| 196 | + *doorbell_off, (uintptr_t)(kfd->doorbell_kernel_ptr + inx)); | |
| 197 | + | |
| 198 | + return kfd->doorbell_kernel_ptr + inx; | |
| 199 | +} | |
| 200 | + | |
| 201 | +void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr) | |
| 202 | +{ | |
| 203 | + unsigned int inx; | |
| 204 | + | |
| 205 | + BUG_ON(!kfd || !db_addr); | |
| 206 | + | |
| 207 | + inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr); | |
| 208 | + | |
| 209 | + mutex_lock(&doorbell_mutex); | |
| 210 | + __clear_bit(inx, doorbell_available_index); | |
| 211 | + mutex_unlock(&doorbell_mutex); | |
| 212 | +} | |
| 213 | + | |
| 214 | +inline void write_kernel_doorbell(u32 __iomem *db, u32 value) | |
| 215 | +{ | |
| 216 | + if (db) { | |
| 217 | + writel(value, db); | |
| 218 | + pr_debug("writing %d to doorbell address 0x%p\n", value, db); | |
| 219 | + } | |
| 220 | +} | |
| 221 | + | |
| 222 | +/* | |
| 223 | + * queue_ids are in the range [0,MAX_PROCESS_QUEUES) and are mapped 1:1 | |
| 224 | + * to doorbells with the process's doorbell page | |
| 225 | + */ | |
| 226 | +unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd, | |
| 227 | + struct kfd_process *process, | |
| 228 | + unsigned int queue_id) | |
| 229 | +{ | |
| 230 | + /* | |
| 231 | + * doorbell_id_offset accounts for doorbells taken by KGD. | |
| 232 | + * pasid * doorbell_process_allocation/sizeof(u32) adjusts | |
| 233 | + * to the process's doorbells | |
| 234 | + */ | |
| 235 | + return kfd->doorbell_id_offset + | |
| 236 | + process->pasid * (doorbell_process_allocation()/sizeof(u32)) + | |
| 237 | + queue_id; | |
| 238 | +} | |
| 239 | + | |
| 240 | +uint64_t kfd_get_number_elems(struct kfd_dev *kfd) | |
| 241 | +{ | |
| 242 | + uint64_t num_of_elems = (kfd->shared_resources.doorbell_aperture_size - | |
| 243 | + kfd->shared_resources.doorbell_start_offset) / | |
| 244 | + doorbell_process_allocation() + 1; | |
| 245 | + | |
| 246 | + return num_of_elems; | |
| 247 | + | |
| 248 | +} | |
| 249 | + | |
| 250 | +phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, | |
| 251 | + struct kfd_process *process) | |
| 252 | +{ | |
| 253 | + return dev->doorbell_base + | |
| 254 | + process->pasid * doorbell_process_allocation(); | |
| 255 | +} | |
| @@ -0,0 +1,355 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + * | |
| 22 | + */ | |
| 23 | + | |
| 24 | +#include <linux/device.h> | |
| 25 | +#include <linux/export.h> | |
| 26 | +#include <linux/err.h> | |
| 27 | +#include <linux/fs.h> | |
| 28 | +#include <linux/sched.h> | |
| 29 | +#include <linux/slab.h> | |
| 30 | +#include <linux/uaccess.h> | |
| 31 | +#include <linux/compat.h> | |
| 32 | +#include <uapi/linux/kfd_ioctl.h> | |
| 33 | +#include <linux/time.h> | |
| 34 | +#include "kfd_priv.h" | |
| 35 | +#include <linux/mm.h> | |
| 36 | +#include <uapi/asm-generic/mman-common.h> | |
| 37 | +#include <asm/processor.h> | |
| 38 | + | |
| 39 | +/* | |
| 40 | + * The primary memory I/O features being added for revisions of gfxip | |
| 41 | + * beyond 7.0 (Kaveri) are: | |
| 42 | + * | |
| 43 | + * Access to ATC/IOMMU mapped memory w/ associated extension of VA to 48b | |
| 44 | + * | |
| 45 | + * “Flat” shader memory access – These are new shader vector memory | |
| 46 | + * operations that do not reference a T#/V# so a “pointer” is what is | |
| 47 | + * sourced from the vector gprs for direct access to memory. | |
| 48 | + * This pointer space has the Shared(LDS) and Private(Scratch) memory | |
| 49 | + * mapped into this pointer space as apertures. | |
| 50 | + * The hardware then determines how to direct the memory request | |
| 51 | + * based on what apertures the request falls in. | |
| 52 | + * | |
| 53 | + * Unaligned support and alignment check | |
| 54 | + * | |
| 55 | + * | |
| 56 | + * System Unified Address - SUA | |
| 57 | + * | |
| 58 | + * The standard usage for GPU virtual addresses are that they are mapped by | |
| 59 | + * a set of page tables we call GPUVM and these page tables are managed by | |
| 60 | + * a combination of vidMM/driver software components. The current virtual | |
| 61 | + * address (VA) range for GPUVM is 40b. | |
| 62 | + * | |
| 63 | + * As of gfxip7.1 and beyond we’re adding the ability for compute memory | |
| 64 | + * clients (CP/RLC, DMA, SHADER(ifetch, scalar, and vector ops)) to access | |
| 65 | + * the same page tables used by host x86 processors and that are managed by | |
| 66 | + * the operating system. This is via a technique and hardware called ATC/IOMMU. | |
| 67 | + * The GPU has the capability of accessing both the GPUVM and ATC address | |
| 68 | + * spaces for a given VMID (process) simultaneously and we call this feature | |
| 69 | + * system unified address (SUA). | |
| 70 | + * | |
| 71 | + * There are three fundamental address modes of operation for a given VMID | |
| 72 | + * (process) on the GPU: | |
| 73 | + * | |
| 74 | + * HSA64 – 64b pointers and the default address space is ATC | |
| 75 | + * HSA32 – 32b pointers and the default address space is ATC | |
| 76 | + * GPUVM – 64b pointers and the default address space is GPUVM (driver | |
| 77 | + * model mode) | |
| 78 | + * | |
| 79 | + * | |
| 80 | + * HSA64 - ATC/IOMMU 64b | |
| 81 | + * | |
| 82 | + * A 64b pointer in the AMD64/IA64 CPU architecture is not fully utilized | |
| 83 | + * by the CPU so an AMD CPU can only access the high area | |
| 84 | + * (VA[63:47] == 0x1FFFF) and low area (VA[63:47 == 0) of the address space | |
| 85 | + * so the actual VA carried to translation is 48b. There is a “hole” in | |
| 86 | + * the middle of the 64b VA space. | |
| 87 | + * | |
| 88 | + * The GPU not only has access to all of the CPU accessible address space via | |
| 89 | + * ATC/IOMMU, but it also has access to the GPUVM address space. The “system | |
| 90 | + * unified address” feature (SUA) is the mapping of GPUVM and ATC address | |
| 91 | + * spaces into a unified pointer space. The method we take for 64b mode is | |
| 92 | + * to map the full 40b GPUVM address space into the hole of the 64b address | |
| 93 | + * space. | |
| 94 | + | |
| 95 | + * The GPUVM_Base/GPUVM_Limit defines the aperture in the 64b space where we | |
| 96 | + * direct requests to be translated via GPUVM page tables instead of the | |
| 97 | + * IOMMU path. | |
| 98 | + * | |
| 99 | + * | |
| 100 | + * 64b to 49b Address conversion | |
| 101 | + * | |
| 102 | + * Note that there are still significant portions of unused regions (holes) | |
| 103 | + * in the 64b address space even for the GPU. There are several places in | |
| 104 | + * the pipeline (sw and hw), we wish to compress the 64b virtual address | |
| 105 | + * to a 49b address. This 49b address is constituted of an “ATC” bit | |
| 106 | + * plus a 48b virtual address. This 49b address is what is passed to the | |
| 107 | + * translation hardware. ATC==0 means the 48b address is a GPUVM address | |
| 108 | + * (max of 2^40 – 1) intended to be translated via GPUVM page tables. | |
| 109 | + * ATC==1 means the 48b address is intended to be translated via IOMMU | |
| 110 | + * page tables. | |
| 111 | + * | |
| 112 | + * A 64b pointer is compared to the apertures that are defined (Base/Limit), in | |
| 113 | + * this case the GPUVM aperture (red) is defined and if a pointer falls in this | |
| 114 | + * aperture, we subtract the GPUVM_Base address and set the ATC bit to zero | |
| 115 | + * as part of the 64b to 49b conversion. | |
| 116 | + * | |
| 117 | + * Where this 64b to 49b conversion is done is a function of the usage. | |
| 118 | + * Most GPU memory access is via memory objects where the driver builds | |
| 119 | + * a descriptor which consists of a base address and a memory access by | |
| 120 | + * the GPU usually consists of some kind of an offset or Cartesian coordinate | |
| 121 | + * that references this memory descriptor. This is the case for shader | |
| 122 | + * instructions that reference the T# or V# constants, or for specified | |
| 123 | + * locations of assets (ex. the shader program location). In these cases | |
| 124 | + * the driver is what handles the 64b to 49b conversion and the base | |
| 125 | + * address in the descriptor (ex. V# or T# or shader program location) | |
| 126 | + * is defined as a 48b address w/ an ATC bit. For this usage a given | |
| 127 | + * memory object cannot straddle multiple apertures in the 64b address | |
| 128 | + * space. For example a shader program cannot jump in/out between ATC | |
| 129 | + * and GPUVM space. | |
| 130 | + * | |
| 131 | + * In some cases we wish to pass a 64b pointer to the GPU hardware and | |
| 132 | + * the GPU hw does the 64b to 49b conversion before passing memory | |
| 133 | + * requests to the cache/memory system. This is the case for the | |
| 134 | + * S_LOAD and FLAT_* shader memory instructions where we have 64b pointers | |
| 135 | + * in scalar and vector GPRs respectively. | |
| 136 | + * | |
| 137 | + * In all cases (no matter where the 64b -> 49b conversion is done), the gfxip | |
| 138 | + * hardware sends a 48b address along w/ an ATC bit, to the memory controller | |
| 139 | + * on the memory request interfaces. | |
| 140 | + * | |
| 141 | + * <client>_MC_rdreq_atc // read request ATC bit | |
| 142 | + * | |
| 143 | + * 0 : <client>_MC_rdreq_addr is a GPUVM VA | |
| 144 | + * | |
| 145 | + * 1 : <client>_MC_rdreq_addr is a ATC VA | |
| 146 | + * | |
| 147 | + * | |
| 148 | + * “Spare” aperture (APE1) | |
| 149 | + * | |
| 150 | + * We use the GPUVM aperture to differentiate ATC vs. GPUVM, but we also use | |
| 151 | + * apertures to set the Mtype field for S_LOAD/FLAT_* ops which is input to the | |
| 152 | + * config tables for setting cache policies. The “spare” (APE1) aperture is | |
| 153 | + * motivated by getting a different Mtype from the default. | |
| 154 | + * The default aperture isn’t an actual base/limit aperture; it is just the | |
| 155 | + * address space that doesn’t hit any defined base/limit apertures. | |
| 156 | + * The following diagram is a complete picture of the gfxip7.x SUA apertures. | |
| 157 | + * The APE1 can be placed either below or above | |
| 158 | + * the hole (cannot be in the hole). | |
| 159 | + * | |
| 160 | + * | |
| 161 | + * General Aperture definitions and rules | |
| 162 | + * | |
| 163 | + * An aperture register definition consists of a Base, Limit, Mtype, and | |
| 164 | + * usually an ATC bit indicating which translation tables that aperture uses. | |
| 165 | + * In all cases (for SUA and DUA apertures discussed later), aperture base | |
| 166 | + * and limit definitions are 64KB aligned. | |
| 167 | + * | |
| 168 | + * <ape>_Base[63:0] = { <ape>_Base_register[63:16], 0x0000 } | |
| 169 | + * | |
| 170 | + * <ape>_Limit[63:0] = { <ape>_Limit_register[63:16], 0xFFFF } | |
| 171 | + * | |
| 172 | + * The base and limit are considered inclusive to an aperture so being | |
| 173 | + * inside an aperture means (address >= Base) AND (address <= Limit). | |
| 174 | + * | |
| 175 | + * In no case is a payload that straddles multiple apertures expected to work. | |
| 176 | + * For example a load_dword_x4 that starts in one aperture and ends in another, | |
| 177 | + * does not work. For the vector FLAT_* ops we have detection capability in | |
| 178 | + * the shader for reporting a “memory violation” back to the | |
| 179 | + * SQ block for use in traps. | |
| 180 | + * A memory violation results when an op falls into the hole, | |
| 181 | + * or a payload straddles multiple apertures. The S_LOAD instruction | |
| 182 | + * does not have this detection. | |
| 183 | + * | |
| 184 | + * Apertures cannot overlap. | |
| 185 | + * | |
| 186 | + * | |
| 187 | + * | |
| 188 | + * HSA32 - ATC/IOMMU 32b | |
| 189 | + * | |
| 190 | + * For HSA32 mode, the pointers are interpreted as 32 bits and use a single GPR | |
| 191 | + * instead of two for the S_LOAD and FLAT_* ops. The entire GPUVM space of 40b | |
| 192 | + * will not fit so there is only partial visibility to the GPUVM | |
| 193 | + * space (defined by the aperture) for S_LOAD and FLAT_* ops. | |
| 194 | + * There is no spare (APE1) aperture for HSA32 mode. | |
| 195 | + * | |
| 196 | + * | |
| 197 | + * GPUVM 64b mode (driver model) | |
| 198 | + * | |
| 199 | + * This mode is related to HSA64 in that the difference really is that | |
| 200 | + * the default aperture is GPUVM (ATC==0) and not ATC space. | |
| 201 | + * We have gfxip7.x hardware that has FLAT_* and S_LOAD support for | |
| 202 | + * SUA GPUVM mode, but does not support HSA32/HSA64. | |
| 203 | + * | |
| 204 | + * | |
| 205 | + * Device Unified Address - DUA | |
| 206 | + * | |
| 207 | + * Device unified address (DUA) is the name of the feature that maps the | |
| 208 | + * Shared(LDS) memory and Private(Scratch) memory into the overall address | |
| 209 | + * space for use by the new FLAT_* vector memory ops. The Shared and | |
| 210 | + * Private memories are mapped as apertures into the address space, | |
| 211 | + * and the hardware detects when a FLAT_* memory request is to be redirected | |
| 212 | + * to the LDS or Scratch memory when it falls into one of these apertures. | |
| 213 | + * Like the SUA apertures, the Shared/Private apertures are 64KB aligned and | |
| 214 | + * the base/limit is “in” the aperture. For both HSA64 and GPUVM SUA modes, | |
| 215 | + * the Shared/Private apertures are always placed in a limited selection of | |
| 216 | + * options in the hole of the 64b address space. For HSA32 mode, the | |
| 217 | + * Shared/Private apertures can be placed anywhere in the 32b space | |
| 218 | + * except at 0. | |
| 219 | + * | |
| 220 | + * | |
| 221 | + * HSA64 Apertures for FLAT_* vector ops | |
| 222 | + * | |
| 223 | + * For HSA64 SUA mode, the Shared and Private apertures are always placed | |
| 224 | + * in the hole w/ a limited selection of possible locations. The requests | |
| 225 | + * that fall in the private aperture are expanded as a function of the | |
| 226 | + * work-item id (tid) and redirected to the location of the | |
| 227 | + * “hidden private memory”. The hidden private can be placed in either GPUVM | |
| 228 | + * or ATC space. The addresses that fall in the shared aperture are | |
| 229 | + * re-directed to the on-chip LDS memory hardware. | |
| 230 | + * | |
| 231 | + * | |
| 232 | + * HSA32 Apertures for FLAT_* vector ops | |
| 233 | + * | |
| 234 | + * In HSA32 mode, the Private and Shared apertures can be placed anywhere | |
| 235 | + * in the 32b space except at 0 (Private or Shared Base at zero disables | |
| 236 | + * the apertures). If the base address of the apertures are non-zero | |
| 237 | + * (ie apertures exists), the size is always 64KB. | |
| 238 | + * | |
| 239 | + * | |
| 240 | + * GPUVM Apertures for FLAT_* vector ops | |
| 241 | + * | |
| 242 | + * In GPUVM mode, the Shared/Private apertures are specified identically | |
| 243 | + * to HSA64 mode where they are always in the hole at a limited selection | |
| 244 | + * of locations. | |
| 245 | + * | |
| 246 | + * | |
| 247 | + * Aperture Definitions for SUA and DUA | |
| 248 | + * | |
| 249 | + * The interpretation of the aperture register definitions for a given | |
| 250 | + * VMID is a function of the “SUA Mode” which is one of HSA64, HSA32, or | |
| 251 | + * GPUVM64 discussed in previous sections. The mode is first decoded, and | |
| 252 | + * then the remaining register decode is a function of the mode. | |
| 253 | + * | |
| 254 | + * | |
| 255 | + * SUA Mode Decode | |
| 256 | + * | |
| 257 | + * For the S_LOAD and FLAT_* shader operations, the SUA mode is decoded from | |
| 258 | + * the COMPUTE_DISPATCH_INITIATOR:DATA_ATC bit and | |
| 259 | + * the SH_MEM_CONFIG:PTR32 bits. | |
| 260 | + * | |
| 261 | + * COMPUTE_DISPATCH_INITIATOR:DATA_ATC SH_MEM_CONFIG:PTR32 Mode | |
| 262 | + * | |
| 263 | + * 1 0 HSA64 | |
| 264 | + * | |
| 265 | + * 1 1 HSA32 | |
| 266 | + * | |
| 267 | + * 0 X GPUVM64 | |
| 268 | + * | |
| 269 | + * In general the hardware will ignore the PTR32 bit and treat | |
| 270 | + * as “0” whenever DATA_ATC = “0”, but sw should set PTR32=0 | |
| 271 | + * when DATA_ATC=0. | |
| 272 | + * | |
| 273 | + * The DATA_ATC bit is only set for compute dispatches. | |
| 274 | + * All “Draw” dispatches are hardcoded to GPUVM64 mode | |
| 275 | + * for FLAT_* / S_LOAD operations. | |
| 276 | + */ | |
| 277 | + | |
| 278 | +#define MAKE_GPUVM_APP_BASE(gpu_num) \ | |
| 279 | + (((uint64_t)(gpu_num) << 61) + 0x1000000000000) | |
| 280 | + | |
| 281 | +#define MAKE_GPUVM_APP_LIMIT(base) \ | |
| 282 | + (((uint64_t)(base) & 0xFFFFFF0000000000) | 0xFFFFFFFFFF) | |
| 283 | + | |
| 284 | +#define MAKE_SCRATCH_APP_BASE(gpu_num) \ | |
| 285 | + (((uint64_t)(gpu_num) << 61) + 0x100000000) | |
| 286 | + | |
| 287 | +#define MAKE_SCRATCH_APP_LIMIT(base) \ | |
| 288 | + (((uint64_t)base & 0xFFFFFFFF00000000) | 0xFFFFFFFF) | |
| 289 | + | |
| 290 | +#define MAKE_LDS_APP_BASE(gpu_num) \ | |
| 291 | + (((uint64_t)(gpu_num) << 61) + 0x0) | |
| 292 | +#define MAKE_LDS_APP_LIMIT(base) \ | |
| 293 | + (((uint64_t)(base) & 0xFFFFFFFF00000000) | 0xFFFFFFFF) | |
| 294 | + | |
| 295 | +int kfd_init_apertures(struct kfd_process *process) | |
| 296 | +{ | |
| 297 | + uint8_t id = 0; | |
| 298 | + struct kfd_dev *dev; | |
| 299 | + struct kfd_process_device *pdd; | |
| 300 | + | |
| 301 | + mutex_lock(&process->mutex); | |
| 302 | + | |
| 303 | + /*Iterating over all devices*/ | |
| 304 | + while ((dev = kfd_topology_enum_kfd_devices(id)) != NULL && | |
| 305 | + id < NUM_OF_SUPPORTED_GPUS) { | |
| 306 | + | |
| 307 | + pdd = kfd_get_process_device_data(dev, process, 1); | |
| 308 | + | |
| 309 | + /* | |
| 310 | + * For 64 bit process aperture will be statically reserved in | |
| 311 | + * the x86_64 non canonical process address space | |
| 312 | + * amdkfd doesn't currently support apertures for 32 bit process | |
| 313 | + */ | |
| 314 | + if (process->is_32bit_user_mode) { | |
| 315 | + pdd->lds_base = pdd->lds_limit = 0; | |
| 316 | + pdd->gpuvm_base = pdd->gpuvm_limit = 0; | |
| 317 | + pdd->scratch_base = pdd->scratch_limit = 0; | |
| 318 | + } else { | |
| 319 | + /* | |
| 320 | + * node id couldn't be 0 - the three MSB bits of | |
| 321 | + * aperture shoudn't be 0 | |
| 322 | + */ | |
| 323 | + pdd->lds_base = MAKE_LDS_APP_BASE(id + 1); | |
| 324 | + | |
| 325 | + pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base); | |
| 326 | + | |
| 327 | + pdd->gpuvm_base = MAKE_GPUVM_APP_BASE(id + 1); | |
| 328 | + | |
| 329 | + pdd->gpuvm_limit = | |
| 330 | + MAKE_GPUVM_APP_LIMIT(pdd->gpuvm_base); | |
| 331 | + | |
| 332 | + pdd->scratch_base = MAKE_SCRATCH_APP_BASE(id + 1); | |
| 333 | + | |
| 334 | + pdd->scratch_limit = | |
| 335 | + MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base); | |
| 336 | + } | |
| 337 | + | |
| 338 | + dev_dbg(kfd_device, "node id %u\n", id); | |
| 339 | + dev_dbg(kfd_device, "gpu id %u\n", pdd->dev->id); | |
| 340 | + dev_dbg(kfd_device, "lds_base %llX\n", pdd->lds_base); | |
| 341 | + dev_dbg(kfd_device, "lds_limit %llX\n", pdd->lds_limit); | |
| 342 | + dev_dbg(kfd_device, "gpuvm_base %llX\n", pdd->gpuvm_base); | |
| 343 | + dev_dbg(kfd_device, "gpuvm_limit %llX\n", pdd->gpuvm_limit); | |
| 344 | + dev_dbg(kfd_device, "scratch_base %llX\n", pdd->scratch_base); | |
| 345 | + dev_dbg(kfd_device, "scratch_limit %llX\n", pdd->scratch_limit); | |
| 346 | + | |
| 347 | + id++; | |
| 348 | + } | |
| 349 | + | |
| 350 | + mutex_unlock(&process->mutex); | |
| 351 | + | |
| 352 | + return 0; | |
| 353 | +} | |
| 354 | + | |
| 355 | + | |
| @@ -0,0 +1,176 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +/* | |
| 24 | + * KFD Interrupts. | |
| 25 | + * | |
| 26 | + * AMD GPUs deliver interrupts by pushing an interrupt description onto the | |
| 27 | + * interrupt ring and then sending an interrupt. KGD receives the interrupt | |
| 28 | + * in ISR and sends us a pointer to each new entry on the interrupt ring. | |
| 29 | + * | |
| 30 | + * We generally can't process interrupt-signaled events from ISR, so we call | |
| 31 | + * out to each interrupt client module (currently only the scheduler) to ask if | |
| 32 | + * each interrupt is interesting. If they return true, then it requires further | |
| 33 | + * processing so we copy it to an internal interrupt ring and call each | |
| 34 | + * interrupt client again from a work-queue. | |
| 35 | + * | |
| 36 | + * There's no acknowledgment for the interrupts we use. The hardware simply | |
| 37 | + * queues a new interrupt each time without waiting. | |
| 38 | + * | |
| 39 | + * The fixed-size internal queue means that it's possible for us to lose | |
| 40 | + * interrupts because we have no back-pressure to the hardware. | |
| 41 | + */ | |
| 42 | + | |
| 43 | +#include <linux/slab.h> | |
| 44 | +#include <linux/device.h> | |
| 45 | +#include "kfd_priv.h" | |
| 46 | + | |
| 47 | +#define KFD_INTERRUPT_RING_SIZE 256 | |
| 48 | + | |
| 49 | +static void interrupt_wq(struct work_struct *); | |
| 50 | + | |
| 51 | +int kfd_interrupt_init(struct kfd_dev *kfd) | |
| 52 | +{ | |
| 53 | + void *interrupt_ring = kmalloc_array(KFD_INTERRUPT_RING_SIZE, | |
| 54 | + kfd->device_info->ih_ring_entry_size, | |
| 55 | + GFP_KERNEL); | |
| 56 | + if (!interrupt_ring) | |
| 57 | + return -ENOMEM; | |
| 58 | + | |
| 59 | + kfd->interrupt_ring = interrupt_ring; | |
| 60 | + kfd->interrupt_ring_size = | |
| 61 | + KFD_INTERRUPT_RING_SIZE * kfd->device_info->ih_ring_entry_size; | |
| 62 | + atomic_set(&kfd->interrupt_ring_wptr, 0); | |
| 63 | + atomic_set(&kfd->interrupt_ring_rptr, 0); | |
| 64 | + | |
| 65 | + spin_lock_init(&kfd->interrupt_lock); | |
| 66 | + | |
| 67 | + INIT_WORK(&kfd->interrupt_work, interrupt_wq); | |
| 68 | + | |
| 69 | + kfd->interrupts_active = true; | |
| 70 | + | |
| 71 | + /* | |
| 72 | + * After this function returns, the interrupt will be enabled. This | |
| 73 | + * barrier ensures that the interrupt running on a different processor | |
| 74 | + * sees all the above writes. | |
| 75 | + */ | |
| 76 | + smp_wmb(); | |
| 77 | + | |
| 78 | + return 0; | |
| 79 | +} | |
| 80 | + | |
| 81 | +void kfd_interrupt_exit(struct kfd_dev *kfd) | |
| 82 | +{ | |
| 83 | + /* | |
| 84 | + * Stop the interrupt handler from writing to the ring and scheduling | |
| 85 | + * workqueue items. The spinlock ensures that any interrupt running | |
| 86 | + * after we have unlocked sees interrupts_active = false. | |
| 87 | + */ | |
| 88 | + unsigned long flags; | |
| 89 | + | |
| 90 | + spin_lock_irqsave(&kfd->interrupt_lock, flags); | |
| 91 | + kfd->interrupts_active = false; | |
| 92 | + spin_unlock_irqrestore(&kfd->interrupt_lock, flags); | |
| 93 | + | |
| 94 | + /* | |
| 95 | + * Flush_scheduled_work ensures that there are no outstanding | |
| 96 | + * work-queue items that will access interrupt_ring. New work items | |
| 97 | + * can't be created because we stopped interrupt handling above. | |
| 98 | + */ | |
| 99 | + flush_scheduled_work(); | |
| 100 | + | |
| 101 | + kfree(kfd->interrupt_ring); | |
| 102 | +} | |
| 103 | + | |
| 104 | +/* | |
| 105 | + * This assumes that it can't be called concurrently with itself | |
| 106 | + * but only with dequeue_ih_ring_entry. | |
| 107 | + */ | |
| 108 | +bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry) | |
| 109 | +{ | |
| 110 | + unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr); | |
| 111 | + unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr); | |
| 112 | + | |
| 113 | + if ((rptr - wptr) % kfd->interrupt_ring_size == | |
| 114 | + kfd->device_info->ih_ring_entry_size) { | |
| 115 | + /* This is very bad, the system is likely to hang. */ | |
| 116 | + dev_err_ratelimited(kfd_chardev(), | |
| 117 | + "Interrupt ring overflow, dropping interrupt.\n"); | |
| 118 | + return false; | |
| 119 | + } | |
| 120 | + | |
| 121 | + memcpy(kfd->interrupt_ring + wptr, ih_ring_entry, | |
| 122 | + kfd->device_info->ih_ring_entry_size); | |
| 123 | + | |
| 124 | + wptr = (wptr + kfd->device_info->ih_ring_entry_size) % | |
| 125 | + kfd->interrupt_ring_size; | |
| 126 | + smp_wmb(); /* Ensure memcpy'd data is visible before wptr update. */ | |
| 127 | + atomic_set(&kfd->interrupt_ring_wptr, wptr); | |
| 128 | + | |
| 129 | + return true; | |
| 130 | +} | |
| 131 | + | |
| 132 | +/* | |
| 133 | + * This assumes that it can't be called concurrently with itself | |
| 134 | + * but only with enqueue_ih_ring_entry. | |
| 135 | + */ | |
| 136 | +static bool dequeue_ih_ring_entry(struct kfd_dev *kfd, void *ih_ring_entry) | |
| 137 | +{ | |
| 138 | + /* | |
| 139 | + * Assume that wait queues have an implicit barrier, i.e. anything that | |
| 140 | + * happened in the ISR before it queued work is visible. | |
| 141 | + */ | |
| 142 | + | |
| 143 | + unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr); | |
| 144 | + unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr); | |
| 145 | + | |
| 146 | + if (rptr == wptr) | |
| 147 | + return false; | |
| 148 | + | |
| 149 | + memcpy(ih_ring_entry, kfd->interrupt_ring + rptr, | |
| 150 | + kfd->device_info->ih_ring_entry_size); | |
| 151 | + | |
| 152 | + rptr = (rptr + kfd->device_info->ih_ring_entry_size) % | |
| 153 | + kfd->interrupt_ring_size; | |
| 154 | + | |
| 155 | + /* | |
| 156 | + * Ensure the rptr write update is not visible until | |
| 157 | + * memcpy has finished reading. | |
| 158 | + */ | |
| 159 | + smp_mb(); | |
| 160 | + atomic_set(&kfd->interrupt_ring_rptr, rptr); | |
| 161 | + | |
| 162 | + return true; | |
| 163 | +} | |
| 164 | + | |
| 165 | +static void interrupt_wq(struct work_struct *work) | |
| 166 | +{ | |
| 167 | + struct kfd_dev *dev = container_of(work, struct kfd_dev, | |
| 168 | + interrupt_work); | |
| 169 | + | |
| 170 | + uint32_t ih_ring_entry[DIV_ROUND_UP( | |
| 171 | + dev->device_info->ih_ring_entry_size, | |
| 172 | + sizeof(uint32_t))]; | |
| 173 | + | |
| 174 | + while (dequeue_ih_ring_entry(dev, ih_ring_entry)) | |
| 175 | + ; | |
| 176 | +} | |
| @@ -0,0 +1,347 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + * | |
| 22 | + */ | |
| 23 | + | |
| 24 | +#include <linux/types.h> | |
| 25 | +#include <linux/mutex.h> | |
| 26 | +#include <linux/slab.h> | |
| 27 | +#include <linux/printk.h> | |
| 28 | +#include "kfd_kernel_queue.h" | |
| 29 | +#include "kfd_priv.h" | |
| 30 | +#include "kfd_device_queue_manager.h" | |
| 31 | +#include "kfd_pm4_headers.h" | |
| 32 | +#include "kfd_pm4_opcodes.h" | |
| 33 | + | |
| 34 | +#define PM4_COUNT_ZERO (((1 << 15) - 1) << 16) | |
| 35 | + | |
| 36 | +static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev, | |
| 37 | + enum kfd_queue_type type, unsigned int queue_size) | |
| 38 | +{ | |
| 39 | + struct queue_properties prop; | |
| 40 | + int retval; | |
| 41 | + union PM4_MES_TYPE_3_HEADER nop; | |
| 42 | + | |
| 43 | + BUG_ON(!kq || !dev); | |
| 44 | + BUG_ON(type != KFD_QUEUE_TYPE_DIQ && type != KFD_QUEUE_TYPE_HIQ); | |
| 45 | + | |
| 46 | + pr_debug("kfd: In func %s initializing queue type %d size %d\n", | |
| 47 | + __func__, KFD_QUEUE_TYPE_HIQ, queue_size); | |
| 48 | + | |
| 49 | + nop.opcode = IT_NOP; | |
| 50 | + nop.type = PM4_TYPE_3; | |
| 51 | + nop.u32all |= PM4_COUNT_ZERO; | |
| 52 | + | |
| 53 | + kq->dev = dev; | |
| 54 | + kq->nop_packet = nop.u32all; | |
| 55 | + switch (type) { | |
| 56 | + case KFD_QUEUE_TYPE_DIQ: | |
| 57 | + case KFD_QUEUE_TYPE_HIQ: | |
| 58 | + kq->mqd = dev->dqm->get_mqd_manager(dev->dqm, | |
| 59 | + KFD_MQD_TYPE_CIK_HIQ); | |
| 60 | + break; | |
| 61 | + default: | |
| 62 | + BUG(); | |
| 63 | + break; | |
| 64 | + } | |
| 65 | + | |
| 66 | + if (kq->mqd == NULL) | |
| 67 | + return false; | |
| 68 | + | |
| 69 | + prop.doorbell_ptr = | |
| 70 | + (uint32_t *)kfd_get_kernel_doorbell(dev, &prop.doorbell_off); | |
| 71 | + | |
| 72 | + if (prop.doorbell_ptr == NULL) | |
| 73 | + goto err_get_kernel_doorbell; | |
| 74 | + | |
| 75 | + retval = kfd2kgd->allocate_mem(dev->kgd, | |
| 76 | + queue_size, | |
| 77 | + PAGE_SIZE, | |
| 78 | + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | |
| 79 | + (struct kgd_mem **) &kq->pq); | |
| 80 | + | |
| 81 | + if (retval != 0) | |
| 82 | + goto err_pq_allocate_vidmem; | |
| 83 | + | |
| 84 | + kq->pq_kernel_addr = kq->pq->cpu_ptr; | |
| 85 | + kq->pq_gpu_addr = kq->pq->gpu_addr; | |
| 86 | + | |
| 87 | + retval = kfd2kgd->allocate_mem(dev->kgd, | |
| 88 | + sizeof(*kq->rptr_kernel), | |
| 89 | + 32, | |
| 90 | + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | |
| 91 | + (struct kgd_mem **) &kq->rptr_mem); | |
| 92 | + | |
| 93 | + if (retval != 0) | |
| 94 | + goto err_rptr_allocate_vidmem; | |
| 95 | + | |
| 96 | + kq->rptr_kernel = kq->rptr_mem->cpu_ptr; | |
| 97 | + kq->rptr_gpu_addr = kq->rptr_mem->gpu_addr; | |
| 98 | + | |
| 99 | + retval = kfd2kgd->allocate_mem(dev->kgd, | |
| 100 | + sizeof(*kq->wptr_kernel), | |
| 101 | + 32, | |
| 102 | + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | |
| 103 | + (struct kgd_mem **) &kq->wptr_mem); | |
| 104 | + | |
| 105 | + if (retval != 0) | |
| 106 | + goto err_wptr_allocate_vidmem; | |
| 107 | + | |
| 108 | + kq->wptr_kernel = kq->wptr_mem->cpu_ptr; | |
| 109 | + kq->wptr_gpu_addr = kq->wptr_mem->gpu_addr; | |
| 110 | + | |
| 111 | + memset(kq->pq_kernel_addr, 0, queue_size); | |
| 112 | + memset(kq->rptr_kernel, 0, sizeof(*kq->rptr_kernel)); | |
| 113 | + memset(kq->wptr_kernel, 0, sizeof(*kq->wptr_kernel)); | |
| 114 | + | |
| 115 | + prop.queue_size = queue_size; | |
| 116 | + prop.is_interop = false; | |
| 117 | + prop.priority = 1; | |
| 118 | + prop.queue_percent = 100; | |
| 119 | + prop.type = type; | |
| 120 | + prop.vmid = 0; | |
| 121 | + prop.queue_address = kq->pq_gpu_addr; | |
| 122 | + prop.read_ptr = (uint32_t *) kq->rptr_gpu_addr; | |
| 123 | + prop.write_ptr = (uint32_t *) kq->wptr_gpu_addr; | |
| 124 | + | |
| 125 | + if (init_queue(&kq->queue, prop) != 0) | |
| 126 | + goto err_init_queue; | |
| 127 | + | |
| 128 | + kq->queue->device = dev; | |
| 129 | + kq->queue->process = kfd_get_process(current); | |
| 130 | + | |
| 131 | + retval = kq->mqd->init_mqd(kq->mqd, &kq->queue->mqd, | |
| 132 | + &kq->queue->mqd_mem_obj, | |
| 133 | + &kq->queue->gart_mqd_addr, | |
| 134 | + &kq->queue->properties); | |
| 135 | + if (retval != 0) | |
| 136 | + goto err_init_mqd; | |
| 137 | + | |
| 138 | + /* assign HIQ to HQD */ | |
| 139 | + if (type == KFD_QUEUE_TYPE_HIQ) { | |
| 140 | + pr_debug("assigning hiq to hqd\n"); | |
| 141 | + kq->queue->pipe = KFD_CIK_HIQ_PIPE; | |
| 142 | + kq->queue->queue = KFD_CIK_HIQ_QUEUE; | |
| 143 | + kq->mqd->load_mqd(kq->mqd, kq->queue->mqd, kq->queue->pipe, | |
| 144 | + kq->queue->queue, NULL); | |
| 145 | + } else { | |
| 146 | + /* allocate fence for DIQ */ | |
| 147 | + | |
| 148 | + retval = kfd2kgd->allocate_mem(dev->kgd, | |
| 149 | + sizeof(uint32_t), | |
| 150 | + 32, | |
| 151 | + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | |
| 152 | + (struct kgd_mem **) &kq->fence_mem_obj); | |
| 153 | + | |
| 154 | + if (retval != 0) | |
| 155 | + goto err_alloc_fence; | |
| 156 | + | |
| 157 | + kq->fence_kernel_address = kq->fence_mem_obj->cpu_ptr; | |
| 158 | + kq->fence_gpu_addr = kq->fence_mem_obj->gpu_addr; | |
| 159 | + } | |
| 160 | + | |
| 161 | + print_queue(kq->queue); | |
| 162 | + | |
| 163 | + return true; | |
| 164 | +err_alloc_fence: | |
| 165 | +err_init_mqd: | |
| 166 | + uninit_queue(kq->queue); | |
| 167 | +err_init_queue: | |
| 168 | + kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->wptr_mem); | |
| 169 | +err_wptr_allocate_vidmem: | |
| 170 | + kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->rptr_mem); | |
| 171 | +err_rptr_allocate_vidmem: | |
| 172 | + kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->pq); | |
| 173 | +err_pq_allocate_vidmem: | |
| 174 | + pr_err("kfd: error init pq\n"); | |
| 175 | + kfd_release_kernel_doorbell(dev, (u32 *)prop.doorbell_ptr); | |
| 176 | +err_get_kernel_doorbell: | |
| 177 | + pr_err("kfd: error init doorbell"); | |
| 178 | + return false; | |
| 179 | + | |
| 180 | +} | |
| 181 | + | |
| 182 | +static void uninitialize(struct kernel_queue *kq) | |
| 183 | +{ | |
| 184 | + BUG_ON(!kq); | |
| 185 | + | |
| 186 | + if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ) | |
| 187 | + kq->mqd->destroy_mqd(kq->mqd, | |
| 188 | + NULL, | |
| 189 | + false, | |
| 190 | + QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS, | |
| 191 | + kq->queue->pipe, | |
| 192 | + kq->queue->queue); | |
| 193 | + | |
| 194 | + kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->rptr_mem); | |
| 195 | + kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->wptr_mem); | |
| 196 | + kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->pq); | |
| 197 | + kfd_release_kernel_doorbell(kq->dev, | |
| 198 | + (u32 *)kq->queue->properties.doorbell_ptr); | |
| 199 | + uninit_queue(kq->queue); | |
| 200 | +} | |
| 201 | + | |
| 202 | +static int acquire_packet_buffer(struct kernel_queue *kq, | |
| 203 | + size_t packet_size_in_dwords, unsigned int **buffer_ptr) | |
| 204 | +{ | |
| 205 | + size_t available_size; | |
| 206 | + size_t queue_size_dwords; | |
| 207 | + uint32_t wptr, rptr; | |
| 208 | + unsigned int *queue_address; | |
| 209 | + | |
| 210 | + BUG_ON(!kq || !buffer_ptr); | |
| 211 | + | |
| 212 | + rptr = *kq->rptr_kernel; | |
| 213 | + wptr = *kq->wptr_kernel; | |
| 214 | + queue_address = (unsigned int *)kq->pq_kernel_addr; | |
| 215 | + queue_size_dwords = kq->queue->properties.queue_size / sizeof(uint32_t); | |
| 216 | + | |
| 217 | + pr_debug("kfd: In func %s\nrptr: %d\nwptr: %d\nqueue_address 0x%p\n", | |
| 218 | + __func__, rptr, wptr, queue_address); | |
| 219 | + | |
| 220 | + available_size = (rptr - 1 - wptr + queue_size_dwords) % | |
| 221 | + queue_size_dwords; | |
| 222 | + | |
| 223 | + if (packet_size_in_dwords >= queue_size_dwords || | |
| 224 | + packet_size_in_dwords >= available_size) | |
| 225 | + return -ENOMEM; | |
| 226 | + | |
| 227 | + if (wptr + packet_size_in_dwords >= queue_size_dwords) { | |
| 228 | + while (wptr > 0) { | |
| 229 | + queue_address[wptr] = kq->nop_packet; | |
| 230 | + wptr = (wptr + 1) % queue_size_dwords; | |
| 231 | + } | |
| 232 | + } | |
| 233 | + | |
| 234 | + *buffer_ptr = &queue_address[wptr]; | |
| 235 | + kq->pending_wptr = wptr + packet_size_in_dwords; | |
| 236 | + | |
| 237 | + return 0; | |
| 238 | +} | |
| 239 | + | |
| 240 | +static void submit_packet(struct kernel_queue *kq) | |
| 241 | +{ | |
| 242 | +#ifdef DEBUG | |
| 243 | + int i; | |
| 244 | +#endif | |
| 245 | + | |
| 246 | + BUG_ON(!kq); | |
| 247 | + | |
| 248 | +#ifdef DEBUG | |
| 249 | + for (i = *kq->wptr_kernel; i < kq->pending_wptr; i++) { | |
| 250 | + pr_debug("0x%2X ", kq->pq_kernel_addr[i]); | |
| 251 | + if (i % 15 == 0) | |
| 252 | + pr_debug("\n"); | |
| 253 | + } | |
| 254 | + pr_debug("\n"); | |
| 255 | +#endif | |
| 256 | + | |
| 257 | + *kq->wptr_kernel = kq->pending_wptr; | |
| 258 | + write_kernel_doorbell((u32 *)kq->queue->properties.doorbell_ptr, | |
| 259 | + kq->pending_wptr); | |
| 260 | +} | |
| 261 | + | |
| 262 | +static int sync_with_hw(struct kernel_queue *kq, unsigned long timeout_ms) | |
| 263 | +{ | |
| 264 | + unsigned long org_timeout_ms; | |
| 265 | + | |
| 266 | + BUG_ON(!kq); | |
| 267 | + | |
| 268 | + org_timeout_ms = timeout_ms; | |
| 269 | + timeout_ms += jiffies * 1000 / HZ; | |
| 270 | + while (*kq->wptr_kernel != *kq->rptr_kernel) { | |
| 271 | + if (time_after(jiffies * 1000 / HZ, timeout_ms)) { | |
| 272 | + pr_err("kfd: kernel_queue %s timeout expired %lu\n", | |
| 273 | + __func__, org_timeout_ms); | |
| 274 | + pr_err("kfd: wptr: %d rptr: %d\n", | |
| 275 | + *kq->wptr_kernel, *kq->rptr_kernel); | |
| 276 | + return -ETIME; | |
| 277 | + } | |
| 278 | + cpu_relax(); | |
| 279 | + } | |
| 280 | + | |
| 281 | + return 0; | |
| 282 | +} | |
| 283 | + | |
| 284 | +static void rollback_packet(struct kernel_queue *kq) | |
| 285 | +{ | |
| 286 | + BUG_ON(!kq); | |
| 287 | + kq->pending_wptr = *kq->queue->properties.write_ptr; | |
| 288 | +} | |
| 289 | + | |
| 290 | +struct kernel_queue *kernel_queue_init(struct kfd_dev *dev, | |
| 291 | + enum kfd_queue_type type) | |
| 292 | +{ | |
| 293 | + struct kernel_queue *kq; | |
| 294 | + | |
| 295 | + BUG_ON(!dev); | |
| 296 | + | |
| 297 | + kq = kzalloc(sizeof(struct kernel_queue), GFP_KERNEL); | |
| 298 | + if (!kq) | |
| 299 | + return NULL; | |
| 300 | + | |
| 301 | + kq->initialize = initialize; | |
| 302 | + kq->uninitialize = uninitialize; | |
| 303 | + kq->acquire_packet_buffer = acquire_packet_buffer; | |
| 304 | + kq->submit_packet = submit_packet; | |
| 305 | + kq->sync_with_hw = sync_with_hw; | |
| 306 | + kq->rollback_packet = rollback_packet; | |
| 307 | + | |
| 308 | + if (kq->initialize(kq, dev, type, KFD_KERNEL_QUEUE_SIZE) == false) { | |
| 309 | + pr_err("kfd: failed to init kernel queue\n"); | |
| 310 | + kfree(kq); | |
| 311 | + return NULL; | |
| 312 | + } | |
| 313 | + return kq; | |
| 314 | +} | |
| 315 | + | |
| 316 | +void kernel_queue_uninit(struct kernel_queue *kq) | |
| 317 | +{ | |
| 318 | + BUG_ON(!kq); | |
| 319 | + | |
| 320 | + kq->uninitialize(kq); | |
| 321 | + kfree(kq); | |
| 322 | +} | |
| 323 | + | |
| 324 | +void test_kq(struct kfd_dev *dev) | |
| 325 | +{ | |
| 326 | + struct kernel_queue *kq; | |
| 327 | + uint32_t *buffer, i; | |
| 328 | + int retval; | |
| 329 | + | |
| 330 | + BUG_ON(!dev); | |
| 331 | + | |
| 332 | + pr_debug("kfd: starting kernel queue test\n"); | |
| 333 | + | |
| 334 | + kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_HIQ); | |
| 335 | + BUG_ON(!kq); | |
| 336 | + | |
| 337 | + retval = kq->acquire_packet_buffer(kq, 5, &buffer); | |
| 338 | + BUG_ON(retval != 0); | |
| 339 | + for (i = 0; i < 5; i++) | |
| 340 | + buffer[i] = kq->nop_packet; | |
| 341 | + kq->submit_packet(kq); | |
| 342 | + kq->sync_with_hw(kq, 1000); | |
| 343 | + | |
| 344 | + pr_debug("kfd: ending kernel queue test\n"); | |
| 345 | +} | |
| 346 | + | |
| 347 | + | |
| @@ -0,0 +1,69 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + * | |
| 22 | + */ | |
| 23 | + | |
| 24 | +#ifndef KFD_KERNEL_QUEUE_H_ | |
| 25 | +#define KFD_KERNEL_QUEUE_H_ | |
| 26 | + | |
| 27 | +#include <linux/list.h> | |
| 28 | +#include <linux/types.h> | |
| 29 | +#include "kfd_priv.h" | |
| 30 | + | |
| 31 | +struct kernel_queue { | |
| 32 | + /* interface */ | |
| 33 | + bool (*initialize)(struct kernel_queue *kq, struct kfd_dev *dev, | |
| 34 | + enum kfd_queue_type type, unsigned int queue_size); | |
| 35 | + void (*uninitialize)(struct kernel_queue *kq); | |
| 36 | + int (*acquire_packet_buffer)(struct kernel_queue *kq, | |
| 37 | + size_t packet_size_in_dwords, | |
| 38 | + unsigned int **buffer_ptr); | |
| 39 | + | |
| 40 | + void (*submit_packet)(struct kernel_queue *kq); | |
| 41 | + int (*sync_with_hw)(struct kernel_queue *kq, | |
| 42 | + unsigned long timeout_ms); | |
| 43 | + void (*rollback_packet)(struct kernel_queue *kq); | |
| 44 | + | |
| 45 | + /* data */ | |
| 46 | + struct kfd_dev *dev; | |
| 47 | + struct mqd_manager *mqd; | |
| 48 | + struct queue *queue; | |
| 49 | + uint32_t pending_wptr; | |
| 50 | + unsigned int nop_packet; | |
| 51 | + | |
| 52 | + struct kfd_mem_obj *rptr_mem; | |
| 53 | + uint32_t *rptr_kernel; | |
| 54 | + uint64_t rptr_gpu_addr; | |
| 55 | + struct kfd_mem_obj *wptr_mem; | |
| 56 | + uint32_t *wptr_kernel; | |
| 57 | + uint64_t wptr_gpu_addr; | |
| 58 | + struct kfd_mem_obj *pq; | |
| 59 | + uint64_t pq_gpu_addr; | |
| 60 | + uint32_t *pq_kernel_addr; | |
| 61 | + | |
| 62 | + struct kfd_mem_obj *fence_mem_obj; | |
| 63 | + uint64_t fence_gpu_addr; | |
| 64 | + void *fence_kernel_address; | |
| 65 | + | |
| 66 | + struct list_head list; | |
| 67 | +}; | |
| 68 | + | |
| 69 | +#endif /* KFD_KERNEL_QUEUE_H_ */ | |
| @@ -0,0 +1,159 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +#include <linux/module.h> | |
| 24 | +#include <linux/sched.h> | |
| 25 | +#include <linux/moduleparam.h> | |
| 26 | +#include <linux/device.h> | |
| 27 | +#include "kfd_priv.h" | |
| 28 | + | |
| 29 | +#define KFD_DRIVER_AUTHOR "AMD Inc. and others" | |
| 30 | + | |
| 31 | +#define KFD_DRIVER_DESC "Standalone HSA driver for AMD's GPUs" | |
| 32 | +#define KFD_DRIVER_DATE "20141113" | |
| 33 | +#define KFD_DRIVER_MAJOR 0 | |
| 34 | +#define KFD_DRIVER_MINOR 7 | |
| 35 | +#define KFD_DRIVER_PATCHLEVEL 0 | |
| 36 | + | |
| 37 | +const struct kfd2kgd_calls *kfd2kgd; | |
| 38 | +static const struct kgd2kfd_calls kgd2kfd = { | |
| 39 | + .exit = kgd2kfd_exit, | |
| 40 | + .probe = kgd2kfd_probe, | |
| 41 | + .device_init = kgd2kfd_device_init, | |
| 42 | + .device_exit = kgd2kfd_device_exit, | |
| 43 | + .interrupt = kgd2kfd_interrupt, | |
| 44 | + .suspend = kgd2kfd_suspend, | |
| 45 | + .resume = kgd2kfd_resume, | |
| 46 | +}; | |
| 47 | + | |
| 48 | +int sched_policy = KFD_SCHED_POLICY_HWS; | |
| 49 | +module_param(sched_policy, int, 0444); | |
| 50 | +MODULE_PARM_DESC(sched_policy, | |
| 51 | + "Kernel cmdline parameter that defines the amdkfd scheduling policy"); | |
| 52 | + | |
| 53 | +int max_num_of_processes = KFD_MAX_NUM_OF_PROCESSES_DEFAULT; | |
| 54 | +module_param(max_num_of_processes, int, 0444); | |
| 55 | +MODULE_PARM_DESC(max_num_of_processes, | |
| 56 | + "Kernel cmdline parameter that defines the amdkfd maximum number of supported processes"); | |
| 57 | + | |
| 58 | +int max_num_of_queues_per_process = KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT; | |
| 59 | +module_param(max_num_of_queues_per_process, int, 0444); | |
| 60 | +MODULE_PARM_DESC(max_num_of_queues_per_process, | |
| 61 | + "Kernel cmdline parameter that defines the amdkfd maximum number of supported queues per process"); | |
| 62 | + | |
| 63 | +bool kgd2kfd_init(unsigned interface_version, | |
| 64 | + const struct kfd2kgd_calls *f2g, | |
| 65 | + const struct kgd2kfd_calls **g2f) | |
| 66 | +{ | |
| 67 | + /* | |
| 68 | + * Only one interface version is supported, | |
| 69 | + * no kfd/kgd version skew allowed. | |
| 70 | + */ | |
| 71 | + if (interface_version != KFD_INTERFACE_VERSION) | |
| 72 | + return false; | |
| 73 | + | |
| 74 | + /* Protection against multiple amd kgd loads */ | |
| 75 | + if (kfd2kgd) | |
| 76 | + return true; | |
| 77 | + | |
| 78 | + kfd2kgd = f2g; | |
| 79 | + *g2f = &kgd2kfd; | |
| 80 | + | |
| 81 | + return true; | |
| 82 | +} | |
| 83 | +EXPORT_SYMBOL(kgd2kfd_init); | |
| 84 | + | |
| 85 | +void kgd2kfd_exit(void) | |
| 86 | +{ | |
| 87 | +} | |
| 88 | + | |
| 89 | +static int __init kfd_module_init(void) | |
| 90 | +{ | |
| 91 | + int err; | |
| 92 | + | |
| 93 | + kfd2kgd = NULL; | |
| 94 | + | |
| 95 | + /* Verify module parameters */ | |
| 96 | + if ((sched_policy < KFD_SCHED_POLICY_HWS) || | |
| 97 | + (sched_policy > KFD_SCHED_POLICY_NO_HWS)) { | |
| 98 | + pr_err("kfd: sched_policy has invalid value\n"); | |
| 99 | + return -1; | |
| 100 | + } | |
| 101 | + | |
| 102 | + /* Verify module parameters */ | |
| 103 | + if ((max_num_of_processes < 0) || | |
| 104 | + (max_num_of_processes > KFD_MAX_NUM_OF_PROCESSES)) { | |
| 105 | + pr_err("kfd: max_num_of_processes must be between 0 to KFD_MAX_NUM_OF_PROCESSES\n"); | |
| 106 | + return -1; | |
| 107 | + } | |
| 108 | + | |
| 109 | + if ((max_num_of_queues_per_process < 0) || | |
| 110 | + (max_num_of_queues_per_process > | |
| 111 | + KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)) { | |
| 112 | + pr_err("kfd: max_num_of_queues_per_process must be between 0 to KFD_MAX_NUM_OF_QUEUES_PER_PROCESS\n"); | |
| 113 | + return -1; | |
| 114 | + } | |
| 115 | + | |
| 116 | + err = kfd_pasid_init(); | |
| 117 | + if (err < 0) | |
| 118 | + goto err_pasid; | |
| 119 | + | |
| 120 | + err = kfd_chardev_init(); | |
| 121 | + if (err < 0) | |
| 122 | + goto err_ioctl; | |
| 123 | + | |
| 124 | + err = kfd_topology_init(); | |
| 125 | + if (err < 0) | |
| 126 | + goto err_topology; | |
| 127 | + | |
| 128 | + kfd_process_create_wq(); | |
| 129 | + | |
| 130 | + dev_info(kfd_device, "Initialized module\n"); | |
| 131 | + | |
| 132 | + return 0; | |
| 133 | + | |
| 134 | +err_topology: | |
| 135 | + kfd_chardev_exit(); | |
| 136 | +err_ioctl: | |
| 137 | + kfd_pasid_exit(); | |
| 138 | +err_pasid: | |
| 139 | + return err; | |
| 140 | +} | |
| 141 | + | |
| 142 | +static void __exit kfd_module_exit(void) | |
| 143 | +{ | |
| 144 | + kfd_process_destroy_wq(); | |
| 145 | + kfd_topology_shutdown(); | |
| 146 | + kfd_chardev_exit(); | |
| 147 | + kfd_pasid_exit(); | |
| 148 | + dev_info(kfd_device, "Removed module\n"); | |
| 149 | +} | |
| 150 | + | |
| 151 | +module_init(kfd_module_init); | |
| 152 | +module_exit(kfd_module_exit); | |
| 153 | + | |
| 154 | +MODULE_AUTHOR(KFD_DRIVER_AUTHOR); | |
| 155 | +MODULE_DESCRIPTION(KFD_DRIVER_DESC); | |
| 156 | +MODULE_LICENSE("GPL and additional rights"); | |
| 157 | +MODULE_VERSION(__stringify(KFD_DRIVER_MAJOR) "." | |
| 158 | + __stringify(KFD_DRIVER_MINOR) "." | |
| 159 | + __stringify(KFD_DRIVER_PATCHLEVEL)); | |
| @@ -0,0 +1,346 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + * | |
| 22 | + */ | |
| 23 | + | |
| 24 | +#include <linux/printk.h> | |
| 25 | +#include <linux/slab.h> | |
| 26 | +#include "kfd_priv.h" | |
| 27 | +#include "kfd_mqd_manager.h" | |
| 28 | +#include "cik_regs.h" | |
| 29 | +#include "../../radeon/cik_reg.h" | |
| 30 | + | |
| 31 | +inline void busy_wait(unsigned long ms) | |
| 32 | +{ | |
| 33 | + while (time_before(jiffies, ms)) | |
| 34 | + cpu_relax(); | |
| 35 | +} | |
| 36 | + | |
| 37 | +static inline struct cik_mqd *get_mqd(void *mqd) | |
| 38 | +{ | |
| 39 | + return (struct cik_mqd *)mqd; | |
| 40 | +} | |
| 41 | + | |
| 42 | +static int init_mqd(struct mqd_manager *mm, void **mqd, | |
| 43 | + struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
| 44 | + struct queue_properties *q) | |
| 45 | +{ | |
| 46 | + uint64_t addr; | |
| 47 | + struct cik_mqd *m; | |
| 48 | + int retval; | |
| 49 | + | |
| 50 | + BUG_ON(!mm || !q || !mqd); | |
| 51 | + | |
| 52 | + pr_debug("kfd: In func %s\n", __func__); | |
| 53 | + | |
| 54 | + retval = kfd2kgd->allocate_mem(mm->dev->kgd, | |
| 55 | + sizeof(struct cik_mqd), | |
| 56 | + 256, | |
| 57 | + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | |
| 58 | + (struct kgd_mem **) mqd_mem_obj); | |
| 59 | + | |
| 60 | + if (retval != 0) | |
| 61 | + return -ENOMEM; | |
| 62 | + | |
| 63 | + m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; | |
| 64 | + addr = (*mqd_mem_obj)->gpu_addr; | |
| 65 | + | |
| 66 | + memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); | |
| 67 | + | |
| 68 | + m->header = 0xC0310800; | |
| 69 | + m->compute_pipelinestat_enable = 1; | |
| 70 | + m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | |
| 71 | + m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | |
| 72 | + m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | |
| 73 | + m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | |
| 74 | + | |
| 75 | + /* | |
| 76 | + * Make sure to use the last queue state saved on mqd when the cp | |
| 77 | + * reassigns the queue, so when queue is switched on/off (e.g over | |
| 78 | + * subscription or quantum timeout) the context will be consistent | |
| 79 | + */ | |
| 80 | + m->cp_hqd_persistent_state = | |
| 81 | + DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ; | |
| 82 | + | |
| 83 | + m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; | |
| 84 | + m->cp_mqd_base_addr_lo = lower_32_bits(addr); | |
| 85 | + m->cp_mqd_base_addr_hi = upper_32_bits(addr); | |
| 86 | + | |
| 87 | + m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN; | |
| 88 | + /* Although WinKFD writes this, I suspect it should not be necessary */ | |
| 89 | + m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE; | |
| 90 | + | |
| 91 | + m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | | |
| 92 | + QUANTUM_DURATION(10); | |
| 93 | + | |
| 94 | + /* | |
| 95 | + * Pipe Priority | |
| 96 | + * Identifies the pipe relative priority when this queue is connected | |
| 97 | + * to the pipeline. The pipe priority is against the GFX pipe and HP3D. | |
| 98 | + * In KFD we are using a fixed pipe priority set to CS_MEDIUM. | |
| 99 | + * 0 = CS_LOW (typically below GFX) | |
| 100 | + * 1 = CS_MEDIUM (typically between HP3D and GFX | |
| 101 | + * 2 = CS_HIGH (typically above HP3D) | |
| 102 | + */ | |
| 103 | + m->cp_hqd_pipe_priority = 1; | |
| 104 | + m->cp_hqd_queue_priority = 15; | |
| 105 | + | |
| 106 | + *mqd = m; | |
| 107 | + if (gart_addr != NULL) | |
| 108 | + *gart_addr = addr; | |
| 109 | + retval = mm->update_mqd(mm, m, q); | |
| 110 | + | |
| 111 | + return retval; | |
| 112 | +} | |
| 113 | + | |
| 114 | +static void uninit_mqd(struct mqd_manager *mm, void *mqd, | |
| 115 | + struct kfd_mem_obj *mqd_mem_obj) | |
| 116 | +{ | |
| 117 | + BUG_ON(!mm || !mqd); | |
| 118 | + kfd2kgd->free_mem(mm->dev->kgd, (struct kgd_mem *) mqd_mem_obj); | |
| 119 | +} | |
| 120 | + | |
| 121 | +static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id, | |
| 122 | + uint32_t queue_id, uint32_t __user *wptr) | |
| 123 | +{ | |
| 124 | + return kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id, wptr); | |
| 125 | + | |
| 126 | +} | |
| 127 | + | |
| 128 | +static int update_mqd(struct mqd_manager *mm, void *mqd, | |
| 129 | + struct queue_properties *q) | |
| 130 | +{ | |
| 131 | + struct cik_mqd *m; | |
| 132 | + | |
| 133 | + BUG_ON(!mm || !q || !mqd); | |
| 134 | + | |
| 135 | + pr_debug("kfd: In func %s\n", __func__); | |
| 136 | + | |
| 137 | + m = get_mqd(mqd); | |
| 138 | + m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | |
| 139 | + DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN; | |
| 140 | + | |
| 141 | + /* | |
| 142 | + * Calculating queue size which is log base 2 of actual queue size -1 | |
| 143 | + * dwords and another -1 for ffs | |
| 144 | + */ | |
| 145 | + m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) | |
| 146 | + - 1 - 1; | |
| 147 | + m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); | |
| 148 | + m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | |
| 149 | + m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
| 150 | + m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
| 151 | + m->cp_hqd_pq_doorbell_control = DOORBELL_EN | | |
| 152 | + DOORBELL_OFFSET(q->doorbell_off); | |
| 153 | + | |
| 154 | + m->cp_hqd_vmid = q->vmid; | |
| 155 | + | |
| 156 | + if (q->format == KFD_QUEUE_FORMAT_AQL) { | |
| 157 | + m->cp_hqd_iq_rptr = AQL_ENABLE; | |
| 158 | + m->cp_hqd_pq_control |= NO_UPDATE_RPTR; | |
| 159 | + } | |
| 160 | + | |
| 161 | + m->cp_hqd_active = 0; | |
| 162 | + q->is_active = false; | |
| 163 | + if (q->queue_size > 0 && | |
| 164 | + q->queue_address != 0 && | |
| 165 | + q->queue_percent > 0) { | |
| 166 | + m->cp_hqd_active = 1; | |
| 167 | + q->is_active = true; | |
| 168 | + } | |
| 169 | + | |
| 170 | + return 0; | |
| 171 | +} | |
| 172 | + | |
| 173 | +static int destroy_mqd(struct mqd_manager *mm, void *mqd, | |
| 174 | + enum kfd_preempt_type type, | |
| 175 | + unsigned int timeout, uint32_t pipe_id, | |
| 176 | + uint32_t queue_id) | |
| 177 | +{ | |
| 178 | + return kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout, | |
| 179 | + pipe_id, queue_id); | |
| 180 | +} | |
| 181 | + | |
| 182 | +bool is_occupied(struct mqd_manager *mm, void *mqd, | |
| 183 | + uint64_t queue_address, uint32_t pipe_id, | |
| 184 | + uint32_t queue_id) | |
| 185 | +{ | |
| 186 | + | |
| 187 | + return kfd2kgd->hqd_is_occupies(mm->dev->kgd, queue_address, | |
| 188 | + pipe_id, queue_id); | |
| 189 | + | |
| 190 | +} | |
| 191 | + | |
| 192 | +/* | |
| 193 | + * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation. | |
| 194 | + * The HIQ queue in Kaveri is using the same MQD structure as all the user mode | |
| 195 | + * queues but with different initial values. | |
| 196 | + */ | |
| 197 | + | |
| 198 | +static int init_mqd_hiq(struct mqd_manager *mm, void **mqd, | |
| 199 | + struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
| 200 | + struct queue_properties *q) | |
| 201 | +{ | |
| 202 | + uint64_t addr; | |
| 203 | + struct cik_mqd *m; | |
| 204 | + int retval; | |
| 205 | + | |
| 206 | + BUG_ON(!mm || !q || !mqd || !mqd_mem_obj); | |
| 207 | + | |
| 208 | + pr_debug("kfd: In func %s\n", __func__); | |
| 209 | + | |
| 210 | + retval = kfd2kgd->allocate_mem(mm->dev->kgd, | |
| 211 | + sizeof(struct cik_mqd), | |
| 212 | + 256, | |
| 213 | + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | |
| 214 | + (struct kgd_mem **) mqd_mem_obj); | |
| 215 | + | |
| 216 | + if (retval != 0) | |
| 217 | + return -ENOMEM; | |
| 218 | + | |
| 219 | + m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; | |
| 220 | + addr = (*mqd_mem_obj)->gpu_addr; | |
| 221 | + | |
| 222 | + memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); | |
| 223 | + | |
| 224 | + m->header = 0xC0310800; | |
| 225 | + m->compute_pipelinestat_enable = 1; | |
| 226 | + m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | |
| 227 | + m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | |
| 228 | + m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | |
| 229 | + m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | |
| 230 | + | |
| 231 | + m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE | | |
| 232 | + PRELOAD_REQ; | |
| 233 | + m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | | |
| 234 | + QUANTUM_DURATION(10); | |
| 235 | + | |
| 236 | + m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; | |
| 237 | + m->cp_mqd_base_addr_lo = lower_32_bits(addr); | |
| 238 | + m->cp_mqd_base_addr_hi = upper_32_bits(addr); | |
| 239 | + | |
| 240 | + m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE; | |
| 241 | + | |
| 242 | + /* | |
| 243 | + * Pipe Priority | |
| 244 | + * Identifies the pipe relative priority when this queue is connected | |
| 245 | + * to the pipeline. The pipe priority is against the GFX pipe and HP3D. | |
| 246 | + * In KFD we are using a fixed pipe priority set to CS_MEDIUM. | |
| 247 | + * 0 = CS_LOW (typically below GFX) | |
| 248 | + * 1 = CS_MEDIUM (typically between HP3D and GFX | |
| 249 | + * 2 = CS_HIGH (typically above HP3D) | |
| 250 | + */ | |
| 251 | + m->cp_hqd_pipe_priority = 1; | |
| 252 | + m->cp_hqd_queue_priority = 15; | |
| 253 | + | |
| 254 | + *mqd = m; | |
| 255 | + if (gart_addr) | |
| 256 | + *gart_addr = addr; | |
| 257 | + retval = mm->update_mqd(mm, m, q); | |
| 258 | + | |
| 259 | + return retval; | |
| 260 | +} | |
| 261 | + | |
| 262 | +static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, | |
| 263 | + struct queue_properties *q) | |
| 264 | +{ | |
| 265 | + struct cik_mqd *m; | |
| 266 | + | |
| 267 | + BUG_ON(!mm || !q || !mqd); | |
| 268 | + | |
| 269 | + pr_debug("kfd: In func %s\n", __func__); | |
| 270 | + | |
| 271 | + m = get_mqd(mqd); | |
| 272 | + m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | |
| 273 | + DEFAULT_MIN_AVAIL_SIZE | | |
| 274 | + PRIV_STATE | | |
| 275 | + KMD_QUEUE; | |
| 276 | + | |
| 277 | + /* | |
| 278 | + * Calculating queue size which is log base 2 of actual queue | |
| 279 | + * size -1 dwords | |
| 280 | + */ | |
| 281 | + m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) | |
| 282 | + - 1 - 1; | |
| 283 | + m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); | |
| 284 | + m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | |
| 285 | + m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
| 286 | + m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
| 287 | + m->cp_hqd_pq_doorbell_control = DOORBELL_EN | | |
| 288 | + DOORBELL_OFFSET(q->doorbell_off); | |
| 289 | + | |
| 290 | + m->cp_hqd_vmid = q->vmid; | |
| 291 | + | |
| 292 | + m->cp_hqd_active = 0; | |
| 293 | + q->is_active = false; | |
| 294 | + if (q->queue_size > 0 && | |
| 295 | + q->queue_address != 0 && | |
| 296 | + q->queue_percent > 0) { | |
| 297 | + m->cp_hqd_active = 1; | |
| 298 | + q->is_active = true; | |
| 299 | + } | |
| 300 | + | |
| 301 | + return 0; | |
| 302 | +} | |
| 303 | + | |
| 304 | +struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, | |
| 305 | + struct kfd_dev *dev) | |
| 306 | +{ | |
| 307 | + struct mqd_manager *mqd; | |
| 308 | + | |
| 309 | + BUG_ON(!dev); | |
| 310 | + BUG_ON(type >= KFD_MQD_TYPE_MAX); | |
| 311 | + | |
| 312 | + pr_debug("kfd: In func %s\n", __func__); | |
| 313 | + | |
| 314 | + mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL); | |
| 315 | + if (!mqd) | |
| 316 | + return NULL; | |
| 317 | + | |
| 318 | + mqd->dev = dev; | |
| 319 | + | |
| 320 | + switch (type) { | |
| 321 | + case KFD_MQD_TYPE_CIK_CP: | |
| 322 | + case KFD_MQD_TYPE_CIK_COMPUTE: | |
| 323 | + mqd->init_mqd = init_mqd; | |
| 324 | + mqd->uninit_mqd = uninit_mqd; | |
| 325 | + mqd->load_mqd = load_mqd; | |
| 326 | + mqd->update_mqd = update_mqd; | |
| 327 | + mqd->destroy_mqd = destroy_mqd; | |
| 328 | + mqd->is_occupied = is_occupied; | |
| 329 | + break; | |
| 330 | + case KFD_MQD_TYPE_CIK_HIQ: | |
| 331 | + mqd->init_mqd = init_mqd_hiq; | |
| 332 | + mqd->uninit_mqd = uninit_mqd; | |
| 333 | + mqd->load_mqd = load_mqd; | |
| 334 | + mqd->update_mqd = update_mqd_hiq; | |
| 335 | + mqd->destroy_mqd = destroy_mqd; | |
| 336 | + mqd->is_occupied = is_occupied; | |
| 337 | + break; | |
| 338 | + default: | |
| 339 | + kfree(mqd); | |
| 340 | + return NULL; | |
| 341 | + } | |
| 342 | + | |
| 343 | + return mqd; | |
| 344 | +} | |
| 345 | + | |
| 346 | +/* SDMA queues should be implemented here when the cp will supports them */ | |
| @@ -0,0 +1,91 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + * | |
| 22 | + */ | |
| 23 | + | |
| 24 | +#ifndef KFD_MQD_MANAGER_H_ | |
| 25 | +#define KFD_MQD_MANAGER_H_ | |
| 26 | + | |
| 27 | +#include "kfd_priv.h" | |
| 28 | + | |
| 29 | +/** | |
| 30 | + * struct mqd_manager | |
| 31 | + * | |
| 32 | + * @init_mqd: Allocates the mqd buffer on local gpu memory and initialize it. | |
| 33 | + * | |
| 34 | + * @load_mqd: Loads the mqd to a concrete hqd slot. Used only for no cp | |
| 35 | + * scheduling mode. | |
| 36 | + * | |
| 37 | + * @update_mqd: Handles a update call for the MQD | |
| 38 | + * | |
| 39 | + * @destroy_mqd: Destroys the HQD slot and by that preempt the relevant queue. | |
| 40 | + * Used only for no cp scheduling. | |
| 41 | + * | |
| 42 | + * @uninit_mqd: Releases the mqd buffer from local gpu memory. | |
| 43 | + * | |
| 44 | + * @is_occupied: Checks if the relevant HQD slot is occupied. | |
| 45 | + * | |
| 46 | + * @mqd_mutex: Mqd manager mutex. | |
| 47 | + * | |
| 48 | + * @dev: The kfd device structure coupled with this module. | |
| 49 | + * | |
| 50 | + * MQD stands for Memory Queue Descriptor which represents the current queue | |
| 51 | + * state in the memory and initiate the HQD (Hardware Queue Descriptor) state. | |
| 52 | + * This structure is actually a base class for the different types of MQDs | |
| 53 | + * structures for the variant ASICs that should be supported in the future. | |
| 54 | + * This base class is also contains all the MQD specific operations. | |
| 55 | + * Another important thing to mention is that each queue has a MQD that keeps | |
| 56 | + * his state (or context) after each preemption or reassignment. | |
| 57 | + * Basically there are a instances of the mqd manager class per MQD type per | |
| 58 | + * ASIC. Currently the kfd driver supports only Kaveri so there are instances | |
| 59 | + * per KFD_MQD_TYPE for each device. | |
| 60 | + * | |
| 61 | + */ | |
| 62 | + | |
| 63 | +struct mqd_manager { | |
| 64 | + int (*init_mqd)(struct mqd_manager *mm, void **mqd, | |
| 65 | + struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
| 66 | + struct queue_properties *q); | |
| 67 | + | |
| 68 | + int (*load_mqd)(struct mqd_manager *mm, void *mqd, | |
| 69 | + uint32_t pipe_id, uint32_t queue_id, | |
| 70 | + uint32_t __user *wptr); | |
| 71 | + | |
| 72 | + int (*update_mqd)(struct mqd_manager *mm, void *mqd, | |
| 73 | + struct queue_properties *q); | |
| 74 | + | |
| 75 | + int (*destroy_mqd)(struct mqd_manager *mm, void *mqd, | |
| 76 | + enum kfd_preempt_type type, | |
| 77 | + unsigned int timeout, uint32_t pipe_id, | |
| 78 | + uint32_t queue_id); | |
| 79 | + | |
| 80 | + void (*uninit_mqd)(struct mqd_manager *mm, void *mqd, | |
| 81 | + struct kfd_mem_obj *mqd_mem_obj); | |
| 82 | + | |
| 83 | + bool (*is_occupied)(struct mqd_manager *mm, void *mqd, | |
| 84 | + uint64_t queue_address, uint32_t pipe_id, | |
| 85 | + uint32_t queue_id); | |
| 86 | + | |
| 87 | + struct mutex mqd_mutex; | |
| 88 | + struct kfd_dev *dev; | |
| 89 | +}; | |
| 90 | + | |
| 91 | +#endif /* KFD_MQD_MANAGER_H_ */ | |
| @@ -0,0 +1,565 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + * | |
| 22 | + */ | |
| 23 | + | |
| 24 | +#include <linux/slab.h> | |
| 25 | +#include <linux/mutex.h> | |
| 26 | +#include "kfd_device_queue_manager.h" | |
| 27 | +#include "kfd_kernel_queue.h" | |
| 28 | +#include "kfd_priv.h" | |
| 29 | +#include "kfd_pm4_headers.h" | |
| 30 | +#include "kfd_pm4_opcodes.h" | |
| 31 | + | |
| 32 | +static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes, | |
| 33 | + unsigned int buffer_size_bytes) | |
| 34 | +{ | |
| 35 | + unsigned int temp = *wptr + increment_bytes / sizeof(uint32_t); | |
| 36 | + | |
| 37 | + BUG_ON((temp * sizeof(uint32_t)) > buffer_size_bytes); | |
| 38 | + *wptr = temp; | |
| 39 | +} | |
| 40 | + | |
| 41 | +static unsigned int build_pm4_header(unsigned int opcode, size_t packet_size) | |
| 42 | +{ | |
| 43 | + union PM4_MES_TYPE_3_HEADER header; | |
| 44 | + | |
| 45 | + header.u32all = 0; | |
| 46 | + header.opcode = opcode; | |
| 47 | + header.count = packet_size/sizeof(uint32_t) - 2; | |
| 48 | + header.type = PM4_TYPE_3; | |
| 49 | + | |
| 50 | + return header.u32all; | |
| 51 | +} | |
| 52 | + | |
| 53 | +static void pm_calc_rlib_size(struct packet_manager *pm, | |
| 54 | + unsigned int *rlib_size, | |
| 55 | + bool *over_subscription) | |
| 56 | +{ | |
| 57 | + unsigned int process_count, queue_count; | |
| 58 | + | |
| 59 | + BUG_ON(!pm || !rlib_size || !over_subscription); | |
| 60 | + | |
| 61 | + process_count = pm->dqm->processes_count; | |
| 62 | + queue_count = pm->dqm->queue_count; | |
| 63 | + | |
| 64 | + /* check if there is over subscription*/ | |
| 65 | + *over_subscription = false; | |
| 66 | + if ((process_count > 1) || | |
| 67 | + queue_count > PIPE_PER_ME_CP_SCHEDULING * QUEUES_PER_PIPE) { | |
| 68 | + *over_subscription = true; | |
| 69 | + pr_debug("kfd: over subscribed runlist\n"); | |
| 70 | + } | |
| 71 | + | |
| 72 | + /* calculate run list ib allocation size */ | |
| 73 | + *rlib_size = process_count * sizeof(struct pm4_map_process) + | |
| 74 | + queue_count * sizeof(struct pm4_map_queues); | |
| 75 | + | |
| 76 | + /* | |
| 77 | + * Increase the allocation size in case we need a chained run list | |
| 78 | + * when over subscription | |
| 79 | + */ | |
| 80 | + if (*over_subscription) | |
| 81 | + *rlib_size += sizeof(struct pm4_runlist); | |
| 82 | + | |
| 83 | + pr_debug("kfd: runlist ib size %d\n", *rlib_size); | |
| 84 | +} | |
| 85 | + | |
| 86 | +static int pm_allocate_runlist_ib(struct packet_manager *pm, | |
| 87 | + unsigned int **rl_buffer, | |
| 88 | + uint64_t *rl_gpu_buffer, | |
| 89 | + unsigned int *rl_buffer_size, | |
| 90 | + bool *is_over_subscription) | |
| 91 | +{ | |
| 92 | + int retval; | |
| 93 | + | |
| 94 | + BUG_ON(!pm); | |
| 95 | + BUG_ON(pm->allocated == true); | |
| 96 | + BUG_ON(is_over_subscription == NULL); | |
| 97 | + | |
| 98 | + pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription); | |
| 99 | + | |
| 100 | + retval = kfd2kgd->allocate_mem(pm->dqm->dev->kgd, | |
| 101 | + *rl_buffer_size, | |
| 102 | + PAGE_SIZE, | |
| 103 | + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | |
| 104 | + (struct kgd_mem **) &pm->ib_buffer_obj); | |
| 105 | + | |
| 106 | + if (retval != 0) { | |
| 107 | + pr_err("kfd: failed to allocate runlist IB\n"); | |
| 108 | + return retval; | |
| 109 | + } | |
| 110 | + | |
| 111 | + *(void **)rl_buffer = pm->ib_buffer_obj->cpu_ptr; | |
| 112 | + *rl_gpu_buffer = pm->ib_buffer_obj->gpu_addr; | |
| 113 | + | |
| 114 | + memset(*rl_buffer, 0, *rl_buffer_size); | |
| 115 | + pm->allocated = true; | |
| 116 | + return retval; | |
| 117 | +} | |
| 118 | + | |
| 119 | +static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer, | |
| 120 | + uint64_t ib, size_t ib_size_in_dwords, bool chain) | |
| 121 | +{ | |
| 122 | + struct pm4_runlist *packet; | |
| 123 | + | |
| 124 | + BUG_ON(!pm || !buffer || !ib); | |
| 125 | + | |
| 126 | + packet = (struct pm4_runlist *)buffer; | |
| 127 | + | |
| 128 | + memset(buffer, 0, sizeof(struct pm4_runlist)); | |
| 129 | + packet->header.u32all = build_pm4_header(IT_RUN_LIST, | |
| 130 | + sizeof(struct pm4_runlist)); | |
| 131 | + | |
| 132 | + packet->bitfields4.ib_size = ib_size_in_dwords; | |
| 133 | + packet->bitfields4.chain = chain ? 1 : 0; | |
| 134 | + packet->bitfields4.offload_polling = 0; | |
| 135 | + packet->bitfields4.valid = 1; | |
| 136 | + packet->ordinal2 = lower_32_bits(ib); | |
| 137 | + packet->bitfields3.ib_base_hi = upper_32_bits(ib); | |
| 138 | + | |
| 139 | + return 0; | |
| 140 | +} | |
| 141 | + | |
| 142 | +static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer, | |
| 143 | + struct qcm_process_device *qpd) | |
| 144 | +{ | |
| 145 | + struct pm4_map_process *packet; | |
| 146 | + struct queue *cur; | |
| 147 | + uint32_t num_queues; | |
| 148 | + | |
| 149 | + BUG_ON(!pm || !buffer || !qpd); | |
| 150 | + | |
| 151 | + packet = (struct pm4_map_process *)buffer; | |
| 152 | + | |
| 153 | + pr_debug("kfd: In func %s\n", __func__); | |
| 154 | + | |
| 155 | + memset(buffer, 0, sizeof(struct pm4_map_process)); | |
| 156 | + | |
| 157 | + packet->header.u32all = build_pm4_header(IT_MAP_PROCESS, | |
| 158 | + sizeof(struct pm4_map_process)); | |
| 159 | + packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0; | |
| 160 | + packet->bitfields2.process_quantum = 1; | |
| 161 | + packet->bitfields2.pasid = qpd->pqm->process->pasid; | |
| 162 | + packet->bitfields3.page_table_base = qpd->page_table_base; | |
| 163 | + packet->bitfields10.gds_size = qpd->gds_size; | |
| 164 | + packet->bitfields10.num_gws = qpd->num_gws; | |
| 165 | + packet->bitfields10.num_oac = qpd->num_oac; | |
| 166 | + num_queues = 0; | |
| 167 | + list_for_each_entry(cur, &qpd->queues_list, list) | |
| 168 | + num_queues++; | |
| 169 | + packet->bitfields10.num_queues = num_queues; | |
| 170 | + | |
| 171 | + packet->sh_mem_config = qpd->sh_mem_config; | |
| 172 | + packet->sh_mem_bases = qpd->sh_mem_bases; | |
| 173 | + packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base; | |
| 174 | + packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit; | |
| 175 | + | |
| 176 | + packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area); | |
| 177 | + packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area); | |
| 178 | + | |
| 179 | + return 0; | |
| 180 | +} | |
| 181 | + | |
| 182 | +static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer, | |
| 183 | + struct queue *q) | |
| 184 | +{ | |
| 185 | + struct pm4_map_queues *packet; | |
| 186 | + | |
| 187 | + BUG_ON(!pm || !buffer || !q); | |
| 188 | + | |
| 189 | + pr_debug("kfd: In func %s\n", __func__); | |
| 190 | + | |
| 191 | + packet = (struct pm4_map_queues *)buffer; | |
| 192 | + memset(buffer, 0, sizeof(struct pm4_map_queues)); | |
| 193 | + | |
| 194 | + packet->header.u32all = build_pm4_header(IT_MAP_QUEUES, | |
| 195 | + sizeof(struct pm4_map_queues)); | |
| 196 | + packet->bitfields2.alloc_format = | |
| 197 | + alloc_format__mes_map_queues__one_per_pipe; | |
| 198 | + packet->bitfields2.num_queues = 1; | |
| 199 | + packet->bitfields2.queue_sel = | |
| 200 | + queue_sel__mes_map_queues__map_to_hws_determined_queue_slots; | |
| 201 | + | |
| 202 | + packet->bitfields2.vidmem = (q->properties.is_interop) ? | |
| 203 | + vidmem__mes_map_queues__uses_video_memory : | |
| 204 | + vidmem__mes_map_queues__uses_no_video_memory; | |
| 205 | + | |
| 206 | + switch (q->properties.type) { | |
| 207 | + case KFD_QUEUE_TYPE_COMPUTE: | |
| 208 | + case KFD_QUEUE_TYPE_DIQ: | |
| 209 | + packet->bitfields2.engine_sel = | |
| 210 | + engine_sel__mes_map_queues__compute; | |
| 211 | + break; | |
| 212 | + case KFD_QUEUE_TYPE_SDMA: | |
| 213 | + packet->bitfields2.engine_sel = | |
| 214 | + engine_sel__mes_map_queues__sdma0; | |
| 215 | + break; | |
| 216 | + default: | |
| 217 | + BUG(); | |
| 218 | + break; | |
| 219 | + } | |
| 220 | + | |
| 221 | + packet->mes_map_queues_ordinals[0].bitfields3.doorbell_offset = | |
| 222 | + q->properties.doorbell_off; | |
| 223 | + | |
| 224 | + packet->mes_map_queues_ordinals[0].mqd_addr_lo = | |
| 225 | + lower_32_bits(q->gart_mqd_addr); | |
| 226 | + | |
| 227 | + packet->mes_map_queues_ordinals[0].mqd_addr_hi = | |
| 228 | + upper_32_bits(q->gart_mqd_addr); | |
| 229 | + | |
| 230 | + packet->mes_map_queues_ordinals[0].wptr_addr_lo = | |
| 231 | + lower_32_bits((uint64_t)q->properties.write_ptr); | |
| 232 | + | |
| 233 | + packet->mes_map_queues_ordinals[0].wptr_addr_hi = | |
| 234 | + upper_32_bits((uint64_t)q->properties.write_ptr); | |
| 235 | + | |
| 236 | + return 0; | |
| 237 | +} | |
| 238 | + | |
| 239 | +static int pm_create_runlist_ib(struct packet_manager *pm, | |
| 240 | + struct list_head *queues, | |
| 241 | + uint64_t *rl_gpu_addr, | |
| 242 | + size_t *rl_size_bytes) | |
| 243 | +{ | |
| 244 | + unsigned int alloc_size_bytes; | |
| 245 | + unsigned int *rl_buffer, rl_wptr, i; | |
| 246 | + int retval, proccesses_mapped; | |
| 247 | + struct device_process_node *cur; | |
| 248 | + struct qcm_process_device *qpd; | |
| 249 | + struct queue *q; | |
| 250 | + struct kernel_queue *kq; | |
| 251 | + bool is_over_subscription; | |
| 252 | + | |
| 253 | + BUG_ON(!pm || !queues || !rl_size_bytes || !rl_gpu_addr); | |
| 254 | + | |
| 255 | + rl_wptr = retval = proccesses_mapped = 0; | |
| 256 | + | |
| 257 | + retval = pm_allocate_runlist_ib(pm, &rl_buffer, rl_gpu_addr, | |
| 258 | + &alloc_size_bytes, &is_over_subscription); | |
| 259 | + if (retval != 0) | |
| 260 | + return retval; | |
| 261 | + | |
| 262 | + *rl_size_bytes = alloc_size_bytes; | |
| 263 | + | |
| 264 | + pr_debug("kfd: In func %s\n", __func__); | |
| 265 | + pr_debug("kfd: building runlist ib process count: %d queues count %d\n", | |
| 266 | + pm->dqm->processes_count, pm->dqm->queue_count); | |
| 267 | + | |
| 268 | + /* build the run list ib packet */ | |
| 269 | + list_for_each_entry(cur, queues, list) { | |
| 270 | + qpd = cur->qpd; | |
| 271 | + /* build map process packet */ | |
| 272 | + if (proccesses_mapped >= pm->dqm->processes_count) { | |
| 273 | + pr_debug("kfd: not enough space left in runlist IB\n"); | |
| 274 | + pm_release_ib(pm); | |
| 275 | + return -ENOMEM; | |
| 276 | + } | |
| 277 | + retval = pm_create_map_process(pm, &rl_buffer[rl_wptr], qpd); | |
| 278 | + if (retval != 0) | |
| 279 | + return retval; | |
| 280 | + proccesses_mapped++; | |
| 281 | + inc_wptr(&rl_wptr, sizeof(struct pm4_map_process), | |
| 282 | + alloc_size_bytes); | |
| 283 | + | |
| 284 | + list_for_each_entry(kq, &qpd->priv_queue_list, list) { | |
| 285 | + if (kq->queue->properties.is_active != true) | |
| 286 | + continue; | |
| 287 | + retval = pm_create_map_queue(pm, &rl_buffer[rl_wptr], | |
| 288 | + kq->queue); | |
| 289 | + if (retval != 0) | |
| 290 | + return retval; | |
| 291 | + inc_wptr(&rl_wptr, sizeof(struct pm4_map_queues), | |
| 292 | + alloc_size_bytes); | |
| 293 | + } | |
| 294 | + | |
| 295 | + list_for_each_entry(q, &qpd->queues_list, list) { | |
| 296 | + if (q->properties.is_active != true) | |
| 297 | + continue; | |
| 298 | + retval = pm_create_map_queue(pm, | |
| 299 | + &rl_buffer[rl_wptr], q); | |
| 300 | + if (retval != 0) | |
| 301 | + return retval; | |
| 302 | + inc_wptr(&rl_wptr, sizeof(struct pm4_map_queues), | |
| 303 | + alloc_size_bytes); | |
| 304 | + } | |
| 305 | + } | |
| 306 | + | |
| 307 | + pr_debug("kfd: finished map process and queues to runlist\n"); | |
| 308 | + | |
| 309 | + if (is_over_subscription) | |
| 310 | + pm_create_runlist(pm, &rl_buffer[rl_wptr], *rl_gpu_addr, | |
| 311 | + alloc_size_bytes / sizeof(uint32_t), true); | |
| 312 | + | |
| 313 | + for (i = 0; i < alloc_size_bytes / sizeof(uint32_t); i++) | |
| 314 | + pr_debug("0x%2X ", rl_buffer[i]); | |
| 315 | + pr_debug("\n"); | |
| 316 | + | |
| 317 | + return 0; | |
| 318 | +} | |
| 319 | + | |
| 320 | +int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm) | |
| 321 | +{ | |
| 322 | + BUG_ON(!dqm); | |
| 323 | + | |
| 324 | + pm->dqm = dqm; | |
| 325 | + mutex_init(&pm->lock); | |
| 326 | + pm->priv_queue = kernel_queue_init(dqm->dev, KFD_QUEUE_TYPE_HIQ); | |
| 327 | + if (pm->priv_queue == NULL) { | |
| 328 | + mutex_destroy(&pm->lock); | |
| 329 | + return -ENOMEM; | |
| 330 | + } | |
| 331 | + pm->allocated = false; | |
| 332 | + | |
| 333 | + return 0; | |
| 334 | +} | |
| 335 | + | |
| 336 | +void pm_uninit(struct packet_manager *pm) | |
| 337 | +{ | |
| 338 | + BUG_ON(!pm); | |
| 339 | + | |
| 340 | + mutex_destroy(&pm->lock); | |
| 341 | + kernel_queue_uninit(pm->priv_queue); | |
| 342 | +} | |
| 343 | + | |
| 344 | +int pm_send_set_resources(struct packet_manager *pm, | |
| 345 | + struct scheduling_resources *res) | |
| 346 | +{ | |
| 347 | + struct pm4_set_resources *packet; | |
| 348 | + | |
| 349 | + BUG_ON(!pm || !res); | |
| 350 | + | |
| 351 | + pr_debug("kfd: In func %s\n", __func__); | |
| 352 | + | |
| 353 | + mutex_lock(&pm->lock); | |
| 354 | + pm->priv_queue->acquire_packet_buffer(pm->priv_queue, | |
| 355 | + sizeof(*packet) / sizeof(uint32_t), | |
| 356 | + (unsigned int **)&packet); | |
| 357 | + if (packet == NULL) { | |
| 358 | + mutex_unlock(&pm->lock); | |
| 359 | + pr_err("kfd: failed to allocate buffer on kernel queue\n"); | |
| 360 | + return -ENOMEM; | |
| 361 | + } | |
| 362 | + | |
| 363 | + memset(packet, 0, sizeof(struct pm4_set_resources)); | |
| 364 | + packet->header.u32all = build_pm4_header(IT_SET_RESOURCES, | |
| 365 | + sizeof(struct pm4_set_resources)); | |
| 366 | + | |
| 367 | + packet->bitfields2.queue_type = | |
| 368 | + queue_type__mes_set_resources__hsa_interface_queue_hiq; | |
| 369 | + packet->bitfields2.vmid_mask = res->vmid_mask; | |
| 370 | + packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY; | |
| 371 | + packet->bitfields7.oac_mask = res->oac_mask; | |
| 372 | + packet->bitfields8.gds_heap_base = res->gds_heap_base; | |
| 373 | + packet->bitfields8.gds_heap_size = res->gds_heap_size; | |
| 374 | + | |
| 375 | + packet->gws_mask_lo = lower_32_bits(res->gws_mask); | |
| 376 | + packet->gws_mask_hi = upper_32_bits(res->gws_mask); | |
| 377 | + | |
| 378 | + packet->queue_mask_lo = lower_32_bits(res->queue_mask); | |
| 379 | + packet->queue_mask_hi = upper_32_bits(res->queue_mask); | |
| 380 | + | |
| 381 | + pm->priv_queue->submit_packet(pm->priv_queue); | |
| 382 | + pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); | |
| 383 | + | |
| 384 | + mutex_unlock(&pm->lock); | |
| 385 | + | |
| 386 | + return 0; | |
| 387 | +} | |
| 388 | + | |
| 389 | +int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues) | |
| 390 | +{ | |
| 391 | + uint64_t rl_gpu_ib_addr; | |
| 392 | + uint32_t *rl_buffer; | |
| 393 | + size_t rl_ib_size, packet_size_dwords; | |
| 394 | + int retval; | |
| 395 | + | |
| 396 | + BUG_ON(!pm || !dqm_queues); | |
| 397 | + | |
| 398 | + retval = pm_create_runlist_ib(pm, dqm_queues, &rl_gpu_ib_addr, | |
| 399 | + &rl_ib_size); | |
| 400 | + if (retval != 0) | |
| 401 | + goto fail_create_runlist_ib; | |
| 402 | + | |
| 403 | + pr_debug("kfd: runlist IB address: 0x%llX\n", rl_gpu_ib_addr); | |
| 404 | + | |
| 405 | + packet_size_dwords = sizeof(struct pm4_runlist) / sizeof(uint32_t); | |
| 406 | + mutex_lock(&pm->lock); | |
| 407 | + | |
| 408 | + retval = pm->priv_queue->acquire_packet_buffer(pm->priv_queue, | |
| 409 | + packet_size_dwords, &rl_buffer); | |
| 410 | + if (retval != 0) | |
| 411 | + goto fail_acquire_packet_buffer; | |
| 412 | + | |
| 413 | + retval = pm_create_runlist(pm, rl_buffer, rl_gpu_ib_addr, | |
| 414 | + rl_ib_size / sizeof(uint32_t), false); | |
| 415 | + if (retval != 0) | |
| 416 | + goto fail_create_runlist; | |
| 417 | + | |
| 418 | + pm->priv_queue->submit_packet(pm->priv_queue); | |
| 419 | + pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); | |
| 420 | + | |
| 421 | + mutex_unlock(&pm->lock); | |
| 422 | + | |
| 423 | + return retval; | |
| 424 | + | |
| 425 | +fail_create_runlist: | |
| 426 | + pm->priv_queue->rollback_packet(pm->priv_queue); | |
| 427 | +fail_acquire_packet_buffer: | |
| 428 | + mutex_unlock(&pm->lock); | |
| 429 | +fail_create_runlist_ib: | |
| 430 | + if (pm->allocated == true) | |
| 431 | + pm_release_ib(pm); | |
| 432 | + return retval; | |
| 433 | +} | |
| 434 | + | |
| 435 | +int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address, | |
| 436 | + uint32_t fence_value) | |
| 437 | +{ | |
| 438 | + int retval; | |
| 439 | + struct pm4_query_status *packet; | |
| 440 | + | |
| 441 | + BUG_ON(!pm || !fence_address); | |
| 442 | + | |
| 443 | + mutex_lock(&pm->lock); | |
| 444 | + retval = pm->priv_queue->acquire_packet_buffer( | |
| 445 | + pm->priv_queue, | |
| 446 | + sizeof(struct pm4_query_status) / sizeof(uint32_t), | |
| 447 | + (unsigned int **)&packet); | |
| 448 | + if (retval != 0) | |
| 449 | + goto fail_acquire_packet_buffer; | |
| 450 | + | |
| 451 | + packet->header.u32all = build_pm4_header(IT_QUERY_STATUS, | |
| 452 | + sizeof(struct pm4_query_status)); | |
| 453 | + | |
| 454 | + packet->bitfields2.context_id = 0; | |
| 455 | + packet->bitfields2.interrupt_sel = | |
| 456 | + interrupt_sel__mes_query_status__completion_status; | |
| 457 | + packet->bitfields2.command = | |
| 458 | + command__mes_query_status__fence_only_after_write_ack; | |
| 459 | + | |
| 460 | + packet->addr_hi = upper_32_bits((uint64_t)fence_address); | |
| 461 | + packet->addr_lo = lower_32_bits((uint64_t)fence_address); | |
| 462 | + packet->data_hi = upper_32_bits((uint64_t)fence_value); | |
| 463 | + packet->data_lo = lower_32_bits((uint64_t)fence_value); | |
| 464 | + | |
| 465 | + pm->priv_queue->submit_packet(pm->priv_queue); | |
| 466 | + pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); | |
| 467 | + mutex_unlock(&pm->lock); | |
| 468 | + | |
| 469 | + return 0; | |
| 470 | + | |
| 471 | +fail_acquire_packet_buffer: | |
| 472 | + mutex_unlock(&pm->lock); | |
| 473 | + return retval; | |
| 474 | +} | |
| 475 | + | |
| 476 | +int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type, | |
| 477 | + enum kfd_preempt_type_filter mode, | |
| 478 | + uint32_t filter_param, bool reset, | |
| 479 | + unsigned int sdma_engine) | |
| 480 | +{ | |
| 481 | + int retval; | |
| 482 | + uint32_t *buffer; | |
| 483 | + struct pm4_unmap_queues *packet; | |
| 484 | + | |
| 485 | + BUG_ON(!pm); | |
| 486 | + | |
| 487 | + mutex_lock(&pm->lock); | |
| 488 | + retval = pm->priv_queue->acquire_packet_buffer( | |
| 489 | + pm->priv_queue, | |
| 490 | + sizeof(struct pm4_unmap_queues) / sizeof(uint32_t), | |
| 491 | + &buffer); | |
| 492 | + if (retval != 0) | |
| 493 | + goto err_acquire_packet_buffer; | |
| 494 | + | |
| 495 | + packet = (struct pm4_unmap_queues *)buffer; | |
| 496 | + memset(buffer, 0, sizeof(struct pm4_unmap_queues)); | |
| 497 | + | |
| 498 | + packet->header.u32all = build_pm4_header(IT_UNMAP_QUEUES, | |
| 499 | + sizeof(struct pm4_unmap_queues)); | |
| 500 | + switch (type) { | |
| 501 | + case KFD_QUEUE_TYPE_COMPUTE: | |
| 502 | + case KFD_QUEUE_TYPE_DIQ: | |
| 503 | + packet->bitfields2.engine_sel = | |
| 504 | + engine_sel__mes_unmap_queues__compute; | |
| 505 | + break; | |
| 506 | + case KFD_QUEUE_TYPE_SDMA: | |
| 507 | + packet->bitfields2.engine_sel = | |
| 508 | + engine_sel__mes_unmap_queues__sdma0 + sdma_engine; | |
| 509 | + break; | |
| 510 | + default: | |
| 511 | + BUG(); | |
| 512 | + break; | |
| 513 | + } | |
| 514 | + | |
| 515 | + if (reset) | |
| 516 | + packet->bitfields2.action = | |
| 517 | + action__mes_unmap_queues__reset_queues; | |
| 518 | + else | |
| 519 | + packet->bitfields2.action = | |
| 520 | + action__mes_unmap_queues__preempt_queues; | |
| 521 | + | |
| 522 | + switch (mode) { | |
| 523 | + case KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE: | |
| 524 | + packet->bitfields2.queue_sel = | |
| 525 | + queue_sel__mes_unmap_queues__perform_request_on_specified_queues; | |
| 526 | + packet->bitfields2.num_queues = 1; | |
| 527 | + packet->bitfields3b.doorbell_offset0 = filter_param; | |
| 528 | + break; | |
| 529 | + case KFD_PREEMPT_TYPE_FILTER_BY_PASID: | |
| 530 | + packet->bitfields2.queue_sel = | |
| 531 | + queue_sel__mes_unmap_queues__perform_request_on_pasid_queues; | |
| 532 | + packet->bitfields3a.pasid = filter_param; | |
| 533 | + break; | |
| 534 | + case KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES: | |
| 535 | + packet->bitfields2.queue_sel = | |
| 536 | + queue_sel__mes_unmap_queues__perform_request_on_all_active_queues; | |
| 537 | + break; | |
| 538 | + default: | |
| 539 | + BUG(); | |
| 540 | + break; | |
| 541 | + }; | |
| 542 | + | |
| 543 | + pm->priv_queue->submit_packet(pm->priv_queue); | |
| 544 | + pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); | |
| 545 | + | |
| 546 | + mutex_unlock(&pm->lock); | |
| 547 | + return 0; | |
| 548 | + | |
| 549 | +err_acquire_packet_buffer: | |
| 550 | + mutex_unlock(&pm->lock); | |
| 551 | + return retval; | |
| 552 | +} | |
| 553 | + | |
| 554 | +void pm_release_ib(struct packet_manager *pm) | |
| 555 | +{ | |
| 556 | + BUG_ON(!pm); | |
| 557 | + | |
| 558 | + mutex_lock(&pm->lock); | |
| 559 | + if (pm->allocated) { | |
| 560 | + kfd2kgd->free_mem(pm->dqm->dev->kgd, | |
| 561 | + (struct kgd_mem *) pm->ib_buffer_obj); | |
| 562 | + pm->allocated = false; | |
| 563 | + } | |
| 564 | + mutex_unlock(&pm->lock); | |
| 565 | +} | |
| @@ -0,0 +1,97 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +#include <linux/slab.h> | |
| 24 | +#include <linux/types.h> | |
| 25 | +#include "kfd_priv.h" | |
| 26 | + | |
| 27 | +static unsigned long *pasid_bitmap; | |
| 28 | +static unsigned int pasid_limit; | |
| 29 | +static DEFINE_MUTEX(pasid_mutex); | |
| 30 | + | |
| 31 | +int kfd_pasid_init(void) | |
| 32 | +{ | |
| 33 | + pasid_limit = max_num_of_processes; | |
| 34 | + | |
| 35 | + pasid_bitmap = kzalloc(DIV_ROUND_UP(pasid_limit, BITS_PER_BYTE), | |
| 36 | + GFP_KERNEL); | |
| 37 | + if (!pasid_bitmap) | |
| 38 | + return -ENOMEM; | |
| 39 | + | |
| 40 | + set_bit(0, pasid_bitmap); /* PASID 0 is reserved. */ | |
| 41 | + | |
| 42 | + return 0; | |
| 43 | +} | |
| 44 | + | |
| 45 | +void kfd_pasid_exit(void) | |
| 46 | +{ | |
| 47 | + kfree(pasid_bitmap); | |
| 48 | +} | |
| 49 | + | |
| 50 | +bool kfd_set_pasid_limit(unsigned int new_limit) | |
| 51 | +{ | |
| 52 | + if (new_limit < pasid_limit) { | |
| 53 | + bool ok; | |
| 54 | + | |
| 55 | + mutex_lock(&pasid_mutex); | |
| 56 | + | |
| 57 | + /* ensure that no pasids >= new_limit are in-use */ | |
| 58 | + ok = (find_next_bit(pasid_bitmap, pasid_limit, new_limit) == | |
| 59 | + pasid_limit); | |
| 60 | + if (ok) | |
| 61 | + pasid_limit = new_limit; | |
| 62 | + | |
| 63 | + mutex_unlock(&pasid_mutex); | |
| 64 | + | |
| 65 | + return ok; | |
| 66 | + } | |
| 67 | + | |
| 68 | + return true; | |
| 69 | +} | |
| 70 | + | |
| 71 | +inline unsigned int kfd_get_pasid_limit(void) | |
| 72 | +{ | |
| 73 | + return pasid_limit; | |
| 74 | +} | |
| 75 | + | |
| 76 | +unsigned int kfd_pasid_alloc(void) | |
| 77 | +{ | |
| 78 | + unsigned int found; | |
| 79 | + | |
| 80 | + mutex_lock(&pasid_mutex); | |
| 81 | + | |
| 82 | + found = find_first_zero_bit(pasid_bitmap, pasid_limit); | |
| 83 | + if (found == pasid_limit) | |
| 84 | + found = 0; | |
| 85 | + else | |
| 86 | + set_bit(found, pasid_bitmap); | |
| 87 | + | |
| 88 | + mutex_unlock(&pasid_mutex); | |
| 89 | + | |
| 90 | + return found; | |
| 91 | +} | |
| 92 | + | |
| 93 | +void kfd_pasid_free(unsigned int pasid) | |
| 94 | +{ | |
| 95 | + BUG_ON(pasid == 0 || pasid >= pasid_limit); | |
| 96 | + clear_bit(pasid, pasid_bitmap); | |
| 97 | +} | |
| @@ -0,0 +1,405 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + * | |
| 22 | + */ | |
| 23 | + | |
| 24 | +#ifndef KFD_PM4_HEADERS_H_ | |
| 25 | +#define KFD_PM4_HEADERS_H_ | |
| 26 | + | |
| 27 | +#ifndef PM4_MES_HEADER_DEFINED | |
| 28 | +#define PM4_MES_HEADER_DEFINED | |
| 29 | +union PM4_MES_TYPE_3_HEADER { | |
| 30 | + struct { | |
| 31 | + uint32_t reserved1:8; /* < reserved */ | |
| 32 | + uint32_t opcode:8; /* < IT opcode */ | |
| 33 | + uint32_t count:14; /* < number of DWORDs - 1 | |
| 34 | + * in the information body. | |
| 35 | + */ | |
| 36 | + uint32_t type:2; /* < packet identifier. | |
| 37 | + * It should be 3 for type 3 packets | |
| 38 | + */ | |
| 39 | + }; | |
| 40 | + uint32_t u32all; | |
| 41 | +}; | |
| 42 | +#endif /* PM4_MES_HEADER_DEFINED */ | |
| 43 | + | |
| 44 | +/* --------------------MES_SET_RESOURCES-------------------- */ | |
| 45 | + | |
| 46 | +#ifndef PM4_MES_SET_RESOURCES_DEFINED | |
| 47 | +#define PM4_MES_SET_RESOURCES_DEFINED | |
| 48 | +enum set_resources_queue_type_enum { | |
| 49 | + queue_type__mes_set_resources__kernel_interface_queue_kiq = 0, | |
| 50 | + queue_type__mes_set_resources__hsa_interface_queue_hiq = 1, | |
| 51 | + queue_type__mes_set_resources__hsa_debug_interface_queue = 4 | |
| 52 | +}; | |
| 53 | + | |
| 54 | +struct pm4_set_resources { | |
| 55 | + union { | |
| 56 | + union PM4_MES_TYPE_3_HEADER header; /* header */ | |
| 57 | + uint32_t ordinal1; | |
| 58 | + }; | |
| 59 | + | |
| 60 | + union { | |
| 61 | + struct { | |
| 62 | + uint32_t vmid_mask:16; | |
| 63 | + uint32_t unmap_latency:8; | |
| 64 | + uint32_t reserved1:5; | |
| 65 | + enum set_resources_queue_type_enum queue_type:3; | |
| 66 | + } bitfields2; | |
| 67 | + uint32_t ordinal2; | |
| 68 | + }; | |
| 69 | + | |
| 70 | + uint32_t queue_mask_lo; | |
| 71 | + uint32_t queue_mask_hi; | |
| 72 | + uint32_t gws_mask_lo; | |
| 73 | + uint32_t gws_mask_hi; | |
| 74 | + | |
| 75 | + union { | |
| 76 | + struct { | |
| 77 | + uint32_t oac_mask:16; | |
| 78 | + uint32_t reserved2:16; | |
| 79 | + } bitfields7; | |
| 80 | + uint32_t ordinal7; | |
| 81 | + }; | |
| 82 | + | |
| 83 | + union { | |
| 84 | + struct { | |
| 85 | + uint32_t gds_heap_base:6; | |
| 86 | + uint32_t reserved3:5; | |
| 87 | + uint32_t gds_heap_size:6; | |
| 88 | + uint32_t reserved4:15; | |
| 89 | + } bitfields8; | |
| 90 | + uint32_t ordinal8; | |
| 91 | + }; | |
| 92 | + | |
| 93 | +}; | |
| 94 | +#endif | |
| 95 | + | |
| 96 | +/*--------------------MES_RUN_LIST-------------------- */ | |
| 97 | + | |
| 98 | +#ifndef PM4_MES_RUN_LIST_DEFINED | |
| 99 | +#define PM4_MES_RUN_LIST_DEFINED | |
| 100 | + | |
| 101 | +struct pm4_runlist { | |
| 102 | + union { | |
| 103 | + union PM4_MES_TYPE_3_HEADER header; /* header */ | |
| 104 | + uint32_t ordinal1; | |
| 105 | + }; | |
| 106 | + | |
| 107 | + union { | |
| 108 | + struct { | |
| 109 | + uint32_t reserved1:2; | |
| 110 | + uint32_t ib_base_lo:30; | |
| 111 | + } bitfields2; | |
| 112 | + uint32_t ordinal2; | |
| 113 | + }; | |
| 114 | + | |
| 115 | + union { | |
| 116 | + struct { | |
| 117 | + uint32_t ib_base_hi:16; | |
| 118 | + uint32_t reserved2:16; | |
| 119 | + } bitfields3; | |
| 120 | + uint32_t ordinal3; | |
| 121 | + }; | |
| 122 | + | |
| 123 | + union { | |
| 124 | + struct { | |
| 125 | + uint32_t ib_size:20; | |
| 126 | + uint32_t chain:1; | |
| 127 | + uint32_t offload_polling:1; | |
| 128 | + uint32_t reserved3:1; | |
| 129 | + uint32_t valid:1; | |
| 130 | + uint32_t reserved4:8; | |
| 131 | + } bitfields4; | |
| 132 | + uint32_t ordinal4; | |
| 133 | + }; | |
| 134 | + | |
| 135 | +}; | |
| 136 | +#endif | |
| 137 | + | |
| 138 | +/*--------------------MES_MAP_PROCESS-------------------- */ | |
| 139 | + | |
| 140 | +#ifndef PM4_MES_MAP_PROCESS_DEFINED | |
| 141 | +#define PM4_MES_MAP_PROCESS_DEFINED | |
| 142 | + | |
| 143 | +struct pm4_map_process { | |
| 144 | + union { | |
| 145 | + union PM4_MES_TYPE_3_HEADER header; /* header */ | |
| 146 | + uint32_t ordinal1; | |
| 147 | + }; | |
| 148 | + | |
| 149 | + union { | |
| 150 | + struct { | |
| 151 | + uint32_t pasid:16; | |
| 152 | + uint32_t reserved1:8; | |
| 153 | + uint32_t diq_enable:1; | |
| 154 | + uint32_t process_quantum:7; | |
| 155 | + } bitfields2; | |
| 156 | + uint32_t ordinal2; | |
| 157 | + }; | |
| 158 | + | |
| 159 | + union { | |
| 160 | + struct { | |
| 161 | + uint32_t page_table_base:28; | |
| 162 | + uint32_t reserved3:4; | |
| 163 | + } bitfields3; | |
| 164 | + uint32_t ordinal3; | |
| 165 | + }; | |
| 166 | + | |
| 167 | + uint32_t sh_mem_bases; | |
| 168 | + uint32_t sh_mem_ape1_base; | |
| 169 | + uint32_t sh_mem_ape1_limit; | |
| 170 | + uint32_t sh_mem_config; | |
| 171 | + uint32_t gds_addr_lo; | |
| 172 | + uint32_t gds_addr_hi; | |
| 173 | + | |
| 174 | + union { | |
| 175 | + struct { | |
| 176 | + uint32_t num_gws:6; | |
| 177 | + uint32_t reserved4:2; | |
| 178 | + uint32_t num_oac:4; | |
| 179 | + uint32_t reserved5:4; | |
| 180 | + uint32_t gds_size:6; | |
| 181 | + uint32_t num_queues:10; | |
| 182 | + } bitfields10; | |
| 183 | + uint32_t ordinal10; | |
| 184 | + }; | |
| 185 | + | |
| 186 | +}; | |
| 187 | +#endif | |
| 188 | + | |
| 189 | +/*--------------------MES_MAP_QUEUES--------------------*/ | |
| 190 | + | |
| 191 | +#ifndef PM4_MES_MAP_QUEUES_DEFINED | |
| 192 | +#define PM4_MES_MAP_QUEUES_DEFINED | |
| 193 | +enum map_queues_queue_sel_enum { | |
| 194 | + queue_sel__mes_map_queues__map_to_specified_queue_slots = 0, | |
| 195 | + queue_sel__mes_map_queues__map_to_hws_determined_queue_slots = 1, | |
| 196 | + queue_sel__mes_map_queues__enable_process_queues = 2 | |
| 197 | +}; | |
| 198 | + | |
| 199 | +enum map_queues_vidmem_enum { | |
| 200 | + vidmem__mes_map_queues__uses_no_video_memory = 0, | |
| 201 | + vidmem__mes_map_queues__uses_video_memory = 1 | |
| 202 | +}; | |
| 203 | + | |
| 204 | +enum map_queues_alloc_format_enum { | |
| 205 | + alloc_format__mes_map_queues__one_per_pipe = 0, | |
| 206 | + alloc_format__mes_map_queues__all_on_one_pipe = 1 | |
| 207 | +}; | |
| 208 | + | |
| 209 | +enum map_queues_engine_sel_enum { | |
| 210 | + engine_sel__mes_map_queues__compute = 0, | |
| 211 | + engine_sel__mes_map_queues__sdma0 = 2, | |
| 212 | + engine_sel__mes_map_queues__sdma1 = 3 | |
| 213 | +}; | |
| 214 | + | |
| 215 | +struct pm4_map_queues { | |
| 216 | + union { | |
| 217 | + union PM4_MES_TYPE_3_HEADER header; /* header */ | |
| 218 | + uint32_t ordinal1; | |
| 219 | + }; | |
| 220 | + | |
| 221 | + union { | |
| 222 | + struct { | |
| 223 | + uint32_t reserved1:4; | |
| 224 | + enum map_queues_queue_sel_enum queue_sel:2; | |
| 225 | + uint32_t reserved2:2; | |
| 226 | + uint32_t vmid:4; | |
| 227 | + uint32_t reserved3:4; | |
| 228 | + enum map_queues_vidmem_enum vidmem:2; | |
| 229 | + uint32_t reserved4:6; | |
| 230 | + enum map_queues_alloc_format_enum alloc_format:2; | |
| 231 | + enum map_queues_engine_sel_enum engine_sel:3; | |
| 232 | + uint32_t num_queues:3; | |
| 233 | + } bitfields2; | |
| 234 | + uint32_t ordinal2; | |
| 235 | + }; | |
| 236 | + | |
| 237 | + struct { | |
| 238 | + union { | |
| 239 | + struct { | |
| 240 | + uint32_t reserved5:2; | |
| 241 | + uint32_t doorbell_offset:21; | |
| 242 | + uint32_t reserved6:3; | |
| 243 | + uint32_t queue:6; | |
| 244 | + } bitfields3; | |
| 245 | + uint32_t ordinal3; | |
| 246 | + }; | |
| 247 | + | |
| 248 | + uint32_t mqd_addr_lo; | |
| 249 | + uint32_t mqd_addr_hi; | |
| 250 | + uint32_t wptr_addr_lo; | |
| 251 | + uint32_t wptr_addr_hi; | |
| 252 | + | |
| 253 | + } mes_map_queues_ordinals[1]; /* 1..N of these ordinal groups */ | |
| 254 | + | |
| 255 | +}; | |
| 256 | +#endif | |
| 257 | + | |
| 258 | +/*--------------------MES_QUERY_STATUS--------------------*/ | |
| 259 | + | |
| 260 | +#ifndef PM4_MES_QUERY_STATUS_DEFINED | |
| 261 | +#define PM4_MES_QUERY_STATUS_DEFINED | |
| 262 | +enum query_status_interrupt_sel_enum { | |
| 263 | + interrupt_sel__mes_query_status__completion_status = 0, | |
| 264 | + interrupt_sel__mes_query_status__process_status = 1, | |
| 265 | + interrupt_sel__mes_query_status__queue_status = 2 | |
| 266 | +}; | |
| 267 | + | |
| 268 | +enum query_status_command_enum { | |
| 269 | + command__mes_query_status__interrupt_only = 0, | |
| 270 | + command__mes_query_status__fence_only_immediate = 1, | |
| 271 | + command__mes_query_status__fence_only_after_write_ack = 2, | |
| 272 | + command__mes_query_status__fence_wait_for_write_ack_send_interrupt = 3 | |
| 273 | +}; | |
| 274 | + | |
| 275 | +enum query_status_engine_sel_enum { | |
| 276 | + engine_sel__mes_query_status__compute = 0, | |
| 277 | + engine_sel__mes_query_status__sdma0_queue = 2, | |
| 278 | + engine_sel__mes_query_status__sdma1_queue = 3 | |
| 279 | +}; | |
| 280 | + | |
| 281 | +struct pm4_query_status { | |
| 282 | + union { | |
| 283 | + union PM4_MES_TYPE_3_HEADER header; /* header */ | |
| 284 | + uint32_t ordinal1; | |
| 285 | + }; | |
| 286 | + | |
| 287 | + union { | |
| 288 | + struct { | |
| 289 | + uint32_t context_id:28; | |
| 290 | + enum query_status_interrupt_sel_enum interrupt_sel:2; | |
| 291 | + enum query_status_command_enum command:2; | |
| 292 | + } bitfields2; | |
| 293 | + uint32_t ordinal2; | |
| 294 | + }; | |
| 295 | + | |
| 296 | + union { | |
| 297 | + struct { | |
| 298 | + uint32_t pasid:16; | |
| 299 | + uint32_t reserved1:16; | |
| 300 | + } bitfields3a; | |
| 301 | + struct { | |
| 302 | + uint32_t reserved2:2; | |
| 303 | + uint32_t doorbell_offset:21; | |
| 304 | + uint32_t reserved3:3; | |
| 305 | + enum query_status_engine_sel_enum engine_sel:3; | |
| 306 | + uint32_t reserved4:3; | |
| 307 | + } bitfields3b; | |
| 308 | + uint32_t ordinal3; | |
| 309 | + }; | |
| 310 | + | |
| 311 | + uint32_t addr_lo; | |
| 312 | + uint32_t addr_hi; | |
| 313 | + uint32_t data_lo; | |
| 314 | + uint32_t data_hi; | |
| 315 | +}; | |
| 316 | +#endif | |
| 317 | + | |
| 318 | +/*--------------------MES_UNMAP_QUEUES--------------------*/ | |
| 319 | + | |
| 320 | +#ifndef PM4_MES_UNMAP_QUEUES_DEFINED | |
| 321 | +#define PM4_MES_UNMAP_QUEUES_DEFINED | |
| 322 | +enum unmap_queues_action_enum { | |
| 323 | + action__mes_unmap_queues__preempt_queues = 0, | |
| 324 | + action__mes_unmap_queues__reset_queues = 1, | |
| 325 | + action__mes_unmap_queues__disable_process_queues = 2 | |
| 326 | +}; | |
| 327 | + | |
| 328 | +enum unmap_queues_queue_sel_enum { | |
| 329 | + queue_sel__mes_unmap_queues__perform_request_on_specified_queues = 0, | |
| 330 | + queue_sel__mes_unmap_queues__perform_request_on_pasid_queues = 1, | |
| 331 | + queue_sel__mes_unmap_queues__perform_request_on_all_active_queues = 2 | |
| 332 | +}; | |
| 333 | + | |
| 334 | +enum unmap_queues_engine_sel_enum { | |
| 335 | + engine_sel__mes_unmap_queues__compute = 0, | |
| 336 | + engine_sel__mes_unmap_queues__sdma0 = 2, | |
| 337 | + engine_sel__mes_unmap_queues__sdma1 = 3 | |
| 338 | +}; | |
| 339 | + | |
| 340 | +struct pm4_unmap_queues { | |
| 341 | + union { | |
| 342 | + union PM4_MES_TYPE_3_HEADER header; /* header */ | |
| 343 | + uint32_t ordinal1; | |
| 344 | + }; | |
| 345 | + | |
| 346 | + union { | |
| 347 | + struct { | |
| 348 | + enum unmap_queues_action_enum action:2; | |
| 349 | + uint32_t reserved1:2; | |
| 350 | + enum unmap_queues_queue_sel_enum queue_sel:2; | |
| 351 | + uint32_t reserved2:20; | |
| 352 | + enum unmap_queues_engine_sel_enum engine_sel:3; | |
| 353 | + uint32_t num_queues:3; | |
| 354 | + } bitfields2; | |
| 355 | + uint32_t ordinal2; | |
| 356 | + }; | |
| 357 | + | |
| 358 | + union { | |
| 359 | + struct { | |
| 360 | + uint32_t pasid:16; | |
| 361 | + uint32_t reserved3:16; | |
| 362 | + } bitfields3a; | |
| 363 | + struct { | |
| 364 | + uint32_t reserved4:2; | |
| 365 | + uint32_t doorbell_offset0:21; | |
| 366 | + uint32_t reserved5:9; | |
| 367 | + } bitfields3b; | |
| 368 | + uint32_t ordinal3; | |
| 369 | + }; | |
| 370 | + | |
| 371 | + union { | |
| 372 | + struct { | |
| 373 | + uint32_t reserved6:2; | |
| 374 | + uint32_t doorbell_offset1:21; | |
| 375 | + uint32_t reserved7:9; | |
| 376 | + } bitfields4; | |
| 377 | + uint32_t ordinal4; | |
| 378 | + }; | |
| 379 | + | |
| 380 | + union { | |
| 381 | + struct { | |
| 382 | + uint32_t reserved8:2; | |
| 383 | + uint32_t doorbell_offset2:21; | |
| 384 | + uint32_t reserved9:9; | |
| 385 | + } bitfields5; | |
| 386 | + uint32_t ordinal5; | |
| 387 | + }; | |
| 388 | + | |
| 389 | + union { | |
| 390 | + struct { | |
| 391 | + uint32_t reserved10:2; | |
| 392 | + uint32_t doorbell_offset3:21; | |
| 393 | + uint32_t reserved11:9; | |
| 394 | + } bitfields6; | |
| 395 | + uint32_t ordinal6; | |
| 396 | + }; | |
| 397 | + | |
| 398 | +}; | |
| 399 | +#endif | |
| 400 | + | |
| 401 | +enum { | |
| 402 | + CACHE_FLUSH_AND_INV_TS_EVENT = 0x00000014 | |
| 403 | +}; | |
| 404 | + | |
| 405 | +#endif /* KFD_PM4_HEADERS_H_ */ | |
| @@ -0,0 +1,107 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + * | |
| 22 | + */ | |
| 23 | + | |
| 24 | + | |
| 25 | +#ifndef KFD_PM4_OPCODES_H | |
| 26 | +#define KFD_PM4_OPCODES_H | |
| 27 | + | |
| 28 | +enum it_opcode_type { | |
| 29 | + IT_NOP = 0x10, | |
| 30 | + IT_SET_BASE = 0x11, | |
| 31 | + IT_CLEAR_STATE = 0x12, | |
| 32 | + IT_INDEX_BUFFER_SIZE = 0x13, | |
| 33 | + IT_DISPATCH_DIRECT = 0x15, | |
| 34 | + IT_DISPATCH_INDIRECT = 0x16, | |
| 35 | + IT_ATOMIC_GDS = 0x1D, | |
| 36 | + IT_OCCLUSION_QUERY = 0x1F, | |
| 37 | + IT_SET_PREDICATION = 0x20, | |
| 38 | + IT_REG_RMW = 0x21, | |
| 39 | + IT_COND_EXEC = 0x22, | |
| 40 | + IT_PRED_EXEC = 0x23, | |
| 41 | + IT_DRAW_INDIRECT = 0x24, | |
| 42 | + IT_DRAW_INDEX_INDIRECT = 0x25, | |
| 43 | + IT_INDEX_BASE = 0x26, | |
| 44 | + IT_DRAW_INDEX_2 = 0x27, | |
| 45 | + IT_CONTEXT_CONTROL = 0x28, | |
| 46 | + IT_INDEX_TYPE = 0x2A, | |
| 47 | + IT_DRAW_INDIRECT_MULTI = 0x2C, | |
| 48 | + IT_DRAW_INDEX_AUTO = 0x2D, | |
| 49 | + IT_NUM_INSTANCES = 0x2F, | |
| 50 | + IT_DRAW_INDEX_MULTI_AUTO = 0x30, | |
| 51 | + IT_INDIRECT_BUFFER_CNST = 0x33, | |
| 52 | + IT_STRMOUT_BUFFER_UPDATE = 0x34, | |
| 53 | + IT_DRAW_INDEX_OFFSET_2 = 0x35, | |
| 54 | + IT_DRAW_PREAMBLE = 0x36, | |
| 55 | + IT_WRITE_DATA = 0x37, | |
| 56 | + IT_DRAW_INDEX_INDIRECT_MULTI = 0x38, | |
| 57 | + IT_MEM_SEMAPHORE = 0x39, | |
| 58 | + IT_COPY_DW = 0x3B, | |
| 59 | + IT_WAIT_REG_MEM = 0x3C, | |
| 60 | + IT_INDIRECT_BUFFER = 0x3F, | |
| 61 | + IT_COPY_DATA = 0x40, | |
| 62 | + IT_PFP_SYNC_ME = 0x42, | |
| 63 | + IT_SURFACE_SYNC = 0x43, | |
| 64 | + IT_COND_WRITE = 0x45, | |
| 65 | + IT_EVENT_WRITE = 0x46, | |
| 66 | + IT_EVENT_WRITE_EOP = 0x47, | |
| 67 | + IT_EVENT_WRITE_EOS = 0x48, | |
| 68 | + IT_RELEASE_MEM = 0x49, | |
| 69 | + IT_PREAMBLE_CNTL = 0x4A, | |
| 70 | + IT_DMA_DATA = 0x50, | |
| 71 | + IT_ACQUIRE_MEM = 0x58, | |
| 72 | + IT_REWIND = 0x59, | |
| 73 | + IT_LOAD_UCONFIG_REG = 0x5E, | |
| 74 | + IT_LOAD_SH_REG = 0x5F, | |
| 75 | + IT_LOAD_CONFIG_REG = 0x60, | |
| 76 | + IT_LOAD_CONTEXT_REG = 0x61, | |
| 77 | + IT_SET_CONFIG_REG = 0x68, | |
| 78 | + IT_SET_CONTEXT_REG = 0x69, | |
| 79 | + IT_SET_CONTEXT_REG_INDIRECT = 0x73, | |
| 80 | + IT_SET_SH_REG = 0x76, | |
| 81 | + IT_SET_SH_REG_OFFSET = 0x77, | |
| 82 | + IT_SET_QUEUE_REG = 0x78, | |
| 83 | + IT_SET_UCONFIG_REG = 0x79, | |
| 84 | + IT_SCRATCH_RAM_WRITE = 0x7D, | |
| 85 | + IT_SCRATCH_RAM_READ = 0x7E, | |
| 86 | + IT_LOAD_CONST_RAM = 0x80, | |
| 87 | + IT_WRITE_CONST_RAM = 0x81, | |
| 88 | + IT_DUMP_CONST_RAM = 0x83, | |
| 89 | + IT_INCREMENT_CE_COUNTER = 0x84, | |
| 90 | + IT_INCREMENT_DE_COUNTER = 0x85, | |
| 91 | + IT_WAIT_ON_CE_COUNTER = 0x86, | |
| 92 | + IT_WAIT_ON_DE_COUNTER_DIFF = 0x88, | |
| 93 | + IT_SWITCH_BUFFER = 0x8B, | |
| 94 | + IT_SET_RESOURCES = 0xA0, | |
| 95 | + IT_MAP_PROCESS = 0xA1, | |
| 96 | + IT_MAP_QUEUES = 0xA2, | |
| 97 | + IT_UNMAP_QUEUES = 0xA3, | |
| 98 | + IT_QUERY_STATUS = 0xA4, | |
| 99 | + IT_RUN_LIST = 0xA5, | |
| 100 | +}; | |
| 101 | + | |
| 102 | +#define PM4_TYPE_0 0 | |
| 103 | +#define PM4_TYPE_2 2 | |
| 104 | +#define PM4_TYPE_3 3 | |
| 105 | + | |
| 106 | +#endif /* KFD_PM4_OPCODES_H */ | |
| 107 | + | |
| @@ -0,0 +1,598 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +#ifndef KFD_PRIV_H_INCLUDED | |
| 24 | +#define KFD_PRIV_H_INCLUDED | |
| 25 | + | |
| 26 | +#include <linux/hashtable.h> | |
| 27 | +#include <linux/mmu_notifier.h> | |
| 28 | +#include <linux/mutex.h> | |
| 29 | +#include <linux/types.h> | |
| 30 | +#include <linux/atomic.h> | |
| 31 | +#include <linux/workqueue.h> | |
| 32 | +#include <linux/spinlock.h> | |
| 33 | +#include <linux/kfd_ioctl.h> | |
| 34 | +#include <kgd_kfd_interface.h> | |
| 35 | + | |
| 36 | +#define KFD_SYSFS_FILE_MODE 0444 | |
| 37 | + | |
| 38 | +/* | |
| 39 | + * When working with cp scheduler we should assign the HIQ manually or via | |
| 40 | + * the radeon driver to a fixed hqd slot, here are the fixed HIQ hqd slot | |
| 41 | + * definitions for Kaveri. In Kaveri only the first ME queues participates | |
| 42 | + * in the cp scheduling taking that in mind we set the HIQ slot in the | |
| 43 | + * second ME. | |
| 44 | + */ | |
| 45 | +#define KFD_CIK_HIQ_PIPE 4 | |
| 46 | +#define KFD_CIK_HIQ_QUEUE 0 | |
| 47 | + | |
| 48 | +/* GPU ID hash width in bits */ | |
| 49 | +#define KFD_GPU_ID_HASH_WIDTH 16 | |
| 50 | + | |
| 51 | +/* Macro for allocating structures */ | |
| 52 | +#define kfd_alloc_struct(ptr_to_struct) \ | |
| 53 | + ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL)) | |
| 54 | + | |
| 55 | +/* Kernel module parameter to specify maximum number of supported processes */ | |
| 56 | +extern int max_num_of_processes; | |
| 57 | + | |
| 58 | +#define KFD_MAX_NUM_OF_PROCESSES_DEFAULT 32 | |
| 59 | +#define KFD_MAX_NUM_OF_PROCESSES 512 | |
| 60 | + | |
| 61 | +/* | |
| 62 | + * Kernel module parameter to specify maximum number of supported queues | |
| 63 | + * per process | |
| 64 | + */ | |
| 65 | +extern int max_num_of_queues_per_process; | |
| 66 | + | |
| 67 | +#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT 128 | |
| 68 | +#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024 | |
| 69 | + | |
| 70 | +#define KFD_KERNEL_QUEUE_SIZE 2048 | |
| 71 | + | |
| 72 | +/* Kernel module parameter to specify the scheduling policy */ | |
| 73 | +extern int sched_policy; | |
| 74 | + | |
| 75 | +/** | |
| 76 | + * enum kfd_sched_policy | |
| 77 | + * | |
| 78 | + * @KFD_SCHED_POLICY_HWS: H/W scheduling policy known as command processor (cp) | |
| 79 | + * scheduling. In this scheduling mode we're using the firmware code to | |
| 80 | + * schedule the user mode queues and kernel queues such as HIQ and DIQ. | |
| 81 | + * the HIQ queue is used as a special queue that dispatches the configuration | |
| 82 | + * to the cp and the user mode queues list that are currently running. | |
| 83 | + * the DIQ queue is a debugging queue that dispatches debugging commands to the | |
| 84 | + * firmware. | |
| 85 | + * in this scheduling mode user mode queues over subscription feature is | |
| 86 | + * enabled. | |
| 87 | + * | |
| 88 | + * @KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: The same as above but the over | |
| 89 | + * subscription feature disabled. | |
| 90 | + * | |
| 91 | + * @KFD_SCHED_POLICY_NO_HWS: no H/W scheduling policy is a mode which directly | |
| 92 | + * set the command processor registers and sets the queues "manually". This | |
| 93 | + * mode is used *ONLY* for debugging proposes. | |
| 94 | + * | |
| 95 | + */ | |
| 96 | +enum kfd_sched_policy { | |
| 97 | + KFD_SCHED_POLICY_HWS = 0, | |
| 98 | + KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION, | |
| 99 | + KFD_SCHED_POLICY_NO_HWS | |
| 100 | +}; | |
| 101 | + | |
| 102 | +enum cache_policy { | |
| 103 | + cache_policy_coherent, | |
| 104 | + cache_policy_noncoherent | |
| 105 | +}; | |
| 106 | + | |
| 107 | +struct kfd_device_info { | |
| 108 | + unsigned int max_pasid_bits; | |
| 109 | + size_t ih_ring_entry_size; | |
| 110 | + uint16_t mqd_size_aligned; | |
| 111 | +}; | |
| 112 | + | |
| 113 | +struct kfd_dev { | |
| 114 | + struct kgd_dev *kgd; | |
| 115 | + | |
| 116 | + const struct kfd_device_info *device_info; | |
| 117 | + struct pci_dev *pdev; | |
| 118 | + | |
| 119 | + unsigned int id; /* topology stub index */ | |
| 120 | + | |
| 121 | + phys_addr_t doorbell_base; /* Start of actual doorbells used by | |
| 122 | + * KFD. It is aligned for mapping | |
| 123 | + * into user mode | |
| 124 | + */ | |
| 125 | + size_t doorbell_id_offset; /* Doorbell offset (from KFD doorbell | |
| 126 | + * to HW doorbell, GFX reserved some | |
| 127 | + * at the start) | |
| 128 | + */ | |
| 129 | + size_t doorbell_process_limit; /* Number of processes we have doorbell | |
| 130 | + * space for. | |
| 131 | + */ | |
| 132 | + u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells | |
| 133 | + * page used by kernel queue | |
| 134 | + */ | |
| 135 | + | |
| 136 | + struct kgd2kfd_shared_resources shared_resources; | |
| 137 | + | |
| 138 | + void *interrupt_ring; | |
| 139 | + size_t interrupt_ring_size; | |
| 140 | + atomic_t interrupt_ring_rptr; | |
| 141 | + atomic_t interrupt_ring_wptr; | |
| 142 | + struct work_struct interrupt_work; | |
| 143 | + spinlock_t interrupt_lock; | |
| 144 | + | |
| 145 | + /* QCM Device instance */ | |
| 146 | + struct device_queue_manager *dqm; | |
| 147 | + | |
| 148 | + bool init_complete; | |
| 149 | + /* | |
| 150 | + * Interrupts of interest to KFD are copied | |
| 151 | + * from the HW ring into a SW ring. | |
| 152 | + */ | |
| 153 | + bool interrupts_active; | |
| 154 | +}; | |
| 155 | + | |
| 156 | +/* KGD2KFD callbacks */ | |
| 157 | +void kgd2kfd_exit(void); | |
| 158 | +struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev); | |
| 159 | +bool kgd2kfd_device_init(struct kfd_dev *kfd, | |
| 160 | + const struct kgd2kfd_shared_resources *gpu_resources); | |
| 161 | +void kgd2kfd_device_exit(struct kfd_dev *kfd); | |
| 162 | + | |
| 163 | +extern const struct kfd2kgd_calls *kfd2kgd; | |
| 164 | + | |
| 165 | +struct kfd_mem_obj { | |
| 166 | + void *bo; | |
| 167 | + uint64_t gpu_addr; | |
| 168 | + uint32_t *cpu_ptr; | |
| 169 | +}; | |
| 170 | + | |
| 171 | +enum kfd_mempool { | |
| 172 | + KFD_MEMPOOL_SYSTEM_CACHEABLE = 1, | |
| 173 | + KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2, | |
| 174 | + KFD_MEMPOOL_FRAMEBUFFER = 3, | |
| 175 | +}; | |
| 176 | + | |
| 177 | +/* Character device interface */ | |
| 178 | +int kfd_chardev_init(void); | |
| 179 | +void kfd_chardev_exit(void); | |
| 180 | +struct device *kfd_chardev(void); | |
| 181 | + | |
| 182 | +/** | |
| 183 | + * enum kfd_preempt_type_filter | |
| 184 | + * | |
| 185 | + * @KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE: Preempts single queue. | |
| 186 | + * | |
| 187 | + * @KFD_PRERMPT_TYPE_FILTER_ALL_QUEUES: Preempts all queues in the | |
| 188 | + * running queues list. | |
| 189 | + * | |
| 190 | + * @KFD_PRERMPT_TYPE_FILTER_BY_PASID: Preempts queues that belongs to | |
| 191 | + * specific process. | |
| 192 | + * | |
| 193 | + */ | |
| 194 | +enum kfd_preempt_type_filter { | |
| 195 | + KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE, | |
| 196 | + KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, | |
| 197 | + KFD_PREEMPT_TYPE_FILTER_BY_PASID | |
| 198 | +}; | |
| 199 | + | |
| 200 | +enum kfd_preempt_type { | |
| 201 | + KFD_PREEMPT_TYPE_WAVEFRONT, | |
| 202 | + KFD_PREEMPT_TYPE_WAVEFRONT_RESET | |
| 203 | +}; | |
| 204 | + | |
| 205 | +/** | |
| 206 | + * enum kfd_queue_type | |
| 207 | + * | |
| 208 | + * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type. | |
| 209 | + * | |
| 210 | + * @KFD_QUEUE_TYPE_SDMA: Sdma user mode queue type. | |
| 211 | + * | |
| 212 | + * @KFD_QUEUE_TYPE_HIQ: HIQ queue type. | |
| 213 | + * | |
| 214 | + * @KFD_QUEUE_TYPE_DIQ: DIQ queue type. | |
| 215 | + */ | |
| 216 | +enum kfd_queue_type { | |
| 217 | + KFD_QUEUE_TYPE_COMPUTE, | |
| 218 | + KFD_QUEUE_TYPE_SDMA, | |
| 219 | + KFD_QUEUE_TYPE_HIQ, | |
| 220 | + KFD_QUEUE_TYPE_DIQ | |
| 221 | +}; | |
| 222 | + | |
| 223 | +enum kfd_queue_format { | |
| 224 | + KFD_QUEUE_FORMAT_PM4, | |
| 225 | + KFD_QUEUE_FORMAT_AQL | |
| 226 | +}; | |
| 227 | + | |
| 228 | +/** | |
| 229 | + * struct queue_properties | |
| 230 | + * | |
| 231 | + * @type: The queue type. | |
| 232 | + * | |
| 233 | + * @queue_id: Queue identifier. | |
| 234 | + * | |
| 235 | + * @queue_address: Queue ring buffer address. | |
| 236 | + * | |
| 237 | + * @queue_size: Queue ring buffer size. | |
| 238 | + * | |
| 239 | + * @priority: Defines the queue priority relative to other queues in the | |
| 240 | + * process. | |
| 241 | + * This is just an indication and HW scheduling may override the priority as | |
| 242 | + * necessary while keeping the relative prioritization. | |
| 243 | + * the priority granularity is from 0 to f which f is the highest priority. | |
| 244 | + * currently all queues are initialized with the highest priority. | |
| 245 | + * | |
| 246 | + * @queue_percent: This field is partially implemented and currently a zero in | |
| 247 | + * this field defines that the queue is non active. | |
| 248 | + * | |
| 249 | + * @read_ptr: User space address which points to the number of dwords the | |
| 250 | + * cp read from the ring buffer. This field updates automatically by the H/W. | |
| 251 | + * | |
| 252 | + * @write_ptr: Defines the number of dwords written to the ring buffer. | |
| 253 | + * | |
| 254 | + * @doorbell_ptr: This field aim is to notify the H/W of new packet written to | |
| 255 | + * the queue ring buffer. This field should be similar to write_ptr and the user | |
| 256 | + * should update this field after he updated the write_ptr. | |
| 257 | + * | |
| 258 | + * @doorbell_off: The doorbell offset in the doorbell pci-bar. | |
| 259 | + * | |
| 260 | + * @is_interop: Defines if this is a interop queue. Interop queue means that the | |
| 261 | + * queue can access both graphics and compute resources. | |
| 262 | + * | |
| 263 | + * @is_active: Defines if the queue is active or not. | |
| 264 | + * | |
| 265 | + * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid | |
| 266 | + * of the queue. | |
| 267 | + * | |
| 268 | + * This structure represents the queue properties for each queue no matter if | |
| 269 | + * it's user mode or kernel mode queue. | |
| 270 | + * | |
| 271 | + */ | |
| 272 | +struct queue_properties { | |
| 273 | + enum kfd_queue_type type; | |
| 274 | + enum kfd_queue_format format; | |
| 275 | + unsigned int queue_id; | |
| 276 | + uint64_t queue_address; | |
| 277 | + uint64_t queue_size; | |
| 278 | + uint32_t priority; | |
| 279 | + uint32_t queue_percent; | |
| 280 | + uint32_t *read_ptr; | |
| 281 | + uint32_t *write_ptr; | |
| 282 | + uint32_t *doorbell_ptr; | |
| 283 | + uint32_t doorbell_off; | |
| 284 | + bool is_interop; | |
| 285 | + bool is_active; | |
| 286 | + /* Not relevant for user mode queues in cp scheduling */ | |
| 287 | + unsigned int vmid; | |
| 288 | +}; | |
| 289 | + | |
| 290 | +/** | |
| 291 | + * struct queue | |
| 292 | + * | |
| 293 | + * @list: Queue linked list. | |
| 294 | + * | |
| 295 | + * @mqd: The queue MQD. | |
| 296 | + * | |
| 297 | + * @mqd_mem_obj: The MQD local gpu memory object. | |
| 298 | + * | |
| 299 | + * @gart_mqd_addr: The MQD gart mc address. | |
| 300 | + * | |
| 301 | + * @properties: The queue properties. | |
| 302 | + * | |
| 303 | + * @mec: Used only in no cp scheduling mode and identifies to micro engine id | |
| 304 | + * that the queue should be execute on. | |
| 305 | + * | |
| 306 | + * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe id. | |
| 307 | + * | |
| 308 | + * @queue: Used only in no cp scheduliong mode and identifies the queue's slot. | |
| 309 | + * | |
| 310 | + * @process: The kfd process that created this queue. | |
| 311 | + * | |
| 312 | + * @device: The kfd device that created this queue. | |
| 313 | + * | |
| 314 | + * This structure represents user mode compute queues. | |
| 315 | + * It contains all the necessary data to handle such queues. | |
| 316 | + * | |
| 317 | + */ | |
| 318 | + | |
| 319 | +struct queue { | |
| 320 | + struct list_head list; | |
| 321 | + void *mqd; | |
| 322 | + struct kfd_mem_obj *mqd_mem_obj; | |
| 323 | + uint64_t gart_mqd_addr; | |
| 324 | + struct queue_properties properties; | |
| 325 | + | |
| 326 | + uint32_t mec; | |
| 327 | + uint32_t pipe; | |
| 328 | + uint32_t queue; | |
| 329 | + | |
| 330 | + struct kfd_process *process; | |
| 331 | + struct kfd_dev *device; | |
| 332 | +}; | |
| 333 | + | |
| 334 | +/* | |
| 335 | + * Please read the kfd_mqd_manager.h description. | |
| 336 | + */ | |
| 337 | +enum KFD_MQD_TYPE { | |
| 338 | + KFD_MQD_TYPE_CIK_COMPUTE = 0, /* for no cp scheduling */ | |
| 339 | + KFD_MQD_TYPE_CIK_HIQ, /* for hiq */ | |
| 340 | + KFD_MQD_TYPE_CIK_CP, /* for cp queues and diq */ | |
| 341 | + KFD_MQD_TYPE_CIK_SDMA, /* for sdma queues */ | |
| 342 | + KFD_MQD_TYPE_MAX | |
| 343 | +}; | |
| 344 | + | |
| 345 | +struct scheduling_resources { | |
| 346 | + unsigned int vmid_mask; | |
| 347 | + enum kfd_queue_type type; | |
| 348 | + uint64_t queue_mask; | |
| 349 | + uint64_t gws_mask; | |
| 350 | + uint32_t oac_mask; | |
| 351 | + uint32_t gds_heap_base; | |
| 352 | + uint32_t gds_heap_size; | |
| 353 | +}; | |
| 354 | + | |
| 355 | +struct process_queue_manager { | |
| 356 | + /* data */ | |
| 357 | + struct kfd_process *process; | |
| 358 | + unsigned int num_concurrent_processes; | |
| 359 | + struct list_head queues; | |
| 360 | + unsigned long *queue_slot_bitmap; | |
| 361 | +}; | |
| 362 | + | |
| 363 | +struct qcm_process_device { | |
| 364 | + /* The Device Queue Manager that owns this data */ | |
| 365 | + struct device_queue_manager *dqm; | |
| 366 | + struct process_queue_manager *pqm; | |
| 367 | + /* Device Queue Manager lock */ | |
| 368 | + struct mutex *lock; | |
| 369 | + /* Queues list */ | |
| 370 | + struct list_head queues_list; | |
| 371 | + struct list_head priv_queue_list; | |
| 372 | + | |
| 373 | + unsigned int queue_count; | |
| 374 | + unsigned int vmid; | |
| 375 | + bool is_debug; | |
| 376 | + /* | |
| 377 | + * All the memory management data should be here too | |
| 378 | + */ | |
| 379 | + uint64_t gds_context_area; | |
| 380 | + uint32_t sh_mem_config; | |
| 381 | + uint32_t sh_mem_bases; | |
| 382 | + uint32_t sh_mem_ape1_base; | |
| 383 | + uint32_t sh_mem_ape1_limit; | |
| 384 | + uint32_t page_table_base; | |
| 385 | + uint32_t gds_size; | |
| 386 | + uint32_t num_gws; | |
| 387 | + uint32_t num_oac; | |
| 388 | +}; | |
| 389 | + | |
| 390 | +/* Data that is per-process-per device. */ | |
| 391 | +struct kfd_process_device { | |
| 392 | + /* | |
| 393 | + * List of all per-device data for a process. | |
| 394 | + * Starts from kfd_process.per_device_data. | |
| 395 | + */ | |
| 396 | + struct list_head per_device_list; | |
| 397 | + | |
| 398 | + /* The device that owns this data. */ | |
| 399 | + struct kfd_dev *dev; | |
| 400 | + | |
| 401 | + | |
| 402 | + /* per-process-per device QCM data structure */ | |
| 403 | + struct qcm_process_device qpd; | |
| 404 | + | |
| 405 | + /*Apertures*/ | |
| 406 | + uint64_t lds_base; | |
| 407 | + uint64_t lds_limit; | |
| 408 | + uint64_t gpuvm_base; | |
| 409 | + uint64_t gpuvm_limit; | |
| 410 | + uint64_t scratch_base; | |
| 411 | + uint64_t scratch_limit; | |
| 412 | + | |
| 413 | + /* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */ | |
| 414 | + bool bound; | |
| 415 | +}; | |
| 416 | + | |
| 417 | +/* Process data */ | |
| 418 | +struct kfd_process { | |
| 419 | + /* | |
| 420 | + * kfd_process are stored in an mm_struct*->kfd_process* | |
| 421 | + * hash table (kfd_processes in kfd_process.c) | |
| 422 | + */ | |
| 423 | + struct hlist_node kfd_processes; | |
| 424 | + | |
| 425 | + struct mm_struct *mm; | |
| 426 | + | |
| 427 | + struct mutex mutex; | |
| 428 | + | |
| 429 | + /* | |
| 430 | + * In any process, the thread that started main() is the lead | |
| 431 | + * thread and outlives the rest. | |
| 432 | + * It is here because amd_iommu_bind_pasid wants a task_struct. | |
| 433 | + */ | |
| 434 | + struct task_struct *lead_thread; | |
| 435 | + | |
| 436 | + /* We want to receive a notification when the mm_struct is destroyed */ | |
| 437 | + struct mmu_notifier mmu_notifier; | |
| 438 | + | |
| 439 | + /* Use for delayed freeing of kfd_process structure */ | |
| 440 | + struct rcu_head rcu; | |
| 441 | + | |
| 442 | + unsigned int pasid; | |
| 443 | + | |
| 444 | + /* | |
| 445 | + * List of kfd_process_device structures, | |
| 446 | + * one for each device the process is using. | |
| 447 | + */ | |
| 448 | + struct list_head per_device_data; | |
| 449 | + | |
| 450 | + struct process_queue_manager pqm; | |
| 451 | + | |
| 452 | + /* The process's queues. */ | |
| 453 | + size_t queue_array_size; | |
| 454 | + | |
| 455 | + /* Size is queue_array_size, up to MAX_PROCESS_QUEUES. */ | |
| 456 | + struct kfd_queue **queues; | |
| 457 | + | |
| 458 | + unsigned long allocated_queue_bitmap[DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)]; | |
| 459 | + | |
| 460 | + /*Is the user space process 32 bit?*/ | |
| 461 | + bool is_32bit_user_mode; | |
| 462 | +}; | |
| 463 | + | |
| 464 | +void kfd_process_create_wq(void); | |
| 465 | +void kfd_process_destroy_wq(void); | |
| 466 | +struct kfd_process *kfd_create_process(const struct task_struct *); | |
| 467 | +struct kfd_process *kfd_get_process(const struct task_struct *); | |
| 468 | + | |
| 469 | +struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, | |
| 470 | + struct kfd_process *p); | |
| 471 | +void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid); | |
| 472 | +struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, | |
| 473 | + struct kfd_process *p, | |
| 474 | + int create_pdd); | |
| 475 | + | |
| 476 | +/* Process device data iterator */ | |
| 477 | +struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p); | |
| 478 | +struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p, | |
| 479 | + struct kfd_process_device *pdd); | |
| 480 | +bool kfd_has_process_device_data(struct kfd_process *p); | |
| 481 | + | |
| 482 | +/* PASIDs */ | |
| 483 | +int kfd_pasid_init(void); | |
| 484 | +void kfd_pasid_exit(void); | |
| 485 | +bool kfd_set_pasid_limit(unsigned int new_limit); | |
| 486 | +unsigned int kfd_get_pasid_limit(void); | |
| 487 | +unsigned int kfd_pasid_alloc(void); | |
| 488 | +void kfd_pasid_free(unsigned int pasid); | |
| 489 | + | |
| 490 | +/* Doorbells */ | |
| 491 | +void kfd_doorbell_init(struct kfd_dev *kfd); | |
| 492 | +int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma); | |
| 493 | +u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, | |
| 494 | + unsigned int *doorbell_off); | |
| 495 | +void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr); | |
| 496 | +u32 read_kernel_doorbell(u32 __iomem *db); | |
| 497 | +void write_kernel_doorbell(u32 __iomem *db, u32 value); | |
| 498 | +unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd, | |
| 499 | + struct kfd_process *process, | |
| 500 | + unsigned int queue_id); | |
| 501 | + | |
| 502 | +extern struct device *kfd_device; | |
| 503 | + | |
| 504 | +/* Topology */ | |
| 505 | +int kfd_topology_init(void); | |
| 506 | +void kfd_topology_shutdown(void); | |
| 507 | +int kfd_topology_add_device(struct kfd_dev *gpu); | |
| 508 | +int kfd_topology_remove_device(struct kfd_dev *gpu); | |
| 509 | +struct kfd_dev *kfd_device_by_id(uint32_t gpu_id); | |
| 510 | +struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev); | |
| 511 | +struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx); | |
| 512 | + | |
| 513 | +/* Interrupts */ | |
| 514 | +int kfd_interrupt_init(struct kfd_dev *dev); | |
| 515 | +void kfd_interrupt_exit(struct kfd_dev *dev); | |
| 516 | +void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry); | |
| 517 | +bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry); | |
| 518 | + | |
| 519 | +/* Power Management */ | |
| 520 | +void kgd2kfd_suspend(struct kfd_dev *kfd); | |
| 521 | +int kgd2kfd_resume(struct kfd_dev *kfd); | |
| 522 | + | |
| 523 | +/* amdkfd Apertures */ | |
| 524 | +int kfd_init_apertures(struct kfd_process *process); | |
| 525 | + | |
| 526 | +/* Queue Context Management */ | |
| 527 | +inline uint32_t lower_32(uint64_t x); | |
| 528 | +inline uint32_t upper_32(uint64_t x); | |
| 529 | + | |
| 530 | +int init_queue(struct queue **q, struct queue_properties properties); | |
| 531 | +void uninit_queue(struct queue *q); | |
| 532 | +void print_queue_properties(struct queue_properties *q); | |
| 533 | +void print_queue(struct queue *q); | |
| 534 | + | |
| 535 | +struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, | |
| 536 | + struct kfd_dev *dev); | |
| 537 | +struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev); | |
| 538 | +void device_queue_manager_uninit(struct device_queue_manager *dqm); | |
| 539 | +struct kernel_queue *kernel_queue_init(struct kfd_dev *dev, | |
| 540 | + enum kfd_queue_type type); | |
| 541 | +void kernel_queue_uninit(struct kernel_queue *kq); | |
| 542 | + | |
| 543 | +/* Process Queue Manager */ | |
| 544 | +struct process_queue_node { | |
| 545 | + struct queue *q; | |
| 546 | + struct kernel_queue *kq; | |
| 547 | + struct list_head process_queue_list; | |
| 548 | +}; | |
| 549 | + | |
| 550 | +int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p); | |
| 551 | +void pqm_uninit(struct process_queue_manager *pqm); | |
| 552 | +int pqm_create_queue(struct process_queue_manager *pqm, | |
| 553 | + struct kfd_dev *dev, | |
| 554 | + struct file *f, | |
| 555 | + struct queue_properties *properties, | |
| 556 | + unsigned int flags, | |
| 557 | + enum kfd_queue_type type, | |
| 558 | + unsigned int *qid); | |
| 559 | +int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid); | |
| 560 | +int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid, | |
| 561 | + struct queue_properties *p); | |
| 562 | + | |
| 563 | +/* Packet Manager */ | |
| 564 | + | |
| 565 | +#define KFD_HIQ_TIMEOUT (500) | |
| 566 | + | |
| 567 | +#define KFD_FENCE_COMPLETED (100) | |
| 568 | +#define KFD_FENCE_INIT (10) | |
| 569 | +#define KFD_UNMAP_LATENCY (150) | |
| 570 | + | |
| 571 | +struct packet_manager { | |
| 572 | + struct device_queue_manager *dqm; | |
| 573 | + struct kernel_queue *priv_queue; | |
| 574 | + struct mutex lock; | |
| 575 | + bool allocated; | |
| 576 | + struct kfd_mem_obj *ib_buffer_obj; | |
| 577 | +}; | |
| 578 | + | |
| 579 | +int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm); | |
| 580 | +void pm_uninit(struct packet_manager *pm); | |
| 581 | +int pm_send_set_resources(struct packet_manager *pm, | |
| 582 | + struct scheduling_resources *res); | |
| 583 | +int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues); | |
| 584 | +int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address, | |
| 585 | + uint32_t fence_value); | |
| 586 | + | |
| 587 | +int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type, | |
| 588 | + enum kfd_preempt_type_filter mode, | |
| 589 | + uint32_t filter_param, bool reset, | |
| 590 | + unsigned int sdma_engine); | |
| 591 | + | |
| 592 | +void pm_release_ib(struct packet_manager *pm); | |
| 593 | + | |
| 594 | +uint64_t kfd_get_number_elems(struct kfd_dev *kfd); | |
| 595 | +phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, | |
| 596 | + struct kfd_process *process); | |
| 597 | + | |
| 598 | +#endif | |
| @@ -0,0 +1,415 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +#include <linux/mutex.h> | |
| 24 | +#include <linux/log2.h> | |
| 25 | +#include <linux/sched.h> | |
| 26 | +#include <linux/slab.h> | |
| 27 | +#include <linux/amd-iommu.h> | |
| 28 | +#include <linux/notifier.h> | |
| 29 | +struct mm_struct; | |
| 30 | + | |
| 31 | +#include "kfd_priv.h" | |
| 32 | + | |
| 33 | +/* | |
| 34 | + * Initial size for the array of queues. | |
| 35 | + * The allocated size is doubled each time | |
| 36 | + * it is exceeded up to MAX_PROCESS_QUEUES. | |
| 37 | + */ | |
| 38 | +#define INITIAL_QUEUE_ARRAY_SIZE 16 | |
| 39 | + | |
| 40 | +/* | |
| 41 | + * List of struct kfd_process (field kfd_process). | |
| 42 | + * Unique/indexed by mm_struct* | |
| 43 | + */ | |
| 44 | +#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */ | |
| 45 | +static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE); | |
| 46 | +static DEFINE_MUTEX(kfd_processes_mutex); | |
| 47 | + | |
| 48 | +DEFINE_STATIC_SRCU(kfd_processes_srcu); | |
| 49 | + | |
| 50 | +static struct workqueue_struct *kfd_process_wq; | |
| 51 | + | |
| 52 | +struct kfd_process_release_work { | |
| 53 | + struct work_struct kfd_work; | |
| 54 | + struct kfd_process *p; | |
| 55 | +}; | |
| 56 | + | |
| 57 | +static struct kfd_process *find_process(const struct task_struct *thread); | |
| 58 | +static struct kfd_process *create_process(const struct task_struct *thread); | |
| 59 | + | |
| 60 | +void kfd_process_create_wq(void) | |
| 61 | +{ | |
| 62 | + if (!kfd_process_wq) | |
| 63 | + kfd_process_wq = create_workqueue("kfd_process_wq"); | |
| 64 | +} | |
| 65 | + | |
| 66 | +void kfd_process_destroy_wq(void) | |
| 67 | +{ | |
| 68 | + if (kfd_process_wq) { | |
| 69 | + flush_workqueue(kfd_process_wq); | |
| 70 | + destroy_workqueue(kfd_process_wq); | |
| 71 | + kfd_process_wq = NULL; | |
| 72 | + } | |
| 73 | +} | |
| 74 | + | |
| 75 | +struct kfd_process *kfd_create_process(const struct task_struct *thread) | |
| 76 | +{ | |
| 77 | + struct kfd_process *process; | |
| 78 | + | |
| 79 | + BUG_ON(!kfd_process_wq); | |
| 80 | + | |
| 81 | + if (thread->mm == NULL) | |
| 82 | + return ERR_PTR(-EINVAL); | |
| 83 | + | |
| 84 | + /* Only the pthreads threading model is supported. */ | |
| 85 | + if (thread->group_leader->mm != thread->mm) | |
| 86 | + return ERR_PTR(-EINVAL); | |
| 87 | + | |
| 88 | + /* Take mmap_sem because we call __mmu_notifier_register inside */ | |
| 89 | + down_write(&thread->mm->mmap_sem); | |
| 90 | + | |
| 91 | + /* | |
| 92 | + * take kfd processes mutex before starting of process creation | |
| 93 | + * so there won't be a case where two threads of the same process | |
| 94 | + * create two kfd_process structures | |
| 95 | + */ | |
| 96 | + mutex_lock(&kfd_processes_mutex); | |
| 97 | + | |
| 98 | + /* A prior open of /dev/kfd could have already created the process. */ | |
| 99 | + process = find_process(thread); | |
| 100 | + if (process) | |
| 101 | + pr_debug("kfd: process already found\n"); | |
| 102 | + | |
| 103 | + if (!process) | |
| 104 | + process = create_process(thread); | |
| 105 | + | |
| 106 | + mutex_unlock(&kfd_processes_mutex); | |
| 107 | + | |
| 108 | + up_write(&thread->mm->mmap_sem); | |
| 109 | + | |
| 110 | + return process; | |
| 111 | +} | |
| 112 | + | |
| 113 | +struct kfd_process *kfd_get_process(const struct task_struct *thread) | |
| 114 | +{ | |
| 115 | + struct kfd_process *process; | |
| 116 | + | |
| 117 | + if (thread->mm == NULL) | |
| 118 | + return ERR_PTR(-EINVAL); | |
| 119 | + | |
| 120 | + /* Only the pthreads threading model is supported. */ | |
| 121 | + if (thread->group_leader->mm != thread->mm) | |
| 122 | + return ERR_PTR(-EINVAL); | |
| 123 | + | |
| 124 | + process = find_process(thread); | |
| 125 | + | |
| 126 | + return process; | |
| 127 | +} | |
| 128 | + | |
| 129 | +static struct kfd_process *find_process_by_mm(const struct mm_struct *mm) | |
| 130 | +{ | |
| 131 | + struct kfd_process *process; | |
| 132 | + | |
| 133 | + hash_for_each_possible_rcu(kfd_processes_table, process, | |
| 134 | + kfd_processes, (uintptr_t)mm) | |
| 135 | + if (process->mm == mm) | |
| 136 | + return process; | |
| 137 | + | |
| 138 | + return NULL; | |
| 139 | +} | |
| 140 | + | |
| 141 | +static struct kfd_process *find_process(const struct task_struct *thread) | |
| 142 | +{ | |
| 143 | + struct kfd_process *p; | |
| 144 | + int idx; | |
| 145 | + | |
| 146 | + idx = srcu_read_lock(&kfd_processes_srcu); | |
| 147 | + p = find_process_by_mm(thread->mm); | |
| 148 | + srcu_read_unlock(&kfd_processes_srcu, idx); | |
| 149 | + | |
| 150 | + return p; | |
| 151 | +} | |
| 152 | + | |
| 153 | +static void kfd_process_wq_release(struct work_struct *work) | |
| 154 | +{ | |
| 155 | + struct kfd_process_release_work *my_work; | |
| 156 | + struct kfd_process_device *pdd, *temp; | |
| 157 | + struct kfd_process *p; | |
| 158 | + | |
| 159 | + my_work = (struct kfd_process_release_work *) work; | |
| 160 | + | |
| 161 | + p = my_work->p; | |
| 162 | + | |
| 163 | + mutex_lock(&p->mutex); | |
| 164 | + | |
| 165 | + list_for_each_entry_safe(pdd, temp, &p->per_device_data, | |
| 166 | + per_device_list) { | |
| 167 | + amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid); | |
| 168 | + list_del(&pdd->per_device_list); | |
| 169 | + | |
| 170 | + kfree(pdd); | |
| 171 | + } | |
| 172 | + | |
| 173 | + kfd_pasid_free(p->pasid); | |
| 174 | + | |
| 175 | + mutex_unlock(&p->mutex); | |
| 176 | + | |
| 177 | + mutex_destroy(&p->mutex); | |
| 178 | + | |
| 179 | + kfree(p->queues); | |
| 180 | + | |
| 181 | + kfree(p); | |
| 182 | + | |
| 183 | + kfree((void *)work); | |
| 184 | +} | |
| 185 | + | |
| 186 | +static void kfd_process_destroy_delayed(struct rcu_head *rcu) | |
| 187 | +{ | |
| 188 | + struct kfd_process_release_work *work; | |
| 189 | + struct kfd_process *p; | |
| 190 | + | |
| 191 | + BUG_ON(!kfd_process_wq); | |
| 192 | + | |
| 193 | + p = container_of(rcu, struct kfd_process, rcu); | |
| 194 | + BUG_ON(atomic_read(&p->mm->mm_count) <= 0); | |
| 195 | + | |
| 196 | + mmdrop(p->mm); | |
| 197 | + | |
| 198 | + work = (struct kfd_process_release_work *) | |
| 199 | + kmalloc(sizeof(struct kfd_process_release_work), GFP_KERNEL); | |
| 200 | + | |
| 201 | + if (work) { | |
| 202 | + INIT_WORK((struct work_struct *) work, kfd_process_wq_release); | |
| 203 | + work->p = p; | |
| 204 | + queue_work(kfd_process_wq, (struct work_struct *) work); | |
| 205 | + } | |
| 206 | +} | |
| 207 | + | |
| 208 | +static void kfd_process_notifier_release(struct mmu_notifier *mn, | |
| 209 | + struct mm_struct *mm) | |
| 210 | +{ | |
| 211 | + struct kfd_process *p; | |
| 212 | + | |
| 213 | + /* | |
| 214 | + * The kfd_process structure can not be free because the | |
| 215 | + * mmu_notifier srcu is read locked | |
| 216 | + */ | |
| 217 | + p = container_of(mn, struct kfd_process, mmu_notifier); | |
| 218 | + BUG_ON(p->mm != mm); | |
| 219 | + | |
| 220 | + mutex_lock(&kfd_processes_mutex); | |
| 221 | + hash_del_rcu(&p->kfd_processes); | |
| 222 | + mutex_unlock(&kfd_processes_mutex); | |
| 223 | + synchronize_srcu(&kfd_processes_srcu); | |
| 224 | + | |
| 225 | + mutex_lock(&p->mutex); | |
| 226 | + | |
| 227 | + /* In case our notifier is called before IOMMU notifier */ | |
| 228 | + pqm_uninit(&p->pqm); | |
| 229 | + | |
| 230 | + mutex_unlock(&p->mutex); | |
| 231 | + | |
| 232 | + /* | |
| 233 | + * Because we drop mm_count inside kfd_process_destroy_delayed | |
| 234 | + * and because the mmu_notifier_unregister function also drop | |
| 235 | + * mm_count we need to take an extra count here. | |
| 236 | + */ | |
| 237 | + atomic_inc(&p->mm->mm_count); | |
| 238 | + mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm); | |
| 239 | + mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed); | |
| 240 | +} | |
| 241 | + | |
| 242 | +static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = { | |
| 243 | + .release = kfd_process_notifier_release, | |
| 244 | +}; | |
| 245 | + | |
| 246 | +static struct kfd_process *create_process(const struct task_struct *thread) | |
| 247 | +{ | |
| 248 | + struct kfd_process *process; | |
| 249 | + int err = -ENOMEM; | |
| 250 | + | |
| 251 | + process = kzalloc(sizeof(*process), GFP_KERNEL); | |
| 252 | + | |
| 253 | + if (!process) | |
| 254 | + goto err_alloc_process; | |
| 255 | + | |
| 256 | + process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE, | |
| 257 | + sizeof(process->queues[0]), GFP_KERNEL); | |
| 258 | + if (!process->queues) | |
| 259 | + goto err_alloc_queues; | |
| 260 | + | |
| 261 | + process->pasid = kfd_pasid_alloc(); | |
| 262 | + if (process->pasid == 0) | |
| 263 | + goto err_alloc_pasid; | |
| 264 | + | |
| 265 | + mutex_init(&process->mutex); | |
| 266 | + | |
| 267 | + process->mm = thread->mm; | |
| 268 | + | |
| 269 | + /* register notifier */ | |
| 270 | + process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops; | |
| 271 | + err = __mmu_notifier_register(&process->mmu_notifier, process->mm); | |
| 272 | + if (err) | |
| 273 | + goto err_mmu_notifier; | |
| 274 | + | |
| 275 | + hash_add_rcu(kfd_processes_table, &process->kfd_processes, | |
| 276 | + (uintptr_t)process->mm); | |
| 277 | + | |
| 278 | + process->lead_thread = thread->group_leader; | |
| 279 | + | |
| 280 | + process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE; | |
| 281 | + | |
| 282 | + INIT_LIST_HEAD(&process->per_device_data); | |
| 283 | + | |
| 284 | + err = pqm_init(&process->pqm, process); | |
| 285 | + if (err != 0) | |
| 286 | + goto err_process_pqm_init; | |
| 287 | + | |
| 288 | + return process; | |
| 289 | + | |
| 290 | +err_process_pqm_init: | |
| 291 | + hash_del_rcu(&process->kfd_processes); | |
| 292 | + synchronize_rcu(); | |
| 293 | + mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm); | |
| 294 | +err_mmu_notifier: | |
| 295 | + kfd_pasid_free(process->pasid); | |
| 296 | +err_alloc_pasid: | |
| 297 | + kfree(process->queues); | |
| 298 | +err_alloc_queues: | |
| 299 | + kfree(process); | |
| 300 | +err_alloc_process: | |
| 301 | + return ERR_PTR(err); | |
| 302 | +} | |
| 303 | + | |
| 304 | +struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, | |
| 305 | + struct kfd_process *p, | |
| 306 | + int create_pdd) | |
| 307 | +{ | |
| 308 | + struct kfd_process_device *pdd = NULL; | |
| 309 | + | |
| 310 | + list_for_each_entry(pdd, &p->per_device_data, per_device_list) | |
| 311 | + if (pdd->dev == dev) | |
| 312 | + return pdd; | |
| 313 | + | |
| 314 | + if (create_pdd) { | |
| 315 | + pdd = kzalloc(sizeof(*pdd), GFP_KERNEL); | |
| 316 | + if (pdd != NULL) { | |
| 317 | + pdd->dev = dev; | |
| 318 | + INIT_LIST_HEAD(&pdd->qpd.queues_list); | |
| 319 | + INIT_LIST_HEAD(&pdd->qpd.priv_queue_list); | |
| 320 | + pdd->qpd.dqm = dev->dqm; | |
| 321 | + list_add(&pdd->per_device_list, &p->per_device_data); | |
| 322 | + } | |
| 323 | + } | |
| 324 | + | |
| 325 | + return pdd; | |
| 326 | +} | |
| 327 | + | |
| 328 | +/* | |
| 329 | + * Direct the IOMMU to bind the process (specifically the pasid->mm) | |
| 330 | + * to the device. | |
| 331 | + * Unbinding occurs when the process dies or the device is removed. | |
| 332 | + * | |
| 333 | + * Assumes that the process lock is held. | |
| 334 | + */ | |
| 335 | +struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, | |
| 336 | + struct kfd_process *p) | |
| 337 | +{ | |
| 338 | + struct kfd_process_device *pdd = kfd_get_process_device_data(dev, p, 1); | |
| 339 | + int err; | |
| 340 | + | |
| 341 | + if (pdd == NULL) | |
| 342 | + return ERR_PTR(-ENOMEM); | |
| 343 | + | |
| 344 | + if (pdd->bound) | |
| 345 | + return pdd; | |
| 346 | + | |
| 347 | + err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread); | |
| 348 | + if (err < 0) | |
| 349 | + return ERR_PTR(err); | |
| 350 | + | |
| 351 | + if (err < 0) { | |
| 352 | + amd_iommu_unbind_pasid(dev->pdev, p->pasid); | |
| 353 | + return ERR_PTR(err); | |
| 354 | + } | |
| 355 | + | |
| 356 | + pdd->bound = true; | |
| 357 | + | |
| 358 | + return pdd; | |
| 359 | +} | |
| 360 | + | |
| 361 | +void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid) | |
| 362 | +{ | |
| 363 | + struct kfd_process *p; | |
| 364 | + struct kfd_process_device *pdd; | |
| 365 | + int idx, i; | |
| 366 | + | |
| 367 | + BUG_ON(dev == NULL); | |
| 368 | + | |
| 369 | + idx = srcu_read_lock(&kfd_processes_srcu); | |
| 370 | + | |
| 371 | + hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes) | |
| 372 | + if (p->pasid == pasid) | |
| 373 | + break; | |
| 374 | + | |
| 375 | + srcu_read_unlock(&kfd_processes_srcu, idx); | |
| 376 | + | |
| 377 | + BUG_ON(p->pasid != pasid); | |
| 378 | + | |
| 379 | + mutex_lock(&p->mutex); | |
| 380 | + | |
| 381 | + pqm_uninit(&p->pqm); | |
| 382 | + | |
| 383 | + pdd = kfd_get_process_device_data(dev, p, 0); | |
| 384 | + | |
| 385 | + /* | |
| 386 | + * Just mark pdd as unbound, because we still need it to call | |
| 387 | + * amd_iommu_unbind_pasid() in when the process exits. | |
| 388 | + * We don't call amd_iommu_unbind_pasid() here | |
| 389 | + * because the IOMMU called us. | |
| 390 | + */ | |
| 391 | + if (pdd) | |
| 392 | + pdd->bound = false; | |
| 393 | + | |
| 394 | + mutex_unlock(&p->mutex); | |
| 395 | +} | |
| 396 | + | |
| 397 | +struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p) | |
| 398 | +{ | |
| 399 | + return list_first_entry(&p->per_device_data, | |
| 400 | + struct kfd_process_device, | |
| 401 | + per_device_list); | |
| 402 | +} | |
| 403 | + | |
| 404 | +struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p, | |
| 405 | + struct kfd_process_device *pdd) | |
| 406 | +{ | |
| 407 | + if (list_is_last(&pdd->per_device_list, &p->per_device_data)) | |
| 408 | + return NULL; | |
| 409 | + return list_next_entry(pdd, per_device_list); | |
| 410 | +} | |
| 411 | + | |
| 412 | +bool kfd_has_process_device_data(struct kfd_process *p) | |
| 413 | +{ | |
| 414 | + return !(list_empty(&p->per_device_data)); | |
| 415 | +} | |
| @@ -0,0 +1,342 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + * | |
| 22 | + */ | |
| 23 | + | |
| 24 | +#include <linux/slab.h> | |
| 25 | +#include <linux/list.h> | |
| 26 | +#include "kfd_device_queue_manager.h" | |
| 27 | +#include "kfd_priv.h" | |
| 28 | +#include "kfd_kernel_queue.h" | |
| 29 | + | |
| 30 | +static inline struct process_queue_node *get_queue_by_qid( | |
| 31 | + struct process_queue_manager *pqm, unsigned int qid) | |
| 32 | +{ | |
| 33 | + struct process_queue_node *pqn; | |
| 34 | + | |
| 35 | + BUG_ON(!pqm); | |
| 36 | + | |
| 37 | + list_for_each_entry(pqn, &pqm->queues, process_queue_list) { | |
| 38 | + if (pqn->q && pqn->q->properties.queue_id == qid) | |
| 39 | + return pqn; | |
| 40 | + if (pqn->kq && pqn->kq->queue->properties.queue_id == qid) | |
| 41 | + return pqn; | |
| 42 | + } | |
| 43 | + | |
| 44 | + return NULL; | |
| 45 | +} | |
| 46 | + | |
| 47 | +static int find_available_queue_slot(struct process_queue_manager *pqm, | |
| 48 | + unsigned int *qid) | |
| 49 | +{ | |
| 50 | + unsigned long found; | |
| 51 | + | |
| 52 | + BUG_ON(!pqm || !qid); | |
| 53 | + | |
| 54 | + pr_debug("kfd: in %s\n", __func__); | |
| 55 | + | |
| 56 | + found = find_first_zero_bit(pqm->queue_slot_bitmap, | |
| 57 | + max_num_of_queues_per_process); | |
| 58 | + | |
| 59 | + pr_debug("kfd: the new slot id %lu\n", found); | |
| 60 | + | |
| 61 | + if (found >= max_num_of_queues_per_process) { | |
| 62 | + pr_info("amdkfd: Can not open more queues for process with pasid %d\n", | |
| 63 | + pqm->process->pasid); | |
| 64 | + return -ENOMEM; | |
| 65 | + } | |
| 66 | + | |
| 67 | + set_bit(found, pqm->queue_slot_bitmap); | |
| 68 | + *qid = found; | |
| 69 | + | |
| 70 | + return 0; | |
| 71 | +} | |
| 72 | + | |
| 73 | +int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p) | |
| 74 | +{ | |
| 75 | + BUG_ON(!pqm); | |
| 76 | + | |
| 77 | + INIT_LIST_HEAD(&pqm->queues); | |
| 78 | + pqm->queue_slot_bitmap = | |
| 79 | + kzalloc(DIV_ROUND_UP(max_num_of_queues_per_process, | |
| 80 | + BITS_PER_BYTE), GFP_KERNEL); | |
| 81 | + if (pqm->queue_slot_bitmap == NULL) | |
| 82 | + return -ENOMEM; | |
| 83 | + pqm->process = p; | |
| 84 | + | |
| 85 | + return 0; | |
| 86 | +} | |
| 87 | + | |
| 88 | +void pqm_uninit(struct process_queue_manager *pqm) | |
| 89 | +{ | |
| 90 | + int retval; | |
| 91 | + struct process_queue_node *pqn, *next; | |
| 92 | + | |
| 93 | + BUG_ON(!pqm); | |
| 94 | + | |
| 95 | + pr_debug("In func %s\n", __func__); | |
| 96 | + | |
| 97 | + list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) { | |
| 98 | + retval = pqm_destroy_queue( | |
| 99 | + pqm, | |
| 100 | + (pqn->q != NULL) ? | |
| 101 | + pqn->q->properties.queue_id : | |
| 102 | + pqn->kq->queue->properties.queue_id); | |
| 103 | + | |
| 104 | + if (retval != 0) { | |
| 105 | + pr_err("kfd: failed to destroy queue\n"); | |
| 106 | + return; | |
| 107 | + } | |
| 108 | + } | |
| 109 | + kfree(pqm->queue_slot_bitmap); | |
| 110 | + pqm->queue_slot_bitmap = NULL; | |
| 111 | +} | |
| 112 | + | |
| 113 | +static int create_cp_queue(struct process_queue_manager *pqm, | |
| 114 | + struct kfd_dev *dev, struct queue **q, | |
| 115 | + struct queue_properties *q_properties, | |
| 116 | + struct file *f, unsigned int qid) | |
| 117 | +{ | |
| 118 | + int retval; | |
| 119 | + | |
| 120 | + retval = 0; | |
| 121 | + | |
| 122 | + /* Doorbell initialized in user space*/ | |
| 123 | + q_properties->doorbell_ptr = NULL; | |
| 124 | + | |
| 125 | + q_properties->doorbell_off = | |
| 126 | + kfd_queue_id_to_doorbell(dev, pqm->process, qid); | |
| 127 | + | |
| 128 | + /* let DQM handle it*/ | |
| 129 | + q_properties->vmid = 0; | |
| 130 | + q_properties->queue_id = qid; | |
| 131 | + q_properties->type = KFD_QUEUE_TYPE_COMPUTE; | |
| 132 | + | |
| 133 | + retval = init_queue(q, *q_properties); | |
| 134 | + if (retval != 0) | |
| 135 | + goto err_init_queue; | |
| 136 | + | |
| 137 | + (*q)->device = dev; | |
| 138 | + (*q)->process = pqm->process; | |
| 139 | + | |
| 140 | + pr_debug("kfd: PQM After init queue"); | |
| 141 | + | |
| 142 | + return retval; | |
| 143 | + | |
| 144 | +err_init_queue: | |
| 145 | + return retval; | |
| 146 | +} | |
| 147 | + | |
| 148 | +int pqm_create_queue(struct process_queue_manager *pqm, | |
| 149 | + struct kfd_dev *dev, | |
| 150 | + struct file *f, | |
| 151 | + struct queue_properties *properties, | |
| 152 | + unsigned int flags, | |
| 153 | + enum kfd_queue_type type, | |
| 154 | + unsigned int *qid) | |
| 155 | +{ | |
| 156 | + int retval; | |
| 157 | + struct kfd_process_device *pdd; | |
| 158 | + struct queue_properties q_properties; | |
| 159 | + struct queue *q; | |
| 160 | + struct process_queue_node *pqn; | |
| 161 | + struct kernel_queue *kq; | |
| 162 | + | |
| 163 | + BUG_ON(!pqm || !dev || !properties || !qid); | |
| 164 | + | |
| 165 | + memset(&q_properties, 0, sizeof(struct queue_properties)); | |
| 166 | + memcpy(&q_properties, properties, sizeof(struct queue_properties)); | |
| 167 | + q = NULL; | |
| 168 | + kq = NULL; | |
| 169 | + | |
| 170 | + pdd = kfd_get_process_device_data(dev, pqm->process, 1); | |
| 171 | + BUG_ON(!pdd); | |
| 172 | + | |
| 173 | + retval = find_available_queue_slot(pqm, qid); | |
| 174 | + if (retval != 0) | |
| 175 | + return retval; | |
| 176 | + | |
| 177 | + if (list_empty(&pqm->queues)) { | |
| 178 | + pdd->qpd.pqm = pqm; | |
| 179 | + dev->dqm->register_process(dev->dqm, &pdd->qpd); | |
| 180 | + } | |
| 181 | + | |
| 182 | + pqn = kzalloc(sizeof(struct process_queue_node), GFP_KERNEL); | |
| 183 | + if (!pqn) { | |
| 184 | + retval = -ENOMEM; | |
| 185 | + goto err_allocate_pqn; | |
| 186 | + } | |
| 187 | + | |
| 188 | + switch (type) { | |
| 189 | + case KFD_QUEUE_TYPE_COMPUTE: | |
| 190 | + /* check if there is over subscription */ | |
| 191 | + if ((sched_policy == KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) && | |
| 192 | + ((dev->dqm->processes_count >= VMID_PER_DEVICE) || | |
| 193 | + (dev->dqm->queue_count >= PIPE_PER_ME_CP_SCHEDULING * QUEUES_PER_PIPE))) { | |
| 194 | + pr_err("kfd: over-subscription is not allowed in radeon_kfd.sched_policy == 1\n"); | |
| 195 | + retval = -EPERM; | |
| 196 | + goto err_create_queue; | |
| 197 | + } | |
| 198 | + | |
| 199 | + retval = create_cp_queue(pqm, dev, &q, &q_properties, f, *qid); | |
| 200 | + if (retval != 0) | |
| 201 | + goto err_create_queue; | |
| 202 | + pqn->q = q; | |
| 203 | + pqn->kq = NULL; | |
| 204 | + retval = dev->dqm->create_queue(dev->dqm, q, &pdd->qpd, | |
| 205 | + &q->properties.vmid); | |
| 206 | + print_queue(q); | |
| 207 | + break; | |
| 208 | + case KFD_QUEUE_TYPE_DIQ: | |
| 209 | + kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ); | |
| 210 | + if (kq == NULL) { | |
| 211 | + kernel_queue_uninit(kq); | |
| 212 | + goto err_create_queue; | |
| 213 | + } | |
| 214 | + kq->queue->properties.queue_id = *qid; | |
| 215 | + pqn->kq = kq; | |
| 216 | + pqn->q = NULL; | |
| 217 | + retval = dev->dqm->create_kernel_queue(dev->dqm, kq, &pdd->qpd); | |
| 218 | + break; | |
| 219 | + default: | |
| 220 | + BUG(); | |
| 221 | + break; | |
| 222 | + } | |
| 223 | + | |
| 224 | + if (retval != 0) { | |
| 225 | + pr_err("kfd: error dqm create queue\n"); | |
| 226 | + goto err_create_queue; | |
| 227 | + } | |
| 228 | + | |
| 229 | + pr_debug("kfd: PQM After DQM create queue\n"); | |
| 230 | + | |
| 231 | + list_add(&pqn->process_queue_list, &pqm->queues); | |
| 232 | + | |
| 233 | + if (q) { | |
| 234 | + *properties = q->properties; | |
| 235 | + pr_debug("kfd: PQM done creating queue\n"); | |
| 236 | + print_queue_properties(properties); | |
| 237 | + } | |
| 238 | + | |
| 239 | + return retval; | |
| 240 | + | |
| 241 | +err_create_queue: | |
| 242 | + kfree(pqn); | |
| 243 | +err_allocate_pqn: | |
| 244 | + clear_bit(*qid, pqm->queue_slot_bitmap); | |
| 245 | + return retval; | |
| 246 | +} | |
| 247 | + | |
| 248 | +int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid) | |
| 249 | +{ | |
| 250 | + struct process_queue_node *pqn; | |
| 251 | + struct kfd_process_device *pdd; | |
| 252 | + struct device_queue_manager *dqm; | |
| 253 | + struct kfd_dev *dev; | |
| 254 | + int retval; | |
| 255 | + | |
| 256 | + dqm = NULL; | |
| 257 | + | |
| 258 | + BUG_ON(!pqm); | |
| 259 | + retval = 0; | |
| 260 | + | |
| 261 | + pr_debug("kfd: In Func %s\n", __func__); | |
| 262 | + | |
| 263 | + pqn = get_queue_by_qid(pqm, qid); | |
| 264 | + if (pqn == NULL) { | |
| 265 | + pr_err("kfd: queue id does not match any known queue\n"); | |
| 266 | + return -EINVAL; | |
| 267 | + } | |
| 268 | + | |
| 269 | + dev = NULL; | |
| 270 | + if (pqn->kq) | |
| 271 | + dev = pqn->kq->dev; | |
| 272 | + if (pqn->q) | |
| 273 | + dev = pqn->q->device; | |
| 274 | + BUG_ON(!dev); | |
| 275 | + | |
| 276 | + pdd = kfd_get_process_device_data(dev, pqm->process, 1); | |
| 277 | + BUG_ON(!pdd); | |
| 278 | + | |
| 279 | + if (pqn->kq) { | |
| 280 | + /* destroy kernel queue (DIQ) */ | |
| 281 | + dqm = pqn->kq->dev->dqm; | |
| 282 | + dqm->destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd); | |
| 283 | + kernel_queue_uninit(pqn->kq); | |
| 284 | + } | |
| 285 | + | |
| 286 | + if (pqn->q) { | |
| 287 | + dqm = pqn->q->device->dqm; | |
| 288 | + retval = dqm->destroy_queue(dqm, &pdd->qpd, pqn->q); | |
| 289 | + if (retval != 0) | |
| 290 | + return retval; | |
| 291 | + | |
| 292 | + uninit_queue(pqn->q); | |
| 293 | + } | |
| 294 | + | |
| 295 | + list_del(&pqn->process_queue_list); | |
| 296 | + kfree(pqn); | |
| 297 | + clear_bit(qid, pqm->queue_slot_bitmap); | |
| 298 | + | |
| 299 | + if (list_empty(&pqm->queues)) | |
| 300 | + dqm->unregister_process(dqm, &pdd->qpd); | |
| 301 | + | |
| 302 | + return retval; | |
| 303 | +} | |
| 304 | + | |
| 305 | +int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid, | |
| 306 | + struct queue_properties *p) | |
| 307 | +{ | |
| 308 | + int retval; | |
| 309 | + struct process_queue_node *pqn; | |
| 310 | + | |
| 311 | + BUG_ON(!pqm); | |
| 312 | + | |
| 313 | + pqn = get_queue_by_qid(pqm, qid); | |
| 314 | + BUG_ON(!pqn); | |
| 315 | + | |
| 316 | + pqn->q->properties.queue_address = p->queue_address; | |
| 317 | + pqn->q->properties.queue_size = p->queue_size; | |
| 318 | + pqn->q->properties.queue_percent = p->queue_percent; | |
| 319 | + pqn->q->properties.priority = p->priority; | |
| 320 | + | |
| 321 | + retval = pqn->q->device->dqm->update_queue(pqn->q->device->dqm, pqn->q); | |
| 322 | + if (retval != 0) | |
| 323 | + return retval; | |
| 324 | + | |
| 325 | + return 0; | |
| 326 | +} | |
| 327 | + | |
| 328 | +struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm, | |
| 329 | + unsigned int qid) | |
| 330 | +{ | |
| 331 | + struct process_queue_node *pqn; | |
| 332 | + | |
| 333 | + BUG_ON(!pqm); | |
| 334 | + | |
| 335 | + pqn = get_queue_by_qid(pqm, qid); | |
| 336 | + if (pqn && pqn->kq) | |
| 337 | + return pqn->kq; | |
| 338 | + | |
| 339 | + return NULL; | |
| 340 | +} | |
| 341 | + | |
| 342 | + | |
| @@ -0,0 +1,85 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + * | |
| 22 | + */ | |
| 23 | + | |
| 24 | +#include <linux/slab.h> | |
| 25 | +#include "kfd_priv.h" | |
| 26 | + | |
| 27 | +void print_queue_properties(struct queue_properties *q) | |
| 28 | +{ | |
| 29 | + if (!q) | |
| 30 | + return; | |
| 31 | + | |
| 32 | + pr_debug("Printing queue properties:\n"); | |
| 33 | + pr_debug("Queue Type: %u\n", q->type); | |
| 34 | + pr_debug("Queue Size: %llu\n", q->queue_size); | |
| 35 | + pr_debug("Queue percent: %u\n", q->queue_percent); | |
| 36 | + pr_debug("Queue Address: 0x%llX\n", q->queue_address); | |
| 37 | + pr_debug("Queue Id: %u\n", q->queue_id); | |
| 38 | + pr_debug("Queue Process Vmid: %u\n", q->vmid); | |
| 39 | + pr_debug("Queue Read Pointer: 0x%p\n", q->read_ptr); | |
| 40 | + pr_debug("Queue Write Pointer: 0x%p\n", q->write_ptr); | |
| 41 | + pr_debug("Queue Doorbell Pointer: 0x%p\n", q->doorbell_ptr); | |
| 42 | + pr_debug("Queue Doorbell Offset: %u\n", q->doorbell_off); | |
| 43 | +} | |
| 44 | + | |
| 45 | +void print_queue(struct queue *q) | |
| 46 | +{ | |
| 47 | + if (!q) | |
| 48 | + return; | |
| 49 | + pr_debug("Printing queue:\n"); | |
| 50 | + pr_debug("Queue Type: %u\n", q->properties.type); | |
| 51 | + pr_debug("Queue Size: %llu\n", q->properties.queue_size); | |
| 52 | + pr_debug("Queue percent: %u\n", q->properties.queue_percent); | |
| 53 | + pr_debug("Queue Address: 0x%llX\n", q->properties.queue_address); | |
| 54 | + pr_debug("Queue Id: %u\n", q->properties.queue_id); | |
| 55 | + pr_debug("Queue Process Vmid: %u\n", q->properties.vmid); | |
| 56 | + pr_debug("Queue Read Pointer: 0x%p\n", q->properties.read_ptr); | |
| 57 | + pr_debug("Queue Write Pointer: 0x%p\n", q->properties.write_ptr); | |
| 58 | + pr_debug("Queue Doorbell Pointer: 0x%p\n", q->properties.doorbell_ptr); | |
| 59 | + pr_debug("Queue Doorbell Offset: %u\n", q->properties.doorbell_off); | |
| 60 | + pr_debug("Queue MQD Address: 0x%p\n", q->mqd); | |
| 61 | + pr_debug("Queue MQD Gart: 0x%llX\n", q->gart_mqd_addr); | |
| 62 | + pr_debug("Queue Process Address: 0x%p\n", q->process); | |
| 63 | + pr_debug("Queue Device Address: 0x%p\n", q->device); | |
| 64 | +} | |
| 65 | + | |
| 66 | +int init_queue(struct queue **q, struct queue_properties properties) | |
| 67 | +{ | |
| 68 | + struct queue *tmp; | |
| 69 | + | |
| 70 | + BUG_ON(!q); | |
| 71 | + | |
| 72 | + tmp = kzalloc(sizeof(struct queue), GFP_KERNEL); | |
| 73 | + if (!tmp) | |
| 74 | + return -ENOMEM; | |
| 75 | + | |
| 76 | + memcpy(&tmp->properties, &properties, sizeof(struct queue_properties)); | |
| 77 | + | |
| 78 | + *q = tmp; | |
| 79 | + return 0; | |
| 80 | +} | |
| 81 | + | |
| 82 | +void uninit_queue(struct queue *q) | |
| 83 | +{ | |
| 84 | + kfree(q); | |
| 85 | +} | |
| @@ -0,0 +1,1235 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +#include <linux/types.h> | |
| 24 | +#include <linux/kernel.h> | |
| 25 | +#include <linux/pci.h> | |
| 26 | +#include <linux/errno.h> | |
| 27 | +#include <linux/acpi.h> | |
| 28 | +#include <linux/hash.h> | |
| 29 | +#include <linux/cpufreq.h> | |
| 30 | + | |
| 31 | +#include "kfd_priv.h" | |
| 32 | +#include "kfd_crat.h" | |
| 33 | +#include "kfd_topology.h" | |
| 34 | + | |
| 35 | +static struct list_head topology_device_list; | |
| 36 | +static int topology_crat_parsed; | |
| 37 | +static struct kfd_system_properties sys_props; | |
| 38 | + | |
| 39 | +static DECLARE_RWSEM(topology_lock); | |
| 40 | + | |
| 41 | +struct kfd_dev *kfd_device_by_id(uint32_t gpu_id) | |
| 42 | +{ | |
| 43 | + struct kfd_topology_device *top_dev; | |
| 44 | + struct kfd_dev *device = NULL; | |
| 45 | + | |
| 46 | + down_read(&topology_lock); | |
| 47 | + | |
| 48 | + list_for_each_entry(top_dev, &topology_device_list, list) | |
| 49 | + if (top_dev->gpu_id == gpu_id) { | |
| 50 | + device = top_dev->gpu; | |
| 51 | + break; | |
| 52 | + } | |
| 53 | + | |
| 54 | + up_read(&topology_lock); | |
| 55 | + | |
| 56 | + return device; | |
| 57 | +} | |
| 58 | + | |
| 59 | +struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev) | |
| 60 | +{ | |
| 61 | + struct kfd_topology_device *top_dev; | |
| 62 | + struct kfd_dev *device = NULL; | |
| 63 | + | |
| 64 | + down_read(&topology_lock); | |
| 65 | + | |
| 66 | + list_for_each_entry(top_dev, &topology_device_list, list) | |
| 67 | + if (top_dev->gpu->pdev == pdev) { | |
| 68 | + device = top_dev->gpu; | |
| 69 | + break; | |
| 70 | + } | |
| 71 | + | |
| 72 | + up_read(&topology_lock); | |
| 73 | + | |
| 74 | + return device; | |
| 75 | +} | |
| 76 | + | |
| 77 | +static int kfd_topology_get_crat_acpi(void *crat_image, size_t *size) | |
| 78 | +{ | |
| 79 | + struct acpi_table_header *crat_table; | |
| 80 | + acpi_status status; | |
| 81 | + | |
| 82 | + if (!size) | |
| 83 | + return -EINVAL; | |
| 84 | + | |
| 85 | + /* | |
| 86 | + * Fetch the CRAT table from ACPI | |
| 87 | + */ | |
| 88 | + status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table); | |
| 89 | + if (status == AE_NOT_FOUND) { | |
| 90 | + pr_warn("CRAT table not found\n"); | |
| 91 | + return -ENODATA; | |
| 92 | + } else if (ACPI_FAILURE(status)) { | |
| 93 | + const char *err = acpi_format_exception(status); | |
| 94 | + | |
| 95 | + pr_err("CRAT table error: %s\n", err); | |
| 96 | + return -EINVAL; | |
| 97 | + } | |
| 98 | + | |
| 99 | + if (*size >= crat_table->length && crat_image != 0) | |
| 100 | + memcpy(crat_image, crat_table, crat_table->length); | |
| 101 | + | |
| 102 | + *size = crat_table->length; | |
| 103 | + | |
| 104 | + return 0; | |
| 105 | +} | |
| 106 | + | |
| 107 | +static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev, | |
| 108 | + struct crat_subtype_computeunit *cu) | |
| 109 | +{ | |
| 110 | + BUG_ON(!dev); | |
| 111 | + BUG_ON(!cu); | |
| 112 | + | |
| 113 | + dev->node_props.cpu_cores_count = cu->num_cpu_cores; | |
| 114 | + dev->node_props.cpu_core_id_base = cu->processor_id_low; | |
| 115 | + if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT) | |
| 116 | + dev->node_props.capability |= HSA_CAP_ATS_PRESENT; | |
| 117 | + | |
| 118 | + pr_info("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores, | |
| 119 | + cu->processor_id_low); | |
| 120 | +} | |
| 121 | + | |
| 122 | +static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev, | |
| 123 | + struct crat_subtype_computeunit *cu) | |
| 124 | +{ | |
| 125 | + BUG_ON(!dev); | |
| 126 | + BUG_ON(!cu); | |
| 127 | + | |
| 128 | + dev->node_props.simd_id_base = cu->processor_id_low; | |
| 129 | + dev->node_props.simd_count = cu->num_simd_cores; | |
| 130 | + dev->node_props.lds_size_in_kb = cu->lds_size_in_kb; | |
| 131 | + dev->node_props.max_waves_per_simd = cu->max_waves_simd; | |
| 132 | + dev->node_props.wave_front_size = cu->wave_front_size; | |
| 133 | + dev->node_props.mem_banks_count = cu->num_banks; | |
| 134 | + dev->node_props.array_count = cu->num_arrays; | |
| 135 | + dev->node_props.cu_per_simd_array = cu->num_cu_per_array; | |
| 136 | + dev->node_props.simd_per_cu = cu->num_simd_per_cu; | |
| 137 | + dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu; | |
| 138 | + if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE) | |
| 139 | + dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE; | |
| 140 | + pr_info("CU GPU: simds=%d id_base=%d\n", cu->num_simd_cores, | |
| 141 | + cu->processor_id_low); | |
| 142 | +} | |
| 143 | + | |
| 144 | +/* kfd_parse_subtype_cu is called when the topology mutex is already acquired */ | |
| 145 | +static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu) | |
| 146 | +{ | |
| 147 | + struct kfd_topology_device *dev; | |
| 148 | + int i = 0; | |
| 149 | + | |
| 150 | + BUG_ON(!cu); | |
| 151 | + | |
| 152 | + pr_info("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n", | |
| 153 | + cu->proximity_domain, cu->hsa_capability); | |
| 154 | + list_for_each_entry(dev, &topology_device_list, list) { | |
| 155 | + if (cu->proximity_domain == i) { | |
| 156 | + if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT) | |
| 157 | + kfd_populated_cu_info_cpu(dev, cu); | |
| 158 | + | |
| 159 | + if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT) | |
| 160 | + kfd_populated_cu_info_gpu(dev, cu); | |
| 161 | + break; | |
| 162 | + } | |
| 163 | + i++; | |
| 164 | + } | |
| 165 | + | |
| 166 | + return 0; | |
| 167 | +} | |
| 168 | + | |
| 169 | +/* | |
| 170 | + * kfd_parse_subtype_mem is called when the topology mutex is | |
| 171 | + * already acquired | |
| 172 | + */ | |
| 173 | +static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem) | |
| 174 | +{ | |
| 175 | + struct kfd_mem_properties *props; | |
| 176 | + struct kfd_topology_device *dev; | |
| 177 | + int i = 0; | |
| 178 | + | |
| 179 | + BUG_ON(!mem); | |
| 180 | + | |
| 181 | + pr_info("Found memory entry in CRAT table with proximity_domain=%d\n", | |
| 182 | + mem->promixity_domain); | |
| 183 | + list_for_each_entry(dev, &topology_device_list, list) { | |
| 184 | + if (mem->promixity_domain == i) { | |
| 185 | + props = kfd_alloc_struct(props); | |
| 186 | + if (props == 0) | |
| 187 | + return -ENOMEM; | |
| 188 | + | |
| 189 | + if (dev->node_props.cpu_cores_count == 0) | |
| 190 | + props->heap_type = HSA_MEM_HEAP_TYPE_FB_PRIVATE; | |
| 191 | + else | |
| 192 | + props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM; | |
| 193 | + | |
| 194 | + if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE) | |
| 195 | + props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE; | |
| 196 | + if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE) | |
| 197 | + props->flags |= HSA_MEM_FLAGS_NON_VOLATILE; | |
| 198 | + | |
| 199 | + props->size_in_bytes = | |
| 200 | + ((uint64_t)mem->length_high << 32) + | |
| 201 | + mem->length_low; | |
| 202 | + props->width = mem->width; | |
| 203 | + | |
| 204 | + dev->mem_bank_count++; | |
| 205 | + list_add_tail(&props->list, &dev->mem_props); | |
| 206 | + | |
| 207 | + break; | |
| 208 | + } | |
| 209 | + i++; | |
| 210 | + } | |
| 211 | + | |
| 212 | + return 0; | |
| 213 | +} | |
| 214 | + | |
| 215 | +/* | |
| 216 | + * kfd_parse_subtype_cache is called when the topology mutex | |
| 217 | + * is already acquired | |
| 218 | + */ | |
| 219 | +static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache) | |
| 220 | +{ | |
| 221 | + struct kfd_cache_properties *props; | |
| 222 | + struct kfd_topology_device *dev; | |
| 223 | + uint32_t id; | |
| 224 | + | |
| 225 | + BUG_ON(!cache); | |
| 226 | + | |
| 227 | + id = cache->processor_id_low; | |
| 228 | + | |
| 229 | + pr_info("Found cache entry in CRAT table with processor_id=%d\n", id); | |
| 230 | + list_for_each_entry(dev, &topology_device_list, list) | |
| 231 | + if (id == dev->node_props.cpu_core_id_base || | |
| 232 | + id == dev->node_props.simd_id_base) { | |
| 233 | + props = kfd_alloc_struct(props); | |
| 234 | + if (props == 0) | |
| 235 | + return -ENOMEM; | |
| 236 | + | |
| 237 | + props->processor_id_low = id; | |
| 238 | + props->cache_level = cache->cache_level; | |
| 239 | + props->cache_size = cache->cache_size; | |
| 240 | + props->cacheline_size = cache->cache_line_size; | |
| 241 | + props->cachelines_per_tag = cache->lines_per_tag; | |
| 242 | + props->cache_assoc = cache->associativity; | |
| 243 | + props->cache_latency = cache->cache_latency; | |
| 244 | + | |
| 245 | + if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE) | |
| 246 | + props->cache_type |= HSA_CACHE_TYPE_DATA; | |
| 247 | + if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE) | |
| 248 | + props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION; | |
| 249 | + if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE) | |
| 250 | + props->cache_type |= HSA_CACHE_TYPE_CPU; | |
| 251 | + if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE) | |
| 252 | + props->cache_type |= HSA_CACHE_TYPE_HSACU; | |
| 253 | + | |
| 254 | + dev->cache_count++; | |
| 255 | + dev->node_props.caches_count++; | |
| 256 | + list_add_tail(&props->list, &dev->cache_props); | |
| 257 | + | |
| 258 | + break; | |
| 259 | + } | |
| 260 | + | |
| 261 | + return 0; | |
| 262 | +} | |
| 263 | + | |
| 264 | +/* | |
| 265 | + * kfd_parse_subtype_iolink is called when the topology mutex | |
| 266 | + * is already acquired | |
| 267 | + */ | |
| 268 | +static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink) | |
| 269 | +{ | |
| 270 | + struct kfd_iolink_properties *props; | |
| 271 | + struct kfd_topology_device *dev; | |
| 272 | + uint32_t i = 0; | |
| 273 | + uint32_t id_from; | |
| 274 | + uint32_t id_to; | |
| 275 | + | |
| 276 | + BUG_ON(!iolink); | |
| 277 | + | |
| 278 | + id_from = iolink->proximity_domain_from; | |
| 279 | + id_to = iolink->proximity_domain_to; | |
| 280 | + | |
| 281 | + pr_info("Found IO link entry in CRAT table with id_from=%d\n", id_from); | |
| 282 | + list_for_each_entry(dev, &topology_device_list, list) { | |
| 283 | + if (id_from == i) { | |
| 284 | + props = kfd_alloc_struct(props); | |
| 285 | + if (props == 0) | |
| 286 | + return -ENOMEM; | |
| 287 | + | |
| 288 | + props->node_from = id_from; | |
| 289 | + props->node_to = id_to; | |
| 290 | + props->ver_maj = iolink->version_major; | |
| 291 | + props->ver_min = iolink->version_minor; | |
| 292 | + | |
| 293 | + /* | |
| 294 | + * weight factor (derived from CDIR), currently always 1 | |
| 295 | + */ | |
| 296 | + props->weight = 1; | |
| 297 | + | |
| 298 | + props->min_latency = iolink->minimum_latency; | |
| 299 | + props->max_latency = iolink->maximum_latency; | |
| 300 | + props->min_bandwidth = iolink->minimum_bandwidth_mbs; | |
| 301 | + props->max_bandwidth = iolink->maximum_bandwidth_mbs; | |
| 302 | + props->rec_transfer_size = | |
| 303 | + iolink->recommended_transfer_size; | |
| 304 | + | |
| 305 | + dev->io_link_count++; | |
| 306 | + dev->node_props.io_links_count++; | |
| 307 | + list_add_tail(&props->list, &dev->io_link_props); | |
| 308 | + | |
| 309 | + break; | |
| 310 | + } | |
| 311 | + i++; | |
| 312 | + } | |
| 313 | + | |
| 314 | + return 0; | |
| 315 | +} | |
| 316 | + | |
| 317 | +static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr) | |
| 318 | +{ | |
| 319 | + struct crat_subtype_computeunit *cu; | |
| 320 | + struct crat_subtype_memory *mem; | |
| 321 | + struct crat_subtype_cache *cache; | |
| 322 | + struct crat_subtype_iolink *iolink; | |
| 323 | + int ret = 0; | |
| 324 | + | |
| 325 | + BUG_ON(!sub_type_hdr); | |
| 326 | + | |
| 327 | + switch (sub_type_hdr->type) { | |
| 328 | + case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY: | |
| 329 | + cu = (struct crat_subtype_computeunit *)sub_type_hdr; | |
| 330 | + ret = kfd_parse_subtype_cu(cu); | |
| 331 | + break; | |
| 332 | + case CRAT_SUBTYPE_MEMORY_AFFINITY: | |
| 333 | + mem = (struct crat_subtype_memory *)sub_type_hdr; | |
| 334 | + ret = kfd_parse_subtype_mem(mem); | |
| 335 | + break; | |
| 336 | + case CRAT_SUBTYPE_CACHE_AFFINITY: | |
| 337 | + cache = (struct crat_subtype_cache *)sub_type_hdr; | |
| 338 | + ret = kfd_parse_subtype_cache(cache); | |
| 339 | + break; | |
| 340 | + case CRAT_SUBTYPE_TLB_AFFINITY: | |
| 341 | + /* | |
| 342 | + * For now, nothing to do here | |
| 343 | + */ | |
| 344 | + pr_info("Found TLB entry in CRAT table (not processing)\n"); | |
| 345 | + break; | |
| 346 | + case CRAT_SUBTYPE_CCOMPUTE_AFFINITY: | |
| 347 | + /* | |
| 348 | + * For now, nothing to do here | |
| 349 | + */ | |
| 350 | + pr_info("Found CCOMPUTE entry in CRAT table (not processing)\n"); | |
| 351 | + break; | |
| 352 | + case CRAT_SUBTYPE_IOLINK_AFFINITY: | |
| 353 | + iolink = (struct crat_subtype_iolink *)sub_type_hdr; | |
| 354 | + ret = kfd_parse_subtype_iolink(iolink); | |
| 355 | + break; | |
| 356 | + default: | |
| 357 | + pr_warn("Unknown subtype (%d) in CRAT\n", | |
| 358 | + sub_type_hdr->type); | |
| 359 | + } | |
| 360 | + | |
| 361 | + return ret; | |
| 362 | +} | |
| 363 | + | |
| 364 | +static void kfd_release_topology_device(struct kfd_topology_device *dev) | |
| 365 | +{ | |
| 366 | + struct kfd_mem_properties *mem; | |
| 367 | + struct kfd_cache_properties *cache; | |
| 368 | + struct kfd_iolink_properties *iolink; | |
| 369 | + | |
| 370 | + BUG_ON(!dev); | |
| 371 | + | |
| 372 | + list_del(&dev->list); | |
| 373 | + | |
| 374 | + while (dev->mem_props.next != &dev->mem_props) { | |
| 375 | + mem = container_of(dev->mem_props.next, | |
| 376 | + struct kfd_mem_properties, list); | |
| 377 | + list_del(&mem->list); | |
| 378 | + kfree(mem); | |
| 379 | + } | |
| 380 | + | |
| 381 | + while (dev->cache_props.next != &dev->cache_props) { | |
| 382 | + cache = container_of(dev->cache_props.next, | |
| 383 | + struct kfd_cache_properties, list); | |
| 384 | + list_del(&cache->list); | |
| 385 | + kfree(cache); | |
| 386 | + } | |
| 387 | + | |
| 388 | + while (dev->io_link_props.next != &dev->io_link_props) { | |
| 389 | + iolink = container_of(dev->io_link_props.next, | |
| 390 | + struct kfd_iolink_properties, list); | |
| 391 | + list_del(&iolink->list); | |
| 392 | + kfree(iolink); | |
| 393 | + } | |
| 394 | + | |
| 395 | + kfree(dev); | |
| 396 | + | |
| 397 | + sys_props.num_devices--; | |
| 398 | +} | |
| 399 | + | |
| 400 | +static void kfd_release_live_view(void) | |
| 401 | +{ | |
| 402 | + struct kfd_topology_device *dev; | |
| 403 | + | |
| 404 | + while (topology_device_list.next != &topology_device_list) { | |
| 405 | + dev = container_of(topology_device_list.next, | |
| 406 | + struct kfd_topology_device, list); | |
| 407 | + kfd_release_topology_device(dev); | |
| 408 | +} | |
| 409 | + | |
| 410 | + memset(&sys_props, 0, sizeof(sys_props)); | |
| 411 | +} | |
| 412 | + | |
| 413 | +static struct kfd_topology_device *kfd_create_topology_device(void) | |
| 414 | +{ | |
| 415 | + struct kfd_topology_device *dev; | |
| 416 | + | |
| 417 | + dev = kfd_alloc_struct(dev); | |
| 418 | + if (dev == 0) { | |
| 419 | + pr_err("No memory to allocate a topology device"); | |
| 420 | + return 0; | |
| 421 | + } | |
| 422 | + | |
| 423 | + INIT_LIST_HEAD(&dev->mem_props); | |
| 424 | + INIT_LIST_HEAD(&dev->cache_props); | |
| 425 | + INIT_LIST_HEAD(&dev->io_link_props); | |
| 426 | + | |
| 427 | + list_add_tail(&dev->list, &topology_device_list); | |
| 428 | + sys_props.num_devices++; | |
| 429 | + | |
| 430 | + return dev; | |
| 431 | + } | |
| 432 | + | |
| 433 | +static int kfd_parse_crat_table(void *crat_image) | |
| 434 | +{ | |
| 435 | + struct kfd_topology_device *top_dev; | |
| 436 | + struct crat_subtype_generic *sub_type_hdr; | |
| 437 | + uint16_t node_id; | |
| 438 | + int ret; | |
| 439 | + struct crat_header *crat_table = (struct crat_header *)crat_image; | |
| 440 | + uint16_t num_nodes; | |
| 441 | + uint32_t image_len; | |
| 442 | + | |
| 443 | + if (!crat_image) | |
| 444 | + return -EINVAL; | |
| 445 | + | |
| 446 | + num_nodes = crat_table->num_domains; | |
| 447 | + image_len = crat_table->length; | |
| 448 | + | |
| 449 | + pr_info("Parsing CRAT table with %d nodes\n", num_nodes); | |
| 450 | + | |
| 451 | + for (node_id = 0; node_id < num_nodes; node_id++) { | |
| 452 | + top_dev = kfd_create_topology_device(); | |
| 453 | + if (!top_dev) { | |
| 454 | + kfd_release_live_view(); | |
| 455 | + return -ENOMEM; | |
| 456 | + } | |
| 457 | + } | |
| 458 | + | |
| 459 | + sys_props.platform_id = | |
| 460 | + (*((uint64_t *)crat_table->oem_id)) & CRAT_OEMID_64BIT_MASK; | |
| 461 | + sys_props.platform_oem = *((uint64_t *)crat_table->oem_table_id); | |
| 462 | + sys_props.platform_rev = crat_table->revision; | |
| 463 | + | |
| 464 | + sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1); | |
| 465 | + while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) < | |
| 466 | + ((char *)crat_image) + image_len) { | |
| 467 | + if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) { | |
| 468 | + ret = kfd_parse_subtype(sub_type_hdr); | |
| 469 | + if (ret != 0) { | |
| 470 | + kfd_release_live_view(); | |
| 471 | + return ret; | |
| 472 | + } | |
| 473 | + } | |
| 474 | + | |
| 475 | + sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr + | |
| 476 | + sub_type_hdr->length); | |
| 477 | + } | |
| 478 | + | |
| 479 | + sys_props.generation_count++; | |
| 480 | + topology_crat_parsed = 1; | |
| 481 | + | |
| 482 | + return 0; | |
| 483 | +} | |
| 484 | + | |
| 485 | + | |
| 486 | +#define sysfs_show_gen_prop(buffer, fmt, ...) \ | |
| 487 | + snprintf(buffer, PAGE_SIZE, "%s"fmt, buffer, __VA_ARGS__) | |
| 488 | +#define sysfs_show_32bit_prop(buffer, name, value) \ | |
| 489 | + sysfs_show_gen_prop(buffer, "%s %u\n", name, value) | |
| 490 | +#define sysfs_show_64bit_prop(buffer, name, value) \ | |
| 491 | + sysfs_show_gen_prop(buffer, "%s %llu\n", name, value) | |
| 492 | +#define sysfs_show_32bit_val(buffer, value) \ | |
| 493 | + sysfs_show_gen_prop(buffer, "%u\n", value) | |
| 494 | +#define sysfs_show_str_val(buffer, value) \ | |
| 495 | + sysfs_show_gen_prop(buffer, "%s\n", value) | |
| 496 | + | |
| 497 | +static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr, | |
| 498 | + char *buffer) | |
| 499 | +{ | |
| 500 | + ssize_t ret; | |
| 501 | + | |
| 502 | + /* Making sure that the buffer is an empty string */ | |
| 503 | + buffer[0] = 0; | |
| 504 | + | |
| 505 | + if (attr == &sys_props.attr_genid) { | |
| 506 | + ret = sysfs_show_32bit_val(buffer, sys_props.generation_count); | |
| 507 | + } else if (attr == &sys_props.attr_props) { | |
| 508 | + sysfs_show_64bit_prop(buffer, "platform_oem", | |
| 509 | + sys_props.platform_oem); | |
| 510 | + sysfs_show_64bit_prop(buffer, "platform_id", | |
| 511 | + sys_props.platform_id); | |
| 512 | + ret = sysfs_show_64bit_prop(buffer, "platform_rev", | |
| 513 | + sys_props.platform_rev); | |
| 514 | + } else { | |
| 515 | + ret = -EINVAL; | |
| 516 | + } | |
| 517 | + | |
| 518 | + return ret; | |
| 519 | +} | |
| 520 | + | |
| 521 | +static const struct sysfs_ops sysprops_ops = { | |
| 522 | + .show = sysprops_show, | |
| 523 | +}; | |
| 524 | + | |
| 525 | +static struct kobj_type sysprops_type = { | |
| 526 | + .sysfs_ops = &sysprops_ops, | |
| 527 | +}; | |
| 528 | + | |
| 529 | +static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr, | |
| 530 | + char *buffer) | |
| 531 | +{ | |
| 532 | + ssize_t ret; | |
| 533 | + struct kfd_iolink_properties *iolink; | |
| 534 | + | |
| 535 | + /* Making sure that the buffer is an empty string */ | |
| 536 | + buffer[0] = 0; | |
| 537 | + | |
| 538 | + iolink = container_of(attr, struct kfd_iolink_properties, attr); | |
| 539 | + sysfs_show_32bit_prop(buffer, "type", iolink->iolink_type); | |
| 540 | + sysfs_show_32bit_prop(buffer, "version_major", iolink->ver_maj); | |
| 541 | + sysfs_show_32bit_prop(buffer, "version_minor", iolink->ver_min); | |
| 542 | + sysfs_show_32bit_prop(buffer, "node_from", iolink->node_from); | |
| 543 | + sysfs_show_32bit_prop(buffer, "node_to", iolink->node_to); | |
| 544 | + sysfs_show_32bit_prop(buffer, "weight", iolink->weight); | |
| 545 | + sysfs_show_32bit_prop(buffer, "min_latency", iolink->min_latency); | |
| 546 | + sysfs_show_32bit_prop(buffer, "max_latency", iolink->max_latency); | |
| 547 | + sysfs_show_32bit_prop(buffer, "min_bandwidth", iolink->min_bandwidth); | |
| 548 | + sysfs_show_32bit_prop(buffer, "max_bandwidth", iolink->max_bandwidth); | |
| 549 | + sysfs_show_32bit_prop(buffer, "recommended_transfer_size", | |
| 550 | + iolink->rec_transfer_size); | |
| 551 | + ret = sysfs_show_32bit_prop(buffer, "flags", iolink->flags); | |
| 552 | + | |
| 553 | + return ret; | |
| 554 | +} | |
| 555 | + | |
| 556 | +static const struct sysfs_ops iolink_ops = { | |
| 557 | + .show = iolink_show, | |
| 558 | +}; | |
| 559 | + | |
| 560 | +static struct kobj_type iolink_type = { | |
| 561 | + .sysfs_ops = &iolink_ops, | |
| 562 | +}; | |
| 563 | + | |
| 564 | +static ssize_t mem_show(struct kobject *kobj, struct attribute *attr, | |
| 565 | + char *buffer) | |
| 566 | +{ | |
| 567 | + ssize_t ret; | |
| 568 | + struct kfd_mem_properties *mem; | |
| 569 | + | |
| 570 | + /* Making sure that the buffer is an empty string */ | |
| 571 | + buffer[0] = 0; | |
| 572 | + | |
| 573 | + mem = container_of(attr, struct kfd_mem_properties, attr); | |
| 574 | + sysfs_show_32bit_prop(buffer, "heap_type", mem->heap_type); | |
| 575 | + sysfs_show_64bit_prop(buffer, "size_in_bytes", mem->size_in_bytes); | |
| 576 | + sysfs_show_32bit_prop(buffer, "flags", mem->flags); | |
| 577 | + sysfs_show_32bit_prop(buffer, "width", mem->width); | |
| 578 | + ret = sysfs_show_32bit_prop(buffer, "mem_clk_max", mem->mem_clk_max); | |
| 579 | + | |
| 580 | + return ret; | |
| 581 | +} | |
| 582 | + | |
| 583 | +static const struct sysfs_ops mem_ops = { | |
| 584 | + .show = mem_show, | |
| 585 | +}; | |
| 586 | + | |
| 587 | +static struct kobj_type mem_type = { | |
| 588 | + .sysfs_ops = &mem_ops, | |
| 589 | +}; | |
| 590 | + | |
| 591 | +static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr, | |
| 592 | + char *buffer) | |
| 593 | +{ | |
| 594 | + ssize_t ret; | |
| 595 | + uint32_t i; | |
| 596 | + struct kfd_cache_properties *cache; | |
| 597 | + | |
| 598 | + /* Making sure that the buffer is an empty string */ | |
| 599 | + buffer[0] = 0; | |
| 600 | + | |
| 601 | + cache = container_of(attr, struct kfd_cache_properties, attr); | |
| 602 | + sysfs_show_32bit_prop(buffer, "processor_id_low", | |
| 603 | + cache->processor_id_low); | |
| 604 | + sysfs_show_32bit_prop(buffer, "level", cache->cache_level); | |
| 605 | + sysfs_show_32bit_prop(buffer, "size", cache->cache_size); | |
| 606 | + sysfs_show_32bit_prop(buffer, "cache_line_size", cache->cacheline_size); | |
| 607 | + sysfs_show_32bit_prop(buffer, "cache_lines_per_tag", | |
| 608 | + cache->cachelines_per_tag); | |
| 609 | + sysfs_show_32bit_prop(buffer, "association", cache->cache_assoc); | |
| 610 | + sysfs_show_32bit_prop(buffer, "latency", cache->cache_latency); | |
| 611 | + sysfs_show_32bit_prop(buffer, "type", cache->cache_type); | |
| 612 | + snprintf(buffer, PAGE_SIZE, "%ssibling_map ", buffer); | |
| 613 | + for (i = 0; i < KFD_TOPOLOGY_CPU_SIBLINGS; i++) | |
| 614 | + ret = snprintf(buffer, PAGE_SIZE, "%s%d%s", | |
| 615 | + buffer, cache->sibling_map[i], | |
| 616 | + (i == KFD_TOPOLOGY_CPU_SIBLINGS-1) ? | |
| 617 | + "\n" : ","); | |
| 618 | + | |
| 619 | + return ret; | |
| 620 | +} | |
| 621 | + | |
| 622 | +static const struct sysfs_ops cache_ops = { | |
| 623 | + .show = kfd_cache_show, | |
| 624 | +}; | |
| 625 | + | |
| 626 | +static struct kobj_type cache_type = { | |
| 627 | + .sysfs_ops = &cache_ops, | |
| 628 | +}; | |
| 629 | + | |
| 630 | +static ssize_t node_show(struct kobject *kobj, struct attribute *attr, | |
| 631 | + char *buffer) | |
| 632 | +{ | |
| 633 | + ssize_t ret; | |
| 634 | + struct kfd_topology_device *dev; | |
| 635 | + char public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE]; | |
| 636 | + uint32_t i; | |
| 637 | + | |
| 638 | + /* Making sure that the buffer is an empty string */ | |
| 639 | + buffer[0] = 0; | |
| 640 | + | |
| 641 | + if (strcmp(attr->name, "gpu_id") == 0) { | |
| 642 | + dev = container_of(attr, struct kfd_topology_device, | |
| 643 | + attr_gpuid); | |
| 644 | + ret = sysfs_show_32bit_val(buffer, dev->gpu_id); | |
| 645 | + } else if (strcmp(attr->name, "name") == 0) { | |
| 646 | + dev = container_of(attr, struct kfd_topology_device, | |
| 647 | + attr_name); | |
| 648 | + for (i = 0; i < KFD_TOPOLOGY_PUBLIC_NAME_SIZE; i++) { | |
| 649 | + public_name[i] = | |
| 650 | + (char)dev->node_props.marketing_name[i]; | |
| 651 | + if (dev->node_props.marketing_name[i] == 0) | |
| 652 | + break; | |
| 653 | + } | |
| 654 | + public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE-1] = 0x0; | |
| 655 | + ret = sysfs_show_str_val(buffer, public_name); | |
| 656 | + } else { | |
| 657 | + dev = container_of(attr, struct kfd_topology_device, | |
| 658 | + attr_props); | |
| 659 | + sysfs_show_32bit_prop(buffer, "cpu_cores_count", | |
| 660 | + dev->node_props.cpu_cores_count); | |
| 661 | + sysfs_show_32bit_prop(buffer, "simd_count", | |
| 662 | + dev->node_props.simd_count); | |
| 663 | + | |
| 664 | + if (dev->mem_bank_count < dev->node_props.mem_banks_count) { | |
| 665 | + pr_warn("kfd: mem_banks_count truncated from %d to %d\n", | |
| 666 | + dev->node_props.mem_banks_count, | |
| 667 | + dev->mem_bank_count); | |
| 668 | + sysfs_show_32bit_prop(buffer, "mem_banks_count", | |
| 669 | + dev->mem_bank_count); | |
| 670 | + } else { | |
| 671 | + sysfs_show_32bit_prop(buffer, "mem_banks_count", | |
| 672 | + dev->node_props.mem_banks_count); | |
| 673 | + } | |
| 674 | + | |
| 675 | + sysfs_show_32bit_prop(buffer, "caches_count", | |
| 676 | + dev->node_props.caches_count); | |
| 677 | + sysfs_show_32bit_prop(buffer, "io_links_count", | |
| 678 | + dev->node_props.io_links_count); | |
| 679 | + sysfs_show_32bit_prop(buffer, "cpu_core_id_base", | |
| 680 | + dev->node_props.cpu_core_id_base); | |
| 681 | + sysfs_show_32bit_prop(buffer, "simd_id_base", | |
| 682 | + dev->node_props.simd_id_base); | |
| 683 | + sysfs_show_32bit_prop(buffer, "capability", | |
| 684 | + dev->node_props.capability); | |
| 685 | + sysfs_show_32bit_prop(buffer, "max_waves_per_simd", | |
| 686 | + dev->node_props.max_waves_per_simd); | |
| 687 | + sysfs_show_32bit_prop(buffer, "lds_size_in_kb", | |
| 688 | + dev->node_props.lds_size_in_kb); | |
| 689 | + sysfs_show_32bit_prop(buffer, "gds_size_in_kb", | |
| 690 | + dev->node_props.gds_size_in_kb); | |
| 691 | + sysfs_show_32bit_prop(buffer, "wave_front_size", | |
| 692 | + dev->node_props.wave_front_size); | |
| 693 | + sysfs_show_32bit_prop(buffer, "array_count", | |
| 694 | + dev->node_props.array_count); | |
| 695 | + sysfs_show_32bit_prop(buffer, "simd_arrays_per_engine", | |
| 696 | + dev->node_props.simd_arrays_per_engine); | |
| 697 | + sysfs_show_32bit_prop(buffer, "cu_per_simd_array", | |
| 698 | + dev->node_props.cu_per_simd_array); | |
| 699 | + sysfs_show_32bit_prop(buffer, "simd_per_cu", | |
| 700 | + dev->node_props.simd_per_cu); | |
| 701 | + sysfs_show_32bit_prop(buffer, "max_slots_scratch_cu", | |
| 702 | + dev->node_props.max_slots_scratch_cu); | |
| 703 | + sysfs_show_32bit_prop(buffer, "engine_id", | |
| 704 | + dev->node_props.engine_id); | |
| 705 | + sysfs_show_32bit_prop(buffer, "vendor_id", | |
| 706 | + dev->node_props.vendor_id); | |
| 707 | + sysfs_show_32bit_prop(buffer, "device_id", | |
| 708 | + dev->node_props.device_id); | |
| 709 | + sysfs_show_32bit_prop(buffer, "location_id", | |
| 710 | + dev->node_props.location_id); | |
| 711 | + | |
| 712 | + if (dev->gpu) { | |
| 713 | + sysfs_show_32bit_prop(buffer, "max_engine_clk_fcompute", | |
| 714 | + kfd2kgd->get_max_engine_clock_in_mhz( | |
| 715 | + dev->gpu->kgd)); | |
| 716 | + sysfs_show_64bit_prop(buffer, "local_mem_size", | |
| 717 | + kfd2kgd->get_vmem_size(dev->gpu->kgd)); | |
| 718 | + } | |
| 719 | + | |
| 720 | + ret = sysfs_show_32bit_prop(buffer, "max_engine_clk_ccompute", | |
| 721 | + cpufreq_quick_get_max(0)/1000); | |
| 722 | + } | |
| 723 | + | |
| 724 | + return ret; | |
| 725 | +} | |
| 726 | + | |
| 727 | +static const struct sysfs_ops node_ops = { | |
| 728 | + .show = node_show, | |
| 729 | +}; | |
| 730 | + | |
| 731 | +static struct kobj_type node_type = { | |
| 732 | + .sysfs_ops = &node_ops, | |
| 733 | +}; | |
| 734 | + | |
| 735 | +static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr) | |
| 736 | +{ | |
| 737 | + sysfs_remove_file(kobj, attr); | |
| 738 | + kobject_del(kobj); | |
| 739 | + kobject_put(kobj); | |
| 740 | +} | |
| 741 | + | |
| 742 | +static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev) | |
| 743 | +{ | |
| 744 | + struct kfd_iolink_properties *iolink; | |
| 745 | + struct kfd_cache_properties *cache; | |
| 746 | + struct kfd_mem_properties *mem; | |
| 747 | + | |
| 748 | + BUG_ON(!dev); | |
| 749 | + | |
| 750 | + if (dev->kobj_iolink) { | |
| 751 | + list_for_each_entry(iolink, &dev->io_link_props, list) | |
| 752 | + if (iolink->kobj) { | |
| 753 | + kfd_remove_sysfs_file(iolink->kobj, | |
| 754 | + &iolink->attr); | |
| 755 | + iolink->kobj = 0; | |
| 756 | + } | |
| 757 | + kobject_del(dev->kobj_iolink); | |
| 758 | + kobject_put(dev->kobj_iolink); | |
| 759 | + dev->kobj_iolink = 0; | |
| 760 | + } | |
| 761 | + | |
| 762 | + if (dev->kobj_cache) { | |
| 763 | + list_for_each_entry(cache, &dev->cache_props, list) | |
| 764 | + if (cache->kobj) { | |
| 765 | + kfd_remove_sysfs_file(cache->kobj, | |
| 766 | + &cache->attr); | |
| 767 | + cache->kobj = 0; | |
| 768 | + } | |
| 769 | + kobject_del(dev->kobj_cache); | |
| 770 | + kobject_put(dev->kobj_cache); | |
| 771 | + dev->kobj_cache = 0; | |
| 772 | + } | |
| 773 | + | |
| 774 | + if (dev->kobj_mem) { | |
| 775 | + list_for_each_entry(mem, &dev->mem_props, list) | |
| 776 | + if (mem->kobj) { | |
| 777 | + kfd_remove_sysfs_file(mem->kobj, &mem->attr); | |
| 778 | + mem->kobj = 0; | |
| 779 | + } | |
| 780 | + kobject_del(dev->kobj_mem); | |
| 781 | + kobject_put(dev->kobj_mem); | |
| 782 | + dev->kobj_mem = 0; | |
| 783 | + } | |
| 784 | + | |
| 785 | + if (dev->kobj_node) { | |
| 786 | + sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid); | |
| 787 | + sysfs_remove_file(dev->kobj_node, &dev->attr_name); | |
| 788 | + sysfs_remove_file(dev->kobj_node, &dev->attr_props); | |
| 789 | + kobject_del(dev->kobj_node); | |
| 790 | + kobject_put(dev->kobj_node); | |
| 791 | + dev->kobj_node = 0; | |
| 792 | + } | |
| 793 | +} | |
| 794 | + | |
| 795 | +static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev, | |
| 796 | + uint32_t id) | |
| 797 | +{ | |
| 798 | + struct kfd_iolink_properties *iolink; | |
| 799 | + struct kfd_cache_properties *cache; | |
| 800 | + struct kfd_mem_properties *mem; | |
| 801 | + int ret; | |
| 802 | + uint32_t i; | |
| 803 | + | |
| 804 | + BUG_ON(!dev); | |
| 805 | + | |
| 806 | + /* | |
| 807 | + * Creating the sysfs folders | |
| 808 | + */ | |
| 809 | + BUG_ON(dev->kobj_node); | |
| 810 | + dev->kobj_node = kfd_alloc_struct(dev->kobj_node); | |
| 811 | + if (!dev->kobj_node) | |
| 812 | + return -ENOMEM; | |
| 813 | + | |
| 814 | + ret = kobject_init_and_add(dev->kobj_node, &node_type, | |
| 815 | + sys_props.kobj_nodes, "%d", id); | |
| 816 | + if (ret < 0) | |
| 817 | + return ret; | |
| 818 | + | |
| 819 | + dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node); | |
| 820 | + if (!dev->kobj_mem) | |
| 821 | + return -ENOMEM; | |
| 822 | + | |
| 823 | + dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node); | |
| 824 | + if (!dev->kobj_cache) | |
| 825 | + return -ENOMEM; | |
| 826 | + | |
| 827 | + dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node); | |
| 828 | + if (!dev->kobj_iolink) | |
| 829 | + return -ENOMEM; | |
| 830 | + | |
| 831 | + /* | |
| 832 | + * Creating sysfs files for node properties | |
| 833 | + */ | |
| 834 | + dev->attr_gpuid.name = "gpu_id"; | |
| 835 | + dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE; | |
| 836 | + sysfs_attr_init(&dev->attr_gpuid); | |
| 837 | + dev->attr_name.name = "name"; | |
| 838 | + dev->attr_name.mode = KFD_SYSFS_FILE_MODE; | |
| 839 | + sysfs_attr_init(&dev->attr_name); | |
| 840 | + dev->attr_props.name = "properties"; | |
| 841 | + dev->attr_props.mode = KFD_SYSFS_FILE_MODE; | |
| 842 | + sysfs_attr_init(&dev->attr_props); | |
| 843 | + ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid); | |
| 844 | + if (ret < 0) | |
| 845 | + return ret; | |
| 846 | + ret = sysfs_create_file(dev->kobj_node, &dev->attr_name); | |
| 847 | + if (ret < 0) | |
| 848 | + return ret; | |
| 849 | + ret = sysfs_create_file(dev->kobj_node, &dev->attr_props); | |
| 850 | + if (ret < 0) | |
| 851 | + return ret; | |
| 852 | + | |
| 853 | + i = 0; | |
| 854 | + list_for_each_entry(mem, &dev->mem_props, list) { | |
| 855 | + mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); | |
| 856 | + if (!mem->kobj) | |
| 857 | + return -ENOMEM; | |
| 858 | + ret = kobject_init_and_add(mem->kobj, &mem_type, | |
| 859 | + dev->kobj_mem, "%d", i); | |
| 860 | + if (ret < 0) | |
| 861 | + return ret; | |
| 862 | + | |
| 863 | + mem->attr.name = "properties"; | |
| 864 | + mem->attr.mode = KFD_SYSFS_FILE_MODE; | |
| 865 | + sysfs_attr_init(&mem->attr); | |
| 866 | + ret = sysfs_create_file(mem->kobj, &mem->attr); | |
| 867 | + if (ret < 0) | |
| 868 | + return ret; | |
| 869 | + i++; | |
| 870 | + } | |
| 871 | + | |
| 872 | + i = 0; | |
| 873 | + list_for_each_entry(cache, &dev->cache_props, list) { | |
| 874 | + cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); | |
| 875 | + if (!cache->kobj) | |
| 876 | + return -ENOMEM; | |
| 877 | + ret = kobject_init_and_add(cache->kobj, &cache_type, | |
| 878 | + dev->kobj_cache, "%d", i); | |
| 879 | + if (ret < 0) | |
| 880 | + return ret; | |
| 881 | + | |
| 882 | + cache->attr.name = "properties"; | |
| 883 | + cache->attr.mode = KFD_SYSFS_FILE_MODE; | |
| 884 | + sysfs_attr_init(&cache->attr); | |
| 885 | + ret = sysfs_create_file(cache->kobj, &cache->attr); | |
| 886 | + if (ret < 0) | |
| 887 | + return ret; | |
| 888 | + i++; | |
| 889 | + } | |
| 890 | + | |
| 891 | + i = 0; | |
| 892 | + list_for_each_entry(iolink, &dev->io_link_props, list) { | |
| 893 | + iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); | |
| 894 | + if (!iolink->kobj) | |
| 895 | + return -ENOMEM; | |
| 896 | + ret = kobject_init_and_add(iolink->kobj, &iolink_type, | |
| 897 | + dev->kobj_iolink, "%d", i); | |
| 898 | + if (ret < 0) | |
| 899 | + return ret; | |
| 900 | + | |
| 901 | + iolink->attr.name = "properties"; | |
| 902 | + iolink->attr.mode = KFD_SYSFS_FILE_MODE; | |
| 903 | + sysfs_attr_init(&iolink->attr); | |
| 904 | + ret = sysfs_create_file(iolink->kobj, &iolink->attr); | |
| 905 | + if (ret < 0) | |
| 906 | + return ret; | |
| 907 | + i++; | |
| 908 | +} | |
| 909 | + | |
| 910 | + return 0; | |
| 911 | +} | |
| 912 | + | |
| 913 | +static int kfd_build_sysfs_node_tree(void) | |
| 914 | +{ | |
| 915 | + struct kfd_topology_device *dev; | |
| 916 | + int ret; | |
| 917 | + uint32_t i = 0; | |
| 918 | + | |
| 919 | + list_for_each_entry(dev, &topology_device_list, list) { | |
| 920 | + ret = kfd_build_sysfs_node_entry(dev, 0); | |
| 921 | + if (ret < 0) | |
| 922 | + return ret; | |
| 923 | + i++; | |
| 924 | + } | |
| 925 | + | |
| 926 | + return 0; | |
| 927 | +} | |
| 928 | + | |
| 929 | +static void kfd_remove_sysfs_node_tree(void) | |
| 930 | +{ | |
| 931 | + struct kfd_topology_device *dev; | |
| 932 | + | |
| 933 | + list_for_each_entry(dev, &topology_device_list, list) | |
| 934 | + kfd_remove_sysfs_node_entry(dev); | |
| 935 | +} | |
| 936 | + | |
| 937 | +static int kfd_topology_update_sysfs(void) | |
| 938 | +{ | |
| 939 | + int ret; | |
| 940 | + | |
| 941 | + pr_info("Creating topology SYSFS entries\n"); | |
| 942 | + if (sys_props.kobj_topology == 0) { | |
| 943 | + sys_props.kobj_topology = | |
| 944 | + kfd_alloc_struct(sys_props.kobj_topology); | |
| 945 | + if (!sys_props.kobj_topology) | |
| 946 | + return -ENOMEM; | |
| 947 | + | |
| 948 | + ret = kobject_init_and_add(sys_props.kobj_topology, | |
| 949 | + &sysprops_type, &kfd_device->kobj, | |
| 950 | + "topology"); | |
| 951 | + if (ret < 0) | |
| 952 | + return ret; | |
| 953 | + | |
| 954 | + sys_props.kobj_nodes = kobject_create_and_add("nodes", | |
| 955 | + sys_props.kobj_topology); | |
| 956 | + if (!sys_props.kobj_nodes) | |
| 957 | + return -ENOMEM; | |
| 958 | + | |
| 959 | + sys_props.attr_genid.name = "generation_id"; | |
| 960 | + sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE; | |
| 961 | + sysfs_attr_init(&sys_props.attr_genid); | |
| 962 | + ret = sysfs_create_file(sys_props.kobj_topology, | |
| 963 | + &sys_props.attr_genid); | |
| 964 | + if (ret < 0) | |
| 965 | + return ret; | |
| 966 | + | |
| 967 | + sys_props.attr_props.name = "system_properties"; | |
| 968 | + sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE; | |
| 969 | + sysfs_attr_init(&sys_props.attr_props); | |
| 970 | + ret = sysfs_create_file(sys_props.kobj_topology, | |
| 971 | + &sys_props.attr_props); | |
| 972 | + if (ret < 0) | |
| 973 | + return ret; | |
| 974 | + } | |
| 975 | + | |
| 976 | + kfd_remove_sysfs_node_tree(); | |
| 977 | + | |
| 978 | + return kfd_build_sysfs_node_tree(); | |
| 979 | +} | |
| 980 | + | |
| 981 | +static void kfd_topology_release_sysfs(void) | |
| 982 | +{ | |
| 983 | + kfd_remove_sysfs_node_tree(); | |
| 984 | + if (sys_props.kobj_topology) { | |
| 985 | + sysfs_remove_file(sys_props.kobj_topology, | |
| 986 | + &sys_props.attr_genid); | |
| 987 | + sysfs_remove_file(sys_props.kobj_topology, | |
| 988 | + &sys_props.attr_props); | |
| 989 | + if (sys_props.kobj_nodes) { | |
| 990 | + kobject_del(sys_props.kobj_nodes); | |
| 991 | + kobject_put(sys_props.kobj_nodes); | |
| 992 | + sys_props.kobj_nodes = 0; | |
| 993 | + } | |
| 994 | + kobject_del(sys_props.kobj_topology); | |
| 995 | + kobject_put(sys_props.kobj_topology); | |
| 996 | + sys_props.kobj_topology = 0; | |
| 997 | + } | |
| 998 | +} | |
| 999 | + | |
| 1000 | +int kfd_topology_init(void) | |
| 1001 | +{ | |
| 1002 | + void *crat_image = 0; | |
| 1003 | + size_t image_size = 0; | |
| 1004 | + int ret; | |
| 1005 | + | |
| 1006 | + /* | |
| 1007 | + * Initialize the head for the topology device list | |
| 1008 | + */ | |
| 1009 | + INIT_LIST_HEAD(&topology_device_list); | |
| 1010 | + init_rwsem(&topology_lock); | |
| 1011 | + topology_crat_parsed = 0; | |
| 1012 | + | |
| 1013 | + memset(&sys_props, 0, sizeof(sys_props)); | |
| 1014 | + | |
| 1015 | + /* | |
| 1016 | + * Get the CRAT image from the ACPI | |
| 1017 | + */ | |
| 1018 | + ret = kfd_topology_get_crat_acpi(crat_image, &image_size); | |
| 1019 | + if (ret == 0 && image_size > 0) { | |
| 1020 | + pr_info("Found CRAT image with size=%zd\n", image_size); | |
| 1021 | + crat_image = kmalloc(image_size, GFP_KERNEL); | |
| 1022 | + if (!crat_image) { | |
| 1023 | + ret = -ENOMEM; | |
| 1024 | + pr_err("No memory for allocating CRAT image\n"); | |
| 1025 | + goto err; | |
| 1026 | + } | |
| 1027 | + ret = kfd_topology_get_crat_acpi(crat_image, &image_size); | |
| 1028 | + | |
| 1029 | + if (ret == 0) { | |
| 1030 | + down_write(&topology_lock); | |
| 1031 | + ret = kfd_parse_crat_table(crat_image); | |
| 1032 | + if (ret == 0) | |
| 1033 | + ret = kfd_topology_update_sysfs(); | |
| 1034 | + up_write(&topology_lock); | |
| 1035 | + } else { | |
| 1036 | + pr_err("Couldn't get CRAT table size from ACPI\n"); | |
| 1037 | + } | |
| 1038 | + kfree(crat_image); | |
| 1039 | + } else if (ret == -ENODATA) { | |
| 1040 | + ret = 0; | |
| 1041 | + } else { | |
| 1042 | + pr_err("Couldn't get CRAT table size from ACPI\n"); | |
| 1043 | + } | |
| 1044 | + | |
| 1045 | +err: | |
| 1046 | + pr_info("Finished initializing topology ret=%d\n", ret); | |
| 1047 | + return ret; | |
| 1048 | +} | |
| 1049 | + | |
| 1050 | +void kfd_topology_shutdown(void) | |
| 1051 | +{ | |
| 1052 | + kfd_topology_release_sysfs(); | |
| 1053 | + kfd_release_live_view(); | |
| 1054 | +} | |
| 1055 | + | |
| 1056 | +static void kfd_debug_print_topology(void) | |
| 1057 | +{ | |
| 1058 | + struct kfd_topology_device *dev; | |
| 1059 | + uint32_t i = 0; | |
| 1060 | + | |
| 1061 | + pr_info("DEBUG PRINT OF TOPOLOGY:"); | |
| 1062 | + list_for_each_entry(dev, &topology_device_list, list) { | |
| 1063 | + pr_info("Node: %d\n", i); | |
| 1064 | + pr_info("\tGPU assigned: %s\n", (dev->gpu ? "yes" : "no")); | |
| 1065 | + pr_info("\tCPU count: %d\n", dev->node_props.cpu_cores_count); | |
| 1066 | + pr_info("\tSIMD count: %d", dev->node_props.simd_count); | |
| 1067 | + i++; | |
| 1068 | + } | |
| 1069 | +} | |
| 1070 | + | |
| 1071 | +static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu) | |
| 1072 | +{ | |
| 1073 | + uint32_t hashout; | |
| 1074 | + uint32_t buf[7]; | |
| 1075 | + int i; | |
| 1076 | + | |
| 1077 | + if (!gpu) | |
| 1078 | + return 0; | |
| 1079 | + | |
| 1080 | + buf[0] = gpu->pdev->devfn; | |
| 1081 | + buf[1] = gpu->pdev->subsystem_vendor; | |
| 1082 | + buf[2] = gpu->pdev->subsystem_device; | |
| 1083 | + buf[3] = gpu->pdev->device; | |
| 1084 | + buf[4] = gpu->pdev->bus->number; | |
| 1085 | + buf[5] = (uint32_t)(kfd2kgd->get_vmem_size(gpu->kgd) & 0xffffffff); | |
| 1086 | + buf[6] = (uint32_t)(kfd2kgd->get_vmem_size(gpu->kgd) >> 32); | |
| 1087 | + | |
| 1088 | + for (i = 0, hashout = 0; i < 7; i++) | |
| 1089 | + hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH); | |
| 1090 | + | |
| 1091 | + return hashout; | |
| 1092 | +} | |
| 1093 | + | |
| 1094 | +static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu) | |
| 1095 | +{ | |
| 1096 | + struct kfd_topology_device *dev; | |
| 1097 | + struct kfd_topology_device *out_dev = 0; | |
| 1098 | + | |
| 1099 | + BUG_ON(!gpu); | |
| 1100 | + | |
| 1101 | + list_for_each_entry(dev, &topology_device_list, list) | |
| 1102 | + if (dev->gpu == 0 && dev->node_props.simd_count > 0) { | |
| 1103 | + dev->gpu = gpu; | |
| 1104 | + out_dev = dev; | |
| 1105 | + break; | |
| 1106 | + } | |
| 1107 | + | |
| 1108 | + return out_dev; | |
| 1109 | +} | |
| 1110 | + | |
| 1111 | +static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival) | |
| 1112 | +{ | |
| 1113 | + /* | |
| 1114 | + * TODO: Generate an event for thunk about the arrival/removal | |
| 1115 | + * of the GPU | |
| 1116 | + */ | |
| 1117 | +} | |
| 1118 | + | |
| 1119 | +int kfd_topology_add_device(struct kfd_dev *gpu) | |
| 1120 | +{ | |
| 1121 | + uint32_t gpu_id; | |
| 1122 | + struct kfd_topology_device *dev; | |
| 1123 | + int res; | |
| 1124 | + | |
| 1125 | + BUG_ON(!gpu); | |
| 1126 | + | |
| 1127 | + gpu_id = kfd_generate_gpu_id(gpu); | |
| 1128 | + | |
| 1129 | + pr_debug("kfd: Adding new GPU (ID: 0x%x) to topology\n", gpu_id); | |
| 1130 | + | |
| 1131 | + down_write(&topology_lock); | |
| 1132 | + /* | |
| 1133 | + * Try to assign the GPU to existing topology device (generated from | |
| 1134 | + * CRAT table | |
| 1135 | + */ | |
| 1136 | + dev = kfd_assign_gpu(gpu); | |
| 1137 | + if (!dev) { | |
| 1138 | + pr_info("GPU was not found in the current topology. Extending.\n"); | |
| 1139 | + kfd_debug_print_topology(); | |
| 1140 | + dev = kfd_create_topology_device(); | |
| 1141 | + if (!dev) { | |
| 1142 | + res = -ENOMEM; | |
| 1143 | + goto err; | |
| 1144 | + } | |
| 1145 | + dev->gpu = gpu; | |
| 1146 | + | |
| 1147 | + /* | |
| 1148 | + * TODO: Make a call to retrieve topology information from the | |
| 1149 | + * GPU vBIOS | |
| 1150 | + */ | |
| 1151 | + | |
| 1152 | + /* | |
| 1153 | + * Update the SYSFS tree, since we added another topology device | |
| 1154 | + */ | |
| 1155 | + if (kfd_topology_update_sysfs() < 0) | |
| 1156 | + kfd_topology_release_sysfs(); | |
| 1157 | + | |
| 1158 | + } | |
| 1159 | + | |
| 1160 | + dev->gpu_id = gpu_id; | |
| 1161 | + gpu->id = gpu_id; | |
| 1162 | + dev->node_props.vendor_id = gpu->pdev->vendor; | |
| 1163 | + dev->node_props.device_id = gpu->pdev->device; | |
| 1164 | + dev->node_props.location_id = (gpu->pdev->bus->number << 24) + | |
| 1165 | + (gpu->pdev->devfn & 0xffffff); | |
| 1166 | + /* | |
| 1167 | + * TODO: Retrieve max engine clock values from KGD | |
| 1168 | + */ | |
| 1169 | + | |
| 1170 | + res = 0; | |
| 1171 | + | |
| 1172 | +err: | |
| 1173 | + up_write(&topology_lock); | |
| 1174 | + | |
| 1175 | + if (res == 0) | |
| 1176 | + kfd_notify_gpu_change(gpu_id, 1); | |
| 1177 | + | |
| 1178 | + return res; | |
| 1179 | +} | |
| 1180 | + | |
| 1181 | +int kfd_topology_remove_device(struct kfd_dev *gpu) | |
| 1182 | +{ | |
| 1183 | + struct kfd_topology_device *dev; | |
| 1184 | + uint32_t gpu_id; | |
| 1185 | + int res = -ENODEV; | |
| 1186 | + | |
| 1187 | + BUG_ON(!gpu); | |
| 1188 | + | |
| 1189 | + down_write(&topology_lock); | |
| 1190 | + | |
| 1191 | + list_for_each_entry(dev, &topology_device_list, list) | |
| 1192 | + if (dev->gpu == gpu) { | |
| 1193 | + gpu_id = dev->gpu_id; | |
| 1194 | + kfd_remove_sysfs_node_entry(dev); | |
| 1195 | + kfd_release_topology_device(dev); | |
| 1196 | + res = 0; | |
| 1197 | + if (kfd_topology_update_sysfs() < 0) | |
| 1198 | + kfd_topology_release_sysfs(); | |
| 1199 | + break; | |
| 1200 | + } | |
| 1201 | + | |
| 1202 | + up_write(&topology_lock); | |
| 1203 | + | |
| 1204 | + if (res == 0) | |
| 1205 | + kfd_notify_gpu_change(gpu_id, 0); | |
| 1206 | + | |
| 1207 | + return res; | |
| 1208 | +} | |
| 1209 | + | |
| 1210 | +/* | |
| 1211 | + * When idx is out of bounds, the function will return NULL | |
| 1212 | + */ | |
| 1213 | +struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx) | |
| 1214 | +{ | |
| 1215 | + | |
| 1216 | + struct kfd_topology_device *top_dev; | |
| 1217 | + struct kfd_dev *device = NULL; | |
| 1218 | + uint8_t device_idx = 0; | |
| 1219 | + | |
| 1220 | + down_read(&topology_lock); | |
| 1221 | + | |
| 1222 | + list_for_each_entry(top_dev, &topology_device_list, list) { | |
| 1223 | + if (device_idx == idx) { | |
| 1224 | + device = top_dev->gpu; | |
| 1225 | + break; | |
| 1226 | + } | |
| 1227 | + | |
| 1228 | + device_idx++; | |
| 1229 | + } | |
| 1230 | + | |
| 1231 | + up_read(&topology_lock); | |
| 1232 | + | |
| 1233 | + return device; | |
| 1234 | + | |
| 1235 | +} | |
| @@ -0,0 +1,168 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +#ifndef __KFD_TOPOLOGY_H__ | |
| 24 | +#define __KFD_TOPOLOGY_H__ | |
| 25 | + | |
| 26 | +#include <linux/types.h> | |
| 27 | +#include <linux/list.h> | |
| 28 | +#include "kfd_priv.h" | |
| 29 | + | |
| 30 | +#define KFD_TOPOLOGY_PUBLIC_NAME_SIZE 128 | |
| 31 | + | |
| 32 | +#define HSA_CAP_HOT_PLUGGABLE 0x00000001 | |
| 33 | +#define HSA_CAP_ATS_PRESENT 0x00000002 | |
| 34 | +#define HSA_CAP_SHARED_WITH_GRAPHICS 0x00000004 | |
| 35 | +#define HSA_CAP_QUEUE_SIZE_POW2 0x00000008 | |
| 36 | +#define HSA_CAP_QUEUE_SIZE_32BIT 0x00000010 | |
| 37 | +#define HSA_CAP_QUEUE_IDLE_EVENT 0x00000020 | |
| 38 | +#define HSA_CAP_VA_LIMIT 0x00000040 | |
| 39 | +#define HSA_CAP_WATCH_POINTS_SUPPORTED 0x00000080 | |
| 40 | +#define HSA_CAP_WATCH_POINTS_TOTALBITS_MASK 0x00000f00 | |
| 41 | +#define HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT 8 | |
| 42 | +#define HSA_CAP_RESERVED 0xfffff000 | |
| 43 | + | |
| 44 | +struct kfd_node_properties { | |
| 45 | + uint32_t cpu_cores_count; | |
| 46 | + uint32_t simd_count; | |
| 47 | + uint32_t mem_banks_count; | |
| 48 | + uint32_t caches_count; | |
| 49 | + uint32_t io_links_count; | |
| 50 | + uint32_t cpu_core_id_base; | |
| 51 | + uint32_t simd_id_base; | |
| 52 | + uint32_t capability; | |
| 53 | + uint32_t max_waves_per_simd; | |
| 54 | + uint32_t lds_size_in_kb; | |
| 55 | + uint32_t gds_size_in_kb; | |
| 56 | + uint32_t wave_front_size; | |
| 57 | + uint32_t array_count; | |
| 58 | + uint32_t simd_arrays_per_engine; | |
| 59 | + uint32_t cu_per_simd_array; | |
| 60 | + uint32_t simd_per_cu; | |
| 61 | + uint32_t max_slots_scratch_cu; | |
| 62 | + uint32_t engine_id; | |
| 63 | + uint32_t vendor_id; | |
| 64 | + uint32_t device_id; | |
| 65 | + uint32_t location_id; | |
| 66 | + uint32_t max_engine_clk_fcompute; | |
| 67 | + uint32_t max_engine_clk_ccompute; | |
| 68 | + uint16_t marketing_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE]; | |
| 69 | +}; | |
| 70 | + | |
| 71 | +#define HSA_MEM_HEAP_TYPE_SYSTEM 0 | |
| 72 | +#define HSA_MEM_HEAP_TYPE_FB_PUBLIC 1 | |
| 73 | +#define HSA_MEM_HEAP_TYPE_FB_PRIVATE 2 | |
| 74 | +#define HSA_MEM_HEAP_TYPE_GPU_GDS 3 | |
| 75 | +#define HSA_MEM_HEAP_TYPE_GPU_LDS 4 | |
| 76 | +#define HSA_MEM_HEAP_TYPE_GPU_SCRATCH 5 | |
| 77 | + | |
| 78 | +#define HSA_MEM_FLAGS_HOT_PLUGGABLE 0x00000001 | |
| 79 | +#define HSA_MEM_FLAGS_NON_VOLATILE 0x00000002 | |
| 80 | +#define HSA_MEM_FLAGS_RESERVED 0xfffffffc | |
| 81 | + | |
| 82 | +struct kfd_mem_properties { | |
| 83 | + struct list_head list; | |
| 84 | + uint32_t heap_type; | |
| 85 | + uint64_t size_in_bytes; | |
| 86 | + uint32_t flags; | |
| 87 | + uint32_t width; | |
| 88 | + uint32_t mem_clk_max; | |
| 89 | + struct kobject *kobj; | |
| 90 | + struct attribute attr; | |
| 91 | +}; | |
| 92 | + | |
| 93 | +#define KFD_TOPOLOGY_CPU_SIBLINGS 256 | |
| 94 | + | |
| 95 | +#define HSA_CACHE_TYPE_DATA 0x00000001 | |
| 96 | +#define HSA_CACHE_TYPE_INSTRUCTION 0x00000002 | |
| 97 | +#define HSA_CACHE_TYPE_CPU 0x00000004 | |
| 98 | +#define HSA_CACHE_TYPE_HSACU 0x00000008 | |
| 99 | +#define HSA_CACHE_TYPE_RESERVED 0xfffffff0 | |
| 100 | + | |
| 101 | +struct kfd_cache_properties { | |
| 102 | + struct list_head list; | |
| 103 | + uint32_t processor_id_low; | |
| 104 | + uint32_t cache_level; | |
| 105 | + uint32_t cache_size; | |
| 106 | + uint32_t cacheline_size; | |
| 107 | + uint32_t cachelines_per_tag; | |
| 108 | + uint32_t cache_assoc; | |
| 109 | + uint32_t cache_latency; | |
| 110 | + uint32_t cache_type; | |
| 111 | + uint8_t sibling_map[KFD_TOPOLOGY_CPU_SIBLINGS]; | |
| 112 | + struct kobject *kobj; | |
| 113 | + struct attribute attr; | |
| 114 | +}; | |
| 115 | + | |
| 116 | +struct kfd_iolink_properties { | |
| 117 | + struct list_head list; | |
| 118 | + uint32_t iolink_type; | |
| 119 | + uint32_t ver_maj; | |
| 120 | + uint32_t ver_min; | |
| 121 | + uint32_t node_from; | |
| 122 | + uint32_t node_to; | |
| 123 | + uint32_t weight; | |
| 124 | + uint32_t min_latency; | |
| 125 | + uint32_t max_latency; | |
| 126 | + uint32_t min_bandwidth; | |
| 127 | + uint32_t max_bandwidth; | |
| 128 | + uint32_t rec_transfer_size; | |
| 129 | + uint32_t flags; | |
| 130 | + struct kobject *kobj; | |
| 131 | + struct attribute attr; | |
| 132 | +}; | |
| 133 | + | |
| 134 | +struct kfd_topology_device { | |
| 135 | + struct list_head list; | |
| 136 | + uint32_t gpu_id; | |
| 137 | + struct kfd_node_properties node_props; | |
| 138 | + uint32_t mem_bank_count; | |
| 139 | + struct list_head mem_props; | |
| 140 | + uint32_t cache_count; | |
| 141 | + struct list_head cache_props; | |
| 142 | + uint32_t io_link_count; | |
| 143 | + struct list_head io_link_props; | |
| 144 | + struct kfd_dev *gpu; | |
| 145 | + struct kobject *kobj_node; | |
| 146 | + struct kobject *kobj_mem; | |
| 147 | + struct kobject *kobj_cache; | |
| 148 | + struct kobject *kobj_iolink; | |
| 149 | + struct attribute attr_gpuid; | |
| 150 | + struct attribute attr_name; | |
| 151 | + struct attribute attr_props; | |
| 152 | +}; | |
| 153 | + | |
| 154 | +struct kfd_system_properties { | |
| 155 | + uint32_t num_devices; /* Number of H-NUMA nodes */ | |
| 156 | + uint32_t generation_count; | |
| 157 | + uint64_t platform_oem; | |
| 158 | + uint64_t platform_id; | |
| 159 | + uint64_t platform_rev; | |
| 160 | + struct kobject *kobj_topology; | |
| 161 | + struct kobject *kobj_nodes; | |
| 162 | + struct attribute attr_genid; | |
| 163 | + struct attribute attr_props; | |
| 164 | +}; | |
| 165 | + | |
| 166 | + | |
| 167 | + | |
| 168 | +#endif /* __KFD_TOPOLOGY_H__ */ | |
| @@ -0,0 +1,185 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +/* | |
| 24 | + * This file defines the private interface between the | |
| 25 | + * AMD kernel graphics drivers and the AMD KFD. | |
| 26 | + */ | |
| 27 | + | |
| 28 | +#ifndef KGD_KFD_INTERFACE_H_INCLUDED | |
| 29 | +#define KGD_KFD_INTERFACE_H_INCLUDED | |
| 30 | + | |
| 31 | +#include <linux/types.h> | |
| 32 | + | |
| 33 | +struct pci_dev; | |
| 34 | + | |
| 35 | +#define KFD_INTERFACE_VERSION 1 | |
| 36 | + | |
| 37 | +struct kfd_dev; | |
| 38 | +struct kgd_dev; | |
| 39 | + | |
| 40 | +struct kgd_mem; | |
| 41 | + | |
| 42 | +enum kgd_memory_pool { | |
| 43 | + KGD_POOL_SYSTEM_CACHEABLE = 1, | |
| 44 | + KGD_POOL_SYSTEM_WRITECOMBINE = 2, | |
| 45 | + KGD_POOL_FRAMEBUFFER = 3, | |
| 46 | +}; | |
| 47 | + | |
| 48 | +struct kgd2kfd_shared_resources { | |
| 49 | + /* Bit n == 1 means VMID n is available for KFD. */ | |
| 50 | + unsigned int compute_vmid_bitmap; | |
| 51 | + | |
| 52 | + /* Compute pipes are counted starting from MEC0/pipe0 as 0. */ | |
| 53 | + unsigned int first_compute_pipe; | |
| 54 | + | |
| 55 | + /* Number of MEC pipes available for KFD. */ | |
| 56 | + unsigned int compute_pipe_count; | |
| 57 | + | |
| 58 | + /* Base address of doorbell aperture. */ | |
| 59 | + phys_addr_t doorbell_physical_address; | |
| 60 | + | |
| 61 | + /* Size in bytes of doorbell aperture. */ | |
| 62 | + size_t doorbell_aperture_size; | |
| 63 | + | |
| 64 | + /* Number of bytes at start of aperture reserved for KGD. */ | |
| 65 | + size_t doorbell_start_offset; | |
| 66 | +}; | |
| 67 | + | |
| 68 | +/** | |
| 69 | + * struct kgd2kfd_calls | |
| 70 | + * | |
| 71 | + * @exit: Notifies amdkfd that kgd module is unloaded | |
| 72 | + * | |
| 73 | + * @probe: Notifies amdkfd about a probe done on a device in the kgd driver. | |
| 74 | + * | |
| 75 | + * @device_init: Initialize the newly probed device (if it is a device that | |
| 76 | + * amdkfd supports) | |
| 77 | + * | |
| 78 | + * @device_exit: Notifies amdkfd about a removal of a kgd device | |
| 79 | + * | |
| 80 | + * @suspend: Notifies amdkfd about a suspend action done to a kgd device | |
| 81 | + * | |
| 82 | + * @resume: Notifies amdkfd about a resume action done to a kgd device | |
| 83 | + * | |
| 84 | + * This structure contains function callback pointers so the kgd driver | |
| 85 | + * will notify to the amdkfd about certain status changes. | |
| 86 | + * | |
| 87 | + */ | |
| 88 | +struct kgd2kfd_calls { | |
| 89 | + void (*exit)(void); | |
| 90 | + struct kfd_dev* (*probe)(struct kgd_dev *kgd, struct pci_dev *pdev); | |
| 91 | + bool (*device_init)(struct kfd_dev *kfd, | |
| 92 | + const struct kgd2kfd_shared_resources *gpu_resources); | |
| 93 | + void (*device_exit)(struct kfd_dev *kfd); | |
| 94 | + void (*interrupt)(struct kfd_dev *kfd, const void *ih_ring_entry); | |
| 95 | + void (*suspend)(struct kfd_dev *kfd); | |
| 96 | + int (*resume)(struct kfd_dev *kfd); | |
| 97 | +}; | |
| 98 | + | |
| 99 | +/** | |
| 100 | + * struct kfd2kgd_calls | |
| 101 | + * | |
| 102 | + * @init_sa_manager: Initialize an instance of the sa manager, used by | |
| 103 | + * amdkfd for all system memory allocations that are mapped to the GART | |
| 104 | + * address space | |
| 105 | + * | |
| 106 | + * @fini_sa_manager: Releases all memory allocations for amdkfd that are | |
| 107 | + * handled by kgd sa manager | |
| 108 | + * | |
| 109 | + * @allocate_mem: Allocate a buffer from amdkfd's sa manager. The buffer can | |
| 110 | + * be used for mqds, hpds, kernel queue, fence and runlists | |
| 111 | + * | |
| 112 | + * @free_mem: Frees a buffer that was allocated by amdkfd's sa manager | |
| 113 | + * | |
| 114 | + * @get_vmem_size: Retrieves (physical) size of VRAM | |
| 115 | + * | |
| 116 | + * @get_gpu_clock_counter: Retrieves GPU clock counter | |
| 117 | + * | |
| 118 | + * @get_max_engine_clock_in_mhz: Retrieves maximum GPU clock in MHz | |
| 119 | + * | |
| 120 | + * @program_sh_mem_settings: A function that should initiate the memory | |
| 121 | + * properties such as main aperture memory type (cache / non cached) and | |
| 122 | + * secondary aperture base address, size and memory type. | |
| 123 | + * This function is used only for no cp scheduling mode. | |
| 124 | + * | |
| 125 | + * @set_pasid_vmid_mapping: Exposes pasid/vmid pair to the H/W for no cp | |
| 126 | + * scheduling mode. Only used for no cp scheduling mode. | |
| 127 | + * | |
| 128 | + * @init_memory: Initializes memory apertures to fixed base/limit address | |
| 129 | + * and non cached memory types. | |
| 130 | + * | |
| 131 | + * @init_pipeline: Initialized the compute pipelines. | |
| 132 | + * | |
| 133 | + * @hqd_load: Loads the mqd structure to a H/W hqd slot. used only for no cp | |
| 134 | + * sceduling mode. | |
| 135 | + * | |
| 136 | + * @hqd_is_occupies: Checks if a hqd slot is occupied. | |
| 137 | + * | |
| 138 | + * @hqd_destroy: Destructs and preempts the queue assigned to that hqd slot. | |
| 139 | + * | |
| 140 | + * This structure contains function pointers to services that the kgd driver | |
| 141 | + * provides to amdkfd driver. | |
| 142 | + * | |
| 143 | + */ | |
| 144 | +struct kfd2kgd_calls { | |
| 145 | + /* Memory management. */ | |
| 146 | + int (*init_sa_manager)(struct kgd_dev *kgd, unsigned int size); | |
| 147 | + void (*fini_sa_manager)(struct kgd_dev *kgd); | |
| 148 | + int (*allocate_mem)(struct kgd_dev *kgd, size_t size, size_t alignment, | |
| 149 | + enum kgd_memory_pool pool, struct kgd_mem **mem); | |
| 150 | + | |
| 151 | + void (*free_mem)(struct kgd_dev *kgd, struct kgd_mem *mem); | |
| 152 | + | |
| 153 | + uint64_t (*get_vmem_size)(struct kgd_dev *kgd); | |
| 154 | + uint64_t (*get_gpu_clock_counter)(struct kgd_dev *kgd); | |
| 155 | + | |
| 156 | + uint32_t (*get_max_engine_clock_in_mhz)(struct kgd_dev *kgd); | |
| 157 | + | |
| 158 | + /* Register access functions */ | |
| 159 | + void (*program_sh_mem_settings)(struct kgd_dev *kgd, uint32_t vmid, | |
| 160 | + uint32_t sh_mem_config, uint32_t sh_mem_ape1_base, | |
| 161 | + uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases); | |
| 162 | + | |
| 163 | + int (*set_pasid_vmid_mapping)(struct kgd_dev *kgd, unsigned int pasid, | |
| 164 | + unsigned int vmid); | |
| 165 | + | |
| 166 | + int (*init_memory)(struct kgd_dev *kgd); | |
| 167 | + int (*init_pipeline)(struct kgd_dev *kgd, uint32_t pipe_id, | |
| 168 | + uint32_t hpd_size, uint64_t hpd_gpu_addr); | |
| 169 | + | |
| 170 | + int (*hqd_load)(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, | |
| 171 | + uint32_t queue_id, uint32_t __user *wptr); | |
| 172 | + | |
| 173 | + bool (*hqd_is_occupies)(struct kgd_dev *kgd, uint64_t queue_address, | |
| 174 | + uint32_t pipe_id, uint32_t queue_id); | |
| 175 | + | |
| 176 | + int (*hqd_destroy)(struct kgd_dev *kgd, uint32_t reset_type, | |
| 177 | + unsigned int timeout, uint32_t pipe_id, | |
| 178 | + uint32_t queue_id); | |
| 179 | +}; | |
| 180 | + | |
| 181 | +bool kgd2kfd_init(unsigned interface_version, | |
| 182 | + const struct kfd2kgd_calls *f2g, | |
| 183 | + const struct kgd2kfd_calls **g2f); | |
| 184 | + | |
| 185 | +#endif /* KGD_KFD_INTERFACE_H_INCLUDED */ | |
| @@ -104,6 +104,7 @@ radeon-y += \ · 103 unmodified lines | ||
| 104 | 104 | radeon_vce.o \ |
| 105 | 105 | vce_v1_0.o \ |
| 106 | 106 | vce_v2_0.o \ |
| 107 | + radeon_kfd.o | |
| 107 | 108 | |
| 108 | 109 | radeon-$(CONFIG_COMPAT) += radeon_ioc32.o |
| 109 | 110 | radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o |
| @@ -32,6 +32,7 @@ · 31 unmodified lines | ||
| 32 | 32 | #include "cik_blit_shaders.h" |
| 33 | 33 | #include "radeon_ucode.h" |
| 34 | 34 | #include "clearstate_ci.h" |
| 35 | +#include "radeon_kfd.h" | |
| 35 | 36 | |
| 36 | 37 | MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin"); |
| 37 | 38 | MODULE_FIRMWARE("radeon/BONAIRE_me.bin"); |
| @@ -1563,6 +1564,8 @@ static const u32 godavari_golden_registers[] = · 1525 unmodified lines | ||
| 1563 | 1564 | |
| 1564 | 1565 | static void cik_init_golden_registers(struct radeon_device *rdev) |
| 1565 | 1566 | { |
| 1567 | + /* Some of the registers might be dependent on GRBM_GFX_INDEX */ | |
| 1568 | + mutex_lock(&rdev->grbm_idx_mutex); | |
| 1566 | 1569 | switch (rdev->family) { |
| 1567 | 1570 | case CHIP_BONAIRE: |
| 1568 | 1571 | radeon_program_register_sequence(rdev, |
| @@ -1637,6 +1640,7 @@ static void cik_init_golden_registers(struct radeon_device *rdev) · 68 unmodified lines | ||
| 1637 | 1640 | default: |
| 1638 | 1641 | break; |
| 1639 | 1642 | } |
| 1643 | + mutex_unlock(&rdev->grbm_idx_mutex); | |
| 1640 | 1644 | } |
| 1641 | 1645 | |
| 1642 | 1646 | /** |
| @@ -3428,6 +3432,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 1785 unmodified lines | ||
| 3428 | 3432 | u32 disabled_rbs = 0; |
| 3429 | 3433 | u32 enabled_rbs = 0; |
| 3430 | 3434 | |
| 3435 | + mutex_lock(&rdev->grbm_idx_mutex); | |
| 3431 | 3436 | for (i = 0; i < se_num; i++) { |
| 3432 | 3437 | for (j = 0; j < sh_per_se; j++) { |
| 3433 | 3438 | cik_select_se_sh(rdev, i, j); |
| @@ -3439,6 +3444,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 5 unmodified lines | ||
| 3439 | 3444 | } |
| 3440 | 3445 | } |
| 3441 | 3446 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 3447 | + mutex_unlock(&rdev->grbm_idx_mutex); | |
| 3442 | 3448 | |
| 3443 | 3449 | mask = 1; |
| 3444 | 3450 | for (i = 0; i < max_rb_num_per_se * se_num; i++) { |
| @@ -3449,6 +3455,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 4 unmodified lines | ||
| 3449 | 3455 | |
| 3450 | 3456 | rdev->config.cik.backend_enable_mask = enabled_rbs; |
| 3451 | 3457 | |
| 3458 | + mutex_lock(&rdev->grbm_idx_mutex); | |
| 3452 | 3459 | for (i = 0; i < se_num; i++) { |
| 3453 | 3460 | cik_select_se_sh(rdev, i, 0xffffffff); |
| 3454 | 3461 | data = 0; |
| @@ -3476,6 +3483,7 @@ static void cik_setup_rb(struct radeon_device *rdev, · 21 unmodified lines | ||
| 3476 | 3483 | WREG32(PA_SC_RASTER_CONFIG, data); |
| 3477 | 3484 | } |
| 3478 | 3485 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 3486 | + mutex_unlock(&rdev->grbm_idx_mutex); | |
| 3479 | 3487 | } |
| 3480 | 3488 | |
| 3481 | 3489 | /** |
| @@ -3693,6 +3701,12 @@ static void cik_gpu_init(struct radeon_device *rdev) · 211 unmodified lines | ||
| 3693 | 3701 | /* set HW defaults for 3D engine */ |
| 3694 | 3702 | WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); |
| 3695 | 3703 | |
| 3704 | + mutex_lock(&rdev->grbm_idx_mutex); | |
| 3705 | + /* | |
| 3706 | + * making sure that the following register writes will be broadcasted | |
| 3707 | + * to all the shaders | |
| 3708 | + */ | |
| 3709 | + cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); | |
| 3696 | 3710 | WREG32(SX_DEBUG_1, 0x20); |
| 3697 | 3711 | |
| 3698 | 3712 | WREG32(TA_CNTL_AUX, 0x00010000); |
| @@ -3748,6 +3762,7 @@ static void cik_gpu_init(struct radeon_device *rdev) · 49 unmodified lines | ||
| 3748 | 3762 | |
| 3749 | 3763 | WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); |
| 3750 | 3764 | WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER); |
| 3765 | + mutex_unlock(&rdev->grbm_idx_mutex); | |
| 3751 | 3766 | |
| 3752 | 3767 | udelay(50); |
| 3753 | 3768 | } |
| @@ -4684,12 +4699,11 @@ static int cik_mec_init(struct radeon_device *rdev) · 930 unmodified lines | ||
| 4684 | 4699 | /* |
| 4685 | 4700 | * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total |
| 4686 | 4701 | * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total |
| 4702 | + * Nonetheless, we assign only 1 pipe because all other pipes will | |
| 4703 | + * be handled by KFD | |
| 4687 | 4704 | */ |
| 4688 | − if (rdev->family == CHIP_KAVERI) | |
| 4689 | − rdev->mec.num_mec = 2; | |
| 4690 | − else | |
| 4691 | − rdev->mec.num_mec = 1; | |
| 4692 | − rdev->mec.num_pipe = 4; | |
| 4705 | + rdev->mec.num_mec = 1; | |
| 4706 | + rdev->mec.num_pipe = 1; | |
| 4693 | 4707 | rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8; |
| 4694 | 4708 | |
| 4695 | 4709 | if (rdev->mec.hpd_eop_obj == NULL) { |
| @@ -4831,28 +4845,24 @@ static int cik_cp_compute_resume(struct radeon_device *rdev) · 135 unmodified lines | ||
| 4831 | 4845 | |
| 4832 | 4846 | /* init the pipes */ |
| 4833 | 4847 | mutex_lock(&rdev->srbm_mutex); |
| 4834 | − for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) { | |
| 4835 | − int me = (i < 4) ? 1 : 2; | |
| 4836 | − int pipe = (i < 4) ? i : (i - 4); | |
| 4837 | 4848 | |
| 4838 | − eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2); | |
| 4849 | + eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr; | |
| 4839 | 4850 | |
| 4840 | − cik_srbm_select(rdev, me, pipe, 0, 0); | |
| 4851 | + cik_srbm_select(rdev, 0, 0, 0, 0); | |
| 4841 | 4852 | |
| 4842 | − /* write the EOP addr */ | |
| 4843 | − WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8); | |
| 4844 | − WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8); | |
| 4853 | + /* write the EOP addr */ | |
| 4854 | + WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8); | |
| 4855 | + WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8); | |
| 4845 | 4856 | |
| 4846 | − /* set the VMID assigned */ | |
| 4847 | − WREG32(CP_HPD_EOP_VMID, 0); | |
| 4857 | + /* set the VMID assigned */ | |
| 4858 | + WREG32(CP_HPD_EOP_VMID, 0); | |
| 4859 | + | |
| 4860 | + /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ | |
| 4861 | + tmp = RREG32(CP_HPD_EOP_CONTROL); | |
| 4862 | + tmp &= ~EOP_SIZE_MASK; | |
| 4863 | + tmp |= order_base_2(MEC_HPD_SIZE / 8); | |
| 4864 | + WREG32(CP_HPD_EOP_CONTROL, tmp); | |
| 4848 | 4865 | |
| 4849 | − /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ | |
| 4850 | − tmp = RREG32(CP_HPD_EOP_CONTROL); | |
| 4851 | − tmp &= ~EOP_SIZE_MASK; | |
| 4852 | − tmp |= order_base_2(MEC_HPD_SIZE / 8); | |
| 4853 | − WREG32(CP_HPD_EOP_CONTROL, tmp); | |
| 4854 | − } | |
| 4855 | − cik_srbm_select(rdev, 0, 0, 0, 0); | |
| 4856 | 4866 | mutex_unlock(&rdev->srbm_mutex); |
| 4857 | 4867 | |
| 4858 | 4868 | /* init the queues. Just two for now. */ |
| @@ -5906,8 +5916,13 @@ int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) · 1047 unmodified lines | ||
| 5906 | 5916 | */ |
| 5907 | 5917 | int cik_vm_init(struct radeon_device *rdev) |
| 5908 | 5918 | { |
| 5909 | − /* number of VMs */ | |
| 5910 | − rdev->vm_manager.nvm = 16; | |
| 5919 | + /* | |
| 5920 | + * number of VMs | |
| 5921 | + * VMID 0 is reserved for System | |
| 5922 | + * radeon graphics/compute will use VMIDs 1-7 | |
| 5923 | + * amdkfd will use VMIDs 8-15 | |
| 5924 | + */ | |
| 5925 | + rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS; | |
| 5911 | 5926 | /* base offset of vram pages */ |
| 5912 | 5927 | if (rdev->flags & RADEON_IS_IGP) { |
| 5913 | 5928 | u64 tmp = RREG32(MC_VM_FB_OFFSET); |
| @@ -6068,6 +6083,7 @@ static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) · 154 unmodified lines | ||
| 6068 | 6083 | u32 i, j, k; |
| 6069 | 6084 | u32 mask; |
| 6070 | 6085 | |
| 6086 | + mutex_lock(&rdev->grbm_idx_mutex); | |
| 6071 | 6087 | for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { |
| 6072 | 6088 | for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { |
| 6073 | 6089 | cik_select_se_sh(rdev, i, j); |
| @@ -6079,6 +6095,7 @@ static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) · 5 unmodified lines | ||
| 6079 | 6095 | } |
| 6080 | 6096 | } |
| 6081 | 6097 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 6098 | + mutex_unlock(&rdev->grbm_idx_mutex); | |
| 6082 | 6099 | |
| 6083 | 6100 | mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY; |
| 6084 | 6101 | for (k = 0; k < rdev->usec_timeout; k++) { |
| @@ -6213,10 +6230,12 @@ static int cik_rlc_resume(struct radeon_device *rdev) · 128 unmodified lines | ||
| 6213 | 6230 | WREG32(RLC_LB_CNTR_INIT, 0); |
| 6214 | 6231 | WREG32(RLC_LB_CNTR_MAX, 0x00008000); |
| 6215 | 6232 | |
| 6233 | + mutex_lock(&rdev->grbm_idx_mutex); | |
| 6216 | 6234 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 6217 | 6235 | WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); |
| 6218 | 6236 | WREG32(RLC_LB_PARAMS, 0x00600408); |
| 6219 | 6237 | WREG32(RLC_LB_CNTL, 0x80000004); |
| 6238 | + mutex_unlock(&rdev->grbm_idx_mutex); | |
| 6220 | 6239 | |
| 6221 | 6240 | WREG32(RLC_MC_CNTL, 0); |
| 6222 | 6241 | WREG32(RLC_UCODE_CNTL, 0); |
| @@ -6283,11 +6302,13 @@ static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) · 60 unmodified lines | ||
| 6283 | 6302 | |
| 6284 | 6303 | tmp = cik_halt_rlc(rdev); |
| 6285 | 6304 | |
| 6305 | + mutex_lock(&rdev->grbm_idx_mutex); | |
| 6286 | 6306 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 6287 | 6307 | WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); |
| 6288 | 6308 | WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); |
| 6289 | 6309 | tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE; |
| 6290 | 6310 | WREG32(RLC_SERDES_WR_CTRL, tmp2); |
| 6311 | + mutex_unlock(&rdev->grbm_idx_mutex); | |
| 6291 | 6312 | |
| 6292 | 6313 | cik_update_rlc(rdev, tmp); |
| 6293 | 6314 | |
| @@ -6329,11 +6350,13 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) · 35 unmodified lines | ||
| 6329 | 6350 | |
| 6330 | 6351 | tmp = cik_halt_rlc(rdev); |
| 6331 | 6352 | |
| 6353 | + mutex_lock(&rdev->grbm_idx_mutex); | |
| 6332 | 6354 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 6333 | 6355 | WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); |
| 6334 | 6356 | WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); |
| 6335 | 6357 | data = BPM_ADDR_MASK | MGCG_OVERRIDE_0; |
| 6336 | 6358 | WREG32(RLC_SERDES_WR_CTRL, data); |
| 6359 | + mutex_unlock(&rdev->grbm_idx_mutex); | |
| 6337 | 6360 | |
| 6338 | 6361 | cik_update_rlc(rdev, tmp); |
| 6339 | 6362 | |
| @@ -6377,11 +6400,13 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) · 37 unmodified lines | ||
| 6377 | 6400 | |
| 6378 | 6401 | tmp = cik_halt_rlc(rdev); |
| 6379 | 6402 | |
| 6403 | + mutex_lock(&rdev->grbm_idx_mutex); | |
| 6380 | 6404 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 6381 | 6405 | WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); |
| 6382 | 6406 | WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); |
| 6383 | 6407 | data = BPM_ADDR_MASK | MGCG_OVERRIDE_1; |
| 6384 | 6408 | WREG32(RLC_SERDES_WR_CTRL, data); |
| 6409 | + mutex_unlock(&rdev->grbm_idx_mutex); | |
| 6385 | 6410 | |
| 6386 | 6411 | cik_update_rlc(rdev, tmp); |
| 6387 | 6412 | } |
| @@ -6810,10 +6835,12 @@ static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) · 422 unmodified lines | ||
| 6810 | 6835 | u32 mask = 0, tmp, tmp1; |
| 6811 | 6836 | int i; |
| 6812 | 6837 | |
| 6838 | + mutex_lock(&rdev->grbm_idx_mutex); | |
| 6813 | 6839 | cik_select_se_sh(rdev, se, sh); |
| 6814 | 6840 | tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); |
| 6815 | 6841 | tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); |
| 6816 | 6842 | cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); |
| 6843 | + mutex_unlock(&rdev->grbm_idx_mutex); | |
| 6817 | 6844 | |
| 6818 | 6845 | tmp &= 0xffff0000; |
| 6819 | 6846 | |
| @@ -7297,8 +7324,7 @@ static int cik_irq_init(struct radeon_device *rdev) · 477 unmodified lines | ||
| 7297 | 7324 | int cik_irq_set(struct radeon_device *rdev) |
| 7298 | 7325 | { |
| 7299 | 7326 | u32 cp_int_cntl; |
| 7300 | − u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3; | |
| 7301 | − u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3; | |
| 7327 | + u32 cp_m1p0; | |
| 7302 | 7328 | u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; |
| 7303 | 7329 | u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; |
| 7304 | 7330 | u32 grbm_int_cntl = 0; |
| @@ -7332,13 +7358,6 @@ int cik_irq_set(struct radeon_device *rdev) · 27 unmodified lines | ||
| 7332 | 7358 | dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; |
| 7333 | 7359 | |
| 7334 | 7360 | cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; |
| 7335 | − cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | |
| 7336 | − cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | |
| 7337 | − cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | |
| 7338 | − cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | |
| 7339 | − cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | |
| 7340 | − cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | |
| 7341 | − cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; | |
| 7342 | 7361 | |
| 7343 | 7362 | if (rdev->flags & RADEON_IS_IGP) |
| 7344 | 7363 | thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) & |
| @@ -7360,33 +7379,6 @@ int cik_irq_set(struct radeon_device *rdev) · 15 unmodified lines | ||
| 7360 | 7379 | case 0: |
| 7361 | 7380 | cp_m1p0 |= TIME_STAMP_INT_ENABLE; |
| 7362 | 7381 | break; |
| 7363 | − case 1: | |
| 7364 | − cp_m1p1 |= TIME_STAMP_INT_ENABLE; | |
| 7365 | − break; | |
| 7366 | − case 2: | |
| 7367 | − cp_m1p2 |= TIME_STAMP_INT_ENABLE; | |
| 7368 | − break; | |
| 7369 | − case 3: | |
| 7370 | − cp_m1p2 |= TIME_STAMP_INT_ENABLE; | |
| 7371 | − break; | |
| 7372 | − default: | |
| 7373 | − DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); | |
| 7374 | − break; | |
| 7375 | − } | |
| 7376 | − } else if (ring->me == 2) { | |
| 7377 | − switch (ring->pipe) { | |
| 7378 | − case 0: | |
| 7379 | − cp_m2p0 |= TIME_STAMP_INT_ENABLE; | |
| 7380 | − break; | |
| 7381 | − case 1: | |
| 7382 | − cp_m2p1 |= TIME_STAMP_INT_ENABLE; | |
| 7383 | − break; | |
| 7384 | − case 2: | |
| 7385 | − cp_m2p2 |= TIME_STAMP_INT_ENABLE; | |
| 7386 | − break; | |
| 7387 | − case 3: | |
| 7388 | − cp_m2p2 |= TIME_STAMP_INT_ENABLE; | |
| 7389 | − break; | |
| 7390 | 7382 | default: |
| 7391 | 7383 | DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); |
| 7392 | 7384 | break; |
| @@ -7403,33 +7395,6 @@ int cik_irq_set(struct radeon_device *rdev) · 10 unmodified lines | ||
| 7403 | 7395 | case 0: |
| 7404 | 7396 | cp_m1p0 |= TIME_STAMP_INT_ENABLE; |
| 7405 | 7397 | break; |
| 7406 | − case 1: | |
| 7407 | − cp_m1p1 |= TIME_STAMP_INT_ENABLE; | |
| 7408 | − break; | |
| 7409 | − case 2: | |
| 7410 | − cp_m1p2 |= TIME_STAMP_INT_ENABLE; | |
| 7411 | − break; | |
| 7412 | − case 3: | |
| 7413 | − cp_m1p2 |= TIME_STAMP_INT_ENABLE; | |
| 7414 | − break; | |
| 7415 | − default: | |
| 7416 | − DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); | |
| 7417 | − break; | |
| 7418 | − } | |
| 7419 | − } else if (ring->me == 2) { | |
| 7420 | − switch (ring->pipe) { | |
| 7421 | − case 0: | |
| 7422 | − cp_m2p0 |= TIME_STAMP_INT_ENABLE; | |
| 7423 | − break; | |
| 7424 | − case 1: | |
| 7425 | − cp_m2p1 |= TIME_STAMP_INT_ENABLE; | |
| 7426 | − break; | |
| 7427 | − case 2: | |
| 7428 | − cp_m2p2 |= TIME_STAMP_INT_ENABLE; | |
| 7429 | − break; | |
| 7430 | − case 3: | |
| 7431 | − cp_m2p2 |= TIME_STAMP_INT_ENABLE; | |
| 7432 | − break; | |
| 7433 | 7398 | default: |
| 7434 | 7399 | DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); |
| 7435 | 7400 | break; |
| @@ -7518,13 +7483,6 @@ int cik_irq_set(struct radeon_device *rdev) · 82 unmodified lines | ||
| 7518 | 7483 | WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1); |
| 7519 | 7484 | |
| 7520 | 7485 | WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0); |
| 7521 | − WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1); | |
| 7522 | − WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2); | |
| 7523 | − WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3); | |
| 7524 | − WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0); | |
| 7525 | − WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1); | |
| 7526 | − WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2); | |
| 7527 | − WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3); | |
| 7528 | 7486 | |
| 7529 | 7487 | WREG32(GRBM_INT_CNTL, grbm_int_cntl); |
| 7530 | 7488 | |
| @@ -7841,6 +7799,10 @@ restart_ih: · 310 unmodified lines | ||
| 7841 | 7799 | while (rptr != wptr) { |
| 7842 | 7800 | /* wptr/rptr are in bytes! */ |
| 7843 | 7801 | ring_index = rptr / 4; |
| 7802 | + | |
| 7803 | + radeon_kfd_interrupt(rdev, | |
| 7804 | + (const void *) &rdev->ih.ring[ring_index]); | |
| 7805 | + | |
| 7844 | 7806 | src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; |
| 7845 | 7807 | src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; |
| 7846 | 7808 | ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; |
| @@ -8530,6 +8492,10 @@ static int cik_startup(struct radeon_device *rdev) · 683 unmodified lines | ||
| 8530 | 8492 | if (r) |
| 8531 | 8493 | return r; |
| 8532 | 8494 | |
| 8495 | + r = radeon_kfd_resume(rdev); | |
| 8496 | + if (r) | |
| 8497 | + return r; | |
| 8498 | + | |
| 8533 | 8499 | return 0; |
| 8534 | 8500 | } |
| 8535 | 8501 | |
| @@ -8578,6 +8544,7 @@ int cik_resume(struct radeon_device *rdev) · 42 unmodified lines | ||
| 8578 | 8544 | */ |
| 8579 | 8545 | int cik_suspend(struct radeon_device *rdev) |
| 8580 | 8546 | { |
| 8547 | + radeon_kfd_suspend(rdev); | |
| 8581 | 8548 | radeon_pm_suspend(rdev); |
| 8582 | 8549 | dce6_audio_fini(rdev); |
| 8583 | 8550 | radeon_vm_manager_fini(rdev); |
| @@ -147,4 +147,140 @@ · 146 unmodified lines | ||
| 147 | 147 | |
| 148 | 148 | #define CIK_LB_DESKTOP_HEIGHT 0x6b0c |
| 149 | 149 | |
| 150 | +#define CP_HQD_IQ_RPTR 0xC970u | |
| 151 | +#define AQL_ENABLE (1U << 0) | |
| 152 | + | |
| 153 | +#define IDLE (1 << 2) | |
| 154 | + | |
| 155 | +struct cik_mqd { | |
| 156 | + uint32_t header; | |
| 157 | + uint32_t compute_dispatch_initiator; | |
| 158 | + uint32_t compute_dim_x; | |
| 159 | + uint32_t compute_dim_y; | |
| 160 | + uint32_t compute_dim_z; | |
| 161 | + uint32_t compute_start_x; | |
| 162 | + uint32_t compute_start_y; | |
| 163 | + uint32_t compute_start_z; | |
| 164 | + uint32_t compute_num_thread_x; | |
| 165 | + uint32_t compute_num_thread_y; | |
| 166 | + uint32_t compute_num_thread_z; | |
| 167 | + uint32_t compute_pipelinestat_enable; | |
| 168 | + uint32_t compute_perfcount_enable; | |
| 169 | + uint32_t compute_pgm_lo; | |
| 170 | + uint32_t compute_pgm_hi; | |
| 171 | + uint32_t compute_tba_lo; | |
| 172 | + uint32_t compute_tba_hi; | |
| 173 | + uint32_t compute_tma_lo; | |
| 174 | + uint32_t compute_tma_hi; | |
| 175 | + uint32_t compute_pgm_rsrc1; | |
| 176 | + uint32_t compute_pgm_rsrc2; | |
| 177 | + uint32_t compute_vmid; | |
| 178 | + uint32_t compute_resource_limits; | |
| 179 | + uint32_t compute_static_thread_mgmt_se0; | |
| 180 | + uint32_t compute_static_thread_mgmt_se1; | |
| 181 | + uint32_t compute_tmpring_size; | |
| 182 | + uint32_t compute_static_thread_mgmt_se2; | |
| 183 | + uint32_t compute_static_thread_mgmt_se3; | |
| 184 | + uint32_t compute_restart_x; | |
| 185 | + uint32_t compute_restart_y; | |
| 186 | + uint32_t compute_restart_z; | |
| 187 | + uint32_t compute_thread_trace_enable; | |
| 188 | + uint32_t compute_misc_reserved; | |
| 189 | + uint32_t compute_user_data_0; | |
| 190 | + uint32_t compute_user_data_1; | |
| 191 | + uint32_t compute_user_data_2; | |
| 192 | + uint32_t compute_user_data_3; | |
| 193 | + uint32_t compute_user_data_4; | |
| 194 | + uint32_t compute_user_data_5; | |
| 195 | + uint32_t compute_user_data_6; | |
| 196 | + uint32_t compute_user_data_7; | |
| 197 | + uint32_t compute_user_data_8; | |
| 198 | + uint32_t compute_user_data_9; | |
| 199 | + uint32_t compute_user_data_10; | |
| 200 | + uint32_t compute_user_data_11; | |
| 201 | + uint32_t compute_user_data_12; | |
| 202 | + uint32_t compute_user_data_13; | |
| 203 | + uint32_t compute_user_data_14; | |
| 204 | + uint32_t compute_user_data_15; | |
| 205 | + uint32_t cp_compute_csinvoc_count_lo; | |
| 206 | + uint32_t cp_compute_csinvoc_count_hi; | |
| 207 | + uint32_t cp_mqd_base_addr_lo; | |
| 208 | + uint32_t cp_mqd_base_addr_hi; | |
| 209 | + uint32_t cp_hqd_active; | |
| 210 | + uint32_t cp_hqd_vmid; | |
| 211 | + uint32_t cp_hqd_persistent_state; | |
| 212 | + uint32_t cp_hqd_pipe_priority; | |
| 213 | + uint32_t cp_hqd_queue_priority; | |
| 214 | + uint32_t cp_hqd_quantum; | |
| 215 | + uint32_t cp_hqd_pq_base_lo; | |
| 216 | + uint32_t cp_hqd_pq_base_hi; | |
| 217 | + uint32_t cp_hqd_pq_rptr; | |
| 218 | + uint32_t cp_hqd_pq_rptr_report_addr_lo; | |
| 219 | + uint32_t cp_hqd_pq_rptr_report_addr_hi; | |
| 220 | + uint32_t cp_hqd_pq_wptr_poll_addr_lo; | |
| 221 | + uint32_t cp_hqd_pq_wptr_poll_addr_hi; | |
| 222 | + uint32_t cp_hqd_pq_doorbell_control; | |
| 223 | + uint32_t cp_hqd_pq_wptr; | |
| 224 | + uint32_t cp_hqd_pq_control; | |
| 225 | + uint32_t cp_hqd_ib_base_addr_lo; | |
| 226 | + uint32_t cp_hqd_ib_base_addr_hi; | |
| 227 | + uint32_t cp_hqd_ib_rptr; | |
| 228 | + uint32_t cp_hqd_ib_control; | |
| 229 | + uint32_t cp_hqd_iq_timer; | |
| 230 | + uint32_t cp_hqd_iq_rptr; | |
| 231 | + uint32_t cp_hqd_dequeue_request; | |
| 232 | + uint32_t cp_hqd_dma_offload; | |
| 233 | + uint32_t cp_hqd_sema_cmd; | |
| 234 | + uint32_t cp_hqd_msg_type; | |
| 235 | + uint32_t cp_hqd_atomic0_preop_lo; | |
| 236 | + uint32_t cp_hqd_atomic0_preop_hi; | |
| 237 | + uint32_t cp_hqd_atomic1_preop_lo; | |
| 238 | + uint32_t cp_hqd_atomic1_preop_hi; | |
| 239 | + uint32_t cp_hqd_hq_status0; | |
| 240 | + uint32_t cp_hqd_hq_control0; | |
| 241 | + uint32_t cp_mqd_control; | |
| 242 | + uint32_t cp_mqd_query_time_lo; | |
| 243 | + uint32_t cp_mqd_query_time_hi; | |
| 244 | + uint32_t cp_mqd_connect_start_time_lo; | |
| 245 | + uint32_t cp_mqd_connect_start_time_hi; | |
| 246 | + uint32_t cp_mqd_connect_end_time_lo; | |
| 247 | + uint32_t cp_mqd_connect_end_time_hi; | |
| 248 | + uint32_t cp_mqd_connect_end_wf_count; | |
| 249 | + uint32_t cp_mqd_connect_end_pq_rptr; | |
| 250 | + uint32_t cp_mqd_connect_end_pq_wptr; | |
| 251 | + uint32_t cp_mqd_connect_end_ib_rptr; | |
| 252 | + uint32_t reserved_96; | |
| 253 | + uint32_t reserved_97; | |
| 254 | + uint32_t reserved_98; | |
| 255 | + uint32_t reserved_99; | |
| 256 | + uint32_t iqtimer_pkt_header; | |
| 257 | + uint32_t iqtimer_pkt_dw0; | |
| 258 | + uint32_t iqtimer_pkt_dw1; | |
| 259 | + uint32_t iqtimer_pkt_dw2; | |
| 260 | + uint32_t iqtimer_pkt_dw3; | |
| 261 | + uint32_t iqtimer_pkt_dw4; | |
| 262 | + uint32_t iqtimer_pkt_dw5; | |
| 263 | + uint32_t iqtimer_pkt_dw6; | |
| 264 | + uint32_t reserved_108; | |
| 265 | + uint32_t reserved_109; | |
| 266 | + uint32_t reserved_110; | |
| 267 | + uint32_t reserved_111; | |
| 268 | + uint32_t queue_doorbell_id0; | |
| 269 | + uint32_t queue_doorbell_id1; | |
| 270 | + uint32_t queue_doorbell_id2; | |
| 271 | + uint32_t queue_doorbell_id3; | |
| 272 | + uint32_t queue_doorbell_id4; | |
| 273 | + uint32_t queue_doorbell_id5; | |
| 274 | + uint32_t queue_doorbell_id6; | |
| 275 | + uint32_t queue_doorbell_id7; | |
| 276 | + uint32_t queue_doorbell_id8; | |
| 277 | + uint32_t queue_doorbell_id9; | |
| 278 | + uint32_t queue_doorbell_id10; | |
| 279 | + uint32_t queue_doorbell_id11; | |
| 280 | + uint32_t queue_doorbell_id12; | |
| 281 | + uint32_t queue_doorbell_id13; | |
| 282 | + uint32_t queue_doorbell_id14; | |
| 283 | + uint32_t queue_doorbell_id15; | |
| 284 | +}; | |
| 285 | + | |
| 150 | 286 | #endif |
| @@ -30,6 +30,8 @@ · 29 unmodified lines | ||
| 30 | 30 | #define CIK_RB_BITMAP_WIDTH_PER_SH 2 |
| 31 | 31 | #define HAWAII_RB_BITMAP_WIDTH_PER_SH 4 |
| 32 | 32 | |
| 33 | +#define RADEON_NUM_OF_VMIDS 8 | |
| 34 | + | |
| 33 | 35 | /* DIDT IND registers */ |
| 34 | 36 | #define DIDT_SQ_CTRL0 0x0 |
| 35 | 37 | # define DIDT_CTRL_EN (1 << 0) |
| @@ -1137,6 +1139,9 @@ · 1101 unmodified lines | ||
| 1137 | 1139 | #define SH_MEM_ALIGNMENT_MODE_UNALIGNED 3 |
| 1138 | 1140 | #define DEFAULT_MTYPE(x) ((x) << 4) |
| 1139 | 1141 | #define APE1_MTYPE(x) ((x) << 7) |
| 1142 | +/* valid for both DEFAULT_MTYPE and APE1_MTYPE */ | |
| 1143 | +#define MTYPE_CACHED 0 | |
| 1144 | +#define MTYPE_NONCACHED 3 | |
| 1140 | 1145 | |
| 1141 | 1146 | #define SX_DEBUG_1 0x9060 |
| 1142 | 1147 | |
| @@ -1447,6 +1452,16 @@ · 304 unmodified lines | ||
| 1447 | 1452 | #define CP_HQD_ACTIVE 0xC91C |
| 1448 | 1453 | #define CP_HQD_VMID 0xC920 |
| 1449 | 1454 | |
| 1455 | +#define CP_HQD_PERSISTENT_STATE 0xC924u | |
| 1456 | +#define DEFAULT_CP_HQD_PERSISTENT_STATE (0x33U << 8) | |
| 1457 | + | |
| 1458 | +#define CP_HQD_PIPE_PRIORITY 0xC928u | |
| 1459 | +#define CP_HQD_QUEUE_PRIORITY 0xC92Cu | |
| 1460 | +#define CP_HQD_QUANTUM 0xC930u | |
| 1461 | +#define QUANTUM_EN 1U | |
| 1462 | +#define QUANTUM_SCALE_1MS (1U << 4) | |
| 1463 | +#define QUANTUM_DURATION(x) ((x) << 8) | |
| 1464 | + | |
| 1450 | 1465 | #define CP_HQD_PQ_BASE 0xC934 |
| 1451 | 1466 | #define CP_HQD_PQ_BASE_HI 0xC938 |
| 1452 | 1467 | #define CP_HQD_PQ_RPTR 0xC93C |
| @@ -1474,12 +1489,32 @@ · 21 unmodified lines | ||
| 1474 | 1489 | #define PRIV_STATE (1 << 30) |
| 1475 | 1490 | #define KMD_QUEUE (1 << 31) |
| 1476 | 1491 | |
| 1477 | −#define CP_HQD_DEQUEUE_REQUEST 0xC974 | |
| 1492 | +#define CP_HQD_IB_BASE_ADDR 0xC95Cu | |
| 1493 | +#define CP_HQD_IB_BASE_ADDR_HI 0xC960u | |
| 1494 | +#define CP_HQD_IB_RPTR 0xC964u | |
| 1495 | +#define CP_HQD_IB_CONTROL 0xC968u | |
| 1496 | +#define IB_ATC_EN (1U << 23) | |
| 1497 | +#define DEFAULT_MIN_IB_AVAIL_SIZE (3U << 20) | |
| 1498 | + | |
| 1499 | +#define CP_HQD_DEQUEUE_REQUEST 0xC974 | |
| 1500 | +#define DEQUEUE_REQUEST_DRAIN 1 | |
| 1501 | +#define DEQUEUE_REQUEST_RESET 2 | |
| 1478 | 1502 | |
| 1479 | 1503 | #define CP_MQD_CONTROL 0xC99C |
| 1480 | 1504 | #define MQD_VMID(x) ((x) << 0) |
| 1481 | 1505 | #define MQD_VMID_MASK (0xf << 0) |
| 1482 | 1506 | |
| 1507 | +#define CP_HQD_SEMA_CMD 0xC97Cu | |
| 1508 | +#define CP_HQD_MSG_TYPE 0xC980u | |
| 1509 | +#define CP_HQD_ATOMIC0_PREOP_LO 0xC984u | |
| 1510 | +#define CP_HQD_ATOMIC0_PREOP_HI 0xC988u | |
| 1511 | +#define CP_HQD_ATOMIC1_PREOP_LO 0xC98Cu | |
| 1512 | +#define CP_HQD_ATOMIC1_PREOP_HI 0xC990u | |
| 1513 | +#define CP_HQD_HQ_SCHEDULER0 0xC994u | |
| 1514 | +#define CP_HQD_HQ_SCHEDULER1 0xC998u | |
| 1515 | + | |
| 1516 | +#define SH_STATIC_MEM_CONFIG 0x9604u | |
| 1517 | + | |
| 1483 | 1518 | #define DB_RENDER_CONTROL 0x28000 |
| 1484 | 1519 | |
| 1485 | 1520 | #define PA_SC_RASTER_CONFIG 0x28350 |
| @@ -2069,4 +2104,20 @@ · 583 unmodified lines | ||
| 2069 | 2104 | #define VCE_CMD_IB_AUTO 0x00000005 |
| 2070 | 2105 | #define VCE_CMD_SEMAPHORE 0x00000006 |
| 2071 | 2106 | |
| 2107 | +#define ATC_VMID0_PASID_MAPPING 0x339Cu | |
| 2108 | +#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS 0x3398u | |
| 2109 | +#define ATC_VMID_PASID_MAPPING_VALID (1U << 31) | |
| 2110 | + | |
| 2111 | +#define ATC_VM_APERTURE0_CNTL 0x3310u | |
| 2112 | +#define ATS_ACCESS_MODE_NEVER 0 | |
| 2113 | +#define ATS_ACCESS_MODE_ALWAYS 1 | |
| 2114 | + | |
| 2115 | +#define ATC_VM_APERTURE0_CNTL2 0x3318u | |
| 2116 | +#define ATC_VM_APERTURE0_HIGH_ADDR 0x3308u | |
| 2117 | +#define ATC_VM_APERTURE0_LOW_ADDR 0x3300u | |
| 2118 | +#define ATC_VM_APERTURE1_CNTL 0x3314u | |
| 2119 | +#define ATC_VM_APERTURE1_CNTL2 0x331Cu | |
| 2120 | +#define ATC_VM_APERTURE1_HIGH_ADDR 0x330Cu | |
| 2121 | +#define ATC_VM_APERTURE1_LOW_ADDR 0x3304u | |
| 2122 | + | |
| 2072 | 2123 | #endif |
| @@ -701,6 +701,10 @@ struct radeon_doorbell { · 700 unmodified lines | ||
| 701 | 701 | |
| 702 | 702 | int radeon_doorbell_get(struct radeon_device *rdev, u32 *page); |
| 703 | 703 | void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell); |
| 704 | +void radeon_doorbell_get_kfd_info(struct radeon_device *rdev, | |
| 705 | + phys_addr_t *aperture_base, | |
| 706 | + size_t *aperture_size, | |
| 707 | + size_t *start_offset); | |
| 704 | 708 | |
| 705 | 709 | /* |
| 706 | 710 | * IRQS. |
| @@ -2393,6 +2397,8 @@ struct radeon_device { · 1686 unmodified lines | ||
| 2393 | 2397 | struct radeon_atcs atcs; |
| 2394 | 2398 | /* srbm instance registers */ |
| 2395 | 2399 | struct mutex srbm_mutex; |
| 2400 | + /* GRBM index mutex. Protects concurrents access to GRBM index */ | |
| 2401 | + struct mutex grbm_idx_mutex; | |
| 2396 | 2402 | /* clock, powergating flags */ |
| 2397 | 2403 | u32 cg_flags; |
| 2398 | 2404 | u32 pg_flags; |
| @@ -2405,6 +2411,10 @@ struct radeon_device { · 6 unmodified lines | ||
| 2405 | 2411 | u64 vram_pin_size; |
| 2406 | 2412 | u64 gart_pin_size; |
| 2407 | 2413 | |
| 2414 | + /* amdkfd interface */ | |
| 2415 | + struct kfd_dev *kfd; | |
| 2416 | + struct radeon_sa_manager kfd_bo; | |
| 2417 | + | |
| 2408 | 2418 | struct mutex mn_lock; |
| 2409 | 2419 | DECLARE_HASHTABLE(mn_hash, 7); |
| 2410 | 2420 | }; |
| @@ -377,6 +377,37 @@ void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell) · 376 unmodified lines | ||
| 377 | 377 | __clear_bit(doorbell, rdev->doorbell.used); |
| 378 | 378 | } |
| 379 | 379 | |
| 380 | +/** | |
| 381 | + * radeon_doorbell_get_kfd_info - Report doorbell configuration required to | |
| 382 | + * setup KFD | |
| 383 | + * | |
| 384 | + * @rdev: radeon_device pointer | |
| 385 | + * @aperture_base: output returning doorbell aperture base physical address | |
| 386 | + * @aperture_size: output returning doorbell aperture size in bytes | |
| 387 | + * @start_offset: output returning # of doorbell bytes reserved for radeon. | |
| 388 | + * | |
| 389 | + * Radeon and the KFD share the doorbell aperture. Radeon sets it up, | |
| 390 | + * takes doorbells required for its own rings and reports the setup to KFD. | |
| 391 | + * Radeon reserved doorbells are at the start of the doorbell aperture. | |
| 392 | + */ | |
| 393 | +void radeon_doorbell_get_kfd_info(struct radeon_device *rdev, | |
| 394 | + phys_addr_t *aperture_base, | |
| 395 | + size_t *aperture_size, | |
| 396 | + size_t *start_offset) | |
| 397 | +{ | |
| 398 | + /* The first num_doorbells are used by radeon. | |
| 399 | + * KFD takes whatever's left in the aperture. */ | |
| 400 | + if (rdev->doorbell.size > rdev->doorbell.num_doorbells * sizeof(u32)) { | |
| 401 | + *aperture_base = rdev->doorbell.base; | |
| 402 | + *aperture_size = rdev->doorbell.size; | |
| 403 | + *start_offset = rdev->doorbell.num_doorbells * sizeof(u32); | |
| 404 | + } else { | |
| 405 | + *aperture_base = 0; | |
| 406 | + *aperture_size = 0; | |
| 407 | + *start_offset = 0; | |
| 408 | + } | |
| 409 | +} | |
| 410 | + | |
| 380 | 411 | /* |
| 381 | 412 | * radeon_wb_*() |
| 382 | 413 | * Writeback is the the method by which the the GPU updates special pages |
| @@ -1272,6 +1303,7 @@ int radeon_device_init(struct radeon_device *rdev, · 889 unmodified lines | ||
| 1272 | 1303 | mutex_init(&rdev->pm.mutex); |
| 1273 | 1304 | mutex_init(&rdev->gpu_clock_mutex); |
| 1274 | 1305 | mutex_init(&rdev->srbm_mutex); |
| 1306 | + mutex_init(&rdev->grbm_idx_mutex); | |
| 1275 | 1307 | init_rwsem(&rdev->pm.mclk_lock); |
| 1276 | 1308 | init_rwsem(&rdev->exclusive_lock); |
| 1277 | 1309 | init_waitqueue_head(&rdev->irq.vblank_queue); |
| @@ -41,6 +41,8 @@ · 40 unmodified lines | ||
| 41 | 41 | #include <drm/drm_gem.h> |
| 42 | 42 | |
| 43 | 43 | #include "drm_crtc_helper.h" |
| 44 | +#include "radeon_kfd.h" | |
| 45 | + | |
| 44 | 46 | /* |
| 45 | 47 | * KMS wrapper. |
| 46 | 48 | * - 2.0.0 - initial interface |
| @@ -654,12 +656,15 @@ static int __init radeon_init(void) · 607 unmodified lines | ||
| 654 | 656 | #endif |
| 655 | 657 | } |
| 656 | 658 | |
| 659 | + radeon_kfd_init(); | |
| 660 | + | |
| 657 | 661 | /* let modprobe override vga console setting */ |
| 658 | 662 | return drm_pci_init(driver, pdriver); |
| 659 | 663 | } |
| 660 | 664 | |
| 661 | 665 | static void __exit radeon_exit(void) |
| 662 | 666 | { |
| 667 | + radeon_kfd_fini(); | |
| 663 | 668 | drm_pci_exit(driver, pdriver); |
| 664 | 669 | radeon_unregister_atpx_handler(); |
| 665 | 670 | } |
| @@ -0,0 +1,563 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +#include <linux/module.h> | |
| 24 | +#include <linux/fdtable.h> | |
| 25 | +#include <linux/uaccess.h> | |
| 26 | +#include <drm/drmP.h> | |
| 27 | +#include "radeon.h" | |
| 28 | +#include "cikd.h" | |
| 29 | +#include "cik_reg.h" | |
| 30 | +#include "radeon_kfd.h" | |
| 31 | + | |
| 32 | +#define CIK_PIPE_PER_MEC (4) | |
| 33 | + | |
| 34 | +struct kgd_mem { | |
| 35 | + struct radeon_sa_bo *sa_bo; | |
| 36 | + uint64_t gpu_addr; | |
| 37 | + void *ptr; | |
| 38 | +}; | |
| 39 | + | |
| 40 | +static int init_sa_manager(struct kgd_dev *kgd, unsigned int size); | |
| 41 | +static void fini_sa_manager(struct kgd_dev *kgd); | |
| 42 | + | |
| 43 | +static int allocate_mem(struct kgd_dev *kgd, size_t size, size_t alignment, | |
| 44 | + enum kgd_memory_pool pool, struct kgd_mem **mem); | |
| 45 | + | |
| 46 | +static void free_mem(struct kgd_dev *kgd, struct kgd_mem *mem); | |
| 47 | + | |
| 48 | +static uint64_t get_vmem_size(struct kgd_dev *kgd); | |
| 49 | +static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd); | |
| 50 | + | |
| 51 | +static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd); | |
| 52 | + | |
| 53 | +/* | |
| 54 | + * Register access functions | |
| 55 | + */ | |
| 56 | + | |
| 57 | +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, | |
| 58 | + uint32_t sh_mem_config, uint32_t sh_mem_ape1_base, | |
| 59 | + uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases); | |
| 60 | + | |
| 61 | +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, | |
| 62 | + unsigned int vmid); | |
| 63 | + | |
| 64 | +static int kgd_init_memory(struct kgd_dev *kgd); | |
| 65 | + | |
| 66 | +static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id, | |
| 67 | + uint32_t hpd_size, uint64_t hpd_gpu_addr); | |
| 68 | + | |
| 69 | +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, | |
| 70 | + uint32_t queue_id, uint32_t __user *wptr); | |
| 71 | + | |
| 72 | +static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address, | |
| 73 | + uint32_t pipe_id, uint32_t queue_id); | |
| 74 | + | |
| 75 | +static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, | |
| 76 | + unsigned int timeout, uint32_t pipe_id, | |
| 77 | + uint32_t queue_id); | |
| 78 | + | |
| 79 | +static const struct kfd2kgd_calls kfd2kgd = { | |
| 80 | + .init_sa_manager = init_sa_manager, | |
| 81 | + .fini_sa_manager = fini_sa_manager, | |
| 82 | + .allocate_mem = allocate_mem, | |
| 83 | + .free_mem = free_mem, | |
| 84 | + .get_vmem_size = get_vmem_size, | |
| 85 | + .get_gpu_clock_counter = get_gpu_clock_counter, | |
| 86 | + .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz, | |
| 87 | + .program_sh_mem_settings = kgd_program_sh_mem_settings, | |
| 88 | + .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, | |
| 89 | + .init_memory = kgd_init_memory, | |
| 90 | + .init_pipeline = kgd_init_pipeline, | |
| 91 | + .hqd_load = kgd_hqd_load, | |
| 92 | + .hqd_is_occupies = kgd_hqd_is_occupies, | |
| 93 | + .hqd_destroy = kgd_hqd_destroy, | |
| 94 | +}; | |
| 95 | + | |
| 96 | +static const struct kgd2kfd_calls *kgd2kfd; | |
| 97 | + | |
| 98 | +bool radeon_kfd_init(void) | |
| 99 | +{ | |
| 100 | + bool (*kgd2kfd_init_p)(unsigned, const struct kfd2kgd_calls*, | |
| 101 | + const struct kgd2kfd_calls**); | |
| 102 | + | |
| 103 | + kgd2kfd_init_p = symbol_request(kgd2kfd_init); | |
| 104 | + | |
| 105 | + if (kgd2kfd_init_p == NULL) | |
| 106 | + return false; | |
| 107 | + | |
| 108 | + if (!kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kfd2kgd, &kgd2kfd)) { | |
| 109 | + symbol_put(kgd2kfd_init); | |
| 110 | + kgd2kfd = NULL; | |
| 111 | + | |
| 112 | + return false; | |
| 113 | + } | |
| 114 | + | |
| 115 | + return true; | |
| 116 | +} | |
| 117 | + | |
| 118 | +void radeon_kfd_fini(void) | |
| 119 | +{ | |
| 120 | + if (kgd2kfd) { | |
| 121 | + kgd2kfd->exit(); | |
| 122 | + symbol_put(kgd2kfd_init); | |
| 123 | + } | |
| 124 | +} | |
| 125 | + | |
| 126 | +void radeon_kfd_device_probe(struct radeon_device *rdev) | |
| 127 | +{ | |
| 128 | + if (kgd2kfd) | |
| 129 | + rdev->kfd = kgd2kfd->probe((struct kgd_dev *)rdev, rdev->pdev); | |
| 130 | +} | |
| 131 | + | |
| 132 | +void radeon_kfd_device_init(struct radeon_device *rdev) | |
| 133 | +{ | |
| 134 | + if (rdev->kfd) { | |
| 135 | + struct kgd2kfd_shared_resources gpu_resources = { | |
| 136 | + .compute_vmid_bitmap = 0xFF00, | |
| 137 | + | |
| 138 | + .first_compute_pipe = 1, | |
| 139 | + .compute_pipe_count = 8 - 1, | |
| 140 | + }; | |
| 141 | + | |
| 142 | + radeon_doorbell_get_kfd_info(rdev, | |
| 143 | + &gpu_resources.doorbell_physical_address, | |
| 144 | + &gpu_resources.doorbell_aperture_size, | |
| 145 | + &gpu_resources.doorbell_start_offset); | |
| 146 | + | |
| 147 | + kgd2kfd->device_init(rdev->kfd, &gpu_resources); | |
| 148 | + } | |
| 149 | +} | |
| 150 | + | |
| 151 | +void radeon_kfd_device_fini(struct radeon_device *rdev) | |
| 152 | +{ | |
| 153 | + if (rdev->kfd) { | |
| 154 | + kgd2kfd->device_exit(rdev->kfd); | |
| 155 | + rdev->kfd = NULL; | |
| 156 | + } | |
| 157 | +} | |
| 158 | + | |
| 159 | +void radeon_kfd_interrupt(struct radeon_device *rdev, const void *ih_ring_entry) | |
| 160 | +{ | |
| 161 | + if (rdev->kfd) | |
| 162 | + kgd2kfd->interrupt(rdev->kfd, ih_ring_entry); | |
| 163 | +} | |
| 164 | + | |
| 165 | +void radeon_kfd_suspend(struct radeon_device *rdev) | |
| 166 | +{ | |
| 167 | + if (rdev->kfd) | |
| 168 | + kgd2kfd->suspend(rdev->kfd); | |
| 169 | +} | |
| 170 | + | |
| 171 | +int radeon_kfd_resume(struct radeon_device *rdev) | |
| 172 | +{ | |
| 173 | + int r = 0; | |
| 174 | + | |
| 175 | + if (rdev->kfd) | |
| 176 | + r = kgd2kfd->resume(rdev->kfd); | |
| 177 | + | |
| 178 | + return r; | |
| 179 | +} | |
| 180 | + | |
| 181 | +static u32 pool_to_domain(enum kgd_memory_pool p) | |
| 182 | +{ | |
| 183 | + switch (p) { | |
| 184 | + case KGD_POOL_FRAMEBUFFER: return RADEON_GEM_DOMAIN_VRAM; | |
| 185 | + default: return RADEON_GEM_DOMAIN_GTT; | |
| 186 | + } | |
| 187 | +} | |
| 188 | + | |
| 189 | +static int init_sa_manager(struct kgd_dev *kgd, unsigned int size) | |
| 190 | +{ | |
| 191 | + struct radeon_device *rdev = (struct radeon_device *)kgd; | |
| 192 | + int r; | |
| 193 | + | |
| 194 | + BUG_ON(kgd == NULL); | |
| 195 | + | |
| 196 | + r = radeon_sa_bo_manager_init(rdev, &rdev->kfd_bo, | |
| 197 | + size, | |
| 198 | + RADEON_GPU_PAGE_SIZE, | |
| 199 | + RADEON_GEM_DOMAIN_GTT, | |
| 200 | + RADEON_GEM_GTT_WC); | |
| 201 | + | |
| 202 | + if (r) | |
| 203 | + return r; | |
| 204 | + | |
| 205 | + r = radeon_sa_bo_manager_start(rdev, &rdev->kfd_bo); | |
| 206 | + if (r) | |
| 207 | + radeon_sa_bo_manager_fini(rdev, &rdev->kfd_bo); | |
| 208 | + | |
| 209 | + return r; | |
| 210 | +} | |
| 211 | + | |
| 212 | +static void fini_sa_manager(struct kgd_dev *kgd) | |
| 213 | +{ | |
| 214 | + struct radeon_device *rdev = (struct radeon_device *)kgd; | |
| 215 | + | |
| 216 | + BUG_ON(kgd == NULL); | |
| 217 | + | |
| 218 | + radeon_sa_bo_manager_suspend(rdev, &rdev->kfd_bo); | |
| 219 | + radeon_sa_bo_manager_fini(rdev, &rdev->kfd_bo); | |
| 220 | +} | |
| 221 | + | |
| 222 | +static int allocate_mem(struct kgd_dev *kgd, size_t size, size_t alignment, | |
| 223 | + enum kgd_memory_pool pool, struct kgd_mem **mem) | |
| 224 | +{ | |
| 225 | + struct radeon_device *rdev = (struct radeon_device *)kgd; | |
| 226 | + u32 domain; | |
| 227 | + int r; | |
| 228 | + | |
| 229 | + BUG_ON(kgd == NULL); | |
| 230 | + | |
| 231 | + domain = pool_to_domain(pool); | |
| 232 | + if (domain != RADEON_GEM_DOMAIN_GTT) { | |
| 233 | + dev_err(rdev->dev, | |
| 234 | + "Only allowed to allocate gart memory for kfd\n"); | |
| 235 | + return -EINVAL; | |
| 236 | + } | |
| 237 | + | |
| 238 | + *mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL); | |
| 239 | + if ((*mem) == NULL) | |
| 240 | + return -ENOMEM; | |
| 241 | + | |
| 242 | + r = radeon_sa_bo_new(rdev, &rdev->kfd_bo, &(*mem)->sa_bo, size, | |
| 243 | + alignment); | |
| 244 | + if (r) { | |
| 245 | + dev_err(rdev->dev, "failed to get memory for kfd (%d)\n", r); | |
| 246 | + return r; | |
| 247 | + } | |
| 248 | + | |
| 249 | + (*mem)->ptr = radeon_sa_bo_cpu_addr((*mem)->sa_bo); | |
| 250 | + (*mem)->gpu_addr = radeon_sa_bo_gpu_addr((*mem)->sa_bo); | |
| 251 | + | |
| 252 | + return 0; | |
| 253 | +} | |
| 254 | + | |
| 255 | +static void free_mem(struct kgd_dev *kgd, struct kgd_mem *mem) | |
| 256 | +{ | |
| 257 | + struct radeon_device *rdev = (struct radeon_device *)kgd; | |
| 258 | + | |
| 259 | + BUG_ON(kgd == NULL); | |
| 260 | + | |
| 261 | + radeon_sa_bo_free(rdev, &mem->sa_bo, NULL); | |
| 262 | + kfree(mem); | |
| 263 | +} | |
| 264 | + | |
| 265 | +static uint64_t get_vmem_size(struct kgd_dev *kgd) | |
| 266 | +{ | |
| 267 | + struct radeon_device *rdev = (struct radeon_device *)kgd; | |
| 268 | + | |
| 269 | + BUG_ON(kgd == NULL); | |
| 270 | + | |
| 271 | + return rdev->mc.real_vram_size; | |
| 272 | +} | |
| 273 | + | |
| 274 | +static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd) | |
| 275 | +{ | |
| 276 | + struct radeon_device *rdev = (struct radeon_device *)kgd; | |
| 277 | + | |
| 278 | + return rdev->asic->get_gpu_clock_counter(rdev); | |
| 279 | +} | |
| 280 | + | |
| 281 | +static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd) | |
| 282 | +{ | |
| 283 | + struct radeon_device *rdev = (struct radeon_device *)kgd; | |
| 284 | + | |
| 285 | + /* The sclk is in quantas of 10kHz */ | |
| 286 | + return rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100; | |
| 287 | +} | |
| 288 | + | |
| 289 | +static inline struct radeon_device *get_radeon_device(struct kgd_dev *kgd) | |
| 290 | +{ | |
| 291 | + return (struct radeon_device *)kgd; | |
| 292 | +} | |
| 293 | + | |
| 294 | +static void write_register(struct kgd_dev *kgd, uint32_t offset, uint32_t value) | |
| 295 | +{ | |
| 296 | + struct radeon_device *rdev = get_radeon_device(kgd); | |
| 297 | + | |
| 298 | + writel(value, (void __iomem *)(rdev->rmmio + offset)); | |
| 299 | +} | |
| 300 | + | |
| 301 | +static uint32_t read_register(struct kgd_dev *kgd, uint32_t offset) | |
| 302 | +{ | |
| 303 | + struct radeon_device *rdev = get_radeon_device(kgd); | |
| 304 | + | |
| 305 | + return readl((void __iomem *)(rdev->rmmio + offset)); | |
| 306 | +} | |
| 307 | + | |
| 308 | +static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe, | |
| 309 | + uint32_t queue, uint32_t vmid) | |
| 310 | +{ | |
| 311 | + struct radeon_device *rdev = get_radeon_device(kgd); | |
| 312 | + uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue); | |
| 313 | + | |
| 314 | + mutex_lock(&rdev->srbm_mutex); | |
| 315 | + write_register(kgd, SRBM_GFX_CNTL, value); | |
| 316 | +} | |
| 317 | + | |
| 318 | +static void unlock_srbm(struct kgd_dev *kgd) | |
| 319 | +{ | |
| 320 | + struct radeon_device *rdev = get_radeon_device(kgd); | |
| 321 | + | |
| 322 | + write_register(kgd, SRBM_GFX_CNTL, 0); | |
| 323 | + mutex_unlock(&rdev->srbm_mutex); | |
| 324 | +} | |
| 325 | + | |
| 326 | +static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id, | |
| 327 | + uint32_t queue_id) | |
| 328 | +{ | |
| 329 | + uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1; | |
| 330 | + uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC); | |
| 331 | + | |
| 332 | + lock_srbm(kgd, mec, pipe, queue_id, 0); | |
| 333 | +} | |
| 334 | + | |
| 335 | +static void release_queue(struct kgd_dev *kgd) | |
| 336 | +{ | |
| 337 | + unlock_srbm(kgd); | |
| 338 | +} | |
| 339 | + | |
| 340 | +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, | |
| 341 | + uint32_t sh_mem_config, | |
| 342 | + uint32_t sh_mem_ape1_base, | |
| 343 | + uint32_t sh_mem_ape1_limit, | |
| 344 | + uint32_t sh_mem_bases) | |
| 345 | +{ | |
| 346 | + lock_srbm(kgd, 0, 0, 0, vmid); | |
| 347 | + | |
| 348 | + write_register(kgd, SH_MEM_CONFIG, sh_mem_config); | |
| 349 | + write_register(kgd, SH_MEM_APE1_BASE, sh_mem_ape1_base); | |
| 350 | + write_register(kgd, SH_MEM_APE1_LIMIT, sh_mem_ape1_limit); | |
| 351 | + write_register(kgd, SH_MEM_BASES, sh_mem_bases); | |
| 352 | + | |
| 353 | + unlock_srbm(kgd); | |
| 354 | +} | |
| 355 | + | |
| 356 | +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, | |
| 357 | + unsigned int vmid) | |
| 358 | +{ | |
| 359 | + /* | |
| 360 | + * We have to assume that there is no outstanding mapping. | |
| 361 | + * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 | |
| 362 | + * because a mapping is in progress or because a mapping finished and | |
| 363 | + * the SW cleared it. | |
| 364 | + * So the protocol is to always wait & clear. | |
| 365 | + */ | |
| 366 | + uint32_t pasid_mapping = (pasid == 0) ? 0 : | |
| 367 | + (uint32_t)pasid | ATC_VMID_PASID_MAPPING_VALID; | |
| 368 | + | |
| 369 | + write_register(kgd, ATC_VMID0_PASID_MAPPING + vmid*sizeof(uint32_t), | |
| 370 | + pasid_mapping); | |
| 371 | + | |
| 372 | + while (!(read_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS) & | |
| 373 | + (1U << vmid))) | |
| 374 | + cpu_relax(); | |
| 375 | + write_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid); | |
| 376 | + | |
| 377 | + return 0; | |
| 378 | +} | |
| 379 | + | |
| 380 | +static int kgd_init_memory(struct kgd_dev *kgd) | |
| 381 | +{ | |
| 382 | + /* | |
| 383 | + * Configure apertures: | |
| 384 | + * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) | |
| 385 | + * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) | |
| 386 | + * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) | |
| 387 | + */ | |
| 388 | + int i; | |
| 389 | + uint32_t sh_mem_bases = PRIVATE_BASE(0x6000) | SHARED_BASE(0x6000); | |
| 390 | + | |
| 391 | + for (i = 8; i < 16; i++) { | |
| 392 | + uint32_t sh_mem_config; | |
| 393 | + | |
| 394 | + lock_srbm(kgd, 0, 0, 0, i); | |
| 395 | + | |
| 396 | + sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED); | |
| 397 | + sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED); | |
| 398 | + | |
| 399 | + write_register(kgd, SH_MEM_CONFIG, sh_mem_config); | |
| 400 | + | |
| 401 | + write_register(kgd, SH_MEM_BASES, sh_mem_bases); | |
| 402 | + | |
| 403 | + /* Scratch aperture is not supported for now. */ | |
| 404 | + write_register(kgd, SH_STATIC_MEM_CONFIG, 0); | |
| 405 | + | |
| 406 | + /* APE1 disabled for now. */ | |
| 407 | + write_register(kgd, SH_MEM_APE1_BASE, 1); | |
| 408 | + write_register(kgd, SH_MEM_APE1_LIMIT, 0); | |
| 409 | + | |
| 410 | + unlock_srbm(kgd); | |
| 411 | + } | |
| 412 | + | |
| 413 | + return 0; | |
| 414 | +} | |
| 415 | + | |
| 416 | +static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id, | |
| 417 | + uint32_t hpd_size, uint64_t hpd_gpu_addr) | |
| 418 | +{ | |
| 419 | + uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1; | |
| 420 | + uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC); | |
| 421 | + | |
| 422 | + lock_srbm(kgd, mec, pipe, 0, 0); | |
| 423 | + write_register(kgd, CP_HPD_EOP_BASE_ADDR, | |
| 424 | + lower_32_bits(hpd_gpu_addr >> 8)); | |
| 425 | + write_register(kgd, CP_HPD_EOP_BASE_ADDR_HI, | |
| 426 | + upper_32_bits(hpd_gpu_addr >> 8)); | |
| 427 | + write_register(kgd, CP_HPD_EOP_VMID, 0); | |
| 428 | + write_register(kgd, CP_HPD_EOP_CONTROL, hpd_size); | |
| 429 | + unlock_srbm(kgd); | |
| 430 | + | |
| 431 | + return 0; | |
| 432 | +} | |
| 433 | + | |
| 434 | +static inline struct cik_mqd *get_mqd(void *mqd) | |
| 435 | +{ | |
| 436 | + return (struct cik_mqd *)mqd; | |
| 437 | +} | |
| 438 | + | |
| 439 | +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, | |
| 440 | + uint32_t queue_id, uint32_t __user *wptr) | |
| 441 | +{ | |
| 442 | + uint32_t wptr_shadow, is_wptr_shadow_valid; | |
| 443 | + struct cik_mqd *m; | |
| 444 | + | |
| 445 | + m = get_mqd(mqd); | |
| 446 | + | |
| 447 | + is_wptr_shadow_valid = !get_user(wptr_shadow, wptr); | |
| 448 | + | |
| 449 | + acquire_queue(kgd, pipe_id, queue_id); | |
| 450 | + write_register(kgd, CP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo); | |
| 451 | + write_register(kgd, CP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi); | |
| 452 | + write_register(kgd, CP_MQD_CONTROL, m->cp_mqd_control); | |
| 453 | + | |
| 454 | + write_register(kgd, CP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo); | |
| 455 | + write_register(kgd, CP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi); | |
| 456 | + write_register(kgd, CP_HQD_PQ_CONTROL, m->cp_hqd_pq_control); | |
| 457 | + | |
| 458 | + write_register(kgd, CP_HQD_IB_CONTROL, m->cp_hqd_ib_control); | |
| 459 | + write_register(kgd, CP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo); | |
| 460 | + write_register(kgd, CP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi); | |
| 461 | + | |
| 462 | + write_register(kgd, CP_HQD_IB_RPTR, m->cp_hqd_ib_rptr); | |
| 463 | + | |
| 464 | + write_register(kgd, CP_HQD_PERSISTENT_STATE, | |
| 465 | + m->cp_hqd_persistent_state); | |
| 466 | + write_register(kgd, CP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd); | |
| 467 | + write_register(kgd, CP_HQD_MSG_TYPE, m->cp_hqd_msg_type); | |
| 468 | + | |
| 469 | + write_register(kgd, CP_HQD_ATOMIC0_PREOP_LO, | |
| 470 | + m->cp_hqd_atomic0_preop_lo); | |
| 471 | + | |
| 472 | + write_register(kgd, CP_HQD_ATOMIC0_PREOP_HI, | |
| 473 | + m->cp_hqd_atomic0_preop_hi); | |
| 474 | + | |
| 475 | + write_register(kgd, CP_HQD_ATOMIC1_PREOP_LO, | |
| 476 | + m->cp_hqd_atomic1_preop_lo); | |
| 477 | + | |
| 478 | + write_register(kgd, CP_HQD_ATOMIC1_PREOP_HI, | |
| 479 | + m->cp_hqd_atomic1_preop_hi); | |
| 480 | + | |
| 481 | + write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR, | |
| 482 | + m->cp_hqd_pq_rptr_report_addr_lo); | |
| 483 | + | |
| 484 | + write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR_HI, | |
| 485 | + m->cp_hqd_pq_rptr_report_addr_hi); | |
| 486 | + | |
| 487 | + write_register(kgd, CP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr); | |
| 488 | + | |
| 489 | + write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR, | |
| 490 | + m->cp_hqd_pq_wptr_poll_addr_lo); | |
| 491 | + | |
| 492 | + write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR_HI, | |
| 493 | + m->cp_hqd_pq_wptr_poll_addr_hi); | |
| 494 | + | |
| 495 | + write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, | |
| 496 | + m->cp_hqd_pq_doorbell_control); | |
| 497 | + | |
| 498 | + write_register(kgd, CP_HQD_VMID, m->cp_hqd_vmid); | |
| 499 | + | |
| 500 | + write_register(kgd, CP_HQD_QUANTUM, m->cp_hqd_quantum); | |
| 501 | + | |
| 502 | + write_register(kgd, CP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority); | |
| 503 | + write_register(kgd, CP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority); | |
| 504 | + | |
| 505 | + write_register(kgd, CP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr); | |
| 506 | + | |
| 507 | + if (is_wptr_shadow_valid) | |
| 508 | + write_register(kgd, CP_HQD_PQ_WPTR, wptr_shadow); | |
| 509 | + | |
| 510 | + write_register(kgd, CP_HQD_ACTIVE, m->cp_hqd_active); | |
| 511 | + release_queue(kgd); | |
| 512 | + | |
| 513 | + return 0; | |
| 514 | +} | |
| 515 | + | |
| 516 | +static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address, | |
| 517 | + uint32_t pipe_id, uint32_t queue_id) | |
| 518 | +{ | |
| 519 | + uint32_t act; | |
| 520 | + bool retval = false; | |
| 521 | + uint32_t low, high; | |
| 522 | + | |
| 523 | + acquire_queue(kgd, pipe_id, queue_id); | |
| 524 | + act = read_register(kgd, CP_HQD_ACTIVE); | |
| 525 | + if (act) { | |
| 526 | + low = lower_32_bits(queue_address >> 8); | |
| 527 | + high = upper_32_bits(queue_address >> 8); | |
| 528 | + | |
| 529 | + if (low == read_register(kgd, CP_HQD_PQ_BASE) && | |
| 530 | + high == read_register(kgd, CP_HQD_PQ_BASE_HI)) | |
| 531 | + retval = true; | |
| 532 | + } | |
| 533 | + release_queue(kgd); | |
| 534 | + return retval; | |
| 535 | +} | |
| 536 | + | |
| 537 | +static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, | |
| 538 | + unsigned int timeout, uint32_t pipe_id, | |
| 539 | + uint32_t queue_id) | |
| 540 | +{ | |
| 541 | + uint32_t temp; | |
| 542 | + | |
| 543 | + acquire_queue(kgd, pipe_id, queue_id); | |
| 544 | + write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, 0); | |
| 545 | + | |
| 546 | + write_register(kgd, CP_HQD_DEQUEUE_REQUEST, reset_type); | |
| 547 | + | |
| 548 | + while (true) { | |
| 549 | + temp = read_register(kgd, CP_HQD_ACTIVE); | |
| 550 | + if (temp & 0x1) | |
| 551 | + break; | |
| 552 | + if (timeout == 0) { | |
| 553 | + pr_err("kfd: cp queue preemption time out (%dms)\n", | |
| 554 | + temp); | |
| 555 | + return -ETIME; | |
| 556 | + } | |
| 557 | + msleep(20); | |
| 558 | + timeout -= 20; | |
| 559 | + } | |
| 560 | + | |
| 561 | + release_queue(kgd); | |
| 562 | + return 0; | |
| 563 | +} | |
| @@ -0,0 +1,47 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +/* | |
| 24 | + * radeon_kfd.h defines the private interface between the | |
| 25 | + * AMD kernel graphics drivers and the AMD KFD. | |
| 26 | + */ | |
| 27 | + | |
| 28 | +#ifndef RADEON_KFD_H_INCLUDED | |
| 29 | +#define RADEON_KFD_H_INCLUDED | |
| 30 | + | |
| 31 | +#include <linux/types.h> | |
| 32 | +#include "../amd/include/kgd_kfd_interface.h" | |
| 33 | + | |
| 34 | +struct radeon_device; | |
| 35 | + | |
| 36 | +bool radeon_kfd_init(void); | |
| 37 | +void radeon_kfd_fini(void); | |
| 38 | + | |
| 39 | +void radeon_kfd_suspend(struct radeon_device *rdev); | |
| 40 | +int radeon_kfd_resume(struct radeon_device *rdev); | |
| 41 | +void radeon_kfd_interrupt(struct radeon_device *rdev, | |
| 42 | + const void *ih_ring_entry); | |
| 43 | +void radeon_kfd_device_probe(struct radeon_device *rdev); | |
| 44 | +void radeon_kfd_device_init(struct radeon_device *rdev); | |
| 45 | +void radeon_kfd_device_fini(struct radeon_device *rdev); | |
| 46 | + | |
| 47 | +#endif /* RADEON_KFD_H_INCLUDED */ | |
| @@ -34,6 +34,8 @@ · 33 unmodified lines | ||
| 34 | 34 | #include <linux/slab.h> |
| 35 | 35 | #include <linux/pm_runtime.h> |
| 36 | 36 | |
| 37 | +#include "radeon_kfd.h" | |
| 38 | + | |
| 37 | 39 | #if defined(CONFIG_VGA_SWITCHEROO) |
| 38 | 40 | bool radeon_has_atpx(void); |
| 39 | 41 | #else |
| @@ -63,6 +65,8 @@ int radeon_driver_unload_kms(struct drm_device *dev) · 23 unmodified lines | ||
| 63 | 65 | |
| 64 | 66 | pm_runtime_get_sync(dev->dev); |
| 65 | 67 | |
| 68 | + radeon_kfd_device_fini(rdev); | |
| 69 | + | |
| 66 | 70 | radeon_acpi_fini(rdev); |
| 67 | 71 | |
| 68 | 72 | radeon_modeset_fini(rdev); |
| @@ -142,6 +146,9 @@ int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags) · 73 unmodified lines | ||
| 142 | 146 | "Error during ACPI methods call\n"); |
| 143 | 147 | } |
| 144 | 148 | |
| 149 | + radeon_kfd_device_probe(rdev); | |
| 150 | + radeon_kfd_device_init(rdev); | |
| 151 | + | |
| 145 | 152 | if (radeon_is_px(dev)) { |
| 146 | 153 | pm_runtime_use_autosuspend(dev->dev); |
| 147 | 154 | pm_runtime_set_autosuspend_delay(dev->dev, 5000); |
| @@ -92,13 +92,6 @@ static spinlock_t state_lock; · 91 unmodified lines | ||
| 92 | 92 | |
| 93 | 93 | static struct workqueue_struct *iommu_wq; |
| 94 | 94 | |
| 95 | −/* | |
| 96 | − * Empty page table - Used between | |
| 97 | − * mmu_notifier_invalidate_range_start and | |
| 98 | − * mmu_notifier_invalidate_range_end | |
| 99 | − */ | |
| 100 | −static u64 *empty_page_table; | |
| 101 | − | |
| 102 | 95 | static void free_pasid_states(struct device_state *dev_state); |
| 103 | 96 | |
| 104 | 97 | static u16 device_id(struct pci_dev *pdev) |
| @@ -279,10 +272,8 @@ static void free_pasid_state(struct pasid_state *pasid_state) · 174 unmodified lines | ||
| 279 | 272 | |
| 280 | 273 | static void put_pasid_state(struct pasid_state *pasid_state) |
| 281 | 274 | { |
| 282 | − if (atomic_dec_and_test(&pasid_state->count)) { | |
| 283 | − put_device_state(pasid_state->device_state); | |
| 275 | + if (atomic_dec_and_test(&pasid_state->count)) | |
| 284 | 276 | wake_up(&pasid_state->wq); |
| 285 | − } | |
| 286 | 277 | } |
| 287 | 278 | |
| 288 | 279 | static void put_pasid_state_wait(struct pasid_state *pasid_state) |
| @@ -291,9 +282,7 @@ static void put_pasid_state_wait(struct pasid_state *pasid_state) · 2 unmodified lines | ||
| 291 | 282 | |
| 292 | 283 | prepare_to_wait(&pasid_state->wq, &wait, TASK_UNINTERRUPTIBLE); |
| 293 | 284 | |
| 294 | − if (atomic_dec_and_test(&pasid_state->count)) | |
| 295 | − put_device_state(pasid_state->device_state); | |
| 296 | − else | |
| 285 | + if (!atomic_dec_and_test(&pasid_state->count)) | |
| 297 | 286 | schedule(); |
| 298 | 287 | |
| 299 | 288 | finish_wait(&pasid_state->wq, &wait); |
| @@ -418,46 +407,21 @@ static void mn_invalidate_page(struct mmu_notifier *mn, · 118 unmodified lines | ||
| 418 | 407 | __mn_flush_page(mn, address); |
| 419 | 408 | } |
| 420 | 409 | |
| 421 | −static void mn_invalidate_range_start(struct mmu_notifier *mn, | |
| 422 | − struct mm_struct *mm, | |
| 423 | − unsigned long start, unsigned long end) | |
| 424 | −{ | |
| 425 | − struct pasid_state *pasid_state; | |
| 426 | − struct device_state *dev_state; | |
| 427 | − unsigned long flags; | |
| 428 | − | |
| 429 | − pasid_state = mn_to_state(mn); | |
| 430 | − dev_state = pasid_state->device_state; | |
| 431 | − | |
| 432 | − spin_lock_irqsave(&pasid_state->lock, flags); | |
| 433 | − if (pasid_state->mmu_notifier_count == 0) { | |
| 434 | − amd_iommu_domain_set_gcr3(dev_state->domain, | |
| 435 | − pasid_state->pasid, | |
| 436 | − __pa(empty_page_table)); | |
| 437 | − } | |
| 438 | − pasid_state->mmu_notifier_count += 1; | |
| 439 | − spin_unlock_irqrestore(&pasid_state->lock, flags); | |
| 440 | −} | |
| 441 | − | |
| 442 | −static void mn_invalidate_range_end(struct mmu_notifier *mn, | |
| 443 | − struct mm_struct *mm, | |
| 444 | − unsigned long start, unsigned long end) | |
| 410 | +static void mn_invalidate_range(struct mmu_notifier *mn, | |
| 411 | + struct mm_struct *mm, | |
| 412 | + unsigned long start, unsigned long end) | |
| 445 | 413 | { |
| 446 | 414 | struct pasid_state *pasid_state; |
| 447 | 415 | struct device_state *dev_state; |
| 448 | − unsigned long flags; | |
| 449 | 416 | |
| 450 | 417 | pasid_state = mn_to_state(mn); |
| 451 | 418 | dev_state = pasid_state->device_state; |
| 452 | 419 | |
| 453 | − spin_lock_irqsave(&pasid_state->lock, flags); | |
| 454 | − pasid_state->mmu_notifier_count -= 1; | |
| 455 | − if (pasid_state->mmu_notifier_count == 0) { | |
| 456 | − amd_iommu_domain_set_gcr3(dev_state->domain, | |
| 457 | − pasid_state->pasid, | |
| 458 | − __pa(pasid_state->mm->pgd)); | |
| 459 | − } | |
| 460 | − spin_unlock_irqrestore(&pasid_state->lock, flags); | |
| 420 | + if ((start ^ (end - 1)) < PAGE_SIZE) | |
| 421 | + amd_iommu_flush_page(dev_state->domain, pasid_state->pasid, | |
| 422 | + start); | |
| 423 | + else | |
| 424 | + amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid); | |
| 461 | 425 | } |
| 462 | 426 | |
| 463 | 427 | static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm) |
| @@ -482,8 +446,7 @@ static struct mmu_notifier_ops iommu_mn = { · 18 unmodified lines | ||
| 482 | 446 | .release = mn_release, |
| 483 | 447 | .clear_flush_young = mn_clear_flush_young, |
| 484 | 448 | .invalidate_page = mn_invalidate_page, |
| 485 | − .invalidate_range_start = mn_invalidate_range_start, | |
| 486 | − .invalidate_range_end = mn_invalidate_range_end, | |
| 449 | + .invalidate_range = mn_invalidate_range, | |
| 487 | 450 | }; |
| 488 | 451 | |
| 489 | 452 | static void set_pri_tag_status(struct pasid_state *pasid_state, |
| @@ -954,18 +917,10 @@ static int __init amd_iommu_v2_init(void) · 464 unmodified lines | ||
| 954 | 917 | if (iommu_wq == NULL) |
| 955 | 918 | goto out; |
| 956 | 919 | |
| 957 | − ret = -ENOMEM; | |
| 958 | − empty_page_table = (u64 *)get_zeroed_page(GFP_KERNEL); | |
| 959 | − if (empty_page_table == NULL) | |
| 960 | − goto out_destroy_wq; | |
| 961 | − | |
| 962 | 920 | amd_iommu_register_ppr_notifier(&ppr_nb); |
| 963 | 921 | |
| 964 | 922 | return 0; |
| 965 | 923 | |
| 966 | −out_destroy_wq: | |
| 967 | − destroy_workqueue(iommu_wq); | |
| 968 | − | |
| 969 | 924 | out: |
| 970 | 925 | return ret; |
| 971 | 926 | } |
| @@ -999,8 +954,6 @@ static void __exit amd_iommu_v2_exit(void) · 27 unmodified lines | ||
| 999 | 954 | } |
| 1000 | 955 | |
| 1001 | 956 | destroy_workqueue(iommu_wq); |
| 1002 | − | |
| 1003 | − free_page((unsigned long)empty_page_table); | |
| 1004 | 957 | } |
| 1005 | 958 | |
| 1006 | 959 | module_init(amd_iommu_v2_init); |
| @@ -98,11 +98,11 @@ struct mmu_notifier_ops { · 97 unmodified lines | ||
| 98 | 98 | /* |
| 99 | 99 | * invalidate_range_start() and invalidate_range_end() must be |
| 100 | 100 | * paired and are called only when the mmap_sem and/or the |
| 101 | − * locks protecting the reverse maps are held. The subsystem | |
| 102 | − * must guarantee that no additional references are taken to | |
| 103 | − * the pages in the range established between the call to | |
| 104 | − * invalidate_range_start() and the matching call to | |
| 105 | − * invalidate_range_end(). | |
| 101 | + * locks protecting the reverse maps are held. If the subsystem | |
| 102 | + * can't guarantee that no additional references are taken to | |
| 103 | + * the pages in the range, it has to implement the | |
| 104 | + * invalidate_range() notifier to remove any references taken | |
| 105 | + * after invalidate_range_start(). | |
| 106 | 106 | * |
| 107 | 107 | * Invalidation of multiple concurrent ranges may be |
| 108 | 108 | * optionally permitted by the driver. Either way the |
| @@ -144,6 +144,29 @@ struct mmu_notifier_ops { · 35 unmodified lines | ||
| 144 | 144 | void (*invalidate_range_end)(struct mmu_notifier *mn, |
| 145 | 145 | struct mm_struct *mm, |
| 146 | 146 | unsigned long start, unsigned long end); |
| 147 | + | |
| 148 | + /* | |
| 149 | + * invalidate_range() is either called between | |
| 150 | + * invalidate_range_start() and invalidate_range_end() when the | |
| 151 | + * VM has to free pages that where unmapped, but before the | |
| 152 | + * pages are actually freed, or outside of _start()/_end() when | |
| 153 | + * a (remote) TLB is necessary. | |
| 154 | + * | |
| 155 | + * If invalidate_range() is used to manage a non-CPU TLB with | |
| 156 | + * shared page-tables, it not necessary to implement the | |
| 157 | + * invalidate_range_start()/end() notifiers, as | |
| 158 | + * invalidate_range() alread catches the points in time when an | |
| 159 | + * external TLB range needs to be flushed. | |
| 160 | + * | |
| 161 | + * The invalidate_range() function is called under the ptl | |
| 162 | + * spin-lock and not allowed to sleep. | |
| 163 | + * | |
| 164 | + * Note that this function might be called with just a sub-range | |
| 165 | + * of what was passed to invalidate_range_start()/end(), if | |
| 166 | + * called between those functions. | |
| 167 | + */ | |
| 168 | + void (*invalidate_range)(struct mmu_notifier *mn, struct mm_struct *mm, | |
| 169 | + unsigned long start, unsigned long end); | |
| 147 | 170 | }; |
| 148 | 171 | |
| 149 | 172 | /* |
| @@ -190,6 +213,8 @@ extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm, · 40 unmodified lines | ||
| 190 | 213 | unsigned long start, unsigned long end); |
| 191 | 214 | extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm, |
| 192 | 215 | unsigned long start, unsigned long end); |
| 216 | +extern void __mmu_notifier_invalidate_range(struct mm_struct *mm, | |
| 217 | + unsigned long start, unsigned long end); | |
| 193 | 218 | |
| 194 | 219 | static inline void mmu_notifier_release(struct mm_struct *mm) |
| 195 | 220 | { |
| @@ -242,6 +267,13 @@ static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 46 unmodified lines | ||
| 242 | 267 | __mmu_notifier_invalidate_range_end(mm, start, end); |
| 243 | 268 | } |
| 244 | 269 | |
| 270 | +static inline void mmu_notifier_invalidate_range(struct mm_struct *mm, | |
| 271 | + unsigned long start, unsigned long end) | |
| 272 | +{ | |
| 273 | + if (mm_has_notifiers(mm)) | |
| 274 | + __mmu_notifier_invalidate_range(mm, start, end); | |
| 275 | +} | |
| 276 | + | |
| 245 | 277 | static inline void mmu_notifier_mm_init(struct mm_struct *mm) |
| 246 | 278 | { |
| 247 | 279 | mm->mmu_notifier_mm = NULL; |
| @@ -279,6 +311,44 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) · 31 unmodified lines | ||
| 279 | 311 | __young; \ |
| 280 | 312 | }) |
| 281 | 313 | |
| 314 | +#define ptep_clear_flush_notify(__vma, __address, __ptep) \ | |
| 315 | +({ \ | |
| 316 | + unsigned long ___addr = __address & PAGE_MASK; \ | |
| 317 | + struct mm_struct *___mm = (__vma)->vm_mm; \ | |
| 318 | + pte_t ___pte; \ | |
| 319 | + \ | |
| 320 | + ___pte = ptep_clear_flush(__vma, __address, __ptep); \ | |
| 321 | + mmu_notifier_invalidate_range(___mm, ___addr, \ | |
| 322 | + ___addr + PAGE_SIZE); \ | |
| 323 | + \ | |
| 324 | + ___pte; \ | |
| 325 | +}) | |
| 326 | + | |
| 327 | +#define pmdp_clear_flush_notify(__vma, __haddr, __pmd) \ | |
| 328 | +({ \ | |
| 329 | + unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \ | |
| 330 | + struct mm_struct *___mm = (__vma)->vm_mm; \ | |
| 331 | + pmd_t ___pmd; \ | |
| 332 | + \ | |
| 333 | + ___pmd = pmdp_clear_flush(__vma, __haddr, __pmd); \ | |
| 334 | + mmu_notifier_invalidate_range(___mm, ___haddr, \ | |
| 335 | + ___haddr + HPAGE_PMD_SIZE); \ | |
| 336 | + \ | |
| 337 | + ___pmd; \ | |
| 338 | +}) | |
| 339 | + | |
| 340 | +#define pmdp_get_and_clear_notify(__mm, __haddr, __pmd) \ | |
| 341 | +({ \ | |
| 342 | + unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \ | |
| 343 | + pmd_t ___pmd; \ | |
| 344 | + \ | |
| 345 | + ___pmd = pmdp_get_and_clear(__mm, __haddr, __pmd); \ | |
| 346 | + mmu_notifier_invalidate_range(__mm, ___haddr, \ | |
| 347 | + ___haddr + HPAGE_PMD_SIZE); \ | |
| 348 | + \ | |
| 349 | + ___pmd; \ | |
| 350 | +}) | |
| 351 | + | |
| 282 | 352 | /* |
| 283 | 353 | * set_pte_at_notify() sets the pte _after_ running the notifier. |
| 284 | 354 | * This is safe to start by updating the secondary MMUs, because the primary MMU |
| @@ -342,6 +412,11 @@ static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 57 unmodified lines | ||
| 342 | 412 | { |
| 343 | 413 | } |
| 344 | 414 | |
| 415 | +static inline void mmu_notifier_invalidate_range(struct mm_struct *mm, | |
| 416 | + unsigned long start, unsigned long end) | |
| 417 | +{ | |
| 418 | +} | |
| 419 | + | |
| 345 | 420 | static inline void mmu_notifier_mm_init(struct mm_struct *mm) |
| 346 | 421 | { |
| 347 | 422 | } |
| @@ -352,6 +427,9 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) · 4 unmodified lines | ||
| 352 | 427 | |
| 353 | 428 | #define ptep_clear_flush_young_notify ptep_clear_flush_young |
| 354 | 429 | #define pmdp_clear_flush_young_notify pmdp_clear_flush_young |
| 430 | +#define ptep_clear_flush_notify ptep_clear_flush | |
| 431 | +#define pmdp_clear_flush_notify pmdp_clear_flush | |
| 432 | +#define pmdp_get_and_clear_notify pmdp_get_and_clear | |
| 355 | 433 | #define set_pte_at_notify set_pte_at |
| 356 | 434 | |
| 357 | 435 | #endif /* CONFIG_MMU_NOTIFIER */ |
| @@ -0,0 +1,154 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright 2014 Advanced Micro Devices, Inc. | |
| 3 | + * | |
| 4 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | + * copy of this software and associated documentation files (the "Software"), | |
| 6 | + * to deal in the Software without restriction, including without limitation | |
| 7 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | + * Software is furnished to do so, subject to the following conditions: | |
| 10 | + * | |
| 11 | + * The above copyright notice and this permission notice shall be included in | |
| 12 | + * all copies or substantial portions of the Software. | |
| 13 | + * | |
| 14 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | + */ | |
| 22 | + | |
| 23 | +#ifndef KFD_IOCTL_H_INCLUDED | |
| 24 | +#define KFD_IOCTL_H_INCLUDED | |
| 25 | + | |
| 26 | +#include <linux/types.h> | |
| 27 | +#include <linux/ioctl.h> | |
| 28 | + | |
| 29 | +#define KFD_IOCTL_MAJOR_VERSION 1 | |
| 30 | +#define KFD_IOCTL_MINOR_VERSION 0 | |
| 31 | + | |
| 32 | +struct kfd_ioctl_get_version_args { | |
| 33 | + uint32_t major_version; /* from KFD */ | |
| 34 | + uint32_t minor_version; /* from KFD */ | |
| 35 | +}; | |
| 36 | + | |
| 37 | +/* For kfd_ioctl_create_queue_args.queue_type. */ | |
| 38 | +#define KFD_IOC_QUEUE_TYPE_COMPUTE 0 | |
| 39 | +#define KFD_IOC_QUEUE_TYPE_SDMA 1 | |
| 40 | +#define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 2 | |
| 41 | + | |
| 42 | +#define KFD_MAX_QUEUE_PERCENTAGE 100 | |
| 43 | +#define KFD_MAX_QUEUE_PRIORITY 15 | |
| 44 | + | |
| 45 | +struct kfd_ioctl_create_queue_args { | |
| 46 | + uint64_t ring_base_address; /* to KFD */ | |
| 47 | + uint64_t write_pointer_address; /* from KFD */ | |
| 48 | + uint64_t read_pointer_address; /* from KFD */ | |
| 49 | + uint64_t doorbell_offset; /* from KFD */ | |
| 50 | + | |
| 51 | + uint32_t ring_size; /* to KFD */ | |
| 52 | + uint32_t gpu_id; /* to KFD */ | |
| 53 | + uint32_t queue_type; /* to KFD */ | |
| 54 | + uint32_t queue_percentage; /* to KFD */ | |
| 55 | + uint32_t queue_priority; /* to KFD */ | |
| 56 | + uint32_t queue_id; /* from KFD */ | |
| 57 | + | |
| 58 | + uint64_t eop_buffer_address; /* to KFD */ | |
| 59 | + uint64_t eop_buffer_size; /* to KFD */ | |
| 60 | + uint64_t ctx_save_restore_address; /* to KFD */ | |
| 61 | + uint64_t ctx_save_restore_size; /* to KFD */ | |
| 62 | +}; | |
| 63 | + | |
| 64 | +struct kfd_ioctl_destroy_queue_args { | |
| 65 | + uint32_t queue_id; /* to KFD */ | |
| 66 | + uint32_t pad; | |
| 67 | +}; | |
| 68 | + | |
| 69 | +struct kfd_ioctl_update_queue_args { | |
| 70 | + uint64_t ring_base_address; /* to KFD */ | |
| 71 | + | |
| 72 | + uint32_t queue_id; /* to KFD */ | |
| 73 | + uint32_t ring_size; /* to KFD */ | |
| 74 | + uint32_t queue_percentage; /* to KFD */ | |
| 75 | + uint32_t queue_priority; /* to KFD */ | |
| 76 | +}; | |
| 77 | + | |
| 78 | +/* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */ | |
| 79 | +#define KFD_IOC_CACHE_POLICY_COHERENT 0 | |
| 80 | +#define KFD_IOC_CACHE_POLICY_NONCOHERENT 1 | |
| 81 | + | |
| 82 | +struct kfd_ioctl_set_memory_policy_args { | |
| 83 | + uint64_t alternate_aperture_base; /* to KFD */ | |
| 84 | + uint64_t alternate_aperture_size; /* to KFD */ | |
| 85 | + | |
| 86 | + uint32_t gpu_id; /* to KFD */ | |
| 87 | + uint32_t default_policy; /* to KFD */ | |
| 88 | + uint32_t alternate_policy; /* to KFD */ | |
| 89 | + uint32_t pad; | |
| 90 | +}; | |
| 91 | + | |
| 92 | +/* | |
| 93 | + * All counters are monotonic. They are used for profiling of compute jobs. | |
| 94 | + * The profiling is done by userspace. | |
| 95 | + * | |
| 96 | + * In case of GPU reset, the counter should not be affected. | |
| 97 | + */ | |
| 98 | + | |
| 99 | +struct kfd_ioctl_get_clock_counters_args { | |
| 100 | + uint64_t gpu_clock_counter; /* from KFD */ | |
| 101 | + uint64_t cpu_clock_counter; /* from KFD */ | |
| 102 | + uint64_t system_clock_counter; /* from KFD */ | |
| 103 | + uint64_t system_clock_freq; /* from KFD */ | |
| 104 | + | |
| 105 | + uint32_t gpu_id; /* to KFD */ | |
| 106 | + uint32_t pad; | |
| 107 | +}; | |
| 108 | + | |
| 109 | +#define NUM_OF_SUPPORTED_GPUS 7 | |
| 110 | + | |
| 111 | +struct kfd_process_device_apertures { | |
| 112 | + uint64_t lds_base; /* from KFD */ | |
| 113 | + uint64_t lds_limit; /* from KFD */ | |
| 114 | + uint64_t scratch_base; /* from KFD */ | |
| 115 | + uint64_t scratch_limit; /* from KFD */ | |
| 116 | + uint64_t gpuvm_base; /* from KFD */ | |
| 117 | + uint64_t gpuvm_limit; /* from KFD */ | |
| 118 | + uint32_t gpu_id; /* from KFD */ | |
| 119 | + uint32_t pad; | |
| 120 | +}; | |
| 121 | + | |
| 122 | +struct kfd_ioctl_get_process_apertures_args { | |
| 123 | + struct kfd_process_device_apertures | |
| 124 | + process_apertures[NUM_OF_SUPPORTED_GPUS];/* from KFD */ | |
| 125 | + | |
| 126 | + /* from KFD, should be in the range [1 - NUM_OF_SUPPORTED_GPUS] */ | |
| 127 | + uint32_t num_of_nodes; | |
| 128 | + uint32_t pad; | |
| 129 | +}; | |
| 130 | + | |
| 131 | +#define KFD_IOC_MAGIC 'K' | |
| 132 | + | |
| 133 | +#define KFD_IOC_GET_VERSION \ | |
| 134 | + _IOR(KFD_IOC_MAGIC, 1, struct kfd_ioctl_get_version_args) | |
| 135 | + | |
| 136 | +#define KFD_IOC_CREATE_QUEUE \ | |
| 137 | + _IOWR(KFD_IOC_MAGIC, 2, struct kfd_ioctl_create_queue_args) | |
| 138 | + | |
| 139 | +#define KFD_IOC_DESTROY_QUEUE \ | |
| 140 | + _IOWR(KFD_IOC_MAGIC, 3, struct kfd_ioctl_destroy_queue_args) | |
| 141 | + | |
| 142 | +#define KFD_IOC_SET_MEMORY_POLICY \ | |
| 143 | + _IOW(KFD_IOC_MAGIC, 4, struct kfd_ioctl_set_memory_policy_args) | |
| 144 | + | |
| 145 | +#define KFD_IOC_GET_CLOCK_COUNTERS \ | |
| 146 | + _IOWR(KFD_IOC_MAGIC, 5, struct kfd_ioctl_get_clock_counters_args) | |
| 147 | + | |
| 148 | +#define KFD_IOC_GET_PROCESS_APERTURES \ | |
| 149 | + _IOR(KFD_IOC_MAGIC, 6, struct kfd_ioctl_get_process_apertures_args) | |
| 150 | + | |
| 151 | +#define KFD_IOC_UPDATE_QUEUE \ | |
| 152 | + _IOW(KFD_IOC_MAGIC, 7, struct kfd_ioctl_update_queue_args) | |
| 153 | + | |
| 154 | +#endif | |
| @@ -193,7 +193,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, · 192 unmodified lines | ||
| 193 | 193 | } |
| 194 | 194 | |
| 195 | 195 | flush_cache_page(vma, addr, pte_pfn(*ptep)); |
| 196 | − ptep_clear_flush(vma, addr, ptep); | |
| 196 | + ptep_clear_flush_notify(vma, addr, ptep); | |
| 197 | 197 | set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot)); |
| 198 | 198 | |
| 199 | 199 | page_remove_rmap(page); |
| @@ -37,7 +37,7 @@ static void zap_pte(struct mm_struct *mm, struct vm_area_struct *vma, · 36 unmodified lines | ||
| 37 | 37 | |
| 38 | 38 | if (pte_present(pte)) { |
| 39 | 39 | flush_cache_page(vma, addr, pte_pfn(pte)); |
| 40 | − pte = ptep_clear_flush(vma, addr, ptep); | |
| 40 | + pte = ptep_clear_flush_notify(vma, addr, ptep); | |
| 41 | 41 | page = vm_normal_page(vma, addr, pte); |
| 42 | 42 | if (page) { |
| 43 | 43 | if (pte_dirty(pte)) |
| @@ -1036,7 +1036,7 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, · 1035 unmodified lines | ||
| 1036 | 1036 | goto out_free_pages; |
| 1037 | 1037 | VM_BUG_ON_PAGE(!PageHead(page), page); |
| 1038 | 1038 | |
| 1039 | − pmdp_clear_flush(vma, haddr, pmd); | |
| 1039 | + pmdp_clear_flush_notify(vma, haddr, pmd); | |
| 1040 | 1040 | /* leave pmd empty until pte is filled */ |
| 1041 | 1041 | |
| 1042 | 1042 | pgtable = pgtable_trans_huge_withdraw(mm, pmd); |
| @@ -1179,7 +1179,7 @@ alloc: · 136 unmodified lines | ||
| 1179 | 1179 | pmd_t entry; |
| 1180 | 1180 | entry = mk_huge_pmd(new_page, vma->vm_page_prot); |
| 1181 | 1181 | entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); |
| 1182 | − pmdp_clear_flush(vma, haddr, pmd); | |
| 1182 | + pmdp_clear_flush_notify(vma, haddr, pmd); | |
| 1183 | 1183 | page_add_new_anon_rmap(new_page, vma, haddr); |
| 1184 | 1184 | mem_cgroup_commit_charge(new_page, memcg, false); |
| 1185 | 1185 | lru_cache_add_active_or_unevictable(new_page, vma); |
| @@ -1512,7 +1512,7 @@ int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, · 326 unmodified lines | ||
| 1512 | 1512 | pmd_t entry; |
| 1513 | 1513 | ret = 1; |
| 1514 | 1514 | if (!prot_numa) { |
| 1515 | − entry = pmdp_get_and_clear(mm, addr, pmd); | |
| 1515 | + entry = pmdp_get_and_clear_notify(mm, addr, pmd); | |
| 1516 | 1516 | if (pmd_numa(entry)) |
| 1517 | 1517 | entry = pmd_mknonnuma(entry); |
| 1518 | 1518 | entry = pmd_modify(entry, newprot); |
| @@ -1644,6 +1644,7 @@ static int __split_huge_page_splitting(struct page *page, · 125 unmodified lines | ||
| 1644 | 1644 | * serialize against split_huge_page*. |
| 1645 | 1645 | */ |
| 1646 | 1646 | pmdp_splitting_flush(vma, address, pmd); |
| 1647 | + | |
| 1647 | 1648 | ret = 1; |
| 1648 | 1649 | spin_unlock(ptl); |
| 1649 | 1650 | } |
| @@ -2834,7 +2835,7 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, · 1184 unmodified lines | ||
| 2834 | 2835 | pmd_t _pmd; |
| 2835 | 2836 | int i; |
| 2836 | 2837 | |
| 2837 | − pmdp_clear_flush(vma, haddr, pmd); | |
| 2838 | + pmdp_clear_flush_notify(vma, haddr, pmd); | |
| 2838 | 2839 | /* leave pmd empty until pte is filled */ |
| 2839 | 2840 | |
| 2840 | 2841 | pgtable = pgtable_trans_huge_withdraw(mm, pmd); |
| @@ -2598,8 +2598,11 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, · 2597 unmodified lines | ||
| 2598 | 2598 | } |
| 2599 | 2599 | set_huge_pte_at(dst, addr, dst_pte, entry); |
| 2600 | 2600 | } else { |
| 2601 | − if (cow) | |
| 2601 | + if (cow) { | |
| 2602 | 2602 | huge_ptep_set_wrprotect(src, addr, src_pte); |
| 2603 | + mmu_notifier_invalidate_range(src, mmun_start, | |
| 2604 | + mmun_end); | |
| 2605 | + } | |
| 2603 | 2606 | entry = huge_ptep_get(src_pte); |
| 2604 | 2607 | ptepage = pte_page(entry); |
| 2605 | 2608 | get_page(ptepage); |
| @@ -2899,6 +2902,7 @@ retry_avoidcopy: · 293 unmodified lines | ||
| 2899 | 2902 | |
| 2900 | 2903 | /* Break COW */ |
| 2901 | 2904 | huge_ptep_clear_flush(vma, address, ptep); |
| 2905 | + mmu_notifier_invalidate_range(mm, mmun_start, mmun_end); | |
| 2902 | 2906 | set_huge_pte_at(mm, address, ptep, |
| 2903 | 2907 | make_huge_pte(vma, new_page, 1)); |
| 2904 | 2908 | page_remove_rmap(old_page); |
| @@ -3374,6 +3378,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, · 469 unmodified lines | ||
| 3374 | 3378 | * and that page table be reused and filled with junk. |
| 3375 | 3379 | */ |
| 3376 | 3380 | flush_tlb_range(vma, start, end); |
| 3381 | + mmu_notifier_invalidate_range(mm, start, end); | |
| 3377 | 3382 | mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex); |
| 3378 | 3383 | mmu_notifier_invalidate_range_end(mm, start, end); |
| 3379 | 3384 | |
| @@ -892,7 +892,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page, · 891 unmodified lines | ||
| 892 | 892 | * this assure us that no O_DIRECT can happen after the check |
| 893 | 893 | * or in the middle of the check. |
| 894 | 894 | */ |
| 895 | − entry = ptep_clear_flush(vma, addr, ptep); | |
| 895 | + entry = ptep_clear_flush_notify(vma, addr, ptep); | |
| 896 | 896 | /* |
| 897 | 897 | * Check that no O_DIRECT or similar I/O is in progress on the |
| 898 | 898 | * page |
| @@ -960,7 +960,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, · 61 unmodified lines | ||
| 960 | 960 | page_add_anon_rmap(kpage, vma, addr); |
| 961 | 961 | |
| 962 | 962 | flush_cache_page(vma, addr, pte_pfn(*ptep)); |
| 963 | − ptep_clear_flush(vma, addr, ptep); | |
| 963 | + ptep_clear_flush_notify(vma, addr, ptep); | |
| 964 | 964 | set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot)); |
| 965 | 965 | |
| 966 | 966 | page_remove_rmap(page); |
| @@ -238,6 +238,7 @@ static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) · 237 unmodified lines | ||
| 238 | 238 | { |
| 239 | 239 | tlb->need_flush = 0; |
| 240 | 240 | tlb_flush(tlb); |
| 241 | + mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end); | |
| 241 | 242 | #ifdef CONFIG_HAVE_RCU_TABLE_FREE |
| 242 | 243 | tlb_table_flush(tlb); |
| 243 | 244 | #endif |
| @@ -2234,7 +2235,7 @@ gotten: · 1990 unmodified lines | ||
| 2234 | 2235 | * seen in the presence of one thread doing SMC and another |
| 2235 | 2236 | * thread doing COW. |
| 2236 | 2237 | */ |
| 2237 | − ptep_clear_flush(vma, address, page_table); | |
| 2238 | + ptep_clear_flush_notify(vma, address, page_table); | |
| 2238 | 2239 | page_add_new_anon_rmap(new_page, vma, address); |
| 2239 | 2240 | mem_cgroup_commit_charge(new_page, memcg, false); |
| 2240 | 2241 | lru_cache_add_active_or_unevictable(new_page, vma); |
| @@ -1854,7 +1854,7 @@ fail_putback: · 1853 unmodified lines | ||
| 1854 | 1854 | */ |
| 1855 | 1855 | flush_cache_range(vma, mmun_start, mmun_end); |
| 1856 | 1856 | page_add_anon_rmap(new_page, vma, mmun_start); |
| 1857 | − pmdp_clear_flush(vma, mmun_start, pmd); | |
| 1857 | + pmdp_clear_flush_notify(vma, mmun_start, pmd); | |
| 1858 | 1858 | set_pmd_at(mm, mmun_start, pmd, entry); |
| 1859 | 1859 | flush_tlb_range(vma, mmun_start, mmun_end); |
| 1860 | 1860 | update_mmu_cache_pmd(vma, address, &entry); |
| @@ -1862,6 +1862,7 @@ fail_putback: · 1 unmodified line | ||
| 1862 | 1862 | if (page_count(page) != 2) { |
| 1863 | 1863 | set_pmd_at(mm, mmun_start, pmd, orig_entry); |
| 1864 | 1864 | flush_tlb_range(vma, mmun_start, mmun_end); |
| 1865 | + mmu_notifier_invalidate_range(mm, mmun_start, mmun_end); | |
| 1865 | 1866 | update_mmu_cache_pmd(vma, address, &entry); |
| 1866 | 1867 | page_remove_rmap(new_page); |
| 1867 | 1868 | goto fail_putback; |
| @@ -193,6 +193,16 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 192 unmodified lines | ||
| 193 | 193 | |
| 194 | 194 | id = srcu_read_lock(&srcu); |
| 195 | 195 | hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) { |
| 196 | + /* | |
| 197 | + * Call invalidate_range here too to avoid the need for the | |
| 198 | + * subsystem of having to register an invalidate_range_end | |
| 199 | + * call-back when there is invalidate_range already. Usually a | |
| 200 | + * subsystem registers either invalidate_range_start()/end() or | |
| 201 | + * invalidate_range(), so this will be no additional overhead | |
| 202 | + * (besides the pointer check). | |
| 203 | + */ | |
| 204 | + if (mn->ops->invalidate_range) | |
| 205 | + mn->ops->invalidate_range(mn, mm, start, end); | |
| 196 | 206 | if (mn->ops->invalidate_range_end) |
| 197 | 207 | mn->ops->invalidate_range_end(mn, mm, start, end); |
| 198 | 208 | } |
| @@ -200,6 +210,21 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm, · 1 unmodified line | ||
| 200 | 210 | } |
| 201 | 211 | EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_end); |
| 202 | 212 | |
| 213 | +void __mmu_notifier_invalidate_range(struct mm_struct *mm, | |
| 214 | + unsigned long start, unsigned long end) | |
| 215 | +{ | |
| 216 | + struct mmu_notifier *mn; | |
| 217 | + int id; | |
| 218 | + | |
| 219 | + id = srcu_read_lock(&srcu); | |
| 220 | + hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) { | |
| 221 | + if (mn->ops->invalidate_range) | |
| 222 | + mn->ops->invalidate_range(mn, mm, start, end); | |
| 223 | + } | |
| 224 | + srcu_read_unlock(&srcu, id); | |
| 225 | +} | |
| 226 | +EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range); | |
| 227 | + | |
| 203 | 228 | static int do_mmu_notifier_register(struct mmu_notifier *mn, |
| 204 | 229 | struct mm_struct *mm, |
| 205 | 230 | int take_mmap_sem) |
| @@ -1378,7 +1378,7 @@ static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount, · 1377 unmodified lines | ||
| 1378 | 1378 | |
| 1379 | 1379 | /* Nuke the page table entry. */ |
| 1380 | 1380 | flush_cache_page(vma, address, pte_pfn(*pte)); |
| 1381 | − pteval = ptep_clear_flush(vma, address, pte); | |
| 1381 | + pteval = ptep_clear_flush_notify(vma, address, pte); | |
| 1382 | 1382 | |
| 1383 | 1383 | /* If nonlinear, store the file page offset in the pte. */ |
| 1384 | 1384 | if (page->index != linear_page_index(vma, address)) { |