History

Merge tag 'arm-smmu-updates' of git://git.kernel.org/pub/scm/linux/kernel/git/will/linux into arm/smmu

Arm SMMU updates for 5.10 - Continued SVM enablement, where page-table is shared with CPU - Groundwork to support integrated SMMU with Adreno GPU - Allow disabling of MSI-based polling on the kernel command-line - Minor driver fixes and cleanups (octal permissions, error messages, ...)

Joerg Roedel committed 6 years ago

Commit 0dd4ce62 parents f75aef3 e2eae0918 files · +1410 −798Browse files

Unified · Split

18 files
  1. MAINTAINERS +1 −2
  2. arch/arm64/include/asm/barrier.h +1 −0
  3. arch/arm64/include/asm/io.h +1 −0
  4. arch/arm64/include/asm/mmu.h +3 −0
  5. arch/arm64/include/asm/mmu_context.h +10 −1
  6. arch/arm64/kernel/cpufeature.c +1 −0
  7. arch/arm64/mm/context.c +99 −6
  8. drivers/iommu/Kconfig +10 −0
  9. drivers/iommu/arm/arm-smmu-v3/Makefile +4 −1
  10. drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +248 −0
  11. drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +154 −689
  12. drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +723 −0
  13. drivers/iommu/arm/arm-smmu/arm-smmu-impl.c +2 −1
  14. drivers/iommu/arm/arm-smmu/arm-smmu.c +41 −61
  15. drivers/iommu/arm/arm-smmu/arm-smmu.h +74 −10
  16. drivers/iommu/arm/arm-smmu/qcom_iommu.c +6 −2
  17. drivers/iommu/io-pgtable-arm.c +2 −25
  18. drivers/iommu/io-pgtable-arm.h +30 −0
MAINTAINERS+1−2
@@ -1506,8 +1506,7 @@ L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) · 1505 unmodified lines
1506S: Maintained1506S: Maintained
1507F: Documentation/devicetree/bindings/iommu/arm,smmu*1507F: Documentation/devicetree/bindings/iommu/arm,smmu*
1508F: drivers/iommu/arm/1508F: drivers/iommu/arm/
1509F: drivers/iommu/io-pgtable-arm-v7s.c1509F: drivers/iommu/io-pgtable-arm*
1510F: drivers/iommu/io-pgtable-arm.c
1511 1510
1512ARM SUB-ARCHITECTURES1511ARM SUB-ARCHITECTURES
1513L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)1512L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
arch/arm64/include/asm/barrier.h+1−0
@@ -45,6 +45,7 @@ · 44 unmodified lines
45#define rmb() dsb(ld)45#define rmb() dsb(ld)
46#define wmb() dsb(st)46#define wmb() dsb(st)
47 47
48#define dma_mb() dmb(osh)
48#define dma_rmb() dmb(oshld)49#define dma_rmb() dmb(oshld)
49#define dma_wmb() dmb(oshst)50#define dma_wmb() dmb(oshst)
50 51
arch/arm64/include/asm/io.h+1−0
@@ -110,6 +110,7 @@ static inline u64 __raw_readq(const volatile void __iomem *addr) · 109 unmodified lines
110 110
111#define __io_par(v) __iormb(v)111#define __io_par(v) __iormb(v)
112#define __iowmb() dma_wmb()112#define __iowmb() dma_wmb()
113#define __iomb() dma_mb()
113 114
114/*115/*
115 * Relaxed I/O memory access primitives. These follow the Device memory116 * Relaxed I/O memory access primitives. These follow the Device memory
arch/arm64/include/asm/mmu.h+3−0
@@ -17,11 +17,14 @@ · 16 unmodified lines
17 17
18#ifndef __ASSEMBLY__18#ifndef __ASSEMBLY__
19 19
20#include <linux/refcount.h>
21
20typedef struct {22typedef struct {
21 atomic64_t id;23 atomic64_t id;
22#ifdef CONFIG_COMPAT24#ifdef CONFIG_COMPAT
23 void *sigpage;25 void *sigpage;
24#endif26#endif
27 refcount_t pinned;
25 void *vdso;28 void *vdso;
26 unsigned long flags;29 unsigned long flags;
27} mm_context_t;30} mm_context_t;
arch/arm64/include/asm/mmu_context.h+10−1
@@ -177,7 +177,13 @@ static inline void cpu_replace_ttbr1(pgd_t *pgdp) · 176 unmodified lines
177#define destroy_context(mm) do { } while(0)177#define destroy_context(mm) do { } while(0)
178void check_and_switch_context(struct mm_struct *mm);178void check_and_switch_context(struct mm_struct *mm);
179 179
180#define init_new_context(tsk,mm) ({ atomic64_set(&(mm)->context.id, 0); 0; })180static inline int
181init_new_context(struct task_struct *tsk, struct mm_struct *mm)
182{
183 atomic64_set(&mm->context.id, 0);
184 refcount_set(&mm->context.pinned, 0);
185 return 0;
186}
181 187
182#ifdef CONFIG_ARM64_SW_TTBR0_PAN188#ifdef CONFIG_ARM64_SW_TTBR0_PAN
183static inline void update_saved_ttbr0(struct task_struct *tsk,189static inline void update_saved_ttbr0(struct task_struct *tsk,
@@ -248,6 +254,9 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, · 64 unmodified lines
248void verify_cpu_asid_bits(void);254void verify_cpu_asid_bits(void);
249void post_ttbr_update_workaround(void);255void post_ttbr_update_workaround(void);
250 256
257unsigned long arm64_mm_context_get(struct mm_struct *mm);
258void arm64_mm_context_put(struct mm_struct *mm);
259
251#endif /* !__ASSEMBLY__ */260#endif /* !__ASSEMBLY__ */
252 261
253#endif /* !__ASM_MMU_CONTEXT_H */262#endif /* !__ASM_MMU_CONTEXT_H */
arch/arm64/kernel/cpufeature.c+1−0
@@ -1111,6 +1111,7 @@ u64 read_sanitised_ftr_reg(u32 id) · 1110 unmodified lines
1111 return 0;1111 return 0;
1112 return regp->sys_val;1112 return regp->sys_val;
1113}1113}
1114EXPORT_SYMBOL_GPL(read_sanitised_ftr_reg);
1114 1115
1115#define read_sysreg_case(r) \1116#define read_sysreg_case(r) \
1116 case r: return read_sysreg_s(r)1117 case r: return read_sysreg_s(r)
arch/arm64/mm/context.c+99−6
@@ -27,6 +27,10 @@ static DEFINE_PER_CPU(atomic64_t, active_asids); · 26 unmodified lines
27static DEFINE_PER_CPU(u64, reserved_asids);27static DEFINE_PER_CPU(u64, reserved_asids);
28static cpumask_t tlb_flush_pending;28static cpumask_t tlb_flush_pending;
29 29
30static unsigned long max_pinned_asids;
31static unsigned long nr_pinned_asids;
32static unsigned long *pinned_asid_map;
33
30#define ASID_MASK (~GENMASK(asid_bits - 1, 0))34#define ASID_MASK (~GENMASK(asid_bits - 1, 0))
31#define ASID_FIRST_VERSION (1UL << asid_bits)35#define ASID_FIRST_VERSION (1UL << asid_bits)
32 36
@@ -72,7 +76,7 @@ void verify_cpu_asid_bits(void) · 39 unmodified lines
72 }76 }
73}77}
74 78
75static void set_kpti_asid_bits(void)79static void set_kpti_asid_bits(unsigned long *map)
76{80{
77 unsigned int len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long);81 unsigned int len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long);
78 /*82 /*
@@ -81,13 +85,15 @@ static void set_kpti_asid_bits(void) · 2 unmodified lines
81 * is set, then the ASID will map only userspace. Thus85 * is set, then the ASID will map only userspace. Thus
82 * mark even as reserved for kernel.86 * mark even as reserved for kernel.
83 */87 */
84 memset(asid_map, 0xaa, len);88 memset(map, 0xaa, len);
85}89}
86 90
87static void set_reserved_asid_bits(void)91static void set_reserved_asid_bits(void)
88{92{
89 if (arm64_kernel_unmapped_at_el0())93 if (pinned_asid_map)
90 set_kpti_asid_bits();94 bitmap_copy(asid_map, pinned_asid_map, NUM_USER_ASIDS);
95 else if (arm64_kernel_unmapped_at_el0())
96 set_kpti_asid_bits(asid_map);
91 else97 else
92 bitmap_clear(asid_map, 0, NUM_USER_ASIDS);98 bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
93}99}
@@ -165,6 +171,14 @@ static u64 new_context(struct mm_struct *mm) · 71 unmodified lines
165 if (check_update_reserved_asid(asid, newasid))171 if (check_update_reserved_asid(asid, newasid))
166 return newasid;172 return newasid;
167 173
174 /*
175 * If it is pinned, we can keep using it. Note that reserved
176 * takes priority, because even if it is also pinned, we need to
177 * update the generation into the reserved_asids.
178 */
179 if (refcount_read(&mm->context.pinned))
180 return newasid;
181
168 /*182 /*
169 * We had a valid ASID in a previous life, so try to re-use183 * We had a valid ASID in a previous life, so try to re-use
170 * it if possible.184 * it if possible.
@@ -256,6 +270,71 @@ switch_mm_fastpath: · 85 unmodified lines
256 cpu_switch_mm(mm->pgd, mm);270 cpu_switch_mm(mm->pgd, mm);
257}271}
258 272
273unsigned long arm64_mm_context_get(struct mm_struct *mm)
274{
275 unsigned long flags;
276 u64 asid;
277
278 if (!pinned_asid_map)
279 return 0;
280
281 raw_spin_lock_irqsave(&cpu_asid_lock, flags);
282
283 asid = atomic64_read(&mm->context.id);
284
285 if (refcount_inc_not_zero(&mm->context.pinned))
286 goto out_unlock;
287
288 if (nr_pinned_asids >= max_pinned_asids) {
289 asid = 0;
290 goto out_unlock;
291 }
292
293 if (!asid_gen_match(asid)) {
294 /*
295 * We went through one or more rollover since that ASID was
296 * used. Ensure that it is still valid, or generate a new one.
297 */
298 asid = new_context(mm);
299 atomic64_set(&mm->context.id, asid);
300 }
301
302 nr_pinned_asids++;
303 __set_bit(asid2idx(asid), pinned_asid_map);
304 refcount_set(&mm->context.pinned, 1);
305
306out_unlock:
307 raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
308
309 asid &= ~ASID_MASK;
310
311 /* Set the equivalent of USER_ASID_BIT */
312 if (asid && arm64_kernel_unmapped_at_el0())
313 asid |= 1;
314
315 return asid;
316}
317EXPORT_SYMBOL_GPL(arm64_mm_context_get);
318
319void arm64_mm_context_put(struct mm_struct *mm)
320{
321 unsigned long flags;
322 u64 asid = atomic64_read(&mm->context.id);
323
324 if (!pinned_asid_map)
325 return;
326
327 raw_spin_lock_irqsave(&cpu_asid_lock, flags);
328
329 if (refcount_dec_and_test(&mm->context.pinned)) {
330 __clear_bit(asid2idx(asid), pinned_asid_map);
331 nr_pinned_asids--;
332 }
333
334 raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
335}
336EXPORT_SYMBOL_GPL(arm64_mm_context_put);
337
259/* Errata workaround post TTBRx_EL1 update. */338/* Errata workaround post TTBRx_EL1 update. */
260asmlinkage void post_ttbr_update_workaround(void)339asmlinkage void post_ttbr_update_workaround(void)
261{340{
@@ -296,8 +375,11 @@ static int asids_update_limit(void) · 34 unmodified lines
296{375{
297 unsigned long num_available_asids = NUM_USER_ASIDS;376 unsigned long num_available_asids = NUM_USER_ASIDS;
298 377
299 if (arm64_kernel_unmapped_at_el0())378 if (arm64_kernel_unmapped_at_el0()) {
300 num_available_asids /= 2;379 num_available_asids /= 2;
380 if (pinned_asid_map)
381 set_kpti_asid_bits(pinned_asid_map);
382 }
301 /*383 /*
302 * Expect allocation after rollover to fail if we don't have at least384 * Expect allocation after rollover to fail if we don't have at least
303 * one more ASID than CPUs. ASID #0 is reserved for init_mm.385 * one more ASID than CPUs. ASID #0 is reserved for init_mm.
@@ -305,6 +387,13 @@ static int asids_update_limit(void) · 1 unmodified line
305 WARN_ON(num_available_asids - 1 <= num_possible_cpus());387 WARN_ON(num_available_asids - 1 <= num_possible_cpus());
306 pr_info("ASID allocator initialised with %lu entries\n",388 pr_info("ASID allocator initialised with %lu entries\n",
307 num_available_asids);389 num_available_asids);
390
391 /*
392 * There must always be an ASID available after rollover. Ensure that,
393 * even if all CPUs have a reserved ASID and the maximum number of ASIDs
394 * are pinned, there still is at least one empty slot in the ASID map.
395 */
396 max_pinned_asids = num_available_asids - num_possible_cpus() - 2;
308 return 0;397 return 0;
309}398}
310arch_initcall(asids_update_limit);399arch_initcall(asids_update_limit);
@@ -319,13 +408,17 @@ static int asids_init(void) · 8 unmodified lines
319 panic("Failed to allocate bitmap for %lu ASIDs\n",408 panic("Failed to allocate bitmap for %lu ASIDs\n",
320 NUM_USER_ASIDS);409 NUM_USER_ASIDS);
321 410
411 pinned_asid_map = kcalloc(BITS_TO_LONGS(NUM_USER_ASIDS),
412 sizeof(*pinned_asid_map), GFP_KERNEL);
413 nr_pinned_asids = 0;
414
322 /*415 /*
323 * We cannot call set_reserved_asid_bits() here because CPU416 * We cannot call set_reserved_asid_bits() here because CPU
324 * caps are not finalized yet, so it is safer to assume KPTI417 * caps are not finalized yet, so it is safer to assume KPTI
325 * and reserve kernel ASID's from beginning.418 * and reserve kernel ASID's from beginning.
326 */419 */
327 if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0))420 if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0))
328 set_kpti_asid_bits();421 set_kpti_asid_bits(asid_map);
329 return 0;422 return 0;
330}423}
331early_initcall(asids_init);424early_initcall(asids_init);
drivers/iommu/Kconfig+10−0
@@ -308,6 +308,16 @@ config ARM_SMMU_V3 · 307 unmodified lines
308 Say Y here if your system includes an IOMMU device implementing308 Say Y here if your system includes an IOMMU device implementing
309 the ARM SMMUv3 architecture.309 the ARM SMMUv3 architecture.
310 310
311config ARM_SMMU_V3_SVA
312 bool "Shared Virtual Addressing support for the ARM SMMUv3"
313 depends on ARM_SMMU_V3
314 help
315 Support for sharing process address spaces with devices using the
316 SMMUv3.
317
318 Say Y here if your system supports SVA extensions such as PCIe PASID
319 and PRI.
320
311config S390_IOMMU321config S390_IOMMU
312 def_bool y if S390 && PCI322 def_bool y if S390 && PCI
313 depends on S390 && PCI323 depends on S390 && PCI
drivers/iommu/arm/arm-smmu-v3/Makefile+4−1
@@ -1,2 +1,5 @@
1# SPDX-License-Identifier: GPL-2.01# SPDX-License-Identifier: GPL-2.0
2obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o2obj-$(CONFIG_ARM_SMMU_V3) += arm_smmu_v3.o
3arm_smmu_v3-objs-y += arm-smmu-v3.o
4arm_smmu_v3-objs-$(CONFIG_ARM_SMMU_V3_SVA) += arm-smmu-v3-sva.o
5arm_smmu_v3-objs := $(arm_smmu_v3-objs-y)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c+248−0
@@ -0,0 +1,248 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Implementation of the IOMMU SVA API for the ARM SMMUv3
4 */
5
6#include <linux/mm.h>
7#include <linux/mmu_context.h>
8#include <linux/slab.h>
9
10#include "arm-smmu-v3.h"
11#include "../../io-pgtable-arm.h"
12
13static DEFINE_MUTEX(sva_lock);
14
15/*
16 * Check if the CPU ASID is available on the SMMU side. If a private context
17 * descriptor is using it, try to replace it.
18 */
19static struct arm_smmu_ctx_desc *
20arm_smmu_share_asid(struct mm_struct *mm, u16 asid)
21{
22 int ret;
23 u32 new_asid;
24 struct arm_smmu_ctx_desc *cd;
25 struct arm_smmu_device *smmu;
26 struct arm_smmu_domain *smmu_domain;
27
28 cd = xa_load(&arm_smmu_asid_xa, asid);
29 if (!cd)
30 return NULL;
31
32 if (cd->mm) {
33 if (WARN_ON(cd->mm != mm))
34 return ERR_PTR(-EINVAL);
35 /* All devices bound to this mm use the same cd struct. */
36 refcount_inc(&cd->refs);
37 return cd;
38 }
39
40 smmu_domain = container_of(cd, struct arm_smmu_domain, s1_cfg.cd);
41 smmu = smmu_domain->smmu;
42
43 ret = xa_alloc(&arm_smmu_asid_xa, &new_asid, cd,
44 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
45 if (ret)
46 return ERR_PTR(-ENOSPC);
47 /*
48 * Race with unmap: TLB invalidations will start targeting the new ASID,
49 * which isn't assigned yet. We'll do an invalidate-all on the old ASID
50 * later, so it doesn't matter.
51 */
52 cd->asid = new_asid;
53 /*
54 * Update ASID and invalidate CD in all associated masters. There will
55 * be some overlap between use of both ASIDs, until we invalidate the
56 * TLB.
57 */
58 arm_smmu_write_ctx_desc(smmu_domain, 0, cd);
59
60 /* Invalidate TLB entries previously associated with that context */
61 arm_smmu_tlb_inv_asid(smmu, asid);
62
63 xa_erase(&arm_smmu_asid_xa, asid);
64 return NULL;
65}
66
67__maybe_unused
68static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm)
69{
70 u16 asid;
71 int err = 0;
72 u64 tcr, par, reg;
73 struct arm_smmu_ctx_desc *cd;
74 struct arm_smmu_ctx_desc *ret = NULL;
75
76 asid = arm64_mm_context_get(mm);
77 if (!asid)
78 return ERR_PTR(-ESRCH);
79
80 cd = kzalloc(sizeof(*cd), GFP_KERNEL);
81 if (!cd) {
82 err = -ENOMEM;
83 goto out_put_context;
84 }
85
86 refcount_set(&cd->refs, 1);
87
88 mutex_lock(&arm_smmu_asid_lock);
89 ret = arm_smmu_share_asid(mm, asid);
90 if (ret) {
91 mutex_unlock(&arm_smmu_asid_lock);
92 goto out_free_cd;
93 }
94
95 err = xa_insert(&arm_smmu_asid_xa, asid, cd, GFP_KERNEL);
96 mutex_unlock(&arm_smmu_asid_lock);
97
98 if (err)
99 goto out_free_asid;
100
101 tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 64ULL - vabits_actual) |
102 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, ARM_LPAE_TCR_RGN_WBWA) |
103 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, ARM_LPAE_TCR_RGN_WBWA) |
104 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS) |
105 CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64;
106
107 switch (PAGE_SIZE) {
108 case SZ_4K:
109 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_4K);
110 break;
111 case SZ_16K:
112 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_16K);
113 break;
114 case SZ_64K:
115 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_64K);
116 break;
117 default:
118 WARN_ON(1);
119 err = -EINVAL;
120 goto out_free_asid;
121 }
122
123 reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
124 par = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_PARANGE_SHIFT);
125 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par);
126
127 cd->ttbr = virt_to_phys(mm->pgd);
128 cd->tcr = tcr;
129 /*
130 * MAIR value is pretty much constant and global, so we can just get it
131 * from the current CPU register
132 */
133 cd->mair = read_sysreg(mair_el1);
134 cd->asid = asid;
135 cd->mm = mm;
136
137 return cd;
138
139out_free_asid:
140 arm_smmu_free_asid(cd);
141out_free_cd:
142 kfree(cd);
143out_put_context:
144 arm64_mm_context_put(mm);
145 return err < 0 ? ERR_PTR(err) : ret;
146}
147
148__maybe_unused
149static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd)
150{
151 if (arm_smmu_free_asid(cd)) {
152 /* Unpin ASID */
153 arm64_mm_context_put(cd->mm);
154 kfree(cd);
155 }
156}
157
158bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
159{
160 unsigned long reg, fld;
161 unsigned long oas;
162 unsigned long asid_bits;
163 u32 feat_mask = ARM_SMMU_FEAT_BTM | ARM_SMMU_FEAT_COHERENCY;
164
165 if (vabits_actual == 52)
166 feat_mask |= ARM_SMMU_FEAT_VAX;
167
168 if ((smmu->features & feat_mask) != feat_mask)
169 return false;
170
171 if (!(smmu->pgsize_bitmap & PAGE_SIZE))
172 return false;
173
174 /*
175 * Get the smallest PA size of all CPUs (sanitized by cpufeature). We're
176 * not even pretending to support AArch32 here. Abort if the MMU outputs
177 * addresses larger than what we support.
178 */
179 reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
180 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_PARANGE_SHIFT);
181 oas = id_aa64mmfr0_parange_to_phys_shift(fld);
182 if (smmu->oas < oas)
183 return false;
184
185 /* We can support bigger ASIDs than the CPU, but not smaller */
186 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_ASID_SHIFT);
187 asid_bits = fld ? 16 : 8;
188 if (smmu->asid_bits < asid_bits)
189 return false;
190
191 /*
192 * See max_pinned_asids in arch/arm64/mm/context.c. The following is
193 * generally the maximum number of bindable processes.
194 */
195 if (arm64_kernel_unmapped_at_el0())
196 asid_bits--;
197 dev_dbg(smmu->dev, "%d shared contexts\n", (1 << asid_bits) -
198 num_possible_cpus() - 2);
199
200 return true;
201}
202
203static bool arm_smmu_iopf_supported(struct arm_smmu_master *master)
204{
205 return false;
206}
207
208bool arm_smmu_master_sva_supported(struct arm_smmu_master *master)
209{
210 if (!(master->smmu->features & ARM_SMMU_FEAT_SVA))
211 return false;
212
213 /* SSID and IOPF support are mandatory for the moment */
214 return master->ssid_bits && arm_smmu_iopf_supported(master);
215}
216
217bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master)
218{
219 bool enabled;
220
221 mutex_lock(&sva_lock);
222 enabled = master->sva_enabled;
223 mutex_unlock(&sva_lock);
224 return enabled;
225}
226
227int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
228{
229 mutex_lock(&sva_lock);
230 master->sva_enabled = true;
231 mutex_unlock(&sva_lock);
232
233 return 0;
234}
235
236int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
237{
238 mutex_lock(&sva_lock);
239 if (!list_empty(&master->bonds)) {
240 dev_err(master->dev, "cannot disable SVA, device is bound\n");
241 mutex_unlock(&sva_lock);
242 return -EBUSY;
243 }
244 master->sva_enabled = false;
245 mutex_unlock(&sva_lock);
246
247 return 0;
248}
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c+154−689
@@ -11,7 +11,6 @@ · 10 unmodified lines
11 11
12#include <linux/acpi.h>12#include <linux/acpi.h>
13#include <linux/acpi_iort.h>13#include <linux/acpi_iort.h>
14#include <linux/bitfield.h>
15#include <linux/bitops.h>14#include <linux/bitops.h>
16#include <linux/crash_dump.h>15#include <linux/crash_dump.h>
17#include <linux/delay.h>16#include <linux/delay.h>
@@ -19,7 +18,6 @@ · 1 unmodified line
19#include <linux/err.h>18#include <linux/err.h>
20#include <linux/interrupt.h>19#include <linux/interrupt.h>
21#include <linux/io-pgtable.h>20#include <linux/io-pgtable.h>
22#include <linux/iommu.h>
23#include <linux/iopoll.h>21#include <linux/iopoll.h>
24#include <linux/module.h>22#include <linux/module.h>
25#include <linux/msi.h>23#include <linux/msi.h>
@@ -33,396 +31,17 @@ · 7 unmodified lines
33 31
34#include <linux/amba/bus.h>32#include <linux/amba/bus.h>
35 33
36/* MMIO registers */34#include "arm-smmu-v3.h"
37#define ARM_SMMU_IDR0 0x0
38#define IDR0_ST_LVL GENMASK(28, 27)
39#define IDR0_ST_LVL_2LVL 1
40#define IDR0_STALL_MODEL GENMASK(25, 24)
41#define IDR0_STALL_MODEL_STALL 0
42#define IDR0_STALL_MODEL_FORCE 2
43#define IDR0_TTENDIAN GENMASK(22, 21)
44#define IDR0_TTENDIAN_MIXED 0
45#define IDR0_TTENDIAN_LE 2
46#define IDR0_TTENDIAN_BE 3
47#define IDR0_CD2L (1 << 19)
48#define IDR0_VMID16 (1 << 18)
49#define IDR0_PRI (1 << 16)
50#define IDR0_SEV (1 << 14)
51#define IDR0_MSI (1 << 13)
52#define IDR0_ASID16 (1 << 12)
53#define IDR0_ATS (1 << 10)
54#define IDR0_HYP (1 << 9)
55#define IDR0_COHACC (1 << 4)
56#define IDR0_TTF GENMASK(3, 2)
57#define IDR0_TTF_AARCH64 2
58#define IDR0_TTF_AARCH32_64 3
59#define IDR0_S1P (1 << 1)
60#define IDR0_S2P (1 << 0)
61
62#define ARM_SMMU_IDR1 0x4
63#define IDR1_TABLES_PRESET (1 << 30)
64#define IDR1_QUEUES_PRESET (1 << 29)
65#define IDR1_REL (1 << 28)
66#define IDR1_CMDQS GENMASK(25, 21)
67#define IDR1_EVTQS GENMASK(20, 16)
68#define IDR1_PRIQS GENMASK(15, 11)
69#define IDR1_SSIDSIZE GENMASK(10, 6)
70#define IDR1_SIDSIZE GENMASK(5, 0)
71
72#define ARM_SMMU_IDR3 0xc
73#define IDR3_RIL (1 << 10)
74
75#define ARM_SMMU_IDR5 0x14
76#define IDR5_STALL_MAX GENMASK(31, 16)
77#define IDR5_GRAN64K (1 << 6)
78#define IDR5_GRAN16K (1 << 5)
79#define IDR5_GRAN4K (1 << 4)
80#define IDR5_OAS GENMASK(2, 0)
81#define IDR5_OAS_32_BIT 0
82#define IDR5_OAS_36_BIT 1
83#define IDR5_OAS_40_BIT 2
84#define IDR5_OAS_42_BIT 3
85#define IDR5_OAS_44_BIT 4
86#define IDR5_OAS_48_BIT 5
87#define IDR5_OAS_52_BIT 6
88#define IDR5_VAX GENMASK(11, 10)
89#define IDR5_VAX_52_BIT 1
90
91#define ARM_SMMU_CR0 0x20
92#define CR0_ATSCHK (1 << 4)
93#define CR0_CMDQEN (1 << 3)
94#define CR0_EVTQEN (1 << 2)
95#define CR0_PRIQEN (1 << 1)
96#define CR0_SMMUEN (1 << 0)
97
98#define ARM_SMMU_CR0ACK 0x24
99
100#define ARM_SMMU_CR1 0x28
101#define CR1_TABLE_SH GENMASK(11, 10)
102#define CR1_TABLE_OC GENMASK(9, 8)
103#define CR1_TABLE_IC GENMASK(7, 6)
104#define CR1_QUEUE_SH GENMASK(5, 4)
105#define CR1_QUEUE_OC GENMASK(3, 2)
106#define CR1_QUEUE_IC GENMASK(1, 0)
107/* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
108#define CR1_CACHE_NC 0
109#define CR1_CACHE_WB 1
110#define CR1_CACHE_WT 2
111
112#define ARM_SMMU_CR2 0x2c
113#define CR2_PTM (1 << 2)
114#define CR2_RECINVSID (1 << 1)
115#define CR2_E2H (1 << 0)
116
117#define ARM_SMMU_GBPA 0x44
118#define GBPA_UPDATE (1 << 31)
119#define GBPA_ABORT (1 << 20)
120
121#define ARM_SMMU_IRQ_CTRL 0x50
122#define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
123#define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
124#define IRQ_CTRL_GERROR_IRQEN (1 << 0)
125
126#define ARM_SMMU_IRQ_CTRLACK 0x54
127
128#define ARM_SMMU_GERROR 0x60
129#define GERROR_SFM_ERR (1 << 8)
130#define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
131#define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
132#define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
133#define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
134#define GERROR_PRIQ_ABT_ERR (1 << 3)
135#define GERROR_EVTQ_ABT_ERR (1 << 2)
136#define GERROR_CMDQ_ERR (1 << 0)
137#define GERROR_ERR_MASK 0xfd
138
139#define ARM_SMMU_GERRORN 0x64
140
141#define ARM_SMMU_GERROR_IRQ_CFG0 0x68
142#define ARM_SMMU_GERROR_IRQ_CFG1 0x70
143#define ARM_SMMU_GERROR_IRQ_CFG2 0x74
144
145#define ARM_SMMU_STRTAB_BASE 0x80
146#define STRTAB_BASE_RA (1UL << 62)
147#define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
148
149#define ARM_SMMU_STRTAB_BASE_CFG 0x88
150#define STRTAB_BASE_CFG_FMT GENMASK(17, 16)
151#define STRTAB_BASE_CFG_FMT_LINEAR 0
152#define STRTAB_BASE_CFG_FMT_2LVL 1
153#define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6)
154#define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0)
155
156#define ARM_SMMU_CMDQ_BASE 0x90
157#define ARM_SMMU_CMDQ_PROD 0x98
158#define ARM_SMMU_CMDQ_CONS 0x9c
159
160#define ARM_SMMU_EVTQ_BASE 0xa0
161#define ARM_SMMU_EVTQ_PROD 0x100a8
162#define ARM_SMMU_EVTQ_CONS 0x100ac
163#define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
164#define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
165#define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
166
167#define ARM_SMMU_PRIQ_BASE 0xc0
168#define ARM_SMMU_PRIQ_PROD 0x100c8
169#define ARM_SMMU_PRIQ_CONS 0x100cc
170#define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
171#define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
172#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
173
174#define ARM_SMMU_REG_SZ 0xe00
175
176/* Common MSI config fields */
177#define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
178#define MSI_CFG2_SH GENMASK(5, 4)
179#define MSI_CFG2_MEMATTR GENMASK(3, 0)
180
181/* Common memory attribute values */
182#define ARM_SMMU_SH_NSH 0
183#define ARM_SMMU_SH_OSH 2
184#define ARM_SMMU_SH_ISH 3
185#define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1
186#define ARM_SMMU_MEMATTR_OIWB 0xf
187
188#define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1))
189#define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift))
190#define Q_OVERFLOW_FLAG (1U << 31)
191#define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG)
192#define Q_ENT(q, p) ((q)->base + \
193 Q_IDX(&((q)->llq), p) * \
194 (q)->ent_dwords)
195
196#define Q_BASE_RWA (1UL << 62)
197#define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
198#define Q_BASE_LOG2SIZE GENMASK(4, 0)
199
200/* Ensure DMA allocations are naturally aligned */
201#ifdef CONFIG_CMA_ALIGNMENT
202#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
203#else
204#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1)
205#endif
206
207/*
208 * Stream table.
209 *
210 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
211 * 2lvl: 128k L1 entries,
212 * 256 lazy entries per table (each table covers a PCI bus)
213 */
214#define STRTAB_L1_SZ_SHIFT 20
215#define STRTAB_SPLIT 8
216
217#define STRTAB_L1_DESC_DWORDS 1
218#define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0)
219#define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6)
220
221#define STRTAB_STE_DWORDS 8
222#define STRTAB_STE_0_V (1UL << 0)
223#define STRTAB_STE_0_CFG GENMASK_ULL(3, 1)
224#define STRTAB_STE_0_CFG_ABORT 0
225#define STRTAB_STE_0_CFG_BYPASS 4
226#define STRTAB_STE_0_CFG_S1_TRANS 5
227#define STRTAB_STE_0_CFG_S2_TRANS 6
228
229#define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)
230#define STRTAB_STE_0_S1FMT_LINEAR 0
231#define STRTAB_STE_0_S1FMT_64K_L2 2
232#define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
233#define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)
234
235#define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0)
236#define STRTAB_STE_1_S1DSS_TERMINATE 0x0
237#define STRTAB_STE_1_S1DSS_BYPASS 0x1
238#define STRTAB_STE_1_S1DSS_SSID0 0x2
239
240#define STRTAB_STE_1_S1C_CACHE_NC 0UL
241#define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
242#define STRTAB_STE_1_S1C_CACHE_WT 2UL
243#define STRTAB_STE_1_S1C_CACHE_WB 3UL
244#define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2)
245#define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4)
246#define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6)
247
248#define STRTAB_STE_1_S1STALLD (1UL << 27)
249
250#define STRTAB_STE_1_EATS GENMASK_ULL(29, 28)
251#define STRTAB_STE_1_EATS_ABT 0UL
252#define STRTAB_STE_1_EATS_TRANS 1UL
253#define STRTAB_STE_1_EATS_S1CHK 2UL
254
255#define STRTAB_STE_1_STRW GENMASK_ULL(31, 30)
256#define STRTAB_STE_1_STRW_NSEL1 0UL
257#define STRTAB_STE_1_STRW_EL2 2UL
258
259#define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44)
260#define STRTAB_STE_1_SHCFG_INCOMING 1UL
261
262#define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
263#define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
264#define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0)
265#define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6)
266#define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8)
267#define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10)
268#define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12)
269#define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14)
270#define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16)
271#define STRTAB_STE_2_S2AA64 (1UL << 51)
272#define STRTAB_STE_2_S2ENDI (1UL << 52)
273#define STRTAB_STE_2_S2PTW (1UL << 54)
274#define STRTAB_STE_2_S2R (1UL << 58)
275
276#define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)
277
278/*
279 * Context descriptors.
280 *
281 * Linear: when less than 1024 SSIDs are supported
282 * 2lvl: at most 1024 L1 entries,
283 * 1024 lazy entries per table.
284 */
285#define CTXDESC_SPLIT 10
286#define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT)
287
288#define CTXDESC_L1_DESC_DWORDS 1
289#define CTXDESC_L1_DESC_V (1UL << 0)
290#define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12)
291
292#define CTXDESC_CD_DWORDS 8
293#define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
294#define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
295#define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
296#define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
297#define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
298#define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)
299#define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)
300
301#define CTXDESC_CD_0_ENDI (1UL << 15)
302#define CTXDESC_CD_0_V (1UL << 31)
303
304#define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)
305#define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)
306
307#define CTXDESC_CD_0_AA64 (1UL << 41)
308#define CTXDESC_CD_0_S (1UL << 44)
309#define CTXDESC_CD_0_R (1UL << 45)
310#define CTXDESC_CD_0_A (1UL << 46)
311#define CTXDESC_CD_0_ASET (1UL << 47)
312#define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48)
313
314#define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
315
316/*
317 * When the SMMU only supports linear context descriptor tables, pick a
318 * reasonable size limit (64kB).
319 */
320#define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
321
322/* Command queue */
323#define CMDQ_ENT_SZ_SHIFT 4
324#define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
325#define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
326
327#define CMDQ_CONS_ERR GENMASK(30, 24)
328#define CMDQ_ERR_CERROR_NONE_IDX 0
329#define CMDQ_ERR_CERROR_ILL_IDX 1
330#define CMDQ_ERR_CERROR_ABT_IDX 2
331#define CMDQ_ERR_CERROR_ATC_INV_IDX 3
332
333#define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG
334
335/*
336 * This is used to size the command queue and therefore must be at least
337 * BITS_PER_LONG so that the valid_map works correctly (it relies on the
338 * total number of queue entries being a multiple of BITS_PER_LONG).
339 */
340#define CMDQ_BATCH_ENTRIES BITS_PER_LONG
341
342#define CMDQ_0_OP GENMASK_ULL(7, 0)
343#define CMDQ_0_SSV (1UL << 11)
344
345#define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32)
346#define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)
347#define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)
348
349#define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12)
350#define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)
351#define CMDQ_CFGI_1_LEAF (1UL << 0)
352#define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)
353
354#define CMDQ_TLBI_0_NUM GENMASK_ULL(16, 12)
355#define CMDQ_TLBI_RANGE_NUM_MAX 31
356#define CMDQ_TLBI_0_SCALE GENMASK_ULL(24, 20)
357#define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32)
358#define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48)
359#define CMDQ_TLBI_1_LEAF (1UL << 0)
360#define CMDQ_TLBI_1_TTL GENMASK_ULL(9, 8)
361#define CMDQ_TLBI_1_TG GENMASK_ULL(11, 10)
362#define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12)
363#define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12)
364
365#define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12)
366#define CMDQ_ATC_0_SID GENMASK_ULL(63, 32)
367#define CMDQ_ATC_0_GLOBAL (1UL << 9)
368#define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0)
369#define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12)
370
371#define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12)
372#define CMDQ_PRI_0_SID GENMASK_ULL(63, 32)
373#define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
374#define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12)
375
376#define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12)
377#define CMDQ_SYNC_0_CS_NONE 0
378#define CMDQ_SYNC_0_CS_IRQ 1
379#define CMDQ_SYNC_0_CS_SEV 2
380#define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22)
381#define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24)
382#define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32)
383#define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2)
384
385/* Event queue */
386#define EVTQ_ENT_SZ_SHIFT 5
387#define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
388#define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
389
390#define EVTQ_0_ID GENMASK_ULL(7, 0)
391
392/* PRI queue */
393#define PRIQ_ENT_SZ_SHIFT 4
394#define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
395#define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
396
397#define PRIQ_0_SID GENMASK_ULL(31, 0)
398#define PRIQ_0_SSID GENMASK_ULL(51, 32)
399#define PRIQ_0_PERM_PRIV (1UL << 58)
400#define PRIQ_0_PERM_EXEC (1UL << 59)
401#define PRIQ_0_PERM_READ (1UL << 60)
402#define PRIQ_0_PERM_WRITE (1UL << 61)
403#define PRIQ_0_PRG_LAST (1UL << 62)
404#define PRIQ_0_SSID_V (1UL << 63)
405
406#define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0)
407#define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12)
408
409/* High-level queue structures */
410#define ARM_SMMU_POLL_TIMEOUT_US 1000000 /* 1s! */
411#define ARM_SMMU_POLL_SPIN_COUNT 10
412
413#define MSI_IOVA_BASE 0x8000000
414#define MSI_IOVA_LENGTH 0x100000
415 35
416static bool disable_bypass = 1;36static bool disable_bypass = 1;
417module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);37module_param(disable_bypass, bool, 0444);
418MODULE_PARM_DESC(disable_bypass,38MODULE_PARM_DESC(disable_bypass,
419 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");39 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
420 40
421enum pri_resp {41static bool disable_msipolling;
422 PRI_RESP_DENY = 0,42module_param(disable_msipolling, bool, 0444);
423 PRI_RESP_FAIL = 1,43MODULE_PARM_DESC(disable_msipolling,
424 PRI_RESP_SUCC = 2,44 "Disable MSI-based polling for CMD_SYNC completion.");
425};
426 45
427enum arm_smmu_msi_index {46enum arm_smmu_msi_index {
428 EVTQ_MSI_INDEX,47 EVTQ_MSI_INDEX,
@@ -449,284 +68,13 @@ static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = { · 20 unmodified lines
449 },68 },
450};69};
451 70
452struct arm_smmu_cmdq_ent {
453 /* Common fields */
454 u8 opcode;
455 bool substream_valid;
456
457 /* Command-specific fields */
458 union {
459 #define CMDQ_OP_PREFETCH_CFG 0x1
460 struct {
461 u32 sid;
462 u8 size;
463 u64 addr;
464 } prefetch;
465
466 #define CMDQ_OP_CFGI_STE 0x3
467 #define CMDQ_OP_CFGI_ALL 0x4
468 #define CMDQ_OP_CFGI_CD 0x5
469 #define CMDQ_OP_CFGI_CD_ALL 0x6
470 struct {
471 u32 sid;
472 u32 ssid;
473 union {
474 bool leaf;
475 u8 span;
476 };
477 } cfgi;
478
479 #define CMDQ_OP_TLBI_NH_ASID 0x11
480 #define CMDQ_OP_TLBI_NH_VA 0x12
481 #define CMDQ_OP_TLBI_EL2_ALL 0x20
482 #define CMDQ_OP_TLBI_S12_VMALL 0x28
483 #define CMDQ_OP_TLBI_S2_IPA 0x2a
484 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
485 struct {
486 u8 num;
487 u8 scale;
488 u16 asid;
489 u16 vmid;
490 bool leaf;
491 u8 ttl;
492 u8 tg;
493 u64 addr;
494 } tlbi;
495
496 #define CMDQ_OP_ATC_INV 0x40
497 #define ATC_INV_SIZE_ALL 52
498 struct {
499 u32 sid;
500 u32 ssid;
501 u64 addr;
502 u8 size;
503 bool global;
504 } atc;
505
506 #define CMDQ_OP_PRI_RESP 0x41
507 struct {
508 u32 sid;
509 u32 ssid;
510 u16 grpid;
511 enum pri_resp resp;
512 } pri;
513
514 #define CMDQ_OP_CMD_SYNC 0x46
515 struct {
516 u64 msiaddr;
517 } sync;
518 };
519};
520
521struct arm_smmu_ll_queue {
522 union {
523 u64 val;
524 struct {
525 u32 prod;
526 u32 cons;
527 };
528 struct {
529 atomic_t prod;
530 atomic_t cons;
531 } atomic;
532 u8 __pad[SMP_CACHE_BYTES];
533 } ____cacheline_aligned_in_smp;
534 u32 max_n_shift;
535};
536
537struct arm_smmu_queue {
538 struct arm_smmu_ll_queue llq;
539 int irq; /* Wired interrupt */
540
541 __le64 *base;
542 dma_addr_t base_dma;
543 u64 q_base;
544
545 size_t ent_dwords;
546
547 u32 __iomem *prod_reg;
548 u32 __iomem *cons_reg;
549};
550
551struct arm_smmu_queue_poll {
552 ktime_t timeout;
553 unsigned int delay;
554 unsigned int spin_cnt;
555 bool wfe;
556};
557
558struct arm_smmu_cmdq {
559 struct arm_smmu_queue q;
560 atomic_long_t *valid_map;
561 atomic_t owner_prod;
562 atomic_t lock;
563};
564
565struct arm_smmu_cmdq_batch {
566 u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
567 int num;
568};
569
570struct arm_smmu_evtq {
571 struct arm_smmu_queue q;
572 u32 max_stalls;
573};
574
575struct arm_smmu_priq {
576 struct arm_smmu_queue q;
577};
578
579/* High-level stream table and context descriptor structures */
580struct arm_smmu_strtab_l1_desc {
581 u8 span;
582
583 __le64 *l2ptr;
584 dma_addr_t l2ptr_dma;
585};
586
587struct arm_smmu_ctx_desc {
588 u16 asid;
589 u64 ttbr;
590 u64 tcr;
591 u64 mair;
592};
593
594struct arm_smmu_l1_ctx_desc {
595 __le64 *l2ptr;
596 dma_addr_t l2ptr_dma;
597};
598
599struct arm_smmu_ctx_desc_cfg {
600 __le64 *cdtab;
601 dma_addr_t cdtab_dma;
602 struct arm_smmu_l1_ctx_desc *l1_desc;
603 unsigned int num_l1_ents;
604};
605
606struct arm_smmu_s1_cfg {
607 struct arm_smmu_ctx_desc_cfg cdcfg;
608 struct arm_smmu_ctx_desc cd;
609 u8 s1fmt;
610 u8 s1cdmax;
611};
612
613struct arm_smmu_s2_cfg {
614 u16 vmid;
615 u64 vttbr;
616 u64 vtcr;
617};
618
619struct arm_smmu_strtab_cfg {
620 __le64 *strtab;
621 dma_addr_t strtab_dma;
622 struct arm_smmu_strtab_l1_desc *l1_desc;
623 unsigned int num_l1_ents;
624
625 u64 strtab_base;
626 u32 strtab_base_cfg;
627};
628
629/* An SMMUv3 instance */
630struct arm_smmu_device {
631 struct device *dev;
632 void __iomem *base;
633 void __iomem *page1;
634
635#define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
636#define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
637#define ARM_SMMU_FEAT_TT_LE (1 << 2)
638#define ARM_SMMU_FEAT_TT_BE (1 << 3)
639#define ARM_SMMU_FEAT_PRI (1 << 4)
640#define ARM_SMMU_FEAT_ATS (1 << 5)
641#define ARM_SMMU_FEAT_SEV (1 << 6)
642#define ARM_SMMU_FEAT_MSI (1 << 7)
643#define ARM_SMMU_FEAT_COHERENCY (1 << 8)
644#define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
645#define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
646#define ARM_SMMU_FEAT_STALLS (1 << 11)
647#define ARM_SMMU_FEAT_HYP (1 << 12)
648#define ARM_SMMU_FEAT_STALL_FORCE (1 << 13)
649#define ARM_SMMU_FEAT_VAX (1 << 14)
650#define ARM_SMMU_FEAT_RANGE_INV (1 << 15)
651 u32 features;
652
653#define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
654#define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1)
655 u32 options;
656
657 struct arm_smmu_cmdq cmdq;
658 struct arm_smmu_evtq evtq;
659 struct arm_smmu_priq priq;
660
661 int gerr_irq;
662 int combined_irq;
663
664 unsigned long ias; /* IPA */
665 unsigned long oas; /* PA */
666 unsigned long pgsize_bitmap;
667
668#define ARM_SMMU_MAX_ASIDS (1 << 16)
669 unsigned int asid_bits;
670
671#define ARM_SMMU_MAX_VMIDS (1 << 16)
672 unsigned int vmid_bits;
673 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
674
675 unsigned int ssid_bits;
676 unsigned int sid_bits;
677
678 struct arm_smmu_strtab_cfg strtab_cfg;
679
680 /* IOMMU core code handle */
681 struct iommu_device iommu;
682};
683
684/* SMMU private data for each master */
685struct arm_smmu_master {
686 struct arm_smmu_device *smmu;
687 struct device *dev;
688 struct arm_smmu_domain *domain;
689 struct list_head domain_head;
690 u32 *sids;
691 unsigned int num_sids;
692 bool ats_enabled;
693 unsigned int ssid_bits;
694};
695
696/* SMMU private data for an IOMMU domain */
697enum arm_smmu_domain_stage {
698 ARM_SMMU_DOMAIN_S1 = 0,
699 ARM_SMMU_DOMAIN_S2,
700 ARM_SMMU_DOMAIN_NESTED,
701 ARM_SMMU_DOMAIN_BYPASS,
702};
703
704struct arm_smmu_domain {
705 struct arm_smmu_device *smmu;
706 struct mutex init_mutex; /* Protects smmu pointer */
707
708 struct io_pgtable_ops *pgtbl_ops;
709 bool non_strict;
710 atomic_t nr_ats_masters;
711
712 enum arm_smmu_domain_stage stage;
713 union {
714 struct arm_smmu_s1_cfg s1_cfg;
715 struct arm_smmu_s2_cfg s2_cfg;
716 };
717
718 struct iommu_domain domain;
719
720 struct list_head devices;
721 spinlock_t devices_lock;
722};
723
724struct arm_smmu_option_prop {71struct arm_smmu_option_prop {
725 u32 opt;72 u32 opt;
726 const char *prop;73 const char *prop;
727};74};
728 75
729static DEFINE_XARRAY_ALLOC1(asid_xa);76DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa);
77DEFINE_MUTEX(arm_smmu_asid_lock);
730 78
731static struct arm_smmu_option_prop arm_smmu_options[] = {79static struct arm_smmu_option_prop arm_smmu_options[] = {
732 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },80 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
@@ -804,7 +152,7 @@ static void queue_sync_cons_out(struct arm_smmu_queue *q) · 71 unmodified lines
804 * Ensure that all CPU accesses (reads and writes) to the queue152 * Ensure that all CPU accesses (reads and writes) to the queue
805 * are complete before we update the cons pointer.153 * are complete before we update the cons pointer.
806 */154 */
807 mb();155 __iomb();
808 writel_relaxed(q->llq.cons, q->cons_reg);156 writel_relaxed(q->llq.cons, q->cons_reg);
809}157}
810 158
@@ -816,8 +164,15 @@ static void queue_inc_cons(struct arm_smmu_ll_queue *q) · 5 unmodified lines
816 164
817static int queue_sync_prod_in(struct arm_smmu_queue *q)165static int queue_sync_prod_in(struct arm_smmu_queue *q)
818{166{
167 u32 prod;
819 int ret = 0;168 int ret = 0;
820 u32 prod = readl_relaxed(q->prod_reg);169
170 /*
171 * We can't use the _relaxed() variant here, as we must prevent
172 * speculative reads of the queue before we have determined that
173 * prod has indeed moved.
174 */
175 prod = readl(q->prod_reg);
821 176
822 if (Q_OVF(prod) != Q_OVF(q->llq.prod))177 if (Q_OVF(prod) != Q_OVF(q->llq.prod))
823 ret = -EOVERFLOW;178 ret = -EOVERFLOW;
@@ -867,7 +222,7 @@ static void queue_write(__le64 *dst, u64 *src, size_t n_dwords) · 43 unmodified lines
867 *dst++ = cpu_to_le64(*src++);222 *dst++ = cpu_to_le64(*src++);
868}223}
869 224
870static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)225static void queue_read(u64 *dst, __le64 *src, size_t n_dwords)
871{226{
872 int i;227 int i;
873 228
@@ -992,8 +347,7 @@ static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu, · 118 unmodified lines
992 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI347 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
993 * payload, so the write will zero the entire command on that platform.348 * payload, so the write will zero the entire command on that platform.
994 */349 */
995 if (smmu->features & ARM_SMMU_FEAT_MSI &&350 if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
996 smmu->features & ARM_SMMU_FEAT_COHERENCY) {
997 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *351 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
998 q->ent_dwords * 8;352 q->ent_dwords * 8;
999 }353 }
@@ -1331,8 +685,7 @@ static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu, · 331 unmodified lines
1331static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,685static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
1332 struct arm_smmu_ll_queue *llq)686 struct arm_smmu_ll_queue *llq)
1333{687{
1334 if (smmu->features & ARM_SMMU_FEAT_MSI &&688 if (smmu->options & ARM_SMMU_OPT_MSIPOLL)
1335 smmu->features & ARM_SMMU_FEAT_COHERENCY)
1336 return __arm_smmu_cmdq_poll_until_msi(smmu, llq);689 return __arm_smmu_cmdq_poll_until_msi(smmu, llq);
1337 690
1338 return __arm_smmu_cmdq_poll_until_consumed(smmu, llq);691 return __arm_smmu_cmdq_poll_until_consumed(smmu, llq);
@@ -1529,6 +882,17 @@ static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, · 190 unmodified lines
1529}882}
1530 883
1531/* Context descriptor manipulation functions */884/* Context descriptor manipulation functions */
885void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
886{
887 struct arm_smmu_cmdq_ent cmd = {
888 .opcode = CMDQ_OP_TLBI_NH_ASID,
889 .tlbi.asid = asid,
890 };
891
892 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
893 arm_smmu_cmdq_issue_sync(smmu);
894}
895
1532static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,896static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
1533 int ssid, bool leaf)897 int ssid, bool leaf)
1534{898{
@@ -1609,8 +973,8 @@ static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_domain *smmu_domain, · 74 unmodified lines
1609 return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS;973 return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS;
1610}974}
1611 975
1612static int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain,976int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
1613 int ssid, struct arm_smmu_ctx_desc *cd)977 struct arm_smmu_ctx_desc *cd)
1614{978{
1615 /*979 /*
1616 * This function handles the following cases:980 * This function handles the following cases:
@@ -1661,7 +1025,8 @@ static int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, · 44 unmodified lines
1661#ifdef __BIG_ENDIAN1025#ifdef __BIG_ENDIAN
1662 CTXDESC_CD_0_ENDI |1026 CTXDESC_CD_0_ENDI |
1663#endif1027#endif
1664 CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET |1028 CTXDESC_CD_0_R | CTXDESC_CD_0_A |
1029 (cd->mm ? 0 : CTXDESC_CD_0_ASET) |
1665 CTXDESC_CD_0_AA64 |1030 CTXDESC_CD_0_AA64 |
1666 FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) |1031 FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) |
1667 CTXDESC_CD_0_V;1032 CTXDESC_CD_0_V;
@@ -1765,12 +1130,20 @@ static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain) · 97 unmodified lines
1765 cdcfg->cdtab = NULL;1130 cdcfg->cdtab = NULL;
1766}1131}
1767 1132
1768static void arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd)1133bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd)
1769{1134{
1135 bool free;
1136 struct arm_smmu_ctx_desc *old_cd;
1137
1770 if (!cd->asid)1138 if (!cd->asid)
1771 return;1139 return false;
1772 1140
1773 xa_erase(&asid_xa, cd->asid);1141 free = refcount_dec_and_test(&cd->refs);
1142 if (free) {
1143 old_cd = xa_erase(&arm_smmu_asid_xa, cd->asid);
1144 WARN_ON(old_cd != cd);
1145 }
1146 return free;
1774}1147}
1775 1148
1776/* Stream table manipulation functions */1149/* Stream table manipulation functions */
@@ -1939,7 +1312,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, · 162 unmodified lines
1939 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);1312 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1940}1313}
1941 1314
1942static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)1315static void arm_smmu_init_bypass_stes(__le64 *strtab, unsigned int nent)
1943{1316{
1944 unsigned int i;1317 unsigned int i;
1945 1318
@@ -2257,15 +1630,6 @@ static void arm_smmu_tlb_inv_context(void *cookie) · 311 unmodified lines
2257 struct arm_smmu_device *smmu = smmu_domain->smmu;1630 struct arm_smmu_device *smmu = smmu_domain->smmu;
2258 struct arm_smmu_cmdq_ent cmd;1631 struct arm_smmu_cmdq_ent cmd;
2259 1632
2260 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2261 cmd.opcode = CMDQ_OP_TLBI_NH_ASID;
2262 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
2263 cmd.tlbi.vmid = 0;
2264 } else {
2265 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
2266 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
2267 }
2268
2269 /*1633 /*
2270 * NOTE: when io-pgtable is in non-strict mode, we may get here with1634 * NOTE: when io-pgtable is in non-strict mode, we may get here with
2271 * PTEs previously cleared by unmaps on the current CPU not yet visible1635 * PTEs previously cleared by unmaps on the current CPU not yet visible
@@ -2273,8 +1637,14 @@ static void arm_smmu_tlb_inv_context(void *cookie) · 1 unmodified line
2273 * insertion to guarantee those are observed before the TLBI. Do be1637 * insertion to guarantee those are observed before the TLBI. Do be
2274 * careful, 007.1638 * careful, 007.
2275 */1639 */
2276 arm_smmu_cmdq_issue_cmd(smmu, &cmd);1640 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2277 arm_smmu_cmdq_issue_sync(smmu);1641 arm_smmu_tlb_inv_asid(smmu, smmu_domain->s1_cfg.cd.asid);
1642 } else {
1643 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
1644 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1645 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1646 arm_smmu_cmdq_issue_sync(smmu);
1647 }
2278 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);1648 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
2279}1649}
2280 1650
@@ -2458,9 +1828,12 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) · 177 unmodified lines
2458 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {1828 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2459 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;1829 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2460 1830
1831 /* Prevent SVA from touching the CD while we're freeing it */
1832 mutex_lock(&arm_smmu_asid_lock);
2461 if (cfg->cdcfg.cdtab)1833 if (cfg->cdcfg.cdtab)
2462 arm_smmu_free_cd_tables(smmu_domain);1834 arm_smmu_free_cd_tables(smmu_domain);
2463 arm_smmu_free_asid(&cfg->cd);1835 arm_smmu_free_asid(&cfg->cd);
1836 mutex_unlock(&arm_smmu_asid_lock);
2464 } else {1837 } else {
2465 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;1838 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2466 if (cfg->vmid)1839 if (cfg->vmid)
@@ -2480,10 +1853,14 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain, · 13 unmodified lines
2480 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;1853 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2481 typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;1854 typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
2482 1855
2483 ret = xa_alloc(&asid_xa, &asid, &cfg->cd,1856 refcount_set(&cfg->cd.refs, 1);
1857
1858 /* Prevent SVA from modifying the ASID until it is written to the CD */
1859 mutex_lock(&arm_smmu_asid_lock);
1860 ret = xa_alloc(&arm_smmu_asid_xa, &asid, &cfg->cd,
2484 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);1861 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
2485 if (ret)1862 if (ret)
2486 return ret;1863 goto out_unlock;
2487 1864
2488 cfg->s1cdmax = master->ssid_bits;1865 cfg->s1cdmax = master->ssid_bits;
2489 1866
@@ -2511,12 +1888,15 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain, · 21 unmodified lines
2511 if (ret)1888 if (ret)
2512 goto out_free_cd_tables;1889 goto out_free_cd_tables;
2513 1890
1891 mutex_unlock(&arm_smmu_asid_lock);
2514 return 0;1892 return 0;
2515 1893
2516out_free_cd_tables:1894out_free_cd_tables:
2517 arm_smmu_free_cd_tables(smmu_domain);1895 arm_smmu_free_cd_tables(smmu_domain);
2518out_free_asid:1896out_free_asid:
2519 arm_smmu_free_asid(&cfg->cd);1897 arm_smmu_free_asid(&cfg->cd);
1898out_unlock:
1899 mutex_unlock(&arm_smmu_asid_lock);
2520 return ret;1900 return ret;
2521}1901}
2522 1902
@@ -2796,6 +2176,16 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) · 273 unmodified lines
2796 master = dev_iommu_priv_get(dev);2176 master = dev_iommu_priv_get(dev);
2797 smmu = master->smmu;2177 smmu = master->smmu;
2798 2178
2179 /*
2180 * Checking that SVA is disabled ensures that this device isn't bound to
2181 * any mm, and can be safely detached from its old domain. Bonds cannot
2182 * be removed concurrently since we're holding the group mutex.
2183 */
2184 if (arm_smmu_master_sva_enabled(master)) {
2185 dev_err(dev, "cannot attach - SVA enabled\n");
2186 return -EBUSY;
2187 }
2188
2799 arm_smmu_detach_dev(master);2189 arm_smmu_detach_dev(master);
2800 2190
2801 mutex_lock(&smmu_domain->init_mutex);2191 mutex_lock(&smmu_domain->init_mutex);
@@ -2943,6 +2333,7 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev) · 141 unmodified lines
2943 master->smmu = smmu;2333 master->smmu = smmu;
2944 master->sids = fwspec->ids;2334 master->sids = fwspec->ids;
2945 master->num_sids = fwspec->num_ids;2335 master->num_sids = fwspec->num_ids;
2336 INIT_LIST_HEAD(&master->bonds);
2946 dev_iommu_priv_set(dev, master);2337 dev_iommu_priv_set(dev, master);
2947 2338
2948 /* Check the SIDs are in range of the SMMU and our stream table */2339 /* Check the SIDs are in range of the SMMU and our stream table */
@@ -2995,6 +2386,7 @@ static void arm_smmu_release_device(struct device *dev) · 46 unmodified lines
2995 return;2386 return;
2996 2387
2997 master = dev_iommu_priv_get(dev);2388 master = dev_iommu_priv_get(dev);
2389 WARN_ON(arm_smmu_master_sva_enabled(master));
2998 arm_smmu_detach_dev(master);2390 arm_smmu_detach_dev(master);
2999 arm_smmu_disable_pasid(master);2391 arm_smmu_disable_pasid(master);
3000 kfree(master);2392 kfree(master);
@@ -3112,6 +2504,69 @@ static void arm_smmu_get_resv_regions(struct device *dev, · 111 unmodified lines
3112 iommu_dma_get_resv_regions(dev, head);2504 iommu_dma_get_resv_regions(dev, head);
3113}2505}
3114 2506
2507static bool arm_smmu_dev_has_feature(struct device *dev,
2508 enum iommu_dev_features feat)
2509{
2510 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2511
2512 if (!master)
2513 return false;
2514
2515 switch (feat) {
2516 case IOMMU_DEV_FEAT_SVA:
2517 return arm_smmu_master_sva_supported(master);
2518 default:
2519 return false;
2520 }
2521}
2522
2523static bool arm_smmu_dev_feature_enabled(struct device *dev,
2524 enum iommu_dev_features feat)
2525{
2526 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2527
2528 if (!master)
2529 return false;
2530
2531 switch (feat) {
2532 case IOMMU_DEV_FEAT_SVA:
2533 return arm_smmu_master_sva_enabled(master);
2534 default:
2535 return false;
2536 }
2537}
2538
2539static int arm_smmu_dev_enable_feature(struct device *dev,
2540 enum iommu_dev_features feat)
2541{
2542 if (!arm_smmu_dev_has_feature(dev, feat))
2543 return -ENODEV;
2544
2545 if (arm_smmu_dev_feature_enabled(dev, feat))
2546 return -EBUSY;
2547
2548 switch (feat) {
2549 case IOMMU_DEV_FEAT_SVA:
2550 return arm_smmu_master_enable_sva(dev_iommu_priv_get(dev));
2551 default:
2552 return -EINVAL;
2553 }
2554}
2555
2556static int arm_smmu_dev_disable_feature(struct device *dev,
2557 enum iommu_dev_features feat)
2558{
2559 if (!arm_smmu_dev_feature_enabled(dev, feat))
2560 return -EINVAL;
2561
2562 switch (feat) {
2563 case IOMMU_DEV_FEAT_SVA:
2564 return arm_smmu_master_disable_sva(dev_iommu_priv_get(dev));
2565 default:
2566 return -EINVAL;
2567 }
2568}
2569
3115static struct iommu_ops arm_smmu_ops = {2570static struct iommu_ops arm_smmu_ops = {
3116 .capable = arm_smmu_capable,2571 .capable = arm_smmu_capable,
3117 .domain_alloc = arm_smmu_domain_alloc,2572 .domain_alloc = arm_smmu_domain_alloc,
@@ -3130,6 +2585,10 @@ static struct iommu_ops arm_smmu_ops = { · 12 unmodified lines
3130 .of_xlate = arm_smmu_of_xlate,2585 .of_xlate = arm_smmu_of_xlate,
3131 .get_resv_regions = arm_smmu_get_resv_regions,2586 .get_resv_regions = arm_smmu_get_resv_regions,
3132 .put_resv_regions = generic_iommu_put_resv_regions,2587 .put_resv_regions = generic_iommu_put_resv_regions,
2588 .dev_has_feat = arm_smmu_dev_has_feature,
2589 .dev_feat_enabled = arm_smmu_dev_feature_enabled,
2590 .dev_enable_feat = arm_smmu_dev_enable_feature,
2591 .dev_disable_feat = arm_smmu_dev_disable_feature,
3133 .pgsize_bitmap = -1UL, /* Restricted during device attach */2592 .pgsize_bitmap = -1UL, /* Restricted during device attach */
3134};2593};
3135 2594
@@ -3280,7 +2739,7 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) · 144 unmodified lines
3280 if (!strtab) {2739 if (!strtab) {
3281 dev_err(smmu->dev,2740 dev_err(smmu->dev,
3282 "failed to allocate l1 stream table (%u bytes)\n",2741 "failed to allocate l1 stream table (%u bytes)\n",
3283 size);2742 l1size);
3284 return -ENOMEM;2743 return -ENOMEM;
3285 }2744 }
3286 cfg->strtab = strtab;2745 cfg->strtab = strtab;
@@ -3740,8 +3199,11 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) · 453 unmodified lines
3740 if (reg & IDR0_SEV)3199 if (reg & IDR0_SEV)
3741 smmu->features |= ARM_SMMU_FEAT_SEV;3200 smmu->features |= ARM_SMMU_FEAT_SEV;
3742 3201
3743 if (reg & IDR0_MSI)3202 if (reg & IDR0_MSI) {
3744 smmu->features |= ARM_SMMU_FEAT_MSI;3203 smmu->features |= ARM_SMMU_FEAT_MSI;
3204 if (coherent && !disable_msipolling)
3205 smmu->options |= ARM_SMMU_OPT_MSIPOLL;
3206 }
3745 3207
3746 if (reg & IDR0_HYP)3208 if (reg & IDR0_HYP)
3747 smmu->features |= ARM_SMMU_FEAT_HYP;3209 smmu->features |= ARM_SMMU_FEAT_HYP;
@@ -3891,6 +3353,9 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) · 143 unmodified lines
3891 3353
3892 smmu->ias = max(smmu->ias, smmu->oas);3354 smmu->ias = max(smmu->ias, smmu->oas);
3893 3355
3356 if (arm_smmu_sva_supported(smmu))
3357 smmu->features |= ARM_SMMU_FEAT_SVA;
3358
3894 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",3359 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
3895 smmu->ias, smmu->oas, smmu->features);3360 smmu->ias, smmu->oas, smmu->features);
3896 return 0;3361 return 0;
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h+723−0
@@ -0,0 +1,723 @@
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * IOMMU API for ARM architected SMMUv3 implementations.
4 *
5 * Copyright (C) 2015 ARM Limited
6 */
7
8#ifndef _ARM_SMMU_V3_H
9#define _ARM_SMMU_V3_H
10
11#include <linux/bitfield.h>
12#include <linux/iommu.h>
13#include <linux/kernel.h>
14#include <linux/mmzone.h>
15#include <linux/sizes.h>
16
17/* MMIO registers */
18#define ARM_SMMU_IDR0 0x0
19#define IDR0_ST_LVL GENMASK(28, 27)
20#define IDR0_ST_LVL_2LVL 1
21#define IDR0_STALL_MODEL GENMASK(25, 24)
22#define IDR0_STALL_MODEL_STALL 0
23#define IDR0_STALL_MODEL_FORCE 2
24#define IDR0_TTENDIAN GENMASK(22, 21)
25#define IDR0_TTENDIAN_MIXED 0
26#define IDR0_TTENDIAN_LE 2
27#define IDR0_TTENDIAN_BE 3
28#define IDR0_CD2L (1 << 19)
29#define IDR0_VMID16 (1 << 18)
30#define IDR0_PRI (1 << 16)
31#define IDR0_SEV (1 << 14)
32#define IDR0_MSI (1 << 13)
33#define IDR0_ASID16 (1 << 12)
34#define IDR0_ATS (1 << 10)
35#define IDR0_HYP (1 << 9)
36#define IDR0_COHACC (1 << 4)
37#define IDR0_TTF GENMASK(3, 2)
38#define IDR0_TTF_AARCH64 2
39#define IDR0_TTF_AARCH32_64 3
40#define IDR0_S1P (1 << 1)
41#define IDR0_S2P (1 << 0)
42
43#define ARM_SMMU_IDR1 0x4
44#define IDR1_TABLES_PRESET (1 << 30)
45#define IDR1_QUEUES_PRESET (1 << 29)
46#define IDR1_REL (1 << 28)
47#define IDR1_CMDQS GENMASK(25, 21)
48#define IDR1_EVTQS GENMASK(20, 16)
49#define IDR1_PRIQS GENMASK(15, 11)
50#define IDR1_SSIDSIZE GENMASK(10, 6)
51#define IDR1_SIDSIZE GENMASK(5, 0)
52
53#define ARM_SMMU_IDR3 0xc
54#define IDR3_RIL (1 << 10)
55
56#define ARM_SMMU_IDR5 0x14
57#define IDR5_STALL_MAX GENMASK(31, 16)
58#define IDR5_GRAN64K (1 << 6)
59#define IDR5_GRAN16K (1 << 5)
60#define IDR5_GRAN4K (1 << 4)
61#define IDR5_OAS GENMASK(2, 0)
62#define IDR5_OAS_32_BIT 0
63#define IDR5_OAS_36_BIT 1
64#define IDR5_OAS_40_BIT 2
65#define IDR5_OAS_42_BIT 3
66#define IDR5_OAS_44_BIT 4
67#define IDR5_OAS_48_BIT 5
68#define IDR5_OAS_52_BIT 6
69#define IDR5_VAX GENMASK(11, 10)
70#define IDR5_VAX_52_BIT 1
71
72#define ARM_SMMU_CR0 0x20
73#define CR0_ATSCHK (1 << 4)
74#define CR0_CMDQEN (1 << 3)
75#define CR0_EVTQEN (1 << 2)
76#define CR0_PRIQEN (1 << 1)
77#define CR0_SMMUEN (1 << 0)
78
79#define ARM_SMMU_CR0ACK 0x24
80
81#define ARM_SMMU_CR1 0x28
82#define CR1_TABLE_SH GENMASK(11, 10)
83#define CR1_TABLE_OC GENMASK(9, 8)
84#define CR1_TABLE_IC GENMASK(7, 6)
85#define CR1_QUEUE_SH GENMASK(5, 4)
86#define CR1_QUEUE_OC GENMASK(3, 2)
87#define CR1_QUEUE_IC GENMASK(1, 0)
88/* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
89#define CR1_CACHE_NC 0
90#define CR1_CACHE_WB 1
91#define CR1_CACHE_WT 2
92
93#define ARM_SMMU_CR2 0x2c
94#define CR2_PTM (1 << 2)
95#define CR2_RECINVSID (1 << 1)
96#define CR2_E2H (1 << 0)
97
98#define ARM_SMMU_GBPA 0x44
99#define GBPA_UPDATE (1 << 31)
100#define GBPA_ABORT (1 << 20)
101
102#define ARM_SMMU_IRQ_CTRL 0x50
103#define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
104#define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
105#define IRQ_CTRL_GERROR_IRQEN (1 << 0)
106
107#define ARM_SMMU_IRQ_CTRLACK 0x54
108
109#define ARM_SMMU_GERROR 0x60
110#define GERROR_SFM_ERR (1 << 8)
111#define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
112#define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
113#define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
114#define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
115#define GERROR_PRIQ_ABT_ERR (1 << 3)
116#define GERROR_EVTQ_ABT_ERR (1 << 2)
117#define GERROR_CMDQ_ERR (1 << 0)
118#define GERROR_ERR_MASK 0xfd
119
120#define ARM_SMMU_GERRORN 0x64
121
122#define ARM_SMMU_GERROR_IRQ_CFG0 0x68
123#define ARM_SMMU_GERROR_IRQ_CFG1 0x70
124#define ARM_SMMU_GERROR_IRQ_CFG2 0x74
125
126#define ARM_SMMU_STRTAB_BASE 0x80
127#define STRTAB_BASE_RA (1UL << 62)
128#define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
129
130#define ARM_SMMU_STRTAB_BASE_CFG 0x88
131#define STRTAB_BASE_CFG_FMT GENMASK(17, 16)
132#define STRTAB_BASE_CFG_FMT_LINEAR 0
133#define STRTAB_BASE_CFG_FMT_2LVL 1
134#define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6)
135#define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0)
136
137#define ARM_SMMU_CMDQ_BASE 0x90
138#define ARM_SMMU_CMDQ_PROD 0x98
139#define ARM_SMMU_CMDQ_CONS 0x9c
140
141#define ARM_SMMU_EVTQ_BASE 0xa0
142#define ARM_SMMU_EVTQ_PROD 0x100a8
143#define ARM_SMMU_EVTQ_CONS 0x100ac
144#define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
145#define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
146#define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
147
148#define ARM_SMMU_PRIQ_BASE 0xc0
149#define ARM_SMMU_PRIQ_PROD 0x100c8
150#define ARM_SMMU_PRIQ_CONS 0x100cc
151#define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
152#define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
153#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
154
155#define ARM_SMMU_REG_SZ 0xe00
156
157/* Common MSI config fields */
158#define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
159#define MSI_CFG2_SH GENMASK(5, 4)
160#define MSI_CFG2_MEMATTR GENMASK(3, 0)
161
162/* Common memory attribute values */
163#define ARM_SMMU_SH_NSH 0
164#define ARM_SMMU_SH_OSH 2
165#define ARM_SMMU_SH_ISH 3
166#define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1
167#define ARM_SMMU_MEMATTR_OIWB 0xf
168
169#define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1))
170#define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift))
171#define Q_OVERFLOW_FLAG (1U << 31)
172#define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG)
173#define Q_ENT(q, p) ((q)->base + \
174 Q_IDX(&((q)->llq), p) * \
175 (q)->ent_dwords)
176
177#define Q_BASE_RWA (1UL << 62)
178#define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
179#define Q_BASE_LOG2SIZE GENMASK(4, 0)
180
181/* Ensure DMA allocations are naturally aligned */
182#ifdef CONFIG_CMA_ALIGNMENT
183#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
184#else
185#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1)
186#endif
187
188/*
189 * Stream table.
190 *
191 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
192 * 2lvl: 128k L1 entries,
193 * 256 lazy entries per table (each table covers a PCI bus)
194 */
195#define STRTAB_L1_SZ_SHIFT 20
196#define STRTAB_SPLIT 8
197
198#define STRTAB_L1_DESC_DWORDS 1
199#define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0)
200#define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6)
201
202#define STRTAB_STE_DWORDS 8
203#define STRTAB_STE_0_V (1UL << 0)
204#define STRTAB_STE_0_CFG GENMASK_ULL(3, 1)
205#define STRTAB_STE_0_CFG_ABORT 0
206#define STRTAB_STE_0_CFG_BYPASS 4
207#define STRTAB_STE_0_CFG_S1_TRANS 5
208#define STRTAB_STE_0_CFG_S2_TRANS 6
209
210#define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)
211#define STRTAB_STE_0_S1FMT_LINEAR 0
212#define STRTAB_STE_0_S1FMT_64K_L2 2
213#define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
214#define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)
215
216#define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0)
217#define STRTAB_STE_1_S1DSS_TERMINATE 0x0
218#define STRTAB_STE_1_S1DSS_BYPASS 0x1
219#define STRTAB_STE_1_S1DSS_SSID0 0x2
220
221#define STRTAB_STE_1_S1C_CACHE_NC 0UL
222#define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
223#define STRTAB_STE_1_S1C_CACHE_WT 2UL
224#define STRTAB_STE_1_S1C_CACHE_WB 3UL
225#define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2)
226#define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4)
227#define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6)
228
229#define STRTAB_STE_1_S1STALLD (1UL << 27)
230
231#define STRTAB_STE_1_EATS GENMASK_ULL(29, 28)
232#define STRTAB_STE_1_EATS_ABT 0UL
233#define STRTAB_STE_1_EATS_TRANS 1UL
234#define STRTAB_STE_1_EATS_S1CHK 2UL
235
236#define STRTAB_STE_1_STRW GENMASK_ULL(31, 30)
237#define STRTAB_STE_1_STRW_NSEL1 0UL
238#define STRTAB_STE_1_STRW_EL2 2UL
239
240#define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44)
241#define STRTAB_STE_1_SHCFG_INCOMING 1UL
242
243#define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
244#define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
245#define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0)
246#define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6)
247#define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8)
248#define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10)
249#define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12)
250#define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14)
251#define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16)
252#define STRTAB_STE_2_S2AA64 (1UL << 51)
253#define STRTAB_STE_2_S2ENDI (1UL << 52)
254#define STRTAB_STE_2_S2PTW (1UL << 54)
255#define STRTAB_STE_2_S2R (1UL << 58)
256
257#define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)
258
259/*
260 * Context descriptors.
261 *
262 * Linear: when less than 1024 SSIDs are supported
263 * 2lvl: at most 1024 L1 entries,
264 * 1024 lazy entries per table.
265 */
266#define CTXDESC_SPLIT 10
267#define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT)
268
269#define CTXDESC_L1_DESC_DWORDS 1
270#define CTXDESC_L1_DESC_V (1UL << 0)
271#define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12)
272
273#define CTXDESC_CD_DWORDS 8
274#define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
275#define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
276#define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
277#define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
278#define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
279#define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)
280#define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)
281
282#define CTXDESC_CD_0_ENDI (1UL << 15)
283#define CTXDESC_CD_0_V (1UL << 31)
284
285#define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)
286#define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)
287
288#define CTXDESC_CD_0_AA64 (1UL << 41)
289#define CTXDESC_CD_0_S (1UL << 44)
290#define CTXDESC_CD_0_R (1UL << 45)
291#define CTXDESC_CD_0_A (1UL << 46)
292#define CTXDESC_CD_0_ASET (1UL << 47)
293#define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48)
294
295#define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
296
297/*
298 * When the SMMU only supports linear context descriptor tables, pick a
299 * reasonable size limit (64kB).
300 */
301#define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
302
303/* Command queue */
304#define CMDQ_ENT_SZ_SHIFT 4
305#define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
306#define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
307
308#define CMDQ_CONS_ERR GENMASK(30, 24)
309#define CMDQ_ERR_CERROR_NONE_IDX 0
310#define CMDQ_ERR_CERROR_ILL_IDX 1
311#define CMDQ_ERR_CERROR_ABT_IDX 2
312#define CMDQ_ERR_CERROR_ATC_INV_IDX 3
313
314#define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG
315
316/*
317 * This is used to size the command queue and therefore must be at least
318 * BITS_PER_LONG so that the valid_map works correctly (it relies on the
319 * total number of queue entries being a multiple of BITS_PER_LONG).
320 */
321#define CMDQ_BATCH_ENTRIES BITS_PER_LONG
322
323#define CMDQ_0_OP GENMASK_ULL(7, 0)
324#define CMDQ_0_SSV (1UL << 11)
325
326#define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32)
327#define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)
328#define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)
329
330#define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12)
331#define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)
332#define CMDQ_CFGI_1_LEAF (1UL << 0)
333#define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)
334
335#define CMDQ_TLBI_0_NUM GENMASK_ULL(16, 12)
336#define CMDQ_TLBI_RANGE_NUM_MAX 31
337#define CMDQ_TLBI_0_SCALE GENMASK_ULL(24, 20)
338#define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32)
339#define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48)
340#define CMDQ_TLBI_1_LEAF (1UL << 0)
341#define CMDQ_TLBI_1_TTL GENMASK_ULL(9, 8)
342#define CMDQ_TLBI_1_TG GENMASK_ULL(11, 10)
343#define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12)
344#define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12)
345
346#define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12)
347#define CMDQ_ATC_0_SID GENMASK_ULL(63, 32)
348#define CMDQ_ATC_0_GLOBAL (1UL << 9)
349#define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0)
350#define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12)
351
352#define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12)
353#define CMDQ_PRI_0_SID GENMASK_ULL(63, 32)
354#define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
355#define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12)
356
357#define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12)
358#define CMDQ_SYNC_0_CS_NONE 0
359#define CMDQ_SYNC_0_CS_IRQ 1
360#define CMDQ_SYNC_0_CS_SEV 2
361#define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22)
362#define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24)
363#define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32)
364#define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2)
365
366/* Event queue */
367#define EVTQ_ENT_SZ_SHIFT 5
368#define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
369#define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
370
371#define EVTQ_0_ID GENMASK_ULL(7, 0)
372
373/* PRI queue */
374#define PRIQ_ENT_SZ_SHIFT 4
375#define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
376#define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
377
378#define PRIQ_0_SID GENMASK_ULL(31, 0)
379#define PRIQ_0_SSID GENMASK_ULL(51, 32)
380#define PRIQ_0_PERM_PRIV (1UL << 58)
381#define PRIQ_0_PERM_EXEC (1UL << 59)
382#define PRIQ_0_PERM_READ (1UL << 60)
383#define PRIQ_0_PERM_WRITE (1UL << 61)
384#define PRIQ_0_PRG_LAST (1UL << 62)
385#define PRIQ_0_SSID_V (1UL << 63)
386
387#define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0)
388#define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12)
389
390/* High-level queue structures */
391#define ARM_SMMU_POLL_TIMEOUT_US 1000000 /* 1s! */
392#define ARM_SMMU_POLL_SPIN_COUNT 10
393
394#define MSI_IOVA_BASE 0x8000000
395#define MSI_IOVA_LENGTH 0x100000
396
397enum pri_resp {
398 PRI_RESP_DENY = 0,
399 PRI_RESP_FAIL = 1,
400 PRI_RESP_SUCC = 2,
401};
402
403struct arm_smmu_cmdq_ent {
404 /* Common fields */
405 u8 opcode;
406 bool substream_valid;
407
408 /* Command-specific fields */
409 union {
410 #define CMDQ_OP_PREFETCH_CFG 0x1
411 struct {
412 u32 sid;
413 u8 size;
414 u64 addr;
415 } prefetch;
416
417 #define CMDQ_OP_CFGI_STE 0x3
418 #define CMDQ_OP_CFGI_ALL 0x4
419 #define CMDQ_OP_CFGI_CD 0x5
420 #define CMDQ_OP_CFGI_CD_ALL 0x6
421 struct {
422 u32 sid;
423 u32 ssid;
424 union {
425 bool leaf;
426 u8 span;
427 };
428 } cfgi;
429
430 #define CMDQ_OP_TLBI_NH_ASID 0x11
431 #define CMDQ_OP_TLBI_NH_VA 0x12
432 #define CMDQ_OP_TLBI_EL2_ALL 0x20
433 #define CMDQ_OP_TLBI_S12_VMALL 0x28
434 #define CMDQ_OP_TLBI_S2_IPA 0x2a
435 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
436 struct {
437 u8 num;
438 u8 scale;
439 u16 asid;
440 u16 vmid;
441 bool leaf;
442 u8 ttl;
443 u8 tg;
444 u64 addr;
445 } tlbi;
446
447 #define CMDQ_OP_ATC_INV 0x40
448 #define ATC_INV_SIZE_ALL 52
449 struct {
450 u32 sid;
451 u32 ssid;
452 u64 addr;
453 u8 size;
454 bool global;
455 } atc;
456
457 #define CMDQ_OP_PRI_RESP 0x41
458 struct {
459 u32 sid;
460 u32 ssid;
461 u16 grpid;
462 enum pri_resp resp;
463 } pri;
464
465 #define CMDQ_OP_CMD_SYNC 0x46
466 struct {
467 u64 msiaddr;
468 } sync;
469 };
470};
471
472struct arm_smmu_ll_queue {
473 union {
474 u64 val;
475 struct {
476 u32 prod;
477 u32 cons;
478 };
479 struct {
480 atomic_t prod;
481 atomic_t cons;
482 } atomic;
483 u8 __pad[SMP_CACHE_BYTES];
484 } ____cacheline_aligned_in_smp;
485 u32 max_n_shift;
486};
487
488struct arm_smmu_queue {
489 struct arm_smmu_ll_queue llq;
490 int irq; /* Wired interrupt */
491
492 __le64 *base;
493 dma_addr_t base_dma;
494 u64 q_base;
495
496 size_t ent_dwords;
497
498 u32 __iomem *prod_reg;
499 u32 __iomem *cons_reg;
500};
501
502struct arm_smmu_queue_poll {
503 ktime_t timeout;
504 unsigned int delay;
505 unsigned int spin_cnt;
506 bool wfe;
507};
508
509struct arm_smmu_cmdq {
510 struct arm_smmu_queue q;
511 atomic_long_t *valid_map;
512 atomic_t owner_prod;
513 atomic_t lock;
514};
515
516struct arm_smmu_cmdq_batch {
517 u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
518 int num;
519};
520
521struct arm_smmu_evtq {
522 struct arm_smmu_queue q;
523 u32 max_stalls;
524};
525
526struct arm_smmu_priq {
527 struct arm_smmu_queue q;
528};
529
530/* High-level stream table and context descriptor structures */
531struct arm_smmu_strtab_l1_desc {
532 u8 span;
533
534 __le64 *l2ptr;
535 dma_addr_t l2ptr_dma;
536};
537
538struct arm_smmu_ctx_desc {
539 u16 asid;
540 u64 ttbr;
541 u64 tcr;
542 u64 mair;
543
544 refcount_t refs;
545 struct mm_struct *mm;
546};
547
548struct arm_smmu_l1_ctx_desc {
549 __le64 *l2ptr;
550 dma_addr_t l2ptr_dma;
551};
552
553struct arm_smmu_ctx_desc_cfg {
554 __le64 *cdtab;
555 dma_addr_t cdtab_dma;
556 struct arm_smmu_l1_ctx_desc *l1_desc;
557 unsigned int num_l1_ents;
558};
559
560struct arm_smmu_s1_cfg {
561 struct arm_smmu_ctx_desc_cfg cdcfg;
562 struct arm_smmu_ctx_desc cd;
563 u8 s1fmt;
564 u8 s1cdmax;
565};
566
567struct arm_smmu_s2_cfg {
568 u16 vmid;
569 u64 vttbr;
570 u64 vtcr;
571};
572
573struct arm_smmu_strtab_cfg {
574 __le64 *strtab;
575 dma_addr_t strtab_dma;
576 struct arm_smmu_strtab_l1_desc *l1_desc;
577 unsigned int num_l1_ents;
578
579 u64 strtab_base;
580 u32 strtab_base_cfg;
581};
582
583/* An SMMUv3 instance */
584struct arm_smmu_device {
585 struct device *dev;
586 void __iomem *base;
587 void __iomem *page1;
588
589#define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
590#define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
591#define ARM_SMMU_FEAT_TT_LE (1 << 2)
592#define ARM_SMMU_FEAT_TT_BE (1 << 3)
593#define ARM_SMMU_FEAT_PRI (1 << 4)
594#define ARM_SMMU_FEAT_ATS (1 << 5)
595#define ARM_SMMU_FEAT_SEV (1 << 6)
596#define ARM_SMMU_FEAT_MSI (1 << 7)
597#define ARM_SMMU_FEAT_COHERENCY (1 << 8)
598#define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
599#define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
600#define ARM_SMMU_FEAT_STALLS (1 << 11)
601#define ARM_SMMU_FEAT_HYP (1 << 12)
602#define ARM_SMMU_FEAT_STALL_FORCE (1 << 13)
603#define ARM_SMMU_FEAT_VAX (1 << 14)
604#define ARM_SMMU_FEAT_RANGE_INV (1 << 15)
605#define ARM_SMMU_FEAT_BTM (1 << 16)
606#define ARM_SMMU_FEAT_SVA (1 << 17)
607 u32 features;
608
609#define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
610#define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1)
611#define ARM_SMMU_OPT_MSIPOLL (1 << 2)
612 u32 options;
613
614 struct arm_smmu_cmdq cmdq;
615 struct arm_smmu_evtq evtq;
616 struct arm_smmu_priq priq;
617
618 int gerr_irq;
619 int combined_irq;
620
621 unsigned long ias; /* IPA */
622 unsigned long oas; /* PA */
623 unsigned long pgsize_bitmap;
624
625#define ARM_SMMU_MAX_ASIDS (1 << 16)
626 unsigned int asid_bits;
627
628#define ARM_SMMU_MAX_VMIDS (1 << 16)
629 unsigned int vmid_bits;
630 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
631
632 unsigned int ssid_bits;
633 unsigned int sid_bits;
634
635 struct arm_smmu_strtab_cfg strtab_cfg;
636
637 /* IOMMU core code handle */
638 struct iommu_device iommu;
639};
640
641/* SMMU private data for each master */
642struct arm_smmu_master {
643 struct arm_smmu_device *smmu;
644 struct device *dev;
645 struct arm_smmu_domain *domain;
646 struct list_head domain_head;
647 u32 *sids;
648 unsigned int num_sids;
649 bool ats_enabled;
650 bool sva_enabled;
651 struct list_head bonds;
652 unsigned int ssid_bits;
653};
654
655/* SMMU private data for an IOMMU domain */
656enum arm_smmu_domain_stage {
657 ARM_SMMU_DOMAIN_S1 = 0,
658 ARM_SMMU_DOMAIN_S2,
659 ARM_SMMU_DOMAIN_NESTED,
660 ARM_SMMU_DOMAIN_BYPASS,
661};
662
663struct arm_smmu_domain {
664 struct arm_smmu_device *smmu;
665 struct mutex init_mutex; /* Protects smmu pointer */
666
667 struct io_pgtable_ops *pgtbl_ops;
668 bool non_strict;
669 atomic_t nr_ats_masters;
670
671 enum arm_smmu_domain_stage stage;
672 union {
673 struct arm_smmu_s1_cfg s1_cfg;
674 struct arm_smmu_s2_cfg s2_cfg;
675 };
676
677 struct iommu_domain domain;
678
679 struct list_head devices;
680 spinlock_t devices_lock;
681};
682
683extern struct xarray arm_smmu_asid_xa;
684extern struct mutex arm_smmu_asid_lock;
685
686int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
687 struct arm_smmu_ctx_desc *cd);
688void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
689bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
690
691#ifdef CONFIG_ARM_SMMU_V3_SVA
692bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
693bool arm_smmu_master_sva_supported(struct arm_smmu_master *master);
694bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master);
695int arm_smmu_master_enable_sva(struct arm_smmu_master *master);
696int arm_smmu_master_disable_sva(struct arm_smmu_master *master);
697#else /* CONFIG_ARM_SMMU_V3_SVA */
698static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
699{
700 return false;
701}
702
703static inline bool arm_smmu_master_sva_supported(struct arm_smmu_master *master)
704{
705 return false;
706}
707
708static inline bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master)
709{
710 return false;
711}
712
713static inline int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
714{
715 return -ENODEV;
716}
717
718static inline int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
719{
720 return -ENODEV;
721}
722#endif /* CONFIG_ARM_SMMU_V3_SVA */
723#endif /* _ARM_SMMU_V3_H */
drivers/iommu/arm/arm-smmu/arm-smmu-impl.c+2−1
@@ -68,7 +68,8 @@ static int cavium_cfg_probe(struct arm_smmu_device *smmu) · 67 unmodified lines
68 return 0;68 return 0;
69}69}
70 70
71static int cavium_init_context(struct arm_smmu_domain *smmu_domain)71static int cavium_init_context(struct arm_smmu_domain *smmu_domain,
72 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
72{73{
73 struct cavium_smmu *cs = container_of(smmu_domain->smmu,74 struct cavium_smmu *cs = container_of(smmu_domain->smmu,
74 struct cavium_smmu, smmu);75 struct cavium_smmu, smmu);
drivers/iommu/arm/arm-smmu/arm-smmu.c+41−61
@@ -65,41 +65,10 @@ module_param(disable_bypass, bool, S_IRUGO); · 64 unmodified lines
65MODULE_PARM_DESC(disable_bypass,65MODULE_PARM_DESC(disable_bypass,
66 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");66 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
67 67
68struct arm_smmu_s2cr {
69 struct iommu_group *group;
70 int count;
71 enum arm_smmu_s2cr_type type;
72 enum arm_smmu_s2cr_privcfg privcfg;
73 u8 cbndx;
74};
75
76#define s2cr_init_val (struct arm_smmu_s2cr){ \68#define s2cr_init_val (struct arm_smmu_s2cr){ \
77 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \69 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
78}70}
79 71
80struct arm_smmu_smr {
81 u16 mask;
82 u16 id;
83 bool valid;
84};
85
86struct arm_smmu_cb {
87 u64 ttbr[2];
88 u32 tcr[2];
89 u32 mair[2];
90 struct arm_smmu_cfg *cfg;
91};
92
93struct arm_smmu_master_cfg {
94 struct arm_smmu_device *smmu;
95 s16 smendx[];
96};
97#define INVALID_SMENDX -1
98#define cfg_smendx(cfg, fw, i) \
99 (i >= fw->num_ids ? INVALID_SMENDX : cfg->smendx[i])
100#define for_each_cfg_sme(cfg, fw, i, idx) \
101 for (i = 0; idx = cfg_smendx(cfg, fw, i), i < fw->num_ids; ++i)
102
103static bool using_legacy_binding, using_generic_binding;72static bool using_legacy_binding, using_generic_binding;
104 73
105static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)74static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
@@ -234,19 +203,6 @@ static int arm_smmu_register_legacy_master(struct device *dev, · 128 unmodified lines
234}203}
235#endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */204#endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
236 205
237static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
238{
239 int idx;
240
241 do {
242 idx = find_next_zero_bit(map, end, start);
243 if (idx == end)
244 return -ENOSPC;
245 } while (test_and_set_bit(idx, map));
246
247 return idx;
248}
249
250static void __arm_smmu_free_bitmap(unsigned long *map, int idx)206static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
251{207{
252 clear_bit(idx, map);208 clear_bit(idx, map);
@@ -552,11 +508,15 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, · 299 unmodified lines
552 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;508 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
553 cb->ttbr[1] = 0;509 cb->ttbr[1] = 0;
554 } else {510 } else {
555 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;511 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
556 cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID,512 cfg->asid);
557 cfg->asid);
558 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,513 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
559 cfg->asid);514 cfg->asid);
515
516 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
517 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
518 else
519 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
560 }520 }
561 } else {521 } else {
562 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;522 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
@@ -574,7 +534,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, · 11 unmodified lines
574 }534 }
575}535}
576 536
577static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)537void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
578{538{
579 u32 reg;539 u32 reg;
580 bool stage1;540 bool stage1;
@@ -660,8 +620,19 @@ static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) · 79 unmodified lines
660 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);620 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
661}621}
662 622
623static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
624 struct arm_smmu_device *smmu,
625 struct device *dev, unsigned int start)
626{
627 if (smmu->impl && smmu->impl->alloc_context_bank)
628 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
629
630 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
631}
632
663static int arm_smmu_init_domain_context(struct iommu_domain *domain,633static int arm_smmu_init_domain_context(struct iommu_domain *domain,
664 struct arm_smmu_device *smmu)634 struct arm_smmu_device *smmu,
635 struct device *dev)
665{636{
666 int irq, start, ret = 0;637 int irq, start, ret = 0;
667 unsigned long ias, oas;638 unsigned long ias, oas;
@@ -776,10 +747,13 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, · 108 unmodified lines
776 ret = -EINVAL;747 ret = -EINVAL;
777 goto out_unlock;748 goto out_unlock;
778 }749 }
779 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,750
780 smmu->num_context_banks);751 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
781 if (ret < 0)752 if (ret < 0) {
782 goto out_unlock;753 goto out_unlock;
754 }
755
756 smmu_domain->smmu = smmu;
783 757
784 cfg->cbndx = ret;758 cfg->cbndx = ret;
785 if (smmu->version < ARM_SMMU_V2) {759 if (smmu->version < ARM_SMMU_V2) {
@@ -794,13 +768,6 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, · 8 unmodified lines
794 else768 else
795 cfg->asid = cfg->cbndx;769 cfg->asid = cfg->cbndx;
796 770
797 smmu_domain->smmu = smmu;
798 if (smmu->impl && smmu->impl->init_context) {
799 ret = smmu->impl->init_context(smmu_domain);
800 if (ret)
801 goto out_unlock;
802 }
803
804 pgtbl_cfg = (struct io_pgtable_cfg) {771 pgtbl_cfg = (struct io_pgtable_cfg) {
805 .pgsize_bitmap = smmu->pgsize_bitmap,772 .pgsize_bitmap = smmu->pgsize_bitmap,
806 .ias = ias,773 .ias = ias,
@@ -810,6 +777,12 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, · 3 unmodified lines
810 .iommu_dev = smmu->dev,777 .iommu_dev = smmu->dev,
811 };778 };
812 779
780 if (smmu->impl && smmu->impl->init_context) {
781 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
782 if (ret)
783 goto out_clear_smmu;
784 }
785
813 if (smmu_domain->non_strict)786 if (smmu_domain->non_strict)
814 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;787 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
815 788
@@ -821,7 +794,14 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, · 5 unmodified lines
821 794
822 /* Update the domain's page sizes to reflect the page table format */795 /* Update the domain's page sizes to reflect the page table format */
823 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;796 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
824 domain->geometry.aperture_end = (1UL << ias) - 1;797
798 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
799 domain->geometry.aperture_start = ~0UL << ias;
800 domain->geometry.aperture_end = ~0UL;
801 } else {
802 domain->geometry.aperture_end = (1UL << ias) - 1;
803 }
804
825 domain->geometry.force_aperture = true;805 domain->geometry.force_aperture = true;
826 806
827 /* Initialise the context bank with our page table cfg */807 /* Initialise the context bank with our page table cfg */
@@ -1182,7 +1162,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) · 354 unmodified lines
1182 return ret;1162 return ret;
1183 1163
1184 /* Ensure that the domain is finalised */1164 /* Ensure that the domain is finalised */
1185 ret = arm_smmu_init_domain_context(domain, smmu);1165 ret = arm_smmu_init_domain_context(domain, smmu, dev);
1186 if (ret < 0)1166 if (ret < 0)
1187 goto rpm_put;1167 goto rpm_put;
1188 1168
drivers/iommu/arm/arm-smmu/arm-smmu.h+74−10
@@ -169,10 +169,12 @@ enum arm_smmu_cbar_type { · 168 unmodified lines
169#define ARM_SMMU_CB_TCR 0x30169#define ARM_SMMU_CB_TCR 0x30
170#define ARM_SMMU_TCR_EAE BIT(31)170#define ARM_SMMU_TCR_EAE BIT(31)
171#define ARM_SMMU_TCR_EPD1 BIT(23)171#define ARM_SMMU_TCR_EPD1 BIT(23)
172#define ARM_SMMU_TCR_A1 BIT(22)
172#define ARM_SMMU_TCR_TG0 GENMASK(15, 14)173#define ARM_SMMU_TCR_TG0 GENMASK(15, 14)
173#define ARM_SMMU_TCR_SH0 GENMASK(13, 12)174#define ARM_SMMU_TCR_SH0 GENMASK(13, 12)
174#define ARM_SMMU_TCR_ORGN0 GENMASK(11, 10)175#define ARM_SMMU_TCR_ORGN0 GENMASK(11, 10)
175#define ARM_SMMU_TCR_IRGN0 GENMASK(9, 8)176#define ARM_SMMU_TCR_IRGN0 GENMASK(9, 8)
177#define ARM_SMMU_TCR_EPD0 BIT(7)
176#define ARM_SMMU_TCR_T0SZ GENMASK(5, 0)178#define ARM_SMMU_TCR_T0SZ GENMASK(5, 0)
177 179
178#define ARM_SMMU_VTCR_RES1 BIT(31)180#define ARM_SMMU_VTCR_RES1 BIT(31)
@@ -254,6 +256,21 @@ enum arm_smmu_implementation { · 75 unmodified lines
254 QCOM_SMMUV2,256 QCOM_SMMUV2,
255};257};
256 258
259struct arm_smmu_s2cr {
260 struct iommu_group *group;
261 int count;
262 enum arm_smmu_s2cr_type type;
263 enum arm_smmu_s2cr_privcfg privcfg;
264 u8 cbndx;
265};
266
267struct arm_smmu_smr {
268 u16 mask;
269 u16 id;
270 bool valid;
271 bool pinned;
272};
273
257struct arm_smmu_device {274struct arm_smmu_device {
258 struct device *dev;275 struct device *dev;
259 276
@@ -329,6 +346,13 @@ struct arm_smmu_cfg { · 69 unmodified lines
329};346};
330#define ARM_SMMU_INVALID_IRPTNDX 0xff347#define ARM_SMMU_INVALID_IRPTNDX 0xff
331 348
349struct arm_smmu_cb {
350 u64 ttbr[2];
351 u32 tcr[2];
352 u32 mair[2];
353 struct arm_smmu_cfg *cfg;
354};
355
332enum arm_smmu_domain_stage {356enum arm_smmu_domain_stage {
333 ARM_SMMU_DOMAIN_S1 = 0,357 ARM_SMMU_DOMAIN_S1 = 0,
334 ARM_SMMU_DOMAIN_S2,358 ARM_SMMU_DOMAIN_S2,
@@ -348,23 +372,39 @@ struct arm_smmu_domain { · 13 unmodified lines
348 struct iommu_domain domain;372 struct iommu_domain domain;
349};373};
350 374
351static inline u32 arm_smmu_lpae_tcr(struct io_pgtable_cfg *cfg)375struct arm_smmu_master_cfg {
376 struct arm_smmu_device *smmu;
377 s16 smendx[];
378};
379
380static inline u32 arm_smmu_lpae_tcr(const struct io_pgtable_cfg *cfg)
352{381{
353 return ARM_SMMU_TCR_EPD1 |382 u32 tcr = FIELD_PREP(ARM_SMMU_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) |
354 FIELD_PREP(ARM_SMMU_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) |383 FIELD_PREP(ARM_SMMU_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) |
355 FIELD_PREP(ARM_SMMU_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) |384 FIELD_PREP(ARM_SMMU_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) |
356 FIELD_PREP(ARM_SMMU_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) |385 FIELD_PREP(ARM_SMMU_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) |
357 FIELD_PREP(ARM_SMMU_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) |386 FIELD_PREP(ARM_SMMU_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz);
358 FIELD_PREP(ARM_SMMU_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz);387
388 /*
389 * When TTBR1 is selected shift the TCR fields by 16 bits and disable
390 * translation in TTBR0
391 */
392 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
393 tcr = (tcr << 16) & ~ARM_SMMU_TCR_A1;
394 tcr |= ARM_SMMU_TCR_EPD0;
395 } else
396 tcr |= ARM_SMMU_TCR_EPD1;
397
398 return tcr;
359}399}
360 400
361static inline u32 arm_smmu_lpae_tcr2(struct io_pgtable_cfg *cfg)401static inline u32 arm_smmu_lpae_tcr2(const struct io_pgtable_cfg *cfg)
362{402{
363 return FIELD_PREP(ARM_SMMU_TCR2_PASIZE, cfg->arm_lpae_s1_cfg.tcr.ips) |403 return FIELD_PREP(ARM_SMMU_TCR2_PASIZE, cfg->arm_lpae_s1_cfg.tcr.ips) |
364 FIELD_PREP(ARM_SMMU_TCR2_SEP, ARM_SMMU_TCR2_SEP_UPSTREAM);404 FIELD_PREP(ARM_SMMU_TCR2_SEP, ARM_SMMU_TCR2_SEP_UPSTREAM);
365}405}
366 406
367static inline u32 arm_smmu_lpae_vtcr(struct io_pgtable_cfg *cfg)407static inline u32 arm_smmu_lpae_vtcr(const struct io_pgtable_cfg *cfg)
368{408{
369 return ARM_SMMU_VTCR_RES1 |409 return ARM_SMMU_VTCR_RES1 |
370 FIELD_PREP(ARM_SMMU_VTCR_PS, cfg->arm_lpae_s2_cfg.vtcr.ps) |410 FIELD_PREP(ARM_SMMU_VTCR_PS, cfg->arm_lpae_s2_cfg.vtcr.ps) |
@@ -386,14 +426,37 @@ struct arm_smmu_impl { · 15 unmodified lines
386 u64 val);426 u64 val);
387 int (*cfg_probe)(struct arm_smmu_device *smmu);427 int (*cfg_probe)(struct arm_smmu_device *smmu);
388 int (*reset)(struct arm_smmu_device *smmu);428 int (*reset)(struct arm_smmu_device *smmu);
389 int (*init_context)(struct arm_smmu_domain *smmu_domain);429 int (*init_context)(struct arm_smmu_domain *smmu_domain,
430 struct io_pgtable_cfg *cfg, struct device *dev);
390 void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync,431 void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync,
391 int status);432 int status);
392 int (*def_domain_type)(struct device *dev);433 int (*def_domain_type)(struct device *dev);
393 irqreturn_t (*global_fault)(int irq, void *dev);434 irqreturn_t (*global_fault)(int irq, void *dev);
394 irqreturn_t (*context_fault)(int irq, void *dev);435 irqreturn_t (*context_fault)(int irq, void *dev);
436 int (*alloc_context_bank)(struct arm_smmu_domain *smmu_domain,
437 struct arm_smmu_device *smmu,
438 struct device *dev, int start);
395};439};
396 440
441#define INVALID_SMENDX -1
442#define cfg_smendx(cfg, fw, i) \
443 (i >= fw->num_ids ? INVALID_SMENDX : cfg->smendx[i])
444#define for_each_cfg_sme(cfg, fw, i, idx) \
445 for (i = 0; idx = cfg_smendx(cfg, fw, i), i < fw->num_ids; ++i)
446
447static inline int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
448{
449 int idx;
450
451 do {
452 idx = find_next_zero_bit(map, end, start);
453 if (idx == end)
454 return -ENOSPC;
455 } while (test_and_set_bit(idx, map));
456
457 return idx;
458}
459
397static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n)460static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n)
398{461{
399 return smmu->base + (n << smmu->pgshift);462 return smmu->base + (n << smmu->pgshift);
@@ -458,6 +521,7 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu); · 58 unmodified lines
458struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu);521struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu);
459struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu);522struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu);
460 523
524void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx);
461int arm_mmu500_reset(struct arm_smmu_device *smmu);525int arm_mmu500_reset(struct arm_smmu_device *smmu);
462 526
463#endif /* _ARM_SMMU_H */527#endif /* _ARM_SMMU_H */
drivers/iommu/arm/arm-smmu/qcom_iommu.c+6−2
@@ -584,8 +584,10 @@ static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) · 583 unmodified lines
584 * index into qcom_iommu->ctxs:584 * index into qcom_iommu->ctxs:
585 */585 */
586 if (WARN_ON(asid < 1) ||586 if (WARN_ON(asid < 1) ||
587 WARN_ON(asid > qcom_iommu->num_ctxs))587 WARN_ON(asid > qcom_iommu->num_ctxs)) {
588 put_device(&iommu_pdev->dev);
588 return -EINVAL;589 return -EINVAL;
590 }
589 591
590 if (!dev_iommu_priv_get(dev)) {592 if (!dev_iommu_priv_get(dev)) {
591 dev_iommu_priv_set(dev, qcom_iommu);593 dev_iommu_priv_set(dev, qcom_iommu);
@@ -594,8 +596,10 @@ static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) · 2 unmodified lines
594 * multiple different iommu devices. Multiple context596 * multiple different iommu devices. Multiple context
595 * banks are ok, but multiple devices are not:597 * banks are ok, but multiple devices are not:
596 */598 */
597 if (WARN_ON(qcom_iommu != dev_iommu_priv_get(dev)))599 if (WARN_ON(qcom_iommu != dev_iommu_priv_get(dev))) {
600 put_device(&iommu_pdev->dev);
598 return -EINVAL;601 return -EINVAL;
602 }
599 }603 }
600 604
601 return iommu_fwspec_add_ids(dev, &asid, 1);605 return iommu_fwspec_add_ids(dev, &asid, 1);
drivers/iommu/io-pgtable-arm.c+2−25
@@ -20,6 +20,8 @@ · 19 unmodified lines
20 20
21#include <asm/barrier.h>21#include <asm/barrier.h>
22 22
23#include "io-pgtable-arm.h"
24
23#define ARM_LPAE_MAX_ADDR_BITS 5225#define ARM_LPAE_MAX_ADDR_BITS 52
24#define ARM_LPAE_S2_MAX_CONCAT_PAGES 1626#define ARM_LPAE_S2_MAX_CONCAT_PAGES 16
25#define ARM_LPAE_MAX_LEVELS 427#define ARM_LPAE_MAX_LEVELS 4
@@ -100,23 +102,6 @@ · 74 unmodified lines
100#define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2)102#define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2)
101 103
102/* Register bits */104/* Register bits */
103#define ARM_LPAE_TCR_TG0_4K 0
104#define ARM_LPAE_TCR_TG0_64K 1
105#define ARM_LPAE_TCR_TG0_16K 2
106
107#define ARM_LPAE_TCR_TG1_16K 1
108#define ARM_LPAE_TCR_TG1_4K 2
109#define ARM_LPAE_TCR_TG1_64K 3
110
111#define ARM_LPAE_TCR_SH_NS 0
112#define ARM_LPAE_TCR_SH_OS 2
113#define ARM_LPAE_TCR_SH_IS 3
114
115#define ARM_LPAE_TCR_RGN_NC 0
116#define ARM_LPAE_TCR_RGN_WBWA 1
117#define ARM_LPAE_TCR_RGN_WT 2
118#define ARM_LPAE_TCR_RGN_WB 3
119
120#define ARM_LPAE_VTCR_SL0_MASK 0x3105#define ARM_LPAE_VTCR_SL0_MASK 0x3
121 106
122#define ARM_LPAE_TCR_T0SZ_SHIFT 0107#define ARM_LPAE_TCR_T0SZ_SHIFT 0
@@ -124,14 +109,6 @@ · 1 unmodified line
124#define ARM_LPAE_VTCR_PS_SHIFT 16109#define ARM_LPAE_VTCR_PS_SHIFT 16
125#define ARM_LPAE_VTCR_PS_MASK 0x7110#define ARM_LPAE_VTCR_PS_MASK 0x7
126 111
127#define ARM_LPAE_TCR_PS_32_BIT 0x0ULL
128#define ARM_LPAE_TCR_PS_36_BIT 0x1ULL
129#define ARM_LPAE_TCR_PS_40_BIT 0x2ULL
130#define ARM_LPAE_TCR_PS_42_BIT 0x3ULL
131#define ARM_LPAE_TCR_PS_44_BIT 0x4ULL
132#define ARM_LPAE_TCR_PS_48_BIT 0x5ULL
133#define ARM_LPAE_TCR_PS_52_BIT 0x6ULL
134
135#define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3)112#define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3)
136#define ARM_LPAE_MAIR_ATTR_MASK 0xff113#define ARM_LPAE_MAIR_ATTR_MASK 0xff
137#define ARM_LPAE_MAIR_ATTR_DEVICE 0x04114#define ARM_LPAE_MAIR_ATTR_DEVICE 0x04
drivers/iommu/io-pgtable-arm.h+30−0
@@ -0,0 +1,30 @@
1/* SPDX-License-Identifier: GPL-2.0-only */
2#ifndef IO_PGTABLE_ARM_H_
3#define IO_PGTABLE_ARM_H_
4
5#define ARM_LPAE_TCR_TG0_4K 0
6#define ARM_LPAE_TCR_TG0_64K 1
7#define ARM_LPAE_TCR_TG0_16K 2
8
9#define ARM_LPAE_TCR_TG1_16K 1
10#define ARM_LPAE_TCR_TG1_4K 2
11#define ARM_LPAE_TCR_TG1_64K 3
12
13#define ARM_LPAE_TCR_SH_NS 0
14#define ARM_LPAE_TCR_SH_OS 2
15#define ARM_LPAE_TCR_SH_IS 3
16
17#define ARM_LPAE_TCR_RGN_NC 0
18#define ARM_LPAE_TCR_RGN_WBWA 1
19#define ARM_LPAE_TCR_RGN_WT 2
20#define ARM_LPAE_TCR_RGN_WB 3
21
22#define ARM_LPAE_TCR_PS_32_BIT 0x0ULL
23#define ARM_LPAE_TCR_PS_36_BIT 0x1ULL
24#define ARM_LPAE_TCR_PS_40_BIT 0x2ULL
25#define ARM_LPAE_TCR_PS_42_BIT 0x3ULL
26#define ARM_LPAE_TCR_PS_44_BIT 0x4ULL
27#define ARM_LPAE_TCR_PS_48_BIT 0x5ULL
28#define ARM_LPAE_TCR_PS_52_BIT 0x6ULL
29
30#endif /* IO_PGTABLE_ARM_H_ */