KVM/arm64 fixes for 7.2, take #3 - Fix a tiny buglet when propagating the deactivation of an interrupt from a nested guest, which happened to trigger a gold plated CPU bug on a particular implementation - Fix a race between LPI unmapping and mapping, resulting in leaked LPIs - Make LPI mapping more robust on memory allocation failure - Fix the handling of the EL2 tracing clock being disabled - A couple of Sashiko-driven fixes for corner cases in the EL2 tracing code - Add missing sysreg tracepoint for the EL2 code - Tidy-up the mutual exclusion of guest-memfd and MTE - Update Fuad's email address to point to @linux.dev
Unified · Split
| @@ -296,6 +296,7 @@ Frank Rowand <frowand.list@gmail.com> <frank.rowand@sony.com> · 295 unmodified lines | ||
| 296 | 296 | Frank Rowand <frowand.list@gmail.com> <frank.rowand@sonymobile.com> |
| 297 | 297 | Frank Rowand <frowand.list@gmail.com> <frowand@mvista.com> |
| 298 | 298 | Frank Zago <fzago@systemfabricworks.com> |
| 299 | +Fuad Tabba <fuad.tabba@linux.dev> <tabba@google.com> | |
| 299 | 300 | Gao Xiang <xiang@kernel.org> <gaoxiang25@huawei.com> |
| 300 | 301 | Gao Xiang <xiang@kernel.org> <hsiangkao@aol.com> |
| 301 | 302 | Gao Xiang <xiang@kernel.org> <hsiangkao@linux.alibaba.com> |
| @@ -55,10 +55,14 @@ stable kernels. · 54 unmodified lines | ||
| 55 | 55 | +----------------+-----------------+-----------------+-----------------------------+ |
| 56 | 56 | | Ampere | AmpereOne | AC03_CPU_38 | AMPERE_ERRATUM_AC03_CPU_38 | |
| 57 | 57 | +----------------+-----------------+-----------------+-----------------------------+ |
| 58 | +| Ampere | AmpereOne | AC03_CPU_57 | N/A | | |
| 59 | ++----------------+-----------------+-----------------+-----------------------------+ | |
| 58 | 60 | | Ampere | AmpereOne AC04 | AC04_CPU_10 | AMPERE_ERRATUM_AC03_CPU_38 | |
| 59 | 61 | +----------------+-----------------+-----------------+-----------------------------+ |
| 60 | 62 | | Ampere | AmpereOne AC04 | AC04_CPU_23 | AMPERE_ERRATUM_AC04_CPU_23 | |
| 61 | 63 | +----------------+-----------------+-----------------+-----------------------------+ |
| 64 | +| Ampere | AmpereOne AC04 | AC04_CPU_29 | N/A | | |
| 65 | ++----------------+-----------------+-----------------+-----------------------------+ | |
| 62 | 66 | +----------------+-----------------+-----------------+-----------------------------+ |
| 63 | 67 | | ARM | Cortex-A510 | #2457168 | ARM64_ERRATUM_2457168 | |
| 64 | 68 | +----------------+-----------------+-----------------+-----------------------------+ |
| @@ -8414,6 +8414,12 @@ When this capability is enabled all memory in memslots must be mapped as · 8413 unmodified lines | ||
| 8414 | 8414 | attempts to create a memslot with an invalid mmap will result in an |
| 8415 | 8415 | -EINVAL return. |
| 8416 | 8416 | |
| 8417 | +``guest_memfd``, even though it is an anonymous file, is not supported with MTE. | |
| 8418 | +Attempting to create a memslot backed by ``guest_memfd`` when the MTE capability | |
| 8419 | +is enabled, or attempting to enable the MTE capability after | |
| 8420 | +``guest_memfd``-backed memslots have been created, will result in an -EINVAL | |
| 8421 | +return. | |
| 8422 | + | |
| 8417 | 8423 | When enabled the VMM may make use of the ``KVM_ARM_MTE_COPY_TAGS`` ioctl to |
| 8418 | 8424 | perform a bulk copy of tags to/from the guest. |
| 8419 | 8425 | |
| @@ -14193,7 +14193,7 @@ F: virt/kvm/* · 14192 unmodified lines | ||
| 14193 | 14193 | KERNEL VIRTUAL MACHINE FOR ARM64 (KVM/arm64) |
| 14194 | 14194 | M: Marc Zyngier <maz@kernel.org> |
| 14195 | 14195 | M: Oliver Upton <oupton@kernel.org> |
| 14196 | −R: Fuad Tabba <tabba@google.com> | |
| 14196 | +R: Fuad Tabba <fuad.tabba@linux.dev> | |
| 14197 | 14197 | R: Joey Gouly <joey.gouly@arm.com> |
| 14198 | 14198 | R: Steffen Eiden <seiden@linux.ibm.com> |
| 14199 | 14199 | R: Suzuki K Poulose <suzuki.poulose@arm.com> |
| @@ -12,6 +12,7 @@ · 11 unmodified lines | ||
| 12 | 12 | enum hyp_enter_exit_reason { |
| 13 | 13 | HYP_REASON_SMC, |
| 14 | 14 | HYP_REASON_HVC, |
| 15 | + HYP_REASON_SYS, | |
| 15 | 16 | HYP_REASON_PSCI, |
| 16 | 17 | HYP_REASON_HOST_ABORT, |
| 17 | 18 | HYP_REASON_GUEST_EXIT, |
| @@ -149,14 +149,27 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, · 148 unmodified lines | ||
| 149 | 149 | set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER, |
| 150 | 150 | &kvm->arch.flags); |
| 151 | 151 | break; |
| 152 | − case KVM_CAP_ARM_MTE: | |
| 153 | − mutex_lock(&kvm->lock); | |
| 154 | − if (system_supports_mte() && !kvm->created_vcpus) { | |
| 155 | − r = 0; | |
| 156 | − set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags); | |
| 152 | + case KVM_CAP_ARM_MTE: { | |
| 153 | + struct kvm_memory_slot *memslot; | |
| 154 | + int bkt; | |
| 155 | + | |
| 156 | + guard(mutex)(&kvm->lock); | |
| 157 | + if (!system_supports_mte() || kvm->created_vcpus) | |
| 158 | + break; | |
| 159 | + | |
| 160 | + r = 0; | |
| 161 | + guard(mutex)(&kvm->slots_lock); | |
| 162 | + kvm_for_each_memslot(memslot, bkt, kvm_memslots(kvm)) { | |
| 163 | + if (kvm_slot_has_gmem(memslot)) { | |
| 164 | + r = -EINVAL; | |
| 165 | + break; | |
| 166 | + } | |
| 157 | 167 | } |
| 158 | − mutex_unlock(&kvm->lock); | |
| 168 | + if (r == 0) | |
| 169 | + set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags); | |
| 159 | 170 | break; |
| 171 | + | |
| 172 | + } | |
| 160 | 173 | case KVM_CAP_ARM_SYSTEM_SUSPEND: |
| 161 | 174 | r = 0; |
| 162 | 175 | set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags); |
| @@ -932,6 +932,7 @@ void handle_trap(struct kvm_cpu_context *host_ctxt) · 931 unmodified lines | ||
| 932 | 932 | handle_host_mem_abort(host_ctxt); |
| 933 | 933 | break; |
| 934 | 934 | case ESR_ELx_EC_SYS64: |
| 935 | + trace_hyp_enter(host_ctxt, HYP_REASON_SYS); | |
| 935 | 936 | if (handle_host_mte(esr)) |
| 936 | 937 | break; |
| 937 | 938 | fallthrough; |
| @@ -37,8 +37,6 @@ static struct hyp_trace_clock { · 36 unmodified lines | ||
| 37 | 37 | u32 shift; |
| 38 | 38 | struct delayed_work work; |
| 39 | 39 | struct completion ready; |
| 40 | − struct mutex lock; | |
| 41 | − bool running; | |
| 42 | 40 | } hyp_clock; |
| 43 | 41 | |
| 44 | 42 | static void __hyp_clock_work(struct work_struct *work) |
| @@ -110,12 +108,9 @@ static void hyp_trace_clock_enable(struct hyp_trace_clock *hyp_clock, bool enabl · 65 unmodified lines | ||
| 110 | 108 | { |
| 111 | 109 | struct system_time_snapshot snap; |
| 112 | 110 | |
| 113 | − if (hyp_clock->running == enable) | |
| 114 | − return; | |
| 115 | − | |
| 116 | 111 | if (!enable) { |
| 117 | 112 | cancel_delayed_work_sync(&hyp_clock->work); |
| 118 | − hyp_clock->running = false; | |
| 113 | + return; | |
| 119 | 114 | } |
| 120 | 115 | |
| 121 | 116 | ktime_get_snapshot_id(CLOCK_BOOTTIME, &snap); |
| @@ -128,7 +123,6 @@ static void hyp_trace_clock_enable(struct hyp_trace_clock *hyp_clock, bool enabl · 6 unmodified lines | ||
| 128 | 123 | INIT_DELAYED_WORK(&hyp_clock->work, __hyp_clock_work); |
| 129 | 124 | schedule_delayed_work(&hyp_clock->work, msecs_to_jiffies(CLOCK_INIT_MS)); |
| 130 | 125 | wait_for_completion(&hyp_clock->ready); |
| 131 | − hyp_clock->running = true; | |
| 132 | 126 | } |
| 133 | 127 | |
| 134 | 128 | /* Access to this struct within the trace_remote_callbacks are protected by the trace_remote lock */ |
| @@ -160,6 +154,7 @@ static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_ · 25 unmodified lines | ||
| 160 | 154 | int nr_bpages = (PAGE_ALIGN(size) / PAGE_SIZE) + 1; |
| 161 | 155 | size_t backing_size; |
| 162 | 156 | void *start; |
| 157 | + int ret; | |
| 163 | 158 | |
| 164 | 159 | backing_size = PAGE_ALIGN(sizeof(struct simple_buffer_page) * nr_bpages * |
| 165 | 160 | num_possible_cpus()); |
| @@ -168,10 +163,16 @@ static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_ · 2 unmodified lines | ||
| 168 | 163 | if (!start) |
| 169 | 164 | return -ENOMEM; |
| 170 | 165 | |
| 166 | + ret = __map_hyp(start, backing_size); | |
| 167 | + if (ret) { | |
| 168 | + free_pages_exact(start, backing_size); | |
| 169 | + return ret; | |
| 170 | + } | |
| 171 | + | |
| 171 | 172 | trace_buffer->desc->bpages_backing_start = (unsigned long)start; |
| 172 | 173 | trace_buffer->desc->bpages_backing_size = backing_size; |
| 173 | 174 | |
| 174 | − return __map_hyp(start, backing_size); | |
| 175 | + return ret; | |
| 175 | 176 | } |
| 176 | 177 | |
| 177 | 178 | static void hyp_trace_buffer_free_bpages_backing(struct hyp_trace_buffer *trace_buffer) |
| @@ -228,18 +229,22 @@ static int hyp_trace_buffer_share_hyp(struct hyp_trace_buffer *trace_buffer) · 50 unmodified lines | ||
| 228 | 229 | static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv) |
| 229 | 230 | { |
| 230 | 231 | struct hyp_trace_buffer *trace_buffer = priv; |
| 232 | + size_t desc_size, tb_desc_size; | |
| 231 | 233 | struct hyp_trace_desc *desc; |
| 232 | − size_t desc_size; | |
| 233 | 234 | int ret; |
| 234 | 235 | |
| 235 | 236 | if (WARN_ON(trace_buffer->desc)) |
| 236 | 237 | return ERR_PTR(-EINVAL); |
| 237 | 238 | |
| 238 | − desc_size = trace_buffer_desc_size(size, num_possible_cpus()); | |
| 239 | + tb_desc_size = trace_buffer_desc_size(size, num_possible_cpus()); | |
| 240 | + desc_size = size_add(tb_desc_size, offsetof(struct hyp_trace_desc, trace_buffer_desc)); | |
| 239 | 241 | if (desc_size == SIZE_MAX) |
| 240 | 242 | return ERR_PTR(-E2BIG); |
| 241 | 243 | |
| 242 | 244 | desc_size = PAGE_ALIGN(desc_size); |
| 245 | + if (!desc_size) | |
| 246 | + return ERR_PTR(-E2BIG); | |
| 247 | + | |
| 243 | 248 | desc = (struct hyp_trace_desc *)alloc_pages_exact(desc_size, GFP_KERNEL); |
| 244 | 249 | if (!desc) |
| 245 | 250 | return ERR_PTR(-ENOMEM); |
| @@ -255,7 +260,7 @@ static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv) · 9 unmodified lines | ||
| 255 | 260 | if (ret) |
| 256 | 261 | goto err_free_desc; |
| 257 | 262 | |
| 258 | − ret = trace_remote_alloc_buffer(&desc->trace_buffer_desc, desc_size, size, | |
| 263 | + ret = trace_remote_alloc_buffer(&desc->trace_buffer_desc, tb_desc_size, size, | |
| 259 | 264 | cpu_possible_mask); |
| 260 | 265 | if (ret) |
| 261 | 266 | goto err_free_backing; |
| @@ -304,9 +309,15 @@ static void hyp_trace_unload(struct trace_buffer_desc *desc, void *priv) · 42 unmodified lines | ||
| 304 | 309 | |
| 305 | 310 | static int hyp_trace_enable_tracing(bool enable, void *priv) |
| 306 | 311 | { |
| 312 | + int ret; | |
| 313 | + | |
| 307 | 314 | hyp_trace_clock_enable(&hyp_clock, enable); |
| 308 | 315 | |
| 309 | − return kvm_call_hyp_nvhe(__tracing_enable, enable); | |
| 316 | + ret = kvm_call_hyp_nvhe(__tracing_enable, enable); | |
| 317 | + if (ret) | |
| 318 | + hyp_trace_clock_enable(&hyp_clock, !enable); | |
| 319 | + | |
| 320 | + return ret; | |
| 310 | 321 | } |
| 311 | 322 | |
| 312 | 323 | static int hyp_trace_swap_reader_page(unsigned int cpu, void *priv) |
| @@ -398,6 +409,7 @@ static const char *__hyp_enter_exit_reason_str(u8 reason) · 85 unmodified lines | ||
| 398 | 409 | static const char strs[][12] = { |
| 399 | 410 | "smc", |
| 400 | 411 | "hvc", |
| 412 | + "sys", | |
| 401 | 413 | "psci", |
| 402 | 414 | "host_abort", |
| 403 | 415 | "guest_exit", |
| @@ -2652,6 +2652,10 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, · 2651 unmodified lines | ||
| 2652 | 2652 | if (kvm_slot_has_gmem(new) && !kvm_memslot_is_gmem_only(new)) |
| 2653 | 2653 | return -EINVAL; |
| 2654 | 2654 | |
| 2655 | + /* guest_memfd is incompatible with MTE. */ | |
| 2656 | + if (kvm_slot_has_gmem(new) && kvm_has_mte(kvm)) | |
| 2657 | + return -EINVAL; | |
| 2658 | + | |
| 2655 | 2659 | hva = new->userspace_addr; |
| 2656 | 2660 | reg_end = hva + (new->npages << PAGE_SHIFT); |
| 2657 | 2661 | |
| @@ -116,18 +116,27 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, · 115 unmodified lines | ||
| 116 | 116 | kfree(irq); |
| 117 | 117 | irq = oldirq; |
| 118 | 118 | } else { |
| 119 | − ret = xa_err(__xa_store(&dist->lpi_xa, intid, irq, 0)); | |
| 119 | + /* | |
| 120 | + * The entry is either empty or contains a dead LPI (refcount=0) | |
| 121 | + * from the deferred release path, pending cleanup by | |
| 122 | + * vgic_release_deleted_lpis(). Evict and free it if present. | |
| 123 | + */ | |
| 124 | + oldirq = __xa_store(&dist->lpi_xa, intid, irq, | |
| 125 | + GFP_NOWAIT | __GFP_ACCOUNT); | |
| 126 | + ret = xa_err(oldirq); | |
| 127 | + if (ret) { | |
| 128 | + xa_unlock_irqrestore(&dist->lpi_xa, flags); | |
| 129 | + kfree(irq); | |
| 130 | + | |
| 131 | + return ERR_PTR(ret); | |
| 132 | + } | |
| 133 | + | |
| 134 | + if (oldirq && !WARN_ON_ONCE(refcount_read(&oldirq->refcount))) | |
| 135 | + kfree_rcu(oldirq, rcu); | |
| 120 | 136 | } |
| 121 | 137 | |
| 122 | 138 | xa_unlock_irqrestore(&dist->lpi_xa, flags); |
| 123 | 139 | |
| 124 | − if (ret) { | |
| 125 | − xa_release(&dist->lpi_xa, intid); | |
| 126 | − kfree(irq); | |
| 127 | − | |
| 128 | − return ERR_PTR(ret); | |
| 129 | − } | |
| 130 | − | |
| 131 | 140 | /* |
| 132 | 141 | * We "cache" the configuration table entries in our struct vgic_irq's. |
| 133 | 142 | * However we only have those structs for mapped IRQs, so we read in |
| @@ -275,7 +275,13 @@ void vgic_v3_deactivate(struct kvm_vcpu *vcpu, u64 val) · 274 unmodified lines | ||
| 275 | 275 | lr = vgic_v3_compute_lr(vcpu, irq) & ~ICH_LR_ACTIVE_BIT; |
| 276 | 276 | } |
| 277 | 277 | |
| 278 | − if (lr & ICH_LR_HW) | |
| 278 | + /* | |
| 279 | + * In the nested state, the irq has already been deactivated via the HW | |
| 280 | + * bit in the LR. Deactivating again would be harmless except AmpereOne | |
| 281 | + * errata AC03_CPU_57, AC04_CPU_29 could cause irq delivery to break if | |
| 282 | + * the deactivation hits the highest priority pending irq. | |
| 283 | + */ | |
| 284 | + if ((lr & ICH_LR_HW) && !vgic_state_is_nested(vcpu)) | |
| 279 | 285 | vgic_v3_deactivate_phys(FIELD_GET(ICH_LR_PHYS_ID_MASK, lr)); |
| 280 | 286 | |
| 281 | 287 | vgic_v3_fold_lr(vcpu, lr); |
| @@ -146,11 +146,7 @@ static __must_check bool __vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) · 145 unmodified lines | ||
| 146 | 146 | |
| 147 | 147 | static __must_check bool vgic_put_irq_norelease(struct kvm *kvm, struct vgic_irq *irq) |
| 148 | 148 | { |
| 149 | − if (!__vgic_put_irq(kvm, irq)) | |
| 150 | − return false; | |
| 151 | − | |
| 152 | − irq->pending_release = true; | |
| 153 | − return true; | |
| 149 | + return __vgic_put_irq(kvm, irq); | |
| 154 | 150 | } |
| 155 | 151 | |
| 156 | 152 | void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) |
| @@ -167,12 +163,14 @@ void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) · 10 unmodified lines | ||
| 167 | 163 | guard(spinlock_irqsave)(&dist->lpi_xa.xa_lock); |
| 168 | 164 | } |
| 169 | 165 | |
| 170 | − if (!__vgic_put_irq(kvm, irq)) | |
| 166 | + if (!irq_is_lpi(kvm, irq->intid)) | |
| 171 | 167 | return; |
| 172 | 168 | |
| 173 | − xa_lock_irqsave(&dist->lpi_xa, flags); | |
| 174 | − vgic_release_lpi_locked(dist, irq); | |
| 175 | − xa_unlock_irqrestore(&dist->lpi_xa, flags); | |
| 169 | + if (refcount_dec_and_lock_irqsave(&irq->refcount, | |
| 170 | + &dist->lpi_xa.xa_lock, &flags)) { | |
| 171 | + vgic_release_lpi_locked(dist, irq); | |
| 172 | + xa_unlock_irqrestore(&dist->lpi_xa, flags); | |
| 173 | + } | |
| 176 | 174 | } |
| 177 | 175 | |
| 178 | 176 | static void vgic_release_deleted_lpis(struct kvm *kvm) |
| @@ -184,7 +182,7 @@ static void vgic_release_deleted_lpis(struct kvm *kvm) · 5 unmodified lines | ||
| 184 | 182 | xa_lock_irqsave(&dist->lpi_xa, flags); |
| 185 | 183 | |
| 186 | 184 | xa_for_each(&dist->lpi_xa, intid, irq) { |
| 187 | − if (irq->pending_release) | |
| 185 | + if (!refcount_read(&irq->refcount)) | |
| 188 | 186 | vgic_release_lpi_locked(dist, irq); |
| 189 | 187 | } |
| 190 | 188 | |
| @@ -247,9 +247,6 @@ struct vgic_irq { · 246 unmodified lines | ||
| 247 | 247 | * affinity reg (v3). |
| 248 | 248 | */ |
| 249 | 249 | |
| 250 | − bool pending_release:1; /* Used for LPIs only, unreferenced IRQ | |
| 251 | − * pending a release */ | |
| 252 | − | |
| 253 | 250 | bool pending_latch:1; /* The pending latch state used to calculate |
| 254 | 251 | * the pending state for both level |
| 255 | 252 | * and edge triggered IRQs. */ |