// SPDX-License-Identifier: GPL-2.0-or-later/* * Copyright 2016,2017 IBM Corporation. */ #define pr_fmt(fmt) "xive: " fmt #include <linux/types.h>#include <linux/threads.h>#include <linux/kernel.h>#include <linux/irq.h>#include <linux/irqdomain.h>#include <linux/debugfs.h>#include <linux/smp.h>#include <linux/interrupt.h>#include <linux/seq_file.h>#include <linux/init.h>#include <linux/cpu.h>#include <linux/of.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/msi.h>#include <linux/vmalloc.h> #include <asm/io.h>#include <asm/smp.h>#include <asm/machdep.h>#include <asm/irq.h>#include <asm/errno.h>#include <asm/xive.h>#include <asm/xive-regs.h>#include <asm/xmon.h> #include "xive-internal.h" #undef DEBUG_FLUSH#undef DEBUG_ALL #ifdef DEBUG_ALL#define DBG_VERBOSE(fmt, ...) pr_devel("cpu %d - " fmt, \ smp_processor_id(), ## __VA_ARGS__)#else#define DBG_VERBOSE(fmt...) do { } while(0)#endif bool __xive_enabled;EXPORT_SYMBOL_GPL(__xive_enabled);bool xive_cmdline_disabled; /* We use only one priority for now */static u8 xive_irq_priority; /* TIMA exported to KVM */void __iomem *xive_tima;EXPORT_SYMBOL_GPL(xive_tima);u32 xive_tima_offset; /* Backend ops */static const struct xive_ops *xive_ops; /* Our global interrupt domain */static struct irq_domain *xive_irq_domain; #ifdef CONFIG_SMP/* The IPIs use the same logical irq number when on the same chip */static struct xive_ipi_desc { unsigned int irq; char name[16]; atomic_t started;} *xive_ipis; /* * Use early_cpu_to_node() for hot-plugged CPUs */static unsigned int xive_ipi_cpu_to_irq(unsigned int cpu){ if (!xive_ipis) return XIVE_BAD_IRQ; return xive_ipis[early_cpu_to_node(cpu)].irq;}#endif /* Xive state for each CPU */static DEFINE_PER_CPU(struct xive_cpu *, xive_cpu); /* An invalid CPU target */#define XIVE_INVALID_TARGET (-1) /* * Global toggle to switch on/off StoreEOI */static bool xive_store_eoi = true; static bool xive_is_store_eoi(struct xive_irq_data *xd){ return xd->flags & XIVE_IRQ_FLAG_STORE_EOI && xive_store_eoi;} /* * Read the next entry in a queue, return its content if it's valid * or 0 if there is no new entry. * * The queue pointer is moved forward unless "just_peek" is set */static u32 xive_read_eq(struct xive_q *q, bool just_peek){ u32 cur; if (!q->qpage) return 0; cur = be32_to_cpup(q->qpage + q->idx); /* Check valid bit (31) vs current toggle polarity */ if ((cur >> 31) == q->toggle) return 0; /* If consuming from the queue ... */ if (!just_peek) { /* Next entry */ q->idx = (q->idx + 1) & q->msk; /* Wrap around: flip valid toggle */ if (q->idx == 0) q->toggle ^= 1; } /* Mask out the valid bit (31) */ return cur & 0x7fffffff;} /* * Scans all the queue that may have interrupts in them * (based on "pending_prio") in priority order until an * interrupt is found or all the queues are empty. * * Then updates the CPPR (Current Processor Priority * Register) based on the most favored interrupt found * (0xff if none) and return what was found (0 if none). * * If just_peek is set, return the most favored pending * interrupt if any but don't update the queue pointers. * * Note: This function can operate generically on any number * of queues (up to 8). The current implementation of the XIVE * driver only uses a single queue however. * * Note2: This will also "flush" "the pending_count" of a queue * into the "count" when that queue is observed to be empty. * This is used to keep track of the amount of interrupts * targetting a queue. When an interrupt is moved away from * a queue, we only decrement that queue count once the queue * has been observed empty to avoid races. */static u32 xive_scan_interrupts(struct xive_cpu *xc, bool just_peek){ u32 irq = 0; u8 prio = 0; /* Find highest pending priority */ while (xc->pending_prio != 0) { struct xive_q *q; prio = ffs(xc->pending_prio) - 1; DBG_VERBOSE("scan_irq: trying prio %d\n", prio); /* Try to fetch */ irq = xive_read_eq(&xc->queue[prio], just_peek); /* Found something ? That's it */ if (irq) { if (just_peek || irq_to_desc(irq)) break; /* * We should never get here; if we do then we must * have failed to synchronize the interrupt properly * when shutting it down. */ pr_crit("xive: got interrupt %d without descriptor, dropping\n", irq); WARN_ON(1); continue; } /* Clear pending bits */ xc->pending_prio &= ~(1 << prio); /* * Check if the queue count needs adjusting due to * interrupts being moved away. See description of * xive_dec_target_count() */ q = &xc->queue[prio]; if (atomic_read(&q->pending_count)) { int p = atomic_xchg(&q->pending_count, 0); if (p) { WARN_ON(p > atomic_read(&q->count)); atomic_sub(p, &q->count); } } } /* If nothing was found, set CPPR to 0xff */ if (irq == 0) prio = 0xff; /* Update HW CPPR to match if necessary */ if (prio != xc->cppr) { DBG_VERBOSE("scan_irq: adjusting CPPR to %d\n", prio); xc->cppr = prio; out_8(xive_tima + xive_tima_offset + TM_CPPR, prio); } return irq;} /* * This is used to perform the magic loads from an ESB * described in xive-regs.h */static notrace u8 xive_esb_read(struct xive_irq_data *xd, u32 offset){ u64 val; if (offset == XIVE_ESB_SET_PQ_10 && xive_is_store_eoi(xd)) offset |= XIVE_ESB_LD_ST_MO; if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw) val = xive_ops->esb_rw(xd->hw_irq, offset, 0, 0); else val = in_be64(xd->eoi_mmio + offset); return (u8)val;} static void xive_esb_write(struct xive_irq_data *xd, u32 offset, u64 data){ if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw) xive_ops->esb_rw(xd->hw_irq, offset, data, 1); else out_be64(xd->eoi_mmio + offset, data);} #if defined(CONFIG_XMON) || defined(CONFIG_DEBUG_FS)static void xive_irq_data_dump(struct xive_irq_data *xd, char *buffer, size_t size){ u64 val = xive_esb_read(xd, XIVE_ESB_GET); snprintf(buffer, size, "flags=%c%c%c PQ=%c%c 0x%016llx 0x%016llx", xive_is_store_eoi(xd) ? 'S' : ' ', xd->flags & XIVE_IRQ_FLAG_LSI ? 'L' : ' ', xd->flags & XIVE_IRQ_FLAG_H_INT_ESB ? 'H' : ' ', val & XIVE_ESB_VAL_P ? 'P' : '-', val & XIVE_ESB_VAL_Q ? 'Q' : '-', xd->trig_page, xd->eoi_page);}#endif #ifdef CONFIG_XMONstatic notrace void xive_dump_eq(const char *name, struct xive_q *q){ u32 i0, i1, idx; if (!q->qpage) return; idx = q->idx; i0 = be32_to_cpup(q->qpage + idx); idx = (idx + 1) & q->msk; i1 = be32_to_cpup(q->qpage + idx); xmon_printf("%s idx=%d T=%d %08x %08x ...", name, q->idx, q->toggle, i0, i1);} notrace void xmon_xive_do_dump(int cpu){ struct xive_cpu *xc = per_cpu(xive_cpu, cpu); xmon_printf("CPU %d:", cpu); if (xc) { xmon_printf("pp=%02x CPPR=%02x ", xc->pending_prio, xc->cppr); #ifdef CONFIG_SMP { char buffer[128]; xive_irq_data_dump(&xc->ipi_data, buffer, sizeof(buffer)); xmon_printf("IPI=0x%08x %s", xc->hw_ipi, buffer); }#endif xive_dump_eq("EQ", &xc->queue[xive_irq_priority]); } xmon_printf("\n");} static struct irq_data *xive_get_irq_data(u32 hw_irq){ unsigned int irq = irq_find_mapping(xive_irq_domain, hw_irq); return irq ? irq_get_irq_data(irq) : NULL;} int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d){ int rc; u32 target; u8 prio; u32 lirq; rc = xive_ops->get_irq_config(hw_irq, &target, &prio, &lirq); if (rc) { xmon_printf("IRQ 0x%08x : no config rc=%d\n", hw_irq, rc); return rc; } xmon_printf("IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ", hw_irq, target, prio, lirq); if (!d) d = xive_get_irq_data(hw_irq); if (d) { char buffer[128]; xive_irq_data_dump(irq_data_get_irq_chip_data(d), buffer, sizeof(buffer)); xmon_printf("%s", buffer); } xmon_printf("\n"); return 0;} void xmon_xive_get_irq_all(void){ unsigned int i; struct irq_desc *desc; for_each_irq_desc(i, desc) { struct irq_data *d = irq_domain_get_irq_data(xive_irq_domain, i); if (d) xmon_xive_get_irq_config(irqd_to_hwirq(d), d); }} #endif /* CONFIG_XMON */ static unsigned int xive_get_irq(void){ struct xive_cpu *xc = __this_cpu_read(xive_cpu); u32 irq; /* * This can be called either as a result of a HW interrupt or * as a "replay" because EOI decided there was still something * in one of the queues. * * First we perform an ACK cycle in order to update our mask * of pending priorities. This will also have the effect of * updating the CPPR to the most favored pending interrupts. * * In the future, if we have a way to differentiate a first * entry (on HW interrupt) from a replay triggered by EOI, * we could skip this on replays unless we soft-mask tells us * that a new HW interrupt occurred. */ xive_ops->update_pending(xc); DBG_VERBOSE("get_irq: pending=%02x\n", xc->pending_prio); /* Scan our queue(s) for interrupts */ irq = xive_scan_interrupts(xc, false); DBG_VERBOSE("get_irq: got irq 0x%x, new pending=0x%02x\n", irq, xc->pending_prio); /* Return pending interrupt if any */ if (irq == XIVE_BAD_IRQ) return 0; return irq;} /* * After EOI'ing an interrupt, we need to re-check the queue * to see if another interrupt is pending since multiple * interrupts can coalesce into a single notification to the * CPU. * * If we find that there is indeed more in there, we call * force_external_irq_replay() to make Linux synthesize an * external interrupt on the next call to local_irq_restore(). */static void xive_do_queue_eoi(struct xive_cpu *xc){ if (xive_scan_interrupts(xc, true) != 0) { DBG_VERBOSE("eoi: pending=0x%02x\n", xc->pending_prio); force_external_irq_replay(); }} /* * EOI an interrupt at the source. There are several methods * to do this depending on the HW version and source type */static void xive_do_source_eoi(struct xive_irq_data *xd){ u8 eoi_val; xd->stale_p = false; /* If the XIVE supports the new "store EOI facility, use it */ if (xive_is_store_eoi(xd)) { xive_esb_write(xd, XIVE_ESB_STORE_EOI, 0); return; } /* * For LSIs, we use the "EOI cycle" special load rather than * PQ bits, as they are automatically re-triggered in HW when * still pending. */ if (xd->flags & XIVE_IRQ_FLAG_LSI) { xive_esb_read(xd, XIVE_ESB_LOAD_EOI); return; } /* * Otherwise, we use the special MMIO that does a clear of * both P and Q and returns the old Q. This allows us to then * do a re-trigger if Q was set rather than synthesizing an * interrupt in software */ eoi_val = xive_esb_read(xd, XIVE_ESB_SET_PQ_00); DBG_VERBOSE("eoi_val=%x\n", eoi_val); /* Re-trigger if needed */ if ((eoi_val & XIVE_ESB_VAL_Q) && xd->trig_mmio) out_be64(xd->trig_mmio, 0);} /* irq_chip eoi callback, called with irq descriptor lock held */static void xive_irq_eoi(struct irq_data *d){ struct xive_irq_data *xd = irq_data_get_irq_chip_data(d); struct xive_cpu *xc = __this_cpu_read(xive_cpu); DBG_VERBOSE("eoi_irq: irq=%d [0x%lx] pending=%02x\n", d->irq, irqd_to_hwirq(d), xc->pending_prio); /* * EOI the source if it hasn't been disabled and hasn't * been passed-through to a KVM guest */ if (!irqd_irq_disabled(d) && !irqd_is_forwarded_to_vcpu(d) && !(xd->flags & XIVE_IRQ_FLAG_NO_EOI)) xive_do_source_eoi(xd); else xd->stale_p = true; /* * Clear saved_p to indicate that it's no longer occupying * a queue slot on the target queue */ xd->saved_p = false; /* Check for more work in the queue */ xive_do_queue_eoi(xc);} /* * Helper used to mask and unmask an interrupt source. */static void xive_do_source_set_mask(struct xive_irq_data *xd, bool mask){ u64 val; pr_debug("%s: HW 0x%x %smask\n", __func__, xd->hw_irq, mask ? "" : "un"); /* * If the interrupt had P set, it may be in a queue. * * We need to make sure we don't re-enable it until it * has been fetched from that queue and EOId. We keep * a copy of that P state and use it to restore the * ESB accordingly on unmask. */ if (mask) { val = xive_esb_read(xd, XIVE_ESB_SET_PQ_01); if (!xd->stale_p && !!(val & XIVE_ESB_VAL_P)) xd->saved_p = true; xd->stale_p = false; } else if (xd->saved_p) { xive_esb_read(xd, XIVE_ESB_SET_PQ_10); xd->saved_p = false; } else { xive_esb_read(xd, XIVE_ESB_SET_PQ_00); xd->stale_p = false; }} /*Lines 1–500 of 1868. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright 2016,2017 IBM Corporation. */ #define pr_fmt(fmt) "xive: " fmt #include <linux/types.h> #include <linux/threads.h> #include <linux/kernel.h> #include <linux/irq.h> #include <linux/irqdomain.h> #include <linux/debugfs.h> #include <linux/smp.h> #include <linux/interrupt.h> #include <linux/seq_file.h> #include <linux/init.h> #include <linux/cpu.h> #include <linux/of.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/msi.h> #include <linux/vmalloc.h> #include <asm/io.h> #include <asm/smp.h> #include <asm/machdep.h> #include <asm/irq.h> #include <asm/errno.h> #include <asm/xive.h> #include <asm/xive-regs.h> #include <asm/xmon.h> #include "xive-internal.h" #undef DEBUG_FLUSH #undef DEBUG_ALL #ifdef DEBUG_ALL #define DBG_VERBOSE(fmt, ...) pr_devel("cpu %d - " fmt, \ smp_processor_id(), ## __VA_ARGS__) #else #define DBG_VERBOSE(fmt...) do { } while(0) #endif bool __xive_enabled; EXPORT_SYMBOL_GPL(__xive_enabled); bool xive_cmdline_disabled; /* We use only one priority for now */ static u8 xive_irq_priority; /* TIMA exported to KVM */ void __iomem *xive_tima; EXPORT_SYMBOL_GPL(xive_tima); u32 xive_tima_offset; /* Backend ops */ static const struct xive_ops *xive_ops; /* Our global interrupt domain */ static struct irq_domain *xive_irq_domain; #ifdef CONFIG_SMP /* The IPIs use the same logical irq number when on the same chip */ static struct xive_ipi_desc { unsigned int irq; char name[16]; atomic_t started; } *xive_ipis; /* * Use early_cpu_to_node() for hot-plugged CPUs */ static unsigned int xive_ipi_cpu_to_irq(unsigned int cpu) { if (!xive_ipis) return XIVE_BAD_IRQ; return xive_ipis[early_cpu_to_node(cpu)].irq; } #endif /* Xive state for each CPU */ static DEFINE_PER_CPU(struct xive_cpu *, xive_cpu); /* An invalid CPU target */ #define XIVE_INVALID_TARGET (-1) /* * Global toggle to switch on/off StoreEOI */ static bool xive_store_eoi = true; static bool xive_is_store_eoi(struct xive_irq_data *xd) { return xd->flags & XIVE_IRQ_FLAG_STORE_EOI && xive_store_eoi; } /* * Read the next entry in a queue, return its content if it's valid * or 0 if there is no new entry. * * The queue pointer is moved forward unless "just_peek" is set */ static u32 xive_read_eq(struct xive_q *q, bool just_peek) { u32 cur; if (!q->qpage) return 0; cur = be32_to_cpup(q->qpage + q->idx); /* Check valid bit (31) vs current toggle polarity */ if ((cur >> 31) == q->toggle) return 0; /* If consuming from the queue ... */ if (!just_peek) { /* Next entry */ q->idx = (q->idx + 1) & q->msk; /* Wrap around: flip valid toggle */ if (q->idx == 0) q->toggle ^= 1; } /* Mask out the valid bit (31) */ return cur & 0x7fffffff; } /* * Scans all the queue that may have interrupts in them * (based on "pending_prio") in priority order until an * interrupt is found or all the queues are empty. * * Then updates the CPPR (Current Processor Priority * Register) based on the most favored interrupt found * (0xff if none) and return what was found (0 if none). * * If just_peek is set, return the most favored pending * interrupt if any but don't update the queue pointers. * * Note: This function can operate generically on any number * of queues (up to 8). The current implementation of the XIVE * driver only uses a single queue however. * * Note2: This will also "flush" "the pending_count" of a queue * into the "count" when that queue is observed to be empty. * This is used to keep track of the amount of interrupts * targetting a queue. When an interrupt is moved away from * a queue, we only decrement that queue count once the queue * has been observed empty to avoid races. */ static u32 xive_scan_interrupts(struct xive_cpu *xc, bool just_peek) { u32 irq = 0; u8 prio = 0; /* Find highest pending priority */ while (xc->pending_prio != 0) { struct xive_q *q; prio = ffs(xc->pending_prio) - 1; DBG_VERBOSE("scan_irq: trying prio %d\n", prio); /* Try to fetch */ irq = xive_read_eq(&xc->queue[prio], just_peek); /* Found something ? That's it */ if (irq) { if (just_peek || irq_to_desc(irq)) break; /* * We should never get here; if we do then we must * have failed to synchronize the interrupt properly * when shutting it down. */ pr_crit("xive: got interrupt %d without descriptor, dropping\n", irq); WARN_ON(1); continue; } /* Clear pending bits */ xc->pending_prio &= ~(1 << prio); /* * Check if the queue count needs adjusting due to * interrupts being moved away. See description of * xive_dec_target_count() */ q = &xc->queue[prio]; if (atomic_read(&q->pending_count)) { int p = atomic_xchg(&q->pending_count, 0); if (p) { WARN_ON(p > atomic_read(&q->count)); atomic_sub(p, &q->count); } } } /* If nothing was found, set CPPR to 0xff */ if (irq == 0) prio = 0xff; /* Update HW CPPR to match if necessary */ if (prio != xc->cppr) { DBG_VERBOSE("scan_irq: adjusting CPPR to %d\n", prio); xc->cppr = prio; out_8(xive_tima + xive_tima_offset + TM_CPPR, prio); } return irq; } /* * This is used to perform the magic loads from an ESB * described in xive-regs.h */ static notrace u8 xive_esb_read(struct xive_irq_data *xd, u32 offset) { u64 val; if (offset == XIVE_ESB_SET_PQ_10 && xive_is_store_eoi(xd)) offset |= XIVE_ESB_LD_ST_MO; if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw) val = xive_ops->esb_rw(xd->hw_irq, offset, 0, 0); else val = in_be64(xd->eoi_mmio + offset); return (u8)val; } static void xive_esb_write(struct xive_irq_data *xd, u32 offset, u64 data) { if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw) xive_ops->esb_rw(xd->hw_irq, offset, data, 1); else out_be64(xd->eoi_mmio + offset, data); } #if defined(CONFIG_XMON) || defined(CONFIG_DEBUG_FS) static void xive_irq_data_dump(struct xive_irq_data *xd, char *buffer, size_t size) { u64 val = xive_esb_read(xd, XIVE_ESB_GET); snprintf(buffer, size, "flags=%c%c%c PQ=%c%c 0x%016llx 0x%016llx", xive_is_store_eoi(xd) ? 'S' : ' ', xd->flags & XIVE_IRQ_FLAG_LSI ? 'L' : ' ', xd->flags & XIVE_IRQ_FLAG_H_INT_ESB ? 'H' : ' ', val & XIVE_ESB_VAL_P ? 'P' : '-', val & XIVE_ESB_VAL_Q ? 'Q' : '-', xd->trig_page, xd->eoi_page); } #endif #ifdef CONFIG_XMON static notrace void xive_dump_eq(const char *name, struct xive_q *q) { u32 i0, i1, idx; if (!q->qpage) return; idx = q->idx; i0 = be32_to_cpup(q->qpage + idx); idx = (idx + 1) & q->msk; i1 = be32_to_cpup(q->qpage + idx); xmon_printf("%s idx=%d T=%d %08x %08x ...", name, q->idx, q->toggle, i0, i1); } notrace void xmon_xive_do_dump(int cpu) { struct xive_cpu *xc = per_cpu(xive_cpu, cpu); xmon_printf("CPU %d:", cpu); if (xc) { xmon_printf("pp=%02x CPPR=%02x ", xc->pending_prio, xc->cppr); #ifdef CONFIG_SMP { char buffer[128]; xive_irq_data_dump(&xc->ipi_data, buffer, sizeof(buffer)); xmon_printf("IPI=0x%08x %s", xc->hw_ipi, buffer); } #endif xive_dump_eq("EQ", &xc->queue[xive_irq_priority]); } xmon_printf("\n"); } static struct irq_data *xive_get_irq_data(u32 hw_irq) { unsigned int irq = irq_find_mapping(xive_irq_domain, hw_irq); return irq ? irq_get_irq_data(irq) : NULL; } int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d) { int rc; u32 target; u8 prio; u32 lirq; rc = xive_ops->get_irq_config(hw_irq, &target, &prio, &lirq); if (rc) { xmon_printf("IRQ 0x%08x : no config rc=%d\n", hw_irq, rc); return rc; } xmon_printf("IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ", hw_irq, target, prio, lirq); if (!d) d = xive_get_irq_data(hw_irq); if (d) { char buffer[128]; xive_irq_data_dump(irq_data_get_irq_chip_data(d), buffer, sizeof(buffer)); xmon_printf("%s", buffer); } xmon_printf("\n"); return 0; } void xmon_xive_get_irq_all(void) { unsigned int i; struct irq_desc *desc; for_each_irq_desc(i, desc) { struct irq_data *d = irq_domain_get_irq_data(xive_irq_domain, i); if (d) xmon_xive_get_irq_config(irqd_to_hwirq(d), d); } } #endif /* CONFIG_XMON */ static unsigned int xive_get_irq(void) { struct xive_cpu *xc = __this_cpu_read(xive_cpu); u32 irq; /* * This can be called either as a result of a HW interrupt or * as a "replay" because EOI decided there was still something * in one of the queues. * * First we perform an ACK cycle in order to update our mask * of pending priorities. This will also have the effect of * updating the CPPR to the most favored pending interrupts. * * In the future, if we have a way to differentiate a first * entry (on HW interrupt) from a replay triggered by EOI, * we could skip this on replays unless we soft-mask tells us * that a new HW interrupt occurred. */ xive_ops->update_pending(xc); DBG_VERBOSE("get_irq: pending=%02x\n", xc->pending_prio); /* Scan our queue(s) for interrupts */ irq = xive_scan_interrupts(xc, false); DBG_VERBOSE("get_irq: got irq 0x%x, new pending=0x%02x\n", irq, xc->pending_prio); /* Return pending interrupt if any */ if (irq == XIVE_BAD_IRQ) return 0; return irq; } /* * After EOI'ing an interrupt, we need to re-check the queue * to see if another interrupt is pending since multiple * interrupts can coalesce into a single notification to the * CPU. * * If we find that there is indeed more in there, we call * force_external_irq_replay() to make Linux synthesize an * external interrupt on the next call to local_irq_restore(). */ static void xive_do_queue_eoi(struct xive_cpu *xc) { if (xive_scan_interrupts(xc, true) != 0) { DBG_VERBOSE("eoi: pending=0x%02x\n", xc->pending_prio); force_external_irq_replay(); } } /* * EOI an interrupt at the source. There are several methods * to do this depending on the HW version and source type */ static void xive_do_source_eoi(struct xive_irq_data *xd) { u8 eoi_val; xd->stale_p = false; /* If the XIVE supports the new "store EOI facility, use it */ if (xive_is_store_eoi(xd)) { xive_esb_write(xd, XIVE_ESB_STORE_EOI, 0); return; } /* * For LSIs, we use the "EOI cycle" special load rather than * PQ bits, as they are automatically re-triggered in HW when * still pending. */ if (xd->flags & XIVE_IRQ_FLAG_LSI) { xive_esb_read(xd, XIVE_ESB_LOAD_EOI); return; } /* * Otherwise, we use the special MMIO that does a clear of * both P and Q and returns the old Q. This allows us to then * do a re-trigger if Q was set rather than synthesizing an * interrupt in software */ eoi_val = xive_esb_read(xd, XIVE_ESB_SET_PQ_00); DBG_VERBOSE("eoi_val=%x\n", eoi_val); /* Re-trigger if needed */ if ((eoi_val & XIVE_ESB_VAL_Q) && xd->trig_mmio) out_be64(xd->trig_mmio, 0); } /* irq_chip eoi callback, called with irq descriptor lock held */ static void xive_irq_eoi(struct irq_data *d) { struct xive_irq_data *xd = irq_data_get_irq_chip_data(d); struct xive_cpu *xc = __this_cpu_read(xive_cpu); DBG_VERBOSE("eoi_irq: irq=%d [0x%lx] pending=%02x\n", d->irq, irqd_to_hwirq(d), xc->pending_prio); /* * EOI the source if it hasn't been disabled and hasn't * been passed-through to a KVM guest */ if (!irqd_irq_disabled(d) && !irqd_is_forwarded_to_vcpu(d) && !(xd->flags & XIVE_IRQ_FLAG_NO_EOI)) xive_do_source_eoi(xd); else xd->stale_p = true; /* * Clear saved_p to indicate that it's no longer occupying * a queue slot on the target queue */ xd->saved_p = false; /* Check for more work in the queue */ xive_do_queue_eoi(xc); } /* * Helper used to mask and unmask an interrupt source. */ static void xive_do_source_set_mask(struct xive_irq_data *xd, bool mask) { u64 val; pr_debug("%s: HW 0x%x %smask\n", __func__, xd->hw_irq, mask ? "" : "un"); /* * If the interrupt had P set, it may be in a queue. * * We need to make sure we don't re-enable it until it * has been fetched from that queue and EOId. We keep * a copy of that P state and use it to restore the * ESB accordingly on unmask. */ if (mask) { val = xive_esb_read(xd, XIVE_ESB_SET_PQ_01); if (!xd->stale_p && !!(val & XIVE_ESB_VAL_P)) xd->saved_p = true; xd->stale_p = false; } else if (xd->saved_p) { xive_esb_read(xd, XIVE_ESB_SET_PQ_10); xd->saved_p = false; } else { xive_esb_read(xd, XIVE_ESB_SET_PQ_00); xd->stale_p = false; } } /*