// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 2014 Intel Corp. * Author: Jiang Liu <jiang.liu@linux.intel.com> * * This file is licensed under GPLv2. * * This file contains common code to support Message Signaled Interrupts for * PCI compatible and non PCI compatible devices. */#include <linux/device.h>#include <linux/irq.h>#include <linux/irqdomain.h>#include <linux/msi.h>#include <linux/mutex.h>#include <linux/pci.h>#include <linux/slab.h>#include <linux/seq_file.h>#include <linux/sysfs.h>#include <linux/types.h>#include <linux/xarray.h> #include "internals.h" /** * struct msi_device_data - MSI per device data * @properties:		MSI properties which are interesting to drivers * @mutex:		Mutex protecting the MSI descriptor store * @__domains:		Internal data for per device MSI domains * @__iter_idx:		Index to search the next entry for iterators */struct msi_device_data {	unsigned long			properties;	struct mutex			mutex;	struct msi_dev_domain		__domains[MSI_MAX_DEVICE_IRQDOMAINS];	unsigned long			__iter_idx;}; /** * struct msi_ctrl - MSI internal management control structure * @domid:	ID of the domain on which management operations should be done * @first:	First (hardware) slot index to operate on * @last:	Last (hardware) slot index to operate on * @nirqs:	The number of Linux interrupts to allocate. Can be larger *		than the range due to PCI/multi-MSI. */struct msi_ctrl {	unsigned int			domid;	unsigned int			first;	unsigned int			last;	unsigned int			nirqs;}; /* Invalid Xarray index which is outside of any searchable range */#define MSI_XA_MAX_INDEX	(ULONG_MAX - 1)/* The maximum domain size */#define MSI_XA_DOMAIN_SIZE	(MSI_MAX_INDEX + 1) static void msi_domain_free_locked(struct device *dev, struct msi_ctrl *ctrl);static unsigned int msi_domain_get_hwsize(struct device *dev, unsigned int domid);static inline int msi_sysfs_create_group(struct device *dev);static int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev,				   int nvec, msi_alloc_info_t *arg); /** * msi_alloc_desc - Allocate an initialized msi_desc * @dev:	Pointer to the device for which this is allocated * @nvec:	The number of vectors used in this entry * @affinity:	Optional pointer to an affinity mask array size of @nvec * * If @affinity is not %NULL then an affinity array[@nvec] is allocated * and the affinity masks and flags from @affinity are copied. * * Return: pointer to allocated &msi_desc on success or %NULL on failure */static struct msi_desc *msi_alloc_desc(struct device *dev, int nvec,				       const struct irq_affinity_desc *affinity){	struct msi_desc *desc = kzalloc_obj(*desc); 	if (!desc)		return NULL; 	desc->dev = dev;	desc->nvec_used = nvec;	if (affinity) {		desc->affinity = kmemdup_array(affinity, nvec, sizeof(*desc->affinity), GFP_KERNEL);		if (!desc->affinity) {			kfree(desc);			return NULL;		}	}	return desc;} static void msi_free_desc(struct msi_desc *desc){	kfree(desc->affinity);	kfree(desc);} static int msi_insert_desc(struct device *dev, struct msi_desc *desc,			   unsigned int domid, unsigned int index){	struct msi_device_data *md = dev->msi.data;	struct xarray *xa = &md->__domains[domid].store;	unsigned int hwsize;	int ret; 	hwsize = msi_domain_get_hwsize(dev, domid); 	if (index == MSI_ANY_INDEX) {		struct xa_limit limit = { .min = 0, .max = hwsize - 1 };		unsigned int index; 		/* Let the xarray allocate a free index within the limit */		ret = xa_alloc(xa, &index, desc, limit, GFP_KERNEL);		if (ret)			goto fail; 		desc->msi_index = index;		return 0;	} else {		if (index >= hwsize) {			ret = -ERANGE;			goto fail;		} 		desc->msi_index = index;		ret = xa_insert(xa, index, desc, GFP_KERNEL);		if (ret)			goto fail;		return 0;	}fail:	msi_free_desc(desc);	return ret;} /** * msi_domain_insert_msi_desc - Allocate and initialize a MSI descriptor and *				insert it at @init_desc->msi_index * * @dev:	Pointer to the device for which the descriptor is allocated * @domid:	The id of the interrupt domain to which the desriptor is added * @init_desc:	Pointer to an MSI descriptor to initialize the new descriptor * * Return: 0 on success or an appropriate failure code. */int msi_domain_insert_msi_desc(struct device *dev, unsigned int domid,			       struct msi_desc *init_desc){	struct msi_desc *desc; 	lockdep_assert_held(&dev->msi.data->mutex); 	desc = msi_alloc_desc(dev, init_desc->nvec_used, init_desc->affinity);	if (!desc)		return -ENOMEM; 	/* Copy type specific data to the new descriptor. */	desc->pci = init_desc->pci; 	return msi_insert_desc(dev, desc, domid, init_desc->msi_index);} static bool msi_desc_match(struct msi_desc *desc, enum msi_desc_filter filter){	switch (filter) {	case MSI_DESC_ALL:		return true;	case MSI_DESC_NOTASSOCIATED:		return !desc->irq;	case MSI_DESC_ASSOCIATED:		return !!desc->irq;	}	WARN_ON_ONCE(1);	return false;} static bool msi_ctrl_valid(struct device *dev, struct msi_ctrl *ctrl){	unsigned int hwsize; 	if (WARN_ON_ONCE(ctrl->domid >= MSI_MAX_DEVICE_IRQDOMAINS ||			 (dev->msi.domain &&			  !dev->msi.data->__domains[ctrl->domid].domain)))		return false; 	hwsize = msi_domain_get_hwsize(dev, ctrl->domid);	if (WARN_ON_ONCE(ctrl->first > ctrl->last ||			 ctrl->first >= hwsize ||			 ctrl->last >= hwsize))		return false;	return true;} static void msi_domain_free_descs(struct device *dev, struct msi_ctrl *ctrl){	struct msi_desc *desc;	struct xarray *xa;	unsigned long idx; 	lockdep_assert_held(&dev->msi.data->mutex); 	if (!msi_ctrl_valid(dev, ctrl))		return; 	xa = &dev->msi.data->__domains[ctrl->domid].store;	xa_for_each_range(xa, idx, desc, ctrl->first, ctrl->last) {		xa_erase(xa, idx); 		/* Leak the descriptor when it is still referenced */		if (WARN_ON_ONCE(msi_desc_match(desc, MSI_DESC_ASSOCIATED)))			continue;		msi_free_desc(desc);	}} /** * msi_domain_free_msi_descs_range - Free a range of MSI descriptors of a device in an irqdomain * @dev:	Device for which to free the descriptors * @domid:	Id of the domain to operate on * @first:	Index to start freeing from (inclusive) * @last:	Last index to be freed (inclusive) */void msi_domain_free_msi_descs_range(struct device *dev, unsigned int domid,				     unsigned int first, unsigned int last){	struct msi_ctrl ctrl = {		.domid	= domid,		.first	= first,		.last	= last,	}; 	msi_domain_free_descs(dev, &ctrl);} /** * msi_domain_add_simple_msi_descs - Allocate and initialize MSI descriptors * @dev:	Pointer to the device for which the descriptors are allocated * @ctrl:	Allocation control struct * * Return: 0 on success or an appropriate failure code. */static int msi_domain_add_simple_msi_descs(struct device *dev, struct msi_ctrl *ctrl){	struct msi_desc *desc;	unsigned int idx;	int ret; 	lockdep_assert_held(&dev->msi.data->mutex); 	if (!msi_ctrl_valid(dev, ctrl))		return -EINVAL; 	for (idx = ctrl->first; idx <= ctrl->last; idx++) {		desc = msi_alloc_desc(dev, 1, NULL);		if (!desc)			goto fail_mem;		ret = msi_insert_desc(dev, desc, ctrl->domid, idx);		if (ret)			goto fail;	}	return 0; fail_mem:	ret = -ENOMEM;fail:	msi_domain_free_descs(dev, ctrl);	return ret;} void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg){	*msg = entry->msg;} void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg){	struct msi_desc *entry = irq_get_msi_desc(irq); 	__get_cached_msi_msg(entry, msg);}EXPORT_SYMBOL_GPL(get_cached_msi_msg); static void msi_device_data_release(struct device *dev, void *res){	struct msi_device_data *md = res;	int i; 	for (i = 0; i < MSI_MAX_DEVICE_IRQDOMAINS; i++) {		msi_remove_device_irq_domain(dev, i);		WARN_ON_ONCE(!xa_empty(&md->__domains[i].store));		xa_destroy(&md->__domains[i].store);	}	dev->msi.data = NULL;} /** * msi_setup_device_data - Setup MSI device data * @dev:	Device for which MSI device data should be set up * * Return: 0 on success, appropriate error code otherwise * * This can be called more than once for @dev. If the MSI device data is * already allocated the call succeeds. The allocated memory is * automatically released when the device is destroyed. */int msi_setup_device_data(struct device *dev){	struct msi_device_data *md;	int ret, i; 	if (dev->msi.data)		return 0; 	md = devres_alloc(msi_device_data_release, sizeof(*md), GFP_KERNEL);	if (!md)		return -ENOMEM; 	ret = msi_sysfs_create_group(dev);	if (ret) {		devres_free(md);		return ret;	} 	for (i = 0; i < MSI_MAX_DEVICE_IRQDOMAINS; i++)		xa_init_flags(&md->__domains[i].store, XA_FLAGS_ALLOC); 	/*	 * If @dev::msi::domain is set and is a global MSI domain, copy the	 * pointer into the domain array so all code can operate on domain	 * ids. The NULL pointer check is required to keep the legacy	 * architecture specific PCI/MSI support working.	 */	if (dev->msi.domain && !irq_domain_is_msi_parent(dev->msi.domain))		md->__domains[MSI_DEFAULT_DOMAIN].domain = dev->msi.domain; 	mutex_init(&md->mutex);	dev->msi.data = md;	devres_add(dev, md);	return 0;} /** * __msi_lock_descs - Lock the MSI descriptor storage of a device * @dev:	Device to operate on * * Internal function for guard(msi_descs_lock). Don't use in code. */void __msi_lock_descs(struct device *dev){	mutex_lock(&dev->msi.data->mutex);}EXPORT_SYMBOL_GPL(__msi_lock_descs); /** * __msi_unlock_descs - Unlock the MSI descriptor storage of a device * @dev:	Device to operate on * * Internal function for guard(msi_descs_lock). Don't use in code. */void __msi_unlock_descs(struct device *dev){	/* Invalidate the index which was cached by the iterator */	dev->msi.data->__iter_idx = MSI_XA_MAX_INDEX;	mutex_unlock(&dev->msi.data->mutex);}EXPORT_SYMBOL_GPL(__msi_unlock_descs); static struct msi_desc *msi_find_desc(struct msi_device_data *md, unsigned int domid,				      enum msi_desc_filter filter){	struct xarray *xa = &md->__domains[domid].store;	struct msi_desc *desc; 	xa_for_each_start(xa, md->__iter_idx, desc, md->__iter_idx) {		if (msi_desc_match(desc, filter))			return desc;	}	md->__iter_idx = MSI_XA_MAX_INDEX;	return NULL;} /** * msi_domain_first_desc - Get the first MSI descriptor of an irqdomain associated to a device * @dev:	Device to operate on * @domid:	The id of the interrupt domain which should be walked. * @filter:	Descriptor state filter * * Must be called with the MSI descriptor mutex held, i.e. msi_lock_descs() * must be invoked before the call. * * Return: Pointer to the first MSI descriptor matching the search *	   criteria, NULL if none found. */struct msi_desc *msi_domain_first_desc(struct device *dev, unsigned int domid,				       enum msi_desc_filter filter){	struct msi_device_data *md = dev->msi.data; 	if (WARN_ON_ONCE(!md || domid >= MSI_MAX_DEVICE_IRQDOMAINS))		return NULL; 	lockdep_assert_held(&md->mutex); 	md->__iter_idx = 0;	return msi_find_desc(md, domid, filter);}EXPORT_SYMBOL_GPL(msi_domain_first_desc); /** * msi_next_desc - Get the next MSI descriptor of a device * @dev:	Device to operate on * @domid:	The id of the interrupt domain which should be walked. * @filter:	Descriptor state filter * * The first invocation of msi_next_desc() has to be preceeded by a * successful invocation of __msi_first_desc(). Consecutive invocations are * only valid if the previous one was successful. All these operations have * to be done within the same MSI mutex held region. * * Return: Pointer to the next MSI descriptor matching the search *	   criteria, NULL if none found. */struct msi_desc *msi_next_desc(struct device *dev, unsigned int domid,			       enum msi_desc_filter filter){	struct msi_device_data *md = dev->msi.data; 	if (WARN_ON_ONCE(!md || domid >= MSI_MAX_DEVICE_IRQDOMAINS))		return NULL; 	lockdep_assert_held(&md->mutex); 	if (md->__iter_idx >= (unsigned long)MSI_MAX_INDEX)		return NULL; 	md->__iter_idx++;	return msi_find_desc(md, domid, filter);}EXPORT_SYMBOL_GPL(msi_next_desc); /** * msi_domain_get_virq - Lookup the Linux interrupt number for a MSI index on a interrupt domain * @dev:	Device to operate on * @domid:	Domain ID of the interrupt domain associated to the device * @index:	MSI interrupt index to look for (0-based) * * Return: The Linux interrupt number on success (> 0), 0 if not found */unsigned int msi_domain_get_virq(struct device *dev, unsigned int domid, unsigned int index){	struct msi_desc *desc;	bool pcimsi = false;	struct xarray *xa; 	if (!dev->msi.data)		return 0; 	if (WARN_ON_ONCE(index > MSI_MAX_INDEX || domid >= MSI_MAX_DEVICE_IRQDOMAINS))		return 0; 	/* This check is only valid for the PCI default MSI domain */	if (dev_is_pci(dev) && domid == MSI_DEFAULT_DOMAIN)		pcimsi = to_pci_dev(dev)->msi_enabled; 	guard(msi_descs_lock)(dev);	xa = &dev->msi.data->__domains[domid].store;	desc = xa_load(xa, pcimsi ? 0 : index);	if (desc && desc->irq) {		/*		 * PCI-MSI has only one descriptor for multiple interrupts.		 * PCI-MSIX and platform MSI use a descriptor per		 * interrupt.		 */		if (!pcimsi)			return desc->irq;		if (index < desc->nvec_used)			return desc->irq + index;	}	return 0;}EXPORT_SYMBOL_GPL(msi_domain_get_virq); #ifdef CONFIG_SYSFSstatic struct attribute *msi_dev_attrs[] = {	NULL}; static const struct attribute_group msi_irqs_group = {	.name	= "msi_irqs",	.attrs	= msi_dev_attrs,}; static inline int msi_sysfs_create_group(struct device *dev){	return devm_device_add_group(dev, &msi_irqs_group);}

Lines 1–500 of 1777. Load more