// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB/* * Copyright (c) 2019 Mellanox Technologies. All rights reserved. */#include <rdma/ib_verbs.h>#include <rdma/rdma_counter.h> #include "core_priv.h"#include "restrack.h" #define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE | RDMA_COUNTER_MASK_PID) static int __counter_set_mode(struct rdma_port_counter *port_counter,			      enum rdma_nl_counter_mode new_mode,			      enum rdma_nl_counter_mask new_mask,			      bool bind_opcnt){	if (new_mode == RDMA_COUNTER_MODE_AUTO) {		if (new_mask & (~ALL_AUTO_MODE_MASKS))			return -EINVAL;		if (port_counter->num_counters)			return -EBUSY;	} 	port_counter->mode.mode = new_mode;	port_counter->mode.mask = new_mask;	port_counter->mode.bind_opcnt = bind_opcnt;	return 0;} /* * rdma_counter_set_auto_mode() - Turn on/off per-port auto mode * * @dev: Device to operate * @port: Port to use * @mask: Mask to configure * @extack: Message to the user * * Return 0 on success. If counter mode wasn't changed then it is considered * as success as well. * Return -EBUSY when changing to auto mode while there are bounded counters. * */int rdma_counter_set_auto_mode(struct ib_device *dev, u32 port,			       enum rdma_nl_counter_mask mask,			       bool bind_opcnt,			       struct netlink_ext_ack *extack){	struct rdma_port_counter *port_counter;	enum rdma_nl_counter_mode mode;	int ret; 	port_counter = &dev->port_data[port].port_counter;	if (!port_counter->hstats)		return -EOPNOTSUPP; 	mutex_lock(&port_counter->lock);	if (mask)		mode = RDMA_COUNTER_MODE_AUTO;	else		mode = (port_counter->num_counters) ? RDMA_COUNTER_MODE_MANUAL :						      RDMA_COUNTER_MODE_NONE; 	if (port_counter->mode.mode == mode &&	    port_counter->mode.mask == mask &&	    port_counter->mode.bind_opcnt == bind_opcnt) {		ret = 0;		goto out;	} 	ret = __counter_set_mode(port_counter, mode, mask, bind_opcnt); out:	mutex_unlock(&port_counter->lock);	if (ret == -EBUSY)		NL_SET_ERR_MSG(			extack,			"Modifying auto mode is not allowed when there is a bound QP");	return ret;} static void auto_mode_init_counter(struct rdma_counter *counter,				   const struct ib_qp *qp,				   enum rdma_nl_counter_mask new_mask){	struct auto_mode_param *param = &counter->mode.param; 	counter->mode.mode = RDMA_COUNTER_MODE_AUTO;	counter->mode.mask = new_mask; 	if (new_mask & RDMA_COUNTER_MASK_QP_TYPE)		param->qp_type = qp->qp_type;} static int __rdma_counter_bind_qp(struct rdma_counter *counter,				  struct ib_qp *qp, u32 port){	int ret; 	if (qp->counter)		return -EINVAL; 	if (!qp->device->ops.counter_bind_qp)		return -EOPNOTSUPP; 	mutex_lock(&counter->lock);	ret = qp->device->ops.counter_bind_qp(counter, qp, port);	mutex_unlock(&counter->lock); 	return ret;} int rdma_counter_modify(struct ib_device *dev, u32 port,			unsigned int index, bool enable){	struct rdma_hw_stats *stats;	int ret = 0; 	if (!dev->ops.modify_hw_stat)		return -EOPNOTSUPP; 	stats = ib_get_hw_stats_port(dev, port);	if (!stats || index >= stats->num_counters ||	    !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL))		return -EINVAL; 	mutex_lock(&stats->lock); 	if (enable != test_bit(index, stats->is_disabled))		goto out; 	ret = dev->ops.modify_hw_stat(dev, port, index, enable);	if (ret)		goto out; 	if (enable)		clear_bit(index, stats->is_disabled);	else		set_bit(index, stats->is_disabled);out:	mutex_unlock(&stats->lock);	return ret;} static struct rdma_counter *alloc_and_bind(struct ib_device *dev, u32 port,					   struct ib_qp *qp,					   enum rdma_nl_counter_mode mode,					   bool bind_opcnt){	struct rdma_port_counter *port_counter;	struct rdma_counter *counter;	int ret; 	if (!dev->ops.counter_dealloc || !dev->ops.counter_alloc_stats)		return NULL; 	counter = rdma_zalloc_drv_obj(dev, rdma_counter);	if (!counter)		return NULL; 	counter->device    = dev;	counter->port      = port; 	dev->ops.counter_init(counter); 	rdma_restrack_new(&counter->res, RDMA_RESTRACK_COUNTER);	counter->stats = dev->ops.counter_alloc_stats(counter);	if (!counter->stats)		goto err_stats; 	port_counter = &dev->port_data[port].port_counter;	mutex_lock(&port_counter->lock);	switch (mode) {	case RDMA_COUNTER_MODE_MANUAL:		ret = __counter_set_mode(port_counter, RDMA_COUNTER_MODE_MANUAL,					 0, bind_opcnt);		if (ret) {			mutex_unlock(&port_counter->lock);			goto err_mode;		}		break;	case RDMA_COUNTER_MODE_AUTO:		auto_mode_init_counter(counter, qp, port_counter->mode.mask);		break;	default:		ret = -EOPNOTSUPP;		mutex_unlock(&port_counter->lock);		goto err_mode;	} 	port_counter->num_counters++;	mutex_unlock(&port_counter->lock); 	counter->mode.mode = mode;	counter->mode.bind_opcnt = bind_opcnt;	kref_init(&counter->kref);	mutex_init(&counter->lock); 	ret = __rdma_counter_bind_qp(counter, qp, port);	if (ret)		goto err_bind; 	rdma_restrack_parent_name(&counter->res, &qp->res);	rdma_restrack_add(&counter->res);	return counter; err_bind:	mutex_lock(&port_counter->lock);	port_counter->num_counters--;	if (!port_counter->num_counters &&	    port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL)		__counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0,				   false);	mutex_unlock(&port_counter->lock);err_mode:	rdma_free_hw_stats_struct(counter->stats);err_stats:	rdma_restrack_put(&counter->res);	kfree(counter);	return NULL;} static void rdma_counter_free(struct rdma_counter *counter){	struct rdma_port_counter *port_counter; 	port_counter = &counter->device->port_data[counter->port].port_counter;	mutex_lock(&port_counter->lock);	port_counter->num_counters--;	if (!port_counter->num_counters &&	    (port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL))		__counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0,				   false); 	mutex_unlock(&port_counter->lock); 	rdma_free_hw_stats_struct(counter->stats);	kfree(counter);} static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter,			    enum rdma_nl_counter_mask auto_mask){	struct auto_mode_param *param = &counter->mode.param;	bool match = true; 	if (auto_mask & RDMA_COUNTER_MASK_QP_TYPE)		match &= (param->qp_type == qp->qp_type); 	if (auto_mask & RDMA_COUNTER_MASK_PID)		match &= (task_pid_nr(counter->res.task) ==			  task_pid_nr(qp->res.task)); 	return match;} static int __rdma_counter_unbind_qp(struct ib_qp *qp, u32 port){	struct rdma_counter *counter = qp->counter;	int ret; 	if (!qp->device->ops.counter_unbind_qp)		return -EOPNOTSUPP; 	mutex_lock(&counter->lock);	ret = qp->device->ops.counter_unbind_qp(qp, port);	mutex_unlock(&counter->lock); 	return ret;} static void counter_history_stat_update(struct rdma_counter *counter){	struct ib_device *dev = counter->device;	struct rdma_port_counter *port_counter;	int i; 	port_counter = &dev->port_data[counter->port].port_counter;	if (!port_counter->hstats)		return; 	rdma_counter_query_stats(counter); 	for (i = 0; i < counter->stats->num_counters; i++)		port_counter->hstats->value[i] += counter->stats->value[i];} /* * rdma_get_counter_auto_mode - Find the counter that @qp should be bound *     with in auto mode * * Return: The counter (with ref-count increased) if found */static struct rdma_counter *rdma_get_counter_auto_mode(struct ib_qp *qp,						       u32 port){	struct rdma_port_counter *port_counter;	struct rdma_counter *counter = NULL;	struct ib_device *dev = qp->device;	struct rdma_restrack_entry *res;	struct rdma_restrack_root *rt;	unsigned long id = 0; 	port_counter = &dev->port_data[port].port_counter;	rt = &dev->res[RDMA_RESTRACK_COUNTER];	xa_lock(&rt->xa);	xa_for_each(&rt->xa, id, res) {		counter = container_of(res, struct rdma_counter, res);		if ((counter->device != qp->device) || (counter->port != port))			goto next; 		if (auto_mode_match(qp, counter, port_counter->mode.mask))			break;next:		counter = NULL;	} 	if (counter && !kref_get_unless_zero(&counter->kref))		counter = NULL; 	xa_unlock(&rt->xa);	return counter;} static void counter_release(struct kref *kref){	struct rdma_counter *counter; 	counter = container_of(kref, struct rdma_counter, kref);	counter_history_stat_update(counter);	rdma_restrack_del(&counter->res);	counter->device->ops.counter_dealloc(counter);	rdma_counter_free(counter);} /* * rdma_counter_bind_qp_auto - Check and bind the QP to a counter base on *   the auto-mode rule */int rdma_counter_bind_qp_auto(struct ib_qp *qp, u32 port){	struct rdma_port_counter *port_counter;	struct ib_device *dev = qp->device;	struct rdma_counter *counter;	int ret; 	if (!rdma_restrack_is_tracked(&qp->res) || rdma_is_kernel_res(&qp->res))		return 0; 	if (!rdma_is_port_valid(dev, port))		return -EINVAL; 	port_counter = &dev->port_data[port].port_counter;	if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO)		return 0; 	counter = rdma_get_counter_auto_mode(qp, port);	if (counter) {		ret = __rdma_counter_bind_qp(counter, qp, port);		if (ret) {			kref_put(&counter->kref, counter_release);			return ret;		}	} else {		counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_AUTO,					 port_counter->mode.bind_opcnt);		if (!counter)			return -ENOMEM;	} 	return 0;} /* * rdma_counter_unbind_qp - Unbind a qp from a counter * @force: *   true - Decrease the counter ref-count anyway (e.g., qp destroy) */int rdma_counter_unbind_qp(struct ib_qp *qp, u32 port, bool force){	struct rdma_counter *counter = qp->counter;	int ret; 	if (!counter)		return -EINVAL; 	ret = __rdma_counter_unbind_qp(qp, port);	if (ret && !force)		return ret; 	kref_put(&counter->kref, counter_release);	return 0;} int rdma_counter_query_stats(struct rdma_counter *counter){	struct ib_device *dev = counter->device;	int ret; 	if (!dev->ops.counter_update_stats)		return -EINVAL; 	mutex_lock(&counter->lock);	ret = dev->ops.counter_update_stats(counter);	mutex_unlock(&counter->lock); 	return ret;} static u64 get_running_counters_hwstat_sum(struct ib_device *dev,					   u32 port, u32 index){	struct rdma_restrack_entry *res;	struct rdma_restrack_root *rt;	struct rdma_counter *counter;	unsigned long id = 0;	u64 sum = 0; 	rt = &dev->res[RDMA_RESTRACK_COUNTER];	xa_lock(&rt->xa);	xa_for_each(&rt->xa, id, res) {		if (!rdma_restrack_get(res))			continue; 		xa_unlock(&rt->xa); 		counter = container_of(res, struct rdma_counter, res);		if ((counter->device != dev) || (counter->port != port) ||		    rdma_counter_query_stats(counter))			goto next; 		sum += counter->stats->value[index]; next:		xa_lock(&rt->xa);		rdma_restrack_put(res);	} 	xa_unlock(&rt->xa);	return sum;} /* * rdma_counter_get_hwstat_value() - Get the sum value of all counters on a *   specific port, including the running ones and history data */u64 rdma_counter_get_hwstat_value(struct ib_device *dev, u32 port, u32 index){	struct rdma_port_counter *port_counter;	u64 sum; 	port_counter = &dev->port_data[port].port_counter;	if (!port_counter->hstats)		return 0; 	sum = get_running_counters_hwstat_sum(dev, port, index);	sum += port_counter->hstats->value[index]; 	return sum;} static struct ib_qp *rdma_counter_get_qp(struct ib_device *dev, u32 qp_num){	struct rdma_restrack_entry *res = NULL;	struct ib_qp *qp = NULL; 	res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_QP, qp_num);	if (IS_ERR(res))		return NULL; 	qp = container_of(res, struct ib_qp, res);	if (qp->qp_type == IB_QPT_RAW_PACKET && !rdma_dev_has_raw_cap(dev))		goto err; 	return qp; err:	rdma_restrack_put(res);	return NULL;} static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev,						   u32 counter_id){	struct rdma_restrack_entry *res;	struct rdma_counter *counter; 	res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_COUNTER, counter_id);	if (IS_ERR(res))		return NULL; 	counter = container_of(res, struct rdma_counter, res);	if (!kref_get_unless_zero(&counter->kref))		counter = NULL;	rdma_restrack_put(res); 	return counter;} /*

Lines 1–500 of 690. Load more