// SPDX-License-Identifier: GPL-2.0-or-later/* * net/sched/act_api.c	Packet action API. * * Author:	Jamal Hadi Salim */ #include <linux/types.h>#include <linux/kernel.h>#include <linux/string.h>#include <linux/errno.h>#include <linux/slab.h>#include <linux/skbuff.h>#include <linux/init.h>#include <linux/kmod.h>#include <linux/err.h>#include <linux/module.h>#include <net/net_namespace.h>#include <net/sock.h>#include <net/sch_generic.h>#include <net/pkt_cls.h>#include <net/tc_act/tc_pedit.h>#include <net/act_api.h>#include <net/netlink.h>#include <net/flow_offload.h>#include <net/tc_wrapper.h> #ifdef CONFIG_INETDEFINE_STATIC_KEY_FALSE(tcf_frag_xmit_count);EXPORT_SYMBOL_GPL(tcf_frag_xmit_count);#endif int tcf_dev_queue_xmit(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb)){#ifdef CONFIG_INET	if (static_branch_unlikely(&tcf_frag_xmit_count))		return sch_frag_xmit_hook(skb, xmit);#endif 	return xmit(skb);}EXPORT_SYMBOL_GPL(tcf_dev_queue_xmit); static void tcf_action_goto_chain_exec(const struct tcf_chain *chain,				       struct tcf_result *res){	res->goto_tp = rcu_dereference_bh(chain->filter_chain);} static void tcf_free_cookie_rcu(struct rcu_head *p){	struct tc_cookie *cookie = container_of(p, struct tc_cookie, rcu); 	kfree(cookie->data);	kfree(cookie);} static void tcf_set_action_cookie(struct tc_cookie __rcu **old_cookie,				  struct tc_cookie *new_cookie){	struct tc_cookie *old; 	old = unrcu_pointer(xchg(old_cookie, RCU_INITIALIZER(new_cookie)));	if (old)		call_rcu(&old->rcu, tcf_free_cookie_rcu);} int tcf_action_check_ctrlact(int action, struct tcf_proto *tp,			     struct tcf_chain **newchain,			     struct netlink_ext_ack *extack){	int opcode = TC_ACT_EXT_OPCODE(action), ret = -EINVAL;	u32 chain_index; 	if (!opcode)		ret = action > TC_ACT_VALUE_MAX ? -EINVAL : 0;	else if (opcode <= TC_ACT_EXT_OPCODE_MAX || action == TC_ACT_UNSPEC)		ret = 0;	if (ret) {		NL_SET_ERR_MSG(extack, "invalid control action");		goto end;	} 	if (TC_ACT_EXT_CMP(action, TC_ACT_GOTO_CHAIN)) {		chain_index = action & TC_ACT_EXT_VAL_MASK;		if (!tp || !newchain) {			ret = -EINVAL;			NL_SET_ERR_MSG(extack,				       "can't goto NULL proto/chain");			goto end;		}		*newchain = tcf_chain_get_by_act(tp->chain->block, chain_index);		if (!*newchain) {			ret = -ENOMEM;			NL_SET_ERR_MSG(extack,				       "can't allocate goto_chain");		}	}end:	return ret;}EXPORT_SYMBOL(tcf_action_check_ctrlact); struct tcf_chain *tcf_action_set_ctrlact(struct tc_action *a, int action,					 struct tcf_chain *goto_chain){	a->tcfa_action = action;	goto_chain = rcu_replace_pointer(a->goto_chain, goto_chain, 1);	return goto_chain;}EXPORT_SYMBOL(tcf_action_set_ctrlact); static void free_tcf(struct tc_action *p){	struct tcf_chain *chain = rcu_dereference_protected(p->goto_chain, 1); 	free_percpu(p->cpu_bstats);	free_percpu(p->cpu_bstats_hw);	free_percpu(p->cpu_qstats); 	tcf_set_action_cookie(&p->user_cookie, NULL);	if (chain)		tcf_chain_put_by_act(chain); 	kfree_rcu(p, tcfa_rcu);} static void offload_action_hw_count_set(struct tc_action *act,					u32 hw_count){	act->in_hw_count = hw_count;} static void offload_action_hw_count_inc(struct tc_action *act,					u32 hw_count){	act->in_hw_count += hw_count;} static void offload_action_hw_count_dec(struct tc_action *act,					u32 hw_count){	act->in_hw_count = act->in_hw_count > hw_count ?			   act->in_hw_count - hw_count : 0;} static unsigned int tcf_offload_act_num_actions_single(struct tc_action *act){	unsigned int count; 	if (is_tcf_pedit(act)) {		spin_lock_bh(&act->tcfa_lock);		count = tcf_pedit_nkeys_locked(act);		spin_unlock_bh(&act->tcfa_lock);		return count;	}	return 1;} static bool tc_act_skip_hw(u32 flags){	return (flags & TCA_ACT_FLAGS_SKIP_HW) ? true : false;} static bool tc_act_skip_sw(u32 flags){	return (flags & TCA_ACT_FLAGS_SKIP_SW) ? true : false;} /* SKIP_HW and SKIP_SW are mutually exclusive flags. */static bool tc_act_flags_valid(u32 flags){	flags &= TCA_ACT_FLAGS_SKIP_HW | TCA_ACT_FLAGS_SKIP_SW; 	return flags ^ (TCA_ACT_FLAGS_SKIP_HW | TCA_ACT_FLAGS_SKIP_SW);} static int offload_action_init(struct flow_offload_action *fl_action,			       struct tc_action *act,			       enum offload_act_command  cmd,			       struct netlink_ext_ack *extack){	int err; 	fl_action->extack = extack;	fl_action->command = cmd;	fl_action->index = act->tcfa_index;	fl_action->cookie = (unsigned long)act; 	if (act->ops->offload_act_setup) {		spin_lock_bh(&act->tcfa_lock);		err = act->ops->offload_act_setup(act, fl_action, NULL,						  false, extack);		spin_unlock_bh(&act->tcfa_lock);		return err;	} 	return -EOPNOTSUPP;} static int tcf_action_offload_cmd_ex(struct flow_offload_action *fl_act,				     u32 *hw_count){	int err; 	err = flow_indr_dev_setup_offload(NULL, NULL, TC_SETUP_ACT,					  fl_act, NULL, NULL);	if (err < 0)		return err; 	if (hw_count)		*hw_count = err; 	return 0;} static int tcf_action_offload_cmd_cb_ex(struct flow_offload_action *fl_act,					u32 *hw_count,					flow_indr_block_bind_cb_t *cb,					void *cb_priv){	int err; 	err = cb(NULL, NULL, cb_priv, TC_SETUP_ACT, NULL, fl_act, NULL);	if (err < 0)		return err; 	if (hw_count)		*hw_count = 1; 	return 0;} static int tcf_action_offload_cmd(struct flow_offload_action *fl_act,				  u32 *hw_count,				  flow_indr_block_bind_cb_t *cb,				  void *cb_priv){	return cb ? tcf_action_offload_cmd_cb_ex(fl_act, hw_count,						 cb, cb_priv) :		    tcf_action_offload_cmd_ex(fl_act, hw_count);} static int tcf_action_offload_add_ex(struct tc_action *action,				     struct netlink_ext_ack *extack,				     flow_indr_block_bind_cb_t *cb,				     void *cb_priv){	bool skip_sw = tc_act_skip_sw(action->tcfa_flags);	struct tc_action *actions[TCA_ACT_MAX_PRIO] = {		[0] = action,	};	struct flow_offload_action *fl_action;	u32 in_hw_count = 0;	int num, err = 0; 	if (tc_act_skip_hw(action->tcfa_flags))		return 0; 	num = tcf_offload_act_num_actions_single(action);	fl_action = offload_action_alloc(num);	if (!fl_action)		return -ENOMEM; 	err = offload_action_init(fl_action, action, FLOW_ACT_REPLACE, extack);	if (err)		goto fl_err; 	err = tc_setup_action(&fl_action->action, actions, 0, extack);	if (err) {		NL_SET_ERR_MSG_MOD(extack,				   "Failed to setup tc actions for offload");		goto fl_err;	} 	err = tcf_action_offload_cmd(fl_action, &in_hw_count, cb, cb_priv);	if (!err)		cb ? offload_action_hw_count_inc(action, in_hw_count) :		     offload_action_hw_count_set(action, in_hw_count); 	if (skip_sw && !tc_act_in_hw(action))		err = -EINVAL; 	tc_cleanup_offload_action(&fl_action->action); fl_err:	kfree(fl_action); 	return err;} /* offload the tc action after it is inserted */static int tcf_action_offload_add(struct tc_action *action,				  struct netlink_ext_ack *extack){	return tcf_action_offload_add_ex(action, extack, NULL, NULL);} int tcf_action_update_hw_stats(struct tc_action *action){	struct flow_offload_action fl_act = {};	int err; 	err = offload_action_init(&fl_act, action, FLOW_ACT_STATS, NULL);	if (err)		return err; 	err = tcf_action_offload_cmd(&fl_act, NULL, NULL, NULL);	if (!err) {		preempt_disable();		tcf_action_stats_update(action, fl_act.stats.bytes,					fl_act.stats.pkts,					fl_act.stats.drops,					fl_act.stats.lastused,					true);		preempt_enable();		action->used_hw_stats = fl_act.stats.used_hw_stats;		action->used_hw_stats_valid = true;	} else {		return -EOPNOTSUPP;	} 	return 0;}EXPORT_SYMBOL(tcf_action_update_hw_stats); static int tcf_action_offload_del_ex(struct tc_action *action,				     flow_indr_block_bind_cb_t *cb,				     void *cb_priv){	struct flow_offload_action fl_act = {};	u32 in_hw_count = 0;	int err = 0; 	if (!tc_act_in_hw(action))		return 0; 	err = offload_action_init(&fl_act, action, FLOW_ACT_DESTROY, NULL);	if (err)		return err; 	err = tcf_action_offload_cmd(&fl_act, &in_hw_count, cb, cb_priv);	if (err < 0)		return err; 	if (!cb && action->in_hw_count != in_hw_count)		return -EINVAL; 	/* do not need to update hw state when deleting action */	if (cb && in_hw_count)		offload_action_hw_count_dec(action, in_hw_count); 	return 0;} static int tcf_action_offload_del(struct tc_action *action){	return tcf_action_offload_del_ex(action, NULL, NULL);} static void tcf_action_cleanup(struct tc_action *p){	tcf_action_offload_del(p);	if (p->ops->cleanup)		p->ops->cleanup(p); 	gen_kill_estimator(&p->tcfa_rate_est);	free_tcf(p);} static int __tcf_action_put(struct tc_action *p, bool bind){	struct tcf_idrinfo *idrinfo = p->idrinfo; 	if (refcount_dec_and_mutex_lock(&p->tcfa_refcnt, &idrinfo->lock)) {		if (bind)			atomic_dec(&p->tcfa_bindcnt);		idr_remove(&idrinfo->action_idr, p->tcfa_index);		mutex_unlock(&idrinfo->lock); 		tcf_action_cleanup(p);		return 1;	} 	if (bind)		atomic_dec(&p->tcfa_bindcnt); 	return 0;} static int __tcf_idr_release(struct tc_action *p, bool bind, bool strict){	int ret = 0; 	/* Release with strict==1 and bind==0 is only called through act API	 * interface (classifiers always bind). Only case when action with	 * positive reference count and zero bind count can exist is when it was	 * also created with act API (unbinding last classifier will destroy the	 * action if it was created by classifier). So only case when bind count	 * can be changed after initial check is when unbound action is	 * destroyed by act API while classifier binds to action with same id	 * concurrently. This result either creation of new action(same behavior	 * as before), or reusing existing action if concurrent process	 * increments reference count before action is deleted. Both scenarios	 * are acceptable.	 */	if (p) {		if (!bind && strict && atomic_read(&p->tcfa_bindcnt) > 0)			return -EPERM; 		if (__tcf_action_put(p, bind))			ret = ACT_P_DELETED;	} 	return ret;} int tcf_idr_release(struct tc_action *a, bool bind){	const struct tc_action_ops *ops = a->ops;	int ret; 	ret = __tcf_idr_release(a, bind, false);	if (ret == ACT_P_DELETED)		module_put(ops->owner);	return ret;}EXPORT_SYMBOL(tcf_idr_release); static size_t tcf_action_shared_attrs_size(const struct tc_action *act){	struct tc_cookie *user_cookie;	u32 cookie_len = 0; 	rcu_read_lock();	user_cookie = rcu_dereference(act->user_cookie); 	if (user_cookie)		cookie_len = nla_total_size(user_cookie->len);	rcu_read_unlock(); 	return  nla_total_size(0) /* action number nested */		+ nla_total_size(IFNAMSIZ) /* TCA_ACT_KIND */		+ cookie_len /* TCA_ACT_COOKIE */		+ nla_total_size(sizeof(struct nla_bitfield32)) /* TCA_ACT_HW_STATS */		/* TCA_ACT_USED_HW_STATS */		+ nla_total_size(sizeof(struct nla_bitfield32))		+ nla_total_size(sizeof(u32)) /* TCA_ACT_IN_HW_COUNT */		+ nla_total_size(0) /* TCA_ACT_STATS nested */		+ nla_total_size(sizeof(struct nla_bitfield32)) /* TCA_ACT_FLAGS */		/* TCA_STATS_BASIC */		+ nla_total_size_64bit(sizeof(struct gnet_stats_basic))		/* TCA_STATS_BASIC_HW */		+ nla_total_size_64bit(sizeof(struct gnet_stats_basic))		/* TCA_STATS_PKT64, emitted by both of the basic copies above */		+ 2 * nla_total_size_64bit(sizeof(u64))		/* TCA_STATS_RATE_EST */		+ nla_total_size_64bit(sizeof(struct gnet_stats_rate_est))		/* TCA_STATS_RATE_EST64 */		+ nla_total_size_64bit(sizeof(struct gnet_stats_rate_est64))		/* TCA_STATS_QUEUE */		+ nla_total_size_64bit(sizeof(struct gnet_stats_queue))		+ nla_total_size(0) /* TCA_ACT_OPTIONS nested */		/* TCA_GACT_TM; actions dump their tcf_t with nla_put_64bit(),		 * which may emit an extra NLA_PAD attribute.		 */		+ nla_total_size_64bit(sizeof(struct tcf_t));} static size_t tcf_action_full_attrs_size(size_t sz){	return NLMSG_HDRLEN                     /* struct nlmsghdr */		+ sizeof(struct tcamsg)		+ nla_total_size(0)             /* TCA_ACT_TAB nested */		+ sz;} static size_t tcf_action_fill_size(const struct tc_action *act){	size_t sz = tcf_action_shared_attrs_size(act); 	if (act->ops->get_fill_size)		return act->ops->get_fill_size(act) + sz;	return sz;} static inttcf_action_dump_terse(struct sk_buff *skb, struct tc_action *a, bool from_act){	unsigned char *b = skb_tail_pointer(skb);	struct tc_cookie *cookie; 	if (nla_put_string(skb, TCA_ACT_KIND, a->ops->kind))		goto nla_put_failure;	if (tcf_action_copy_stats(skb, a, 0))		goto nla_put_failure;	if (from_act && nla_put_u32(skb, TCA_ACT_INDEX, a->tcfa_index))		goto nla_put_failure; 	rcu_read_lock();

Lines 1–500 of 2317. Load more