1056 lines · 30.3 KB · C
RawDownload
/* * Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved. * * This software is available to you under a choice of one of two * licenses.  You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * *     Redistribution and use in source and binary forms, with or *     without modification, are permitted provided that the following *     conditions are met: * *      - Redistributions of source code must retain the above *        copyright notice, this list of conditions and the following *        disclaimer. * *      - Redistributions in binary form must reproduce the above *        copyright notice, this list of conditions and the following *        disclaimer in the documentation and/or other materials *        provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */#include <linux/kernel.h>#include <linux/list.h>#include <linux/slab.h>#include <linux/export.h>#include <net/ipv6.h>#include <net/inet6_hashtables.h>#include <net/addrconf.h> #include "rds.h"#include "loop.h" #define RDS_CONNECTION_HASH_BITS 12#define RDS_CONNECTION_HASH_ENTRIES (1 << RDS_CONNECTION_HASH_BITS)#define RDS_CONNECTION_HASH_MASK (RDS_CONNECTION_HASH_ENTRIES - 1) /* converting this to RCU is a chore for another day.. */static DEFINE_SPINLOCK(rds_conn_lock);static unsigned long rds_conn_count;static struct hlist_head rds_conn_hash[RDS_CONNECTION_HASH_ENTRIES];static struct kmem_cache *rds_conn_slab; static struct hlist_head *rds_conn_bucket(const struct in6_addr *laddr,					  const struct in6_addr *faddr){	static u32 rds6_hash_secret __read_mostly;	static u32 rds_hash_secret __read_mostly; 	__be32 lhash, fhash;	u32 hash; 	net_get_random_once(&rds_hash_secret, sizeof(rds_hash_secret));	net_get_random_once(&rds6_hash_secret, sizeof(rds6_hash_secret)); 	lhash = laddr->s6_addr32[3];#if IS_ENABLED(CONFIG_IPV6)	fhash = (__force __be32)__ipv6_addr_jhash(faddr, rds6_hash_secret);#else	fhash = faddr->s6_addr32[3];#endif	hash = __inet_ehashfn(lhash, 0, fhash, 0, rds_hash_secret); 	return &rds_conn_hash[hash & RDS_CONNECTION_HASH_MASK];} #define rds_conn_info_set(var, test, suffix) do {		\	if (test)						\		var |= RDS_INFO_CONNECTION_FLAG_##suffix;	\} while (0) /* rcu read lock must be held or the connection spinlock */static struct rds_connection *rds_conn_lookup(struct net *net,					      struct hlist_head *head,					      const struct in6_addr *laddr,					      const struct in6_addr *faddr,					      struct rds_transport *trans,					      u8 tos, int dev_if){	struct rds_connection *conn, *ret = NULL; 	hlist_for_each_entry_rcu(conn, head, c_hash_node) {		if (ipv6_addr_equal(&conn->c_faddr, faddr) &&		    ipv6_addr_equal(&conn->c_laddr, laddr) &&		    conn->c_trans == trans &&		    conn->c_tos == tos &&		    net == rds_conn_net(conn) &&		    conn->c_dev_if == dev_if) {			ret = conn;			break;		}	}	rdsdebug("returning conn %p for %pI6c -> %pI6c\n", ret,		 laddr, faddr);	return ret;} /* * This is called by rds_conn_shutdown() once the transport has brought * a path down.  It clears partial message state so that the transport * can start sending and receiving over this path again in the future. * The caller owns RDS_IN_XMIT and RDS_RECV_REFILL across this call, * which is what serializes it against the send and receive-refill * paths. */static void rds_conn_path_reset(struct rds_conn_path *cp){	struct rds_connection *conn = cp->cp_conn; 	rdsdebug("connection %pI6c to %pI6c reset\n",		 &conn->c_laddr, &conn->c_faddr); 	rds_stats_inc(s_conn_reset);	rds_send_path_reset(cp); 	/* Clear the bits the reset is responsible for individually: a	 * blanket cp_flags = 0 is a plain store that can clobber a	 * concurrent atomic read-modify-write on the same word.	 * RDS_IN_XMIT and RDS_RECV_REFILL are held as locks by the	 * caller, rds_conn_shutdown(), which releases them once the	 * teardown is complete.	 */	clear_bit(RDS_LL_SEND_FULL, &cp->cp_flags);	clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags); 	/* Do not clear next_rx_seq here, else we cannot distinguish	 * retransmitted packets from new packets, and will hand all	 * of them to the application. That is not consistent with the	 * reliability guarantees of RDS. */} static void __rds_conn_path_init(struct rds_connection *conn,				 struct rds_conn_path *cp, bool is_outgoing){	spin_lock_init(&cp->cp_lock);	cp->cp_next_tx_seq = 1;	init_waitqueue_head(&cp->cp_waitq);	INIT_LIST_HEAD(&cp->cp_send_queue);	INIT_LIST_HEAD(&cp->cp_retrans); 	cp->cp_conn = conn;	atomic_set(&cp->cp_state, RDS_CONN_DOWN);	cp->cp_send_gen = 0;	cp->cp_reconnect_jiffies = 0;	cp->cp_conn->c_proposed_version = RDS_PROTOCOL_VERSION;	INIT_DELAYED_WORK(&cp->cp_send_w, rds_send_worker);	INIT_DELAYED_WORK(&cp->cp_recv_w, rds_recv_worker);	INIT_DELAYED_WORK(&cp->cp_conn_w, rds_connect_worker);	INIT_WORK(&cp->cp_down_w, rds_shutdown_worker);	mutex_init(&cp->cp_cm_lock);	cp->cp_flags = 0;} /* * There is only every one 'conn' for a given pair of addresses in the * system at a time.  They contain messages to be retransmitted and so * span the lifetime of the actual underlying transport connections. * * For now they are not garbage collected once they're created.  They * are torn down as the module is removed, if ever. */static struct rds_connection *__rds_conn_create(struct net *net,						const struct in6_addr *laddr,						const struct in6_addr *faddr,						struct rds_transport *trans,						gfp_t gfp, u8 tos,						int is_outgoing,						int dev_if){	struct rds_connection *conn, *parent = NULL;	struct hlist_head *head = rds_conn_bucket(laddr, faddr);	struct rds_transport *loop_trans;	struct rds_conn_path *free_cp = NULL;	unsigned long flags;	int ret, i;	int npaths = (trans->t_mp_capable ? RDS_MPATH_WORKERS : 1); 	rcu_read_lock();	conn = rds_conn_lookup(net, head, laddr, faddr, trans, tos, dev_if);	if (conn &&	    conn->c_loopback &&	    conn->c_trans != &rds_loop_transport &&	    ipv6_addr_equal(laddr, faddr) &&	    !is_outgoing) {		/* This is a looped back IB connection, and we're		 * called by the code handling the incoming connect.		 * We need a second connection object into which we		 * can stick the other QP. */		parent = conn;		conn = parent->c_passive;	}	rcu_read_unlock();	if (conn)		goto out; 	conn = kmem_cache_zalloc(rds_conn_slab, gfp);	if (!conn) {		conn = ERR_PTR(-ENOMEM);		goto out;	}	conn->c_path = kzalloc_objs(struct rds_conn_path, npaths, gfp);	if (!conn->c_path) {		kmem_cache_free(rds_conn_slab, conn);		conn = ERR_PTR(-ENOMEM);		goto out;	} 	INIT_HLIST_NODE(&conn->c_hash_node);	conn->c_laddr = *laddr;	conn->c_isv6 = !ipv6_addr_v4mapped(laddr);	conn->c_faddr = *faddr;	conn->c_dev_if = dev_if;	conn->c_tos = tos; #if IS_ENABLED(CONFIG_IPV6)	/* If the local address is link local, set c_bound_if to be the	 * index used for this connection.  Otherwise, set it to 0 as	 * the socket is not bound to an interface.  c_bound_if is used	 * to look up a socket when a packet is received	 */	if (ipv6_addr_type(laddr) & IPV6_ADDR_LINKLOCAL)		conn->c_bound_if = dev_if;	else#endif		conn->c_bound_if = 0; 	rds_conn_net_set(conn, net); 	ret = rds_cong_get_maps(conn);	if (ret) {		kfree(conn->c_path);		kmem_cache_free(rds_conn_slab, conn);		conn = ERR_PTR(ret);		goto out;	} 	/*	 * This is where a connection becomes loopback.  If *any* RDS sockets	 * can bind to the destination address then we'd rather the messages	 * flow through loopback rather than either transport.	 */	loop_trans = rds_trans_get_preferred(net, faddr, conn->c_dev_if);	if (loop_trans) {		rds_trans_put(loop_trans);		conn->c_loopback = 1;		if (trans->t_prefer_loopback) {			if (likely(is_outgoing)) {				/* "outgoing" connection to local address.				 * Protocol says it wants the connection				 * handled by the loopback transport.				 * This is what TCP does.				 */				trans = &rds_loop_transport;			} else {				/* No transport currently in use				 * should end up here, but if it				 * does, reset/destroy the connection.				 */				kfree(conn->c_path);				kmem_cache_free(rds_conn_slab, conn);				conn = ERR_PTR(-EOPNOTSUPP);				goto out;			}		}	} 	conn->c_trans = trans; 	init_waitqueue_head(&conn->c_hs_waitq);	for (i = 0; i < npaths; i++) {		__rds_conn_path_init(conn, &conn->c_path[i],				     is_outgoing);		conn->c_path[i].cp_index = i;		conn->c_path[i].cp_wq =			alloc_ordered_workqueue("krds_cp_wq#%lu/%d", 0,						rds_conn_count, i);		if (!conn->c_path[i].cp_wq)			conn->c_path[i].cp_wq = rds_wq;	}	rcu_read_lock();	if (rds_destroy_pending(conn))		ret = -ENETDOWN;	else		ret = trans->conn_alloc(conn, GFP_ATOMIC);	if (ret) {		rcu_read_unlock();		free_cp = conn->c_path;		kmem_cache_free(rds_conn_slab, conn);		conn = ERR_PTR(ret);		goto out;	} 	rdsdebug("allocated conn %p for %pI6c -> %pI6c over %s %s\n",		 conn, laddr, faddr,		 strnlen(trans->t_name, sizeof(trans->t_name)) ?		 trans->t_name : "[unknown]", is_outgoing ? "(outgoing)" : ""); 	/*	 * Since we ran without holding the conn lock, someone could	 * have created the same conn (either normal or passive) in the	 * interim. We check while holding the lock. If we won, we complete	 * init and return our conn. If we lost, we rollback and return the	 * other one.	 */	spin_lock_irqsave(&rds_conn_lock, flags);	if (parent) {		/* Creating passive conn */		if (parent->c_passive) {			trans->conn_free(conn->c_path[0].cp_transport_data);			free_cp = conn->c_path;			kmem_cache_free(rds_conn_slab, conn);			conn = parent->c_passive;		} else {			parent->c_passive = conn;			rds_cong_add_conn(conn);			rds_conn_count++;		}	} else {		/* Creating normal conn */		struct rds_connection *found; 		found = rds_conn_lookup(net, head, laddr, faddr, trans,					tos, dev_if);		if (found) {			struct rds_conn_path *cp;			int i; 			for (i = 0; i < npaths; i++) {				cp = &conn->c_path[i];				/* The ->conn_alloc invocation may have				 * allocated resource for all paths, so all				 * of them may have to be freed here.				 */				if (cp->cp_transport_data)					trans->conn_free(cp->cp_transport_data);			}			free_cp = conn->c_path;			kmem_cache_free(rds_conn_slab, conn);			conn = found;		} else {			conn->c_my_gen_num = rds_gen_num;			conn->c_peer_gen_num = 0;			hlist_add_head_rcu(&conn->c_hash_node, head);			rds_cong_add_conn(conn);			rds_conn_count++;		}	}	spin_unlock_irqrestore(&rds_conn_lock, flags);	rcu_read_unlock(); out:	if (free_cp) {		for (i = 0; i < npaths; i++)			if (free_cp[i].cp_wq != rds_wq)				destroy_workqueue(free_cp[i].cp_wq);		kfree(free_cp);	} 	return conn;} struct rds_connection *rds_conn_create(struct net *net,				       const struct in6_addr *laddr,				       const struct in6_addr *faddr,				       struct rds_transport *trans, u8 tos,				       gfp_t gfp, int dev_if){	return __rds_conn_create(net, laddr, faddr, trans, gfp, tos, 0, dev_if);}EXPORT_SYMBOL_GPL(rds_conn_create); struct rds_connection *rds_conn_create_outgoing(struct net *net,						const struct in6_addr *laddr,						const struct in6_addr *faddr,						struct rds_transport *trans,						u8 tos, gfp_t gfp, int dev_if){	return __rds_conn_create(net, laddr, faddr, trans, gfp, tos, 1, dev_if);}EXPORT_SYMBOL_GPL(rds_conn_create_outgoing); void rds_conn_shutdown(struct rds_conn_path *cp){	struct rds_connection *conn = cp->cp_conn; 	/* shut it down unless it's down already */	if (!rds_conn_path_transition(cp, RDS_CONN_DOWN, RDS_CONN_DOWN)) {		/*		 * Quiesce the connection mgmt handlers before we start tearing		 * things down. We don't hold the mutex for the entire		 * duration of the shutdown operation, else we may be		 * deadlocking with the CM handler. Instead, the CM event		 * handler is supposed to check for state DISCONNECTING		 */		mutex_lock(&cp->cp_cm_lock);		if (!rds_conn_path_transition(cp, RDS_CONN_UP,					      RDS_CONN_DISCONNECTING) &&		    !rds_conn_path_transition(cp, RDS_CONN_ERROR,					      RDS_CONN_DISCONNECTING) &&		    !rds_conn_path_transition(cp, RDS_CONN_RESETTING,					      RDS_CONN_DISCONNECTING)) {			rds_conn_path_error(cp,					    "shutdown called in state %d\n",					    atomic_read(&cp->cp_state));			mutex_unlock(&cp->cp_cm_lock);			return;		}		mutex_unlock(&cp->cp_cm_lock); 		/* Quiesce the transmit and receive-refill paths by		 * acquiring their bit locks, not merely waiting for		 * them to be released: with a plain wait, either path		 * can re-take its lock the instant after we sample it		 * clear and then run concurrently with the transport		 * shutdown and the path reset below.  Holding both		 * locks across the teardown makes that structurally		 * impossible.		 */		wait_event(cp->cp_waitq,			   !test_and_set_bit_lock(RDS_IN_XMIT, &cp->cp_flags));		wait_event(cp->cp_waitq,			   !test_and_set_bit(RDS_RECV_REFILL, &cp->cp_flags)); 		conn->c_trans->conn_path_shutdown(cp);		rds_conn_path_reset(cp); 		/* Release the two locks and wake any waiter (e.g.		 * rds_tcp_reset_callbacks()) that blocked on them while		 * we held them.  The unlock orders the transport's ring		 * re-initialization and the path reset above before		 * either bit is seen clear.  rds_conn_path_reset() leaves		 * both bits alone: ownership ends here, not inside the		 * reset.		 */		clear_bit_unlock(RDS_IN_XMIT, &cp->cp_flags);		clear_bit_unlock(RDS_RECV_REFILL, &cp->cp_flags);		wake_up_all(&cp->cp_waitq); 		if (!rds_conn_path_transition(cp, RDS_CONN_DISCONNECTING,					      RDS_CONN_DOWN)) {			/* The path was dropped again while we tore it			 * down: by a socket state-change callback in			 * irq context on receipt of a FIN, or by an			 * accept that claimed the path just before a			 * drop put it back to RDS_CONN_ERROR and then			 * installed a fresh socket on it.  Unless a			 * pending destroy suppressed it, the drop also			 * queued another shutdown pass, and that pass			 * must run, because it is what tears down			 * whatever attached to the path after the			 * transport shutdown above sampled its state.			 * Consuming the RDS_CONN_ERROR here would turn			 * that pass into a no-op: leave the state			 * alone, and let the pass finish the job.			 *			 * Quiesce the reconnect timer before bailing			 * out, though.  When a pending destroy did			 * suppress the queue, no later pass runs, and			 * rds_conn_path_destroy() is about to flush			 * cp_down_w and free the path: it must not			 * find cp_conn_w still armed.  A successor			 * pass, when there is one, re-arms the			 * reconnect from its own tail.			 */			cancel_delayed_work_sync(&cp->cp_conn_w);			clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags); 			if (rds_conn_path_state(cp) == RDS_CONN_ERROR)				return;			/* No current cp_state writer leaves a			 * DISCONNECTING path in any state but			 * RDS_CONN_ERROR; report loudly if one ever			 * does.			 */			rds_conn_path_error(cp, "%s: failed to transition "					    "to state DOWN, current state "					    "is %d\n", __func__,					    atomic_read(&cp->cp_state));			return;		}	} 	/* Then reconnect if it's still live.	 * The passive side of an IB loopback connection is never added	 * to the conn hash, so we never trigger a reconnect on this	 * conn - the reconnect is always triggered by the active peer. */	cancel_delayed_work_sync(&cp->cp_conn_w); 	clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags);	rcu_read_lock();

Lines 1–500 of 1056.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t