// SPDX-License-Identifier: GPL-2.0-or-later/* Processing of received RxRPC packets * * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include "ar-internal.h" /* Override priority when generating ACKs for received DATA */static const u8 rxrpc_ack_priority[RXRPC_ACK__INVALID] = {	[RXRPC_ACK_IDLE]		= 1,	[RXRPC_ACK_DELAY]		= 2,	[RXRPC_ACK_REQUESTED]		= 3,	[RXRPC_ACK_DUPLICATE]		= 4,	[RXRPC_ACK_EXCEEDS_WINDOW]	= 5,	[RXRPC_ACK_NOSPACE]		= 6,	[RXRPC_ACK_OUT_OF_SEQUENCE]	= 7,}; static void rxrpc_proto_abort(struct rxrpc_call *call, rxrpc_seq_t seq,			      enum rxrpc_abort_reason why){	rxrpc_abort_call(call, seq, RX_PROTOCOL_ERROR, -EBADMSG, why);} /* * Do TCP-style congestion management [RFC5681]. */static void rxrpc_congestion_management(struct rxrpc_call *call,					struct rxrpc_ack_summary *summary){	summary->change = rxrpc_cong_no_change;	summary->in_flight = rxrpc_tx_in_flight(call); 	if (test_and_clear_bit(RXRPC_CALL_RETRANS_TIMEOUT, &call->flags)) {		summary->retrans_timeo = true;		call->cong_ssthresh = umax(summary->in_flight / 2, 2);		call->cong_cwnd = 1;		if (call->cong_cwnd >= call->cong_ssthresh &&		    call->cong_ca_state == RXRPC_CA_SLOW_START) {			call->cong_ca_state = RXRPC_CA_CONGEST_AVOIDANCE;			call->cong_tstamp = call->acks_latest_ts;			call->cong_cumul_acks = 0;		}	} 	call->cong_cumul_acks += summary->nr_new_sacks;	call->cong_cumul_acks += summary->nr_new_hacks;	if (call->cong_cumul_acks > 255)		call->cong_cumul_acks = 255; 	switch (call->cong_ca_state) {	case RXRPC_CA_SLOW_START:		if (call->acks_nr_snacks > 0)			goto packet_loss_detected;		if (call->cong_cumul_acks > 0)			call->cong_cwnd += 1;		if (call->cong_cwnd >= call->cong_ssthresh) {			call->cong_ca_state = RXRPC_CA_CONGEST_AVOIDANCE;			call->cong_tstamp = call->acks_latest_ts;		}		goto out; 	case RXRPC_CA_CONGEST_AVOIDANCE:		if (call->acks_nr_snacks > 0)			goto packet_loss_detected; 		/* We analyse the number of packets that get ACK'd per RTT		 * period and increase the window if we managed to fill it.		 */		if (call->rtt_count == 0)			goto out;		if (ktime_before(call->acks_latest_ts,				 ktime_add_us(call->cong_tstamp,					      call->srtt_us >> 3)))			goto out_no_clear_ca;		summary->change = rxrpc_cong_rtt_window_end;		call->cong_tstamp = call->acks_latest_ts;		if (call->cong_cumul_acks >= call->cong_cwnd)			call->cong_cwnd++;		goto out; 	case RXRPC_CA_PACKET_LOSS:		if (call->acks_nr_snacks == 0)			goto resume_normality; 		if (summary->new_low_snack) {			summary->change = rxrpc_cong_new_low_nack;			call->cong_dup_acks = 1;			if (call->cong_extra > 1)				call->cong_extra = 1;			goto send_extra_data;		} 		call->cong_dup_acks++;		if (call->cong_dup_acks < 3)			goto send_extra_data; 		summary->change = rxrpc_cong_begin_retransmission;		call->cong_ca_state = RXRPC_CA_FAST_RETRANSMIT;		call->cong_ssthresh = umax(summary->in_flight / 2, 2);		call->cong_cwnd = call->cong_ssthresh + 3;		call->cong_extra = 0;		call->cong_dup_acks = 0;		summary->need_retransmit = true;		summary->in_fast_or_rto_recovery = true;		goto out; 	case RXRPC_CA_FAST_RETRANSMIT:		rxrpc_tlp_init(call);		summary->in_fast_or_rto_recovery = true;		if (!summary->new_low_snack) {			if (summary->nr_new_sacks == 0)				call->cong_cwnd += 1;			call->cong_dup_acks++;			if (call->cong_dup_acks == 2) {				summary->change = rxrpc_cong_retransmit_again;				call->cong_dup_acks = 0;				summary->need_retransmit = true;			}		} else {			summary->change = rxrpc_cong_progress;			call->cong_cwnd = call->cong_ssthresh;			if (call->acks_nr_snacks == 0) {				summary->exiting_fast_or_rto_recovery = true;				goto resume_normality;			}		}		goto out; 	default:		BUG();		goto out;	} resume_normality:	summary->change = rxrpc_cong_cleared_nacks;	call->cong_dup_acks = 0;	call->cong_extra = 0;	call->cong_tstamp = call->acks_latest_ts;	if (call->cong_cwnd < call->cong_ssthresh)		call->cong_ca_state = RXRPC_CA_SLOW_START;	else		call->cong_ca_state = RXRPC_CA_CONGEST_AVOIDANCE;out:	call->cong_cumul_acks = 0;out_no_clear_ca:	if (call->cong_cwnd >= RXRPC_TX_MAX_WINDOW)		call->cong_cwnd = RXRPC_TX_MAX_WINDOW;	trace_rxrpc_congest(call, summary);	return; packet_loss_detected:	summary->change = rxrpc_cong_saw_nack;	call->cong_ca_state = RXRPC_CA_PACKET_LOSS;	call->cong_dup_acks = 0;	goto send_extra_data; send_extra_data:	/* Send some previously unsent DATA if we have some to advance the ACK	 * state.	 */	if (test_bit(RXRPC_CALL_TX_LAST, &call->flags) ||	    call->acks_nr_sacks != call->tx_top - call->tx_bottom) {		call->cong_extra++;		wake_up(&call->waitq);	}	goto out_no_clear_ca;} /* * Degrade the congestion window if we haven't transmitted a packet for >1RTT. */void rxrpc_congestion_degrade(struct rxrpc_call *call){	ktime_t rtt, now, time_since; 	if (call->cong_ca_state != RXRPC_CA_SLOW_START &&	    call->cong_ca_state != RXRPC_CA_CONGEST_AVOIDANCE)		return;	if (__rxrpc_call_state(call) == RXRPC_CALL_CLIENT_AWAIT_ACK ||	    __rxrpc_call_state(call) == RXRPC_CALL_CLIENT_AWAIT_REPLY)		return; 	rtt = ns_to_ktime(call->srtt_us * (NSEC_PER_USEC / 8));	now = ktime_get_real();	time_since = ktime_sub(now, call->tx_last_sent);	if (ktime_before(time_since, rtt))		return; 	trace_rxrpc_reset_cwnd(call, time_since, rtt);	rxrpc_inc_stat(call->rxnet, stat_tx_data_cwnd_reset);	call->tx_last_sent = now;	call->cong_ca_state = RXRPC_CA_SLOW_START;	call->cong_ssthresh = umax(call->cong_ssthresh, call->cong_cwnd * 3 / 4);	call->cong_cwnd = umax(call->cong_cwnd / 2, RXRPC_MIN_CWND);} /* * Add an RTT sample derived from an ACK'd DATA packet. */static void rxrpc_add_data_rtt_sample(struct rxrpc_call *call,				      struct rxrpc_ack_summary *summary,				      struct rxrpc_txqueue *tq,				      int ix){	ktime_t xmit_ts = ktime_add_us(tq->xmit_ts_base, tq->segment_xmit_ts[ix]); 	rxrpc_call_add_rtt(call, rxrpc_rtt_rx_data_ack, -1,			   summary->acked_serial, summary->ack_serial,			   xmit_ts, call->acks_latest_ts);	__clear_bit(ix, &tq->rtt_samples); /* Prevent repeat RTT sample */} /* * Apply a hard ACK by advancing the Tx window. */static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to,				   struct rxrpc_ack_summary *summary){	struct rxrpc_txqueue *tq = call->tx_queue;	rxrpc_seq_t seq = call->tx_bottom + 1;	bool rot_last = false, trace = false; 	_enter("%x,%x", call->tx_bottom, to); 	trace_rxrpc_tx_rotate(call, seq, to);	trace_rxrpc_tq(call, tq, seq, rxrpc_tq_rotate); 	if (call->acks_lowest_nak == call->tx_bottom) {		call->acks_lowest_nak = to;	} else if (after(to, call->acks_lowest_nak)) {		summary->new_low_snack = true;		call->acks_lowest_nak = to;	} 	if (after(seq, to))		return false; 	/* We may have a left over fully-consumed buffer at the front that we	 * couldn't drop before (rotate_and_keep below).	 */	if (seq == call->tx_qbase + RXRPC_NR_TXQUEUE) {		call->tx_qbase += RXRPC_NR_TXQUEUE;		call->tx_queue = tq->next;		trace_rxrpc_tq(call, tq, seq, rxrpc_tq_rotate_and_free);		kfree(tq);		tq = call->tx_queue;	} 	while (before_eq(seq, to)) {		unsigned int ix = seq - call->tx_qbase; 		_debug("tq=%x seq=%x i=%d f=%x", tq->qbase, seq, ix, tq->bufs[ix]->flags);		if (tq->bufs[ix]->flags & RXRPC_LAST_PACKET) {			set_bit(RXRPC_CALL_TX_LAST, &call->flags);			rot_last = true;		} 		if (summary->acked_serial == tq->segment_serial[ix] &&		    test_bit(ix, &tq->rtt_samples))			rxrpc_add_data_rtt_sample(call, summary, tq, ix); 		if (ix == tq->nr_reported_acks) {			/* Packet directly hard ACK'd. */			tq->nr_reported_acks++;			rxrpc_input_rack_one(call, summary, tq, ix);			if (seq == call->tlp_seq)				summary->tlp_probe_acked = true;			summary->nr_new_hacks++;			__set_bit(ix, &tq->segment_acked);			trace_rxrpc_rotate(call, tq, summary, seq, rxrpc_rotate_trace_hack);		} else if (test_bit(ix, &tq->segment_acked)) {			/* Soft ACK -> hard ACK. */			call->acks_nr_sacks--;			trace_rxrpc_rotate(call, tq, summary, seq, rxrpc_rotate_trace_sack);		} else {			/* Soft NAK -> hard ACK. */			call->acks_nr_snacks--;			rxrpc_input_rack_one(call, summary, tq, ix);			if (seq == call->tlp_seq)				summary->tlp_probe_acked = true;			summary->nr_new_hacks++;			__set_bit(ix, &tq->segment_acked);			trace_rxrpc_rotate(call, tq, summary, seq, rxrpc_rotate_trace_snak);		} 		call->tx_nr_sent--;		if (__test_and_clear_bit(ix, &tq->segment_lost))			call->tx_nr_lost--;		if (__test_and_clear_bit(ix, &tq->segment_retransmitted))			call->tx_nr_resent--;		__clear_bit(ix, &tq->ever_retransmitted); 		rxrpc_put_txbuf(tq->bufs[ix], rxrpc_txbuf_put_rotated);		tq->bufs[ix] = NULL; 		WRITE_ONCE(call->tx_bottom, seq);		trace_rxrpc_txqueue(call, (rot_last ?					   rxrpc_txqueue_rotate_last :					   rxrpc_txqueue_rotate)); 		seq++;		trace = true;		if (!(seq & RXRPC_TXQ_MASK)) {			trace_rxrpc_rack_update(call, summary);			trace = false;			prefetch(tq->next);			if (tq != call->tx_qtail) {				call->tx_qbase += RXRPC_NR_TXQUEUE;				call->tx_queue = tq->next;				trace_rxrpc_tq(call, tq, seq, rxrpc_tq_rotate_and_free);				kfree(tq);				tq = call->tx_queue;			} else {				trace_rxrpc_tq(call, tq, seq, rxrpc_tq_rotate_and_keep);				tq = NULL;				break;			}		}	} 	if (trace)		trace_rxrpc_rack_update(call, summary); 	if (rot_last) {		set_bit(RXRPC_CALL_TX_ALL_ACKED, &call->flags);		if (tq) {			trace_rxrpc_tq(call, tq, seq, rxrpc_tq_rotate_and_free);			kfree(tq);			call->tx_queue = NULL;		}	} 	_debug("%x,%x,%x,%d", to, call->tx_bottom, call->tx_top, rot_last); 	wake_up(&call->waitq);	return rot_last;} /* * End the transmission phase of a call. * * This occurs when we get an ACKALL packet, the first DATA packet of a reply, * or a final ACK packet. */static void rxrpc_end_tx_phase(struct rxrpc_call *call, bool reply_begun,			       enum rxrpc_abort_reason abort_why){	ASSERT(test_bit(RXRPC_CALL_TX_LAST, &call->flags)); 	call->rack_timer_mode = RXRPC_CALL_RACKTIMER_OFF;	call->rack_timo_at = KTIME_MAX;	trace_rxrpc_rack_timer(call, 0, false);	trace_rxrpc_timer_can(call, rxrpc_timer_trace_rack_off + call->rack_timer_mode); 	switch (__rxrpc_call_state(call)) {	case RXRPC_CALL_CLIENT_SEND_REQUEST:	case RXRPC_CALL_CLIENT_AWAIT_ACK:	case RXRPC_CALL_CLIENT_AWAIT_REPLY:		if (reply_begun) {			rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_RECV_REPLY);			trace_rxrpc_txqueue(call, rxrpc_txqueue_end);			break;		} 		rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_AWAIT_REPLY);		trace_rxrpc_txqueue(call, rxrpc_txqueue_await_reply);		break; 	case RXRPC_CALL_SERVER_AWAIT_ACK:		rxrpc_call_completed(call);		trace_rxrpc_txqueue(call, rxrpc_txqueue_end);		break; 	default:		kdebug("end_tx %s", rxrpc_call_states[__rxrpc_call_state(call)]);		rxrpc_proto_abort(call, call->tx_top, abort_why);		break;	}} /* * Begin the reply reception phase of a call. */static bool rxrpc_receiving_reply(struct rxrpc_call *call){	struct rxrpc_ack_summary summary = { 0 };	rxrpc_seq_t top = READ_ONCE(call->tx_top); 	if (call->ackr_reason) {		call->delay_ack_at = KTIME_MAX;		trace_rxrpc_timer_can(call, rxrpc_timer_trace_delayed_ack);	} 	/* Deal with an apparent reply coming in before we've got the request	 * queued or transmitted.	 */	if (!test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {		rxrpc_proto_abort(call, top, rxrpc_eproto_early_reply);		return false;	} 	if (!test_bit(RXRPC_CALL_TX_LAST, &call->flags)) {		if (!rxrpc_rotate_tx_window(call, top, &summary)) {			rxrpc_proto_abort(call, top, rxrpc_eproto_early_reply);			return false;		}	} 	rxrpc_end_tx_phase(call, true, rxrpc_eproto_unexpected_reply);	return true;} /* * End the packet reception phase. */static void rxrpc_end_rx_phase(struct rxrpc_call *call, rxrpc_serial_t serial){	rxrpc_seq_t whigh = READ_ONCE(call->rx_highest_seq); 	_enter("%d,%s", call->debug_id, rxrpc_call_states[__rxrpc_call_state(call)]); 	trace_rxrpc_receive(call, rxrpc_receive_end, 0, whigh); 	switch (__rxrpc_call_state(call)) {	case RXRPC_CALL_CLIENT_RECV_REPLY:		rxrpc_propose_delay_ACK(call, serial, rxrpc_propose_ack_terminal_ack);		rxrpc_call_completed(call);		break; 	case RXRPC_CALL_SERVER_RECV_REQUEST:		rxrpc_set_call_state(call, RXRPC_CALL_SERVER_ACK_REQUEST);		call->expect_req_by = KTIME_MAX;		rxrpc_propose_delay_ACK(call, serial, rxrpc_propose_ack_processing_op);		break; 	default:		break;	}} static void rxrpc_input_update_ack_window(struct rxrpc_call *call,					  rxrpc_seq_t window, rxrpc_seq_t wtop){	call->ackr_window = window;	call->ackr_wtop = wtop;} /* * Push a DATA packet onto the Rx queue. */static void rxrpc_input_queue_data(struct rxrpc_call *call, struct sk_buff *skb,				   rxrpc_seq_t window, rxrpc_seq_t wtop,				   enum rxrpc_receive_trace why){	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);	bool last = sp->hdr.flags & RXRPC_LAST_PACKET; 	spin_lock_irq(&call->recvmsg_queue.lock); 	__skb_queue_tail(&call->recvmsg_queue, skb);	rxrpc_input_update_ack_window(call, window, wtop);	trace_rxrpc_receive(call, last ? why + 1 : why, sp->hdr.serial, sp->hdr.seq);	if (last)		/* Change the state inside the lock so that recvmsg syncs		 * correctly with it and using sendmsg() to send a reply		 * doesn't race.		 */		rxrpc_end_rx_phase(call, sp->hdr.serial); 	spin_unlock_irq(&call->recvmsg_queue.lock);} /* * Process a DATA packet. */static void rxrpc_input_data_one(struct rxrpc_call *call, struct sk_buff *skb,				 bool *_notify, rxrpc_serial_t *_ack_serial, int *_ack_reason){	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);	struct sk_buff *oos;	rxrpc_serial_t serial = sp->hdr.serial;	unsigned int sack = call->ackr_sack_base;	rxrpc_seq_t window = call->ackr_window;	rxrpc_seq_t wtop = call->ackr_wtop;	rxrpc_seq_t wlimit = window + call->rx_winsize - 1;	rxrpc_seq_t seq = sp->hdr.seq;	bool last = sp->hdr.flags & RXRPC_LAST_PACKET;	int ack_reason = -1; 	rxrpc_inc_stat(call->rxnet, stat_rx_data);	if (sp->hdr.flags & RXRPC_REQUEST_ACK)		rxrpc_inc_stat(call->rxnet, stat_rx_data_reqack);	if (sp->hdr.flags & RXRPC_JUMBO_PACKET)		rxrpc_inc_stat(call->rxnet, stat_rx_data_jumbo); 

Lines 1–500 of 1332. Load more