Merge tag 'rxrpc-fixes-20190827' of git://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs

David Howells says: ==================== rxrpc: Fix use of skb_cow_data() Here's a series of patches that replaces the use of skb_cow_data() in rxrpc with skb_unshare() early on in the input process. The problem that is being seen is that skb_cow_data() indirectly requires that the maximum usage count on an sk_buff be 1, and it may generate an assertion failure in pskb_expand_head() if not. This can occur because rxrpc_input_data() may be still holding a ref when it has just attached the sk_buff to the rx ring and given that attachment its own ref. If recvmsg happens fast enough, skb_cow_data() can see the ref still held by the softirq handler. Further, a packet may contain multiple subpackets, each of which gets its own attachment to the ring and its own ref - also making skb_cow_data() go bang. Fix this by: (1) The DATA packet is currently parsed for subpackets twice by the input routines. Parse it just once instead and make notes in the sk_buff private data. (2) Use the notes from (1) when attaching the packet to the ring multiple times. Once the packet is attached to the ring, recvmsg can see it and start modifying it, so the softirq handler is not permitted to look inside it from that point. (3) Pass the ref from the input code to the ring rather than getting an extra ref. rxrpc_input_data() uses a ref on the second refcount to prevent the packet from evaporating under it. (4) Call skb_unshare() on secured DATA packets in rxrpc_input_packet() before we take call->input_lock. Other sorts of packets don't get modified and so can be left. A trace is emitted if skb_unshare() eats the skb. Note that skb_share() for our accounting in this regard as we can't see the parameters in the packet to log in a trace line if it releases it. (5) Remove the calls to skb_cow_data(). These are then no longer necessary. There are also patches to improve the rxrpc_skb tracepoint to make sure that Tx-derived buffers are identified separately from Rx-derived buffers in the trace. ====================

David S. Miller committed 7 years ago

Commit a285c1f2 parents 3b25528 d0d5c0c14 files · +319 −268Browse files

Unified · Split

14 files

include/trace/events/rxrpc.h +28 −31
net/rxrpc/ar-internal.h +11 −5
net/rxrpc/call_event.c +4 −4
net/rxrpc/call_object.c +16 −17
net/rxrpc/conn_event.c +3 −3
net/rxrpc/input.c +167 −137
net/rxrpc/local_event.c +2 −2
net/rxrpc/output.c +3 −3
net/rxrpc/peer_event.c +5 −5
net/rxrpc/protocol.h +9 −0
net/rxrpc/recvmsg.c +27 −20
net/rxrpc/rxkad.c +9 −23
net/rxrpc/sendmsg.c +7 −6
net/rxrpc/skbuff.c +28 −12

include/trace/events/rxrpc.h+28−31

@@ -23,20 +23,17 @@ · 22 unmodified lines
23	23	#define __RXRPC_DECLARE_TRACE_ENUMS_ONCE_ONLY
24	24
25	25	enum rxrpc_skb_trace {
26		− rxrpc_skb_rx_cleaned,
27		− rxrpc_skb_rx_freed,
28		− rxrpc_skb_rx_got,
29		− rxrpc_skb_rx_lost,
30		− rxrpc_skb_rx_purged,
31		− rxrpc_skb_rx_received,
32		− rxrpc_skb_rx_rotated,
33		− rxrpc_skb_rx_seen,
34		− rxrpc_skb_tx_cleaned,
35		− rxrpc_skb_tx_freed,
36		− rxrpc_skb_tx_got,
37		− rxrpc_skb_tx_new,
38		− rxrpc_skb_tx_rotated,
39		− rxrpc_skb_tx_seen,
	26	+ rxrpc_skb_cleaned,
	27	+ rxrpc_skb_freed,
	28	+ rxrpc_skb_got,
	29	+ rxrpc_skb_lost,
	30	+ rxrpc_skb_new,
	31	+ rxrpc_skb_purged,
	32	+ rxrpc_skb_received,
	33	+ rxrpc_skb_rotated,
	34	+ rxrpc_skb_seen,
	35	+ rxrpc_skb_unshared,
	36	+ rxrpc_skb_unshared_nomem,
40	37	};
41	38
42	39	enum rxrpc_local_trace {
@@ -228,20 +225,17 @@ enum rxrpc_tx_point { · 185 unmodified lines
228	225	* Declare tracing information enums and their string mappings for display.
229	226	*/
230	227	#define rxrpc_skb_traces \
231		− EM(rxrpc_skb_rx_cleaned, "Rx CLN") \
232		− EM(rxrpc_skb_rx_freed, "Rx FRE") \
233		− EM(rxrpc_skb_rx_got, "Rx GOT") \
234		− EM(rxrpc_skb_rx_lost, "Rx L") \
235		− EM(rxrpc_skb_rx_purged, "Rx PUR") \
236		− EM(rxrpc_skb_rx_received, "Rx RCV") \
237		− EM(rxrpc_skb_rx_rotated, "Rx ROT") \
238		− EM(rxrpc_skb_rx_seen, "Rx SEE") \
239		− EM(rxrpc_skb_tx_cleaned, "Tx CLN") \
240		− EM(rxrpc_skb_tx_freed, "Tx FRE") \
241		− EM(rxrpc_skb_tx_got, "Tx GOT") \
242		− EM(rxrpc_skb_tx_new, "Tx NEW") \
243		− EM(rxrpc_skb_tx_rotated, "Tx ROT") \
244		− E_(rxrpc_skb_tx_seen, "Tx SEE")
	228	+ EM(rxrpc_skb_cleaned, "CLN") \
	229	+ EM(rxrpc_skb_freed, "FRE") \
	230	+ EM(rxrpc_skb_got, "GOT") \
	231	+ EM(rxrpc_skb_lost, "L") \
	232	+ EM(rxrpc_skb_new, "NEW") \
	233	+ EM(rxrpc_skb_purged, "PUR") \
	234	+ EM(rxrpc_skb_received, "RCV") \
	235	+ EM(rxrpc_skb_rotated, "ROT") \
	236	+ EM(rxrpc_skb_seen, "SEE") \
	237	+ EM(rxrpc_skb_unshared, "UNS") \
	238	+ E_(rxrpc_skb_unshared_nomem, "US0")
245	239
246	240	#define rxrpc_local_traces \
247	241	EM(rxrpc_local_got, "GOT") \
@@ -643,13 +637,14 @@ TRACE_EVENT(rxrpc_call, · 395 unmodified lines
643	637
644	638	TRACE_EVENT(rxrpc_skb,
645	639	TP_PROTO(struct sk_buff *skb, enum rxrpc_skb_trace op,
646		− int usage, int mod_count, const void *where),
	640	+ int usage, int mod_count, u8 flags, const void *where),
647	641
648		− TP_ARGS(skb, op, usage, mod_count, where),
	642	+ TP_ARGS(skb, op, usage, mod_count, flags, where),
649	643
650	644	TP_STRUCT__entry(
651	645	__field(struct sk_buff *, skb )
652	646	__field(enum rxrpc_skb_trace, op )
	647	+ __field(u8, flags )
653	648	__field(int, usage )
654	649	__field(int, mod_count )
655	650	__field(const void *, where )
@@ -657,14 +652,16 @@ TRACE_EVENT(rxrpc_skb, · 1 unmodified line
657	652
658	653	TP_fast_assign(
659	654	__entry->skb = skb;
	655	+ __entry->flags = flags;
660	656	__entry->op = op;
661	657	__entry->usage = usage;
662	658	__entry->mod_count = mod_count;
663	659	__entry->where = where;
664	660	),
665	661
666		− TP_printk("s=%p %s u=%d m=%d p=%pSR",
	662	+ TP_printk("s=%p %cx %s u=%d m=%d p=%pSR",
667	663	__entry->skb,
	664	+ __entry->flags & RXRPC_SKB_TX_BUFFER ? 'T' : 'R',
668	665	__print_symbolic(__entry->op, rxrpc_skb_traces),
669	666	__entry->usage,
670	667	__entry->mod_count,

net/rxrpc/ar-internal.h+11−5

@@ -185,11 +185,17 @@ struct rxrpc_host_header { · 184 unmodified lines
185	185	* - max 48 bytes (struct sk_buff::cb)
186	186	*/
187	187	struct rxrpc_skb_priv {
188		− union {
189		− u8 nr_jumbo; /* Number of jumbo subpackets */
190		− };
	188	+ atomic_t nr_ring_pins; /* Number of rxtx ring pins */
	189	+ u8 nr_subpackets; /* Number of subpackets */
	190	+ u8 rx_flags; /* Received packet flags */
	191	+#define RXRPC_SKB_INCL_LAST 0x01 /* - Includes last packet */
	192	+#define RXRPC_SKB_TX_BUFFER 0x02 /* - Is transmit buffer */
191	193	union {
192	194	int remain; /* amount of space remaining for next write */
	195	+
	196	+ /* List of requested ACKs on subpackets */
	197	+ unsigned long rx_req_ack[(RXRPC_MAX_NR_JUMBO + BITS_PER_LONG - 1) /
	198	+ BITS_PER_LONG];
193	199	};
194	200
195	201	struct rxrpc_host_header hdr; /* RxRPC packet header from this packet */
@@ -613,8 +619,7 @@ struct rxrpc_call { · 417 unmodified lines
613	619	#define RXRPC_TX_ANNO_LAST 0x04
614	620	#define RXRPC_TX_ANNO_RESENT 0x08
615	621
616		−#define RXRPC_RX_ANNO_JUMBO 0x3f /* Jumbo subpacket number + 1 if not zero */
617		−#define RXRPC_RX_ANNO_JLAST 0x40 /* Set if last element of a jumbo packet */
	622	+#define RXRPC_RX_ANNO_SUBPACKET 0x3f /* Subpacket number in jumbogram */
618	623	#define RXRPC_RX_ANNO_VERIFIED 0x80 /* Set if verified and decrypted */
619	624	rxrpc_seq_t tx_hard_ack; /* Dead slot in buffer; the first transmitted but
620	625	* not hard-ACK'd packet follows this.
@@ -1105,6 +1110,7 @@ void rxrpc_kernel_data_consumed(struct rxrpc_call , struct sk_buff ); · 484 unmodified lines
1105	1110	void rxrpc_packet_destructor(struct sk_buff *);
1106	1111	void rxrpc_new_skb(struct sk_buff *, enum rxrpc_skb_trace);
1107	1112	void rxrpc_see_skb(struct sk_buff *, enum rxrpc_skb_trace);
	1113	+void rxrpc_eaten_skb(struct sk_buff *, enum rxrpc_skb_trace);
1108	1114	void rxrpc_get_skb(struct sk_buff *, enum rxrpc_skb_trace);
1109	1115	void rxrpc_free_skb(struct sk_buff *, enum rxrpc_skb_trace);
1110	1116	void rxrpc_purge_queue(struct sk_buff_head *);

net/rxrpc/call_event.c+4−4

@@ -199,7 +199,7 @@ static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j) · 198 unmodified lines
199	199	continue;
200	200
201	201	skb = call->rxtx_buffer[ix];
202		− rxrpc_see_skb(skb, rxrpc_skb_tx_seen);
	202	+ rxrpc_see_skb(skb, rxrpc_skb_seen);
203	203
204	204	if (anno_type == RXRPC_TX_ANNO_UNACK) {
205	205	if (ktime_after(skb->tstamp, max_age)) {
@@ -255,18 +255,18 @@ static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j) · 49 unmodified lines
255	255	continue;
256	256
257	257	skb = call->rxtx_buffer[ix];
258		− rxrpc_get_skb(skb, rxrpc_skb_tx_got);
	258	+ rxrpc_get_skb(skb, rxrpc_skb_got);
259	259	spin_unlock_bh(&call->lock);
260	260
261	261	if (rxrpc_send_data_packet(call, skb, true) < 0) {
262		− rxrpc_free_skb(skb, rxrpc_skb_tx_freed);
	262	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
263	263	return;
264	264	}
265	265
266	266	if (rxrpc_is_client_call(call))
267	267	rxrpc_expose_client_call(call);
268	268
269		− rxrpc_free_skb(skb, rxrpc_skb_tx_freed);
	269	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
270	270	spin_lock_bh(&call->lock);
271	271
272	272	/* We need to clear the retransmit state, but there are two

net/rxrpc/call_object.c+16−17

@@ -421,6 +421,19 @@ void rxrpc_get_call(struct rxrpc_call *call, enum rxrpc_call_trace op) · 420 unmodified lines
421	421	trace_rxrpc_call(call, op, n, here, NULL);
422	422	}
423	423
	424	+/*
	425	+ * Clean up the RxTx skb ring.
	426	+ */
	427	+static void rxrpc_cleanup_ring(struct rxrpc_call *call)
	428	+{
	429	+ int i;
	430	+
	431	+ for (i = 0; i < RXRPC_RXTX_BUFF_SIZE; i++) {
	432	+ rxrpc_free_skb(call->rxtx_buffer[i], rxrpc_skb_cleaned);
	433	+ call->rxtx_buffer[i] = NULL;
	434	+ }
	435	+}
	436	+
424	437	/*
425	438	* Detach a call from its owning socket.
426	439	*/
@@ -429,7 +442,6 @@ void rxrpc_release_call(struct rxrpc_sock rx, struct rxrpc_call call) · 2 unmodified lines
429	442	const void *here = __builtin_return_address(0);
430	443	struct rxrpc_connection *conn = call->conn;
431	444	bool put = false;
432		− int i;
433	445
434	446	_enter("{%d,%d}", call->debug_id, atomic_read(&call->usage));
435	447
@@ -479,13 +491,7 @@ void rxrpc_release_call(struct rxrpc_sock rx, struct rxrpc_call call) · 43 unmodified lines
479	491	if (conn)
480	492	rxrpc_disconnect_call(call);
481	493
482		− for (i = 0; i < RXRPC_RXTX_BUFF_SIZE; i++) {
483		− rxrpc_free_skb(call->rxtx_buffer[i],
484		− (call->tx_phase ? rxrpc_skb_tx_cleaned :
485		− rxrpc_skb_rx_cleaned));
486		− call->rxtx_buffer[i] = NULL;
487		− }
488		−
	494	+ rxrpc_cleanup_ring(call);
489	495	_leave("");
490	496	}
491	497
@@ -568,8 +574,6 @@ static void rxrpc_rcu_destroy_call(struct rcu_head *rcu) · 76 unmodified lines
568	574	*/
569	575	void rxrpc_cleanup_call(struct rxrpc_call *call)
570	576	{
571		− int i;
572		−
573	577	_net("DESTROY CALL %d", call->debug_id);
574	578
575	579	memset(&call->sock_node, 0xcd, sizeof(call->sock_node));
@@ -580,13 +584,8 @@ void rxrpc_cleanup_call(struct rxrpc_call *call) · 4 unmodified lines
580	584	ASSERT(test_bit(RXRPC_CALL_RELEASED, &call->flags));
581	585	ASSERTCMP(call->conn, ==, NULL);
582	586
583		− /* Clean up the Rx/Tx buffer */
584		− for (i = 0; i < RXRPC_RXTX_BUFF_SIZE; i++)
585		− rxrpc_free_skb(call->rxtx_buffer[i],
586		− (call->tx_phase ? rxrpc_skb_tx_cleaned :
587		− rxrpc_skb_rx_cleaned));
588		−
589		− rxrpc_free_skb(call->tx_pending, rxrpc_skb_tx_cleaned);
	587	+ rxrpc_cleanup_ring(call);
	588	+ rxrpc_free_skb(call->tx_pending, rxrpc_skb_cleaned);
590	589
591	590	call_rcu(&call->rcu, rxrpc_rcu_destroy_call);
592	591	}

net/rxrpc/conn_event.c+3−3

@@ -472,7 +472,7 @@ void rxrpc_process_connection(struct work_struct *work) · 471 unmodified lines
472	472	/* go through the conn-level event packets, releasing the ref on this
473	473	* connection that each one has when we've finished with it */
474	474	while ((skb = skb_dequeue(&conn->rx_queue))) {
475		− rxrpc_see_skb(skb, rxrpc_skb_rx_seen);
	475	+ rxrpc_see_skb(skb, rxrpc_skb_seen);
476	476	ret = rxrpc_process_event(conn, skb, &abort_code);
477	477	switch (ret) {
478	478	case -EPROTO:
@@ -484,7 +484,7 @@ void rxrpc_process_connection(struct work_struct *work) · 5 unmodified lines
484	484	goto requeue_and_leave;
485	485	case -ECONNABORTED:
486	486	default:
487		− rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
	487	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
488	488	break;
489	489	}
490	490	}
@@ -501,6 +501,6 @@ requeue_and_leave: · 10 unmodified lines
501	501	protocol_error:
502	502	if (rxrpc_abort_connection(conn, ret, abort_code) < 0)
503	503	goto requeue_and_leave;
504		− rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
	504	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
505	505	goto out;
506	506	}

net/rxrpc/input.c+167−137

@@ -233,7 +233,7 @@ static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to, · 232 unmodified lines
233	233	ix = call->tx_hard_ack & RXRPC_RXTX_BUFF_MASK;
234	234	skb = call->rxtx_buffer[ix];
235	235	annotation = call->rxtx_annotations[ix];
236		− rxrpc_see_skb(skb, rxrpc_skb_tx_rotated);
	236	+ rxrpc_see_skb(skb, rxrpc_skb_rotated);
237	237	call->rxtx_buffer[ix] = NULL;
238	238	call->rxtx_annotations[ix] = 0;
239	239	skb->next = list;
@@ -258,7 +258,7 @@ static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to, · 18 unmodified lines
258	258	skb = list;
259	259	list = skb->next;
260	260	skb_mark_not_on_list(skb);
261		− rxrpc_free_skb(skb, rxrpc_skb_tx_freed);
	261	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
262	262	}
263	263
264	264	return rot_last;
@@ -347,7 +347,7 @@ static bool rxrpc_receiving_reply(struct rxrpc_call *call) · 82 unmodified lines
347	347	}
348	348
349	349	/*
350		− * Scan a jumbo packet to validate its structure and to work out how many
	350	+ * Scan a data packet to validate its structure and to work out how many
351	351	* subpackets it contains.
352	352	*
353	353	* A jumbo packet is a collection of consecutive packets glued together with
@@ -358,16 +358,21 @@ static bool rxrpc_receiving_reply(struct rxrpc_call *call) · 4 unmodified lines
358	358	* the last are RXRPC_JUMBO_DATALEN in size. The last subpacket may be of any
359	359	* size.
360	360	*/
361		−static bool rxrpc_validate_jumbo(struct sk_buff *skb)
	361	+static bool rxrpc_validate_data(struct sk_buff *skb)
362	362	{
363	363	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
364	364	unsigned int offset = sizeof(struct rxrpc_wire_header);
365	365	unsigned int len = skb->len;
366		− int nr_jumbo = 1;
367	366	u8 flags = sp->hdr.flags;
368	367
369		− do {
370		− nr_jumbo++;
	368	+ for (;;) {
	369	+ if (flags & RXRPC_REQUEST_ACK)
	370	+ __set_bit(sp->nr_subpackets, sp->rx_req_ack);
	371	+ sp->nr_subpackets++;
	372	+
	373	+ if (!(flags & RXRPC_JUMBO_PACKET))
	374	+ break;
	375	+
371	376	if (len - offset < RXRPC_JUMBO_SUBPKTLEN)
372	377	goto protocol_error;
373	378	if (flags & RXRPC_LAST_PACKET)
@@ -376,9 +381,10 @@ static bool rxrpc_validate_jumbo(struct sk_buff *skb) · 2 unmodified lines
376	381	if (skb_copy_bits(skb, offset, &flags, 1) < 0)
377	382	goto protocol_error;
378	383	offset += sizeof(struct rxrpc_jumbo_header);
379		− } while (flags & RXRPC_JUMBO_PACKET);
	384	+ }
380	385
381		− sp->nr_jumbo = nr_jumbo;
	386	+ if (flags & RXRPC_LAST_PACKET)
	387	+ sp->rx_flags \|= RXRPC_SKB_INCL_LAST;
382	388	return true;
383	389
384	390	protocol_error:
@@ -399,10 +405,10 @@ protocol_error: · 14 unmodified lines
399	405	* (that information is encoded in the ACK packet).
400	406	*/
401	407	static void rxrpc_input_dup_data(struct rxrpc_call *call, rxrpc_seq_t seq,
402		− u8 annotation, bool *_jumbo_bad)
	408	+ bool is_jumbo, bool *_jumbo_bad)
403	409	{
404	410	/* Discard normal packets that are duplicates. */
405		− if (annotation == 0)
	411	+ if (is_jumbo)
406	412	return;
407	413
408	414	/* Skip jumbo subpackets that are duplicates. When we've had three or
@@ -416,29 +422,30 @@ static void rxrpc_input_dup_data(struct rxrpc_call *call, rxrpc_seq_t seq, · 7 unmodified lines
416	422	}
417	423
418	424	/*
419		− * Process a DATA packet, adding the packet to the Rx ring.
	425	+ * Process a DATA packet, adding the packet to the Rx ring. The caller's
	426	+ * packet ref must be passed on or discarded.
420	427	*/
421	428	static void rxrpc_input_data(struct rxrpc_call call, struct sk_buff skb)
422	429	{
423	430	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
424	431	enum rxrpc_call_state state;
425		− unsigned int offset = sizeof(struct rxrpc_wire_header);
426		− unsigned int ix;
	432	+ unsigned int j;
427	433	rxrpc_serial_t serial = sp->hdr.serial, ack_serial = 0;
428		− rxrpc_seq_t seq = sp->hdr.seq, hard_ack;
429		− bool immediate_ack = false, jumbo_bad = false, queued;
430		− u16 len;
431		− u8 ack = 0, flags, annotation = 0;
	434	+ rxrpc_seq_t seq0 = sp->hdr.seq, hard_ack;
	435	+ bool immediate_ack = false, jumbo_bad = false;
	436	+ u8 ack = 0;
432	437
433	438	_enter("{%u,%u},{%u,%u}",
434		− call->rx_hard_ack, call->rx_top, skb->len, seq);
	439	+ call->rx_hard_ack, call->rx_top, skb->len, seq0);
435	440
436		− _proto("Rx DATA %%%u { #%u f=%02x }",
437		− sp->hdr.serial, seq, sp->hdr.flags);
	441	+ _proto("Rx DATA %%%u { #%u f=%02x n=%u }",
	442	+ sp->hdr.serial, seq0, sp->hdr.flags, sp->nr_subpackets);
438	443
439	444	state = READ_ONCE(call->state);
440		− if (state >= RXRPC_CALL_COMPLETE)
	445	+ if (state >= RXRPC_CALL_COMPLETE) {
	446	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
441	447	return;
	448	+ }
442	449
443	450	if (call->state == RXRPC_CALL_SERVER_RECV_REQUEST) {
444	451	unsigned long timo = READ_ONCE(call->next_req_timo);
@@ -463,137 +470,137 @@ static void rxrpc_input_data(struct rxrpc_call call, struct sk_buff skb) · 18 unmodified lines
463	470	!rxrpc_receiving_reply(call))
464	471	goto unlock;
465	472
466		− call->ackr_prev_seq = seq;
467		−
	473	+ call->ackr_prev_seq = seq0;
468	474	hard_ack = READ_ONCE(call->rx_hard_ack);
469		− if (after(seq, hard_ack + call->rx_winsize)) {
470		− ack = RXRPC_ACK_EXCEEDS_WINDOW;
471		− ack_serial = serial;
472		− goto ack;
473		− }
474	475
475		− flags = sp->hdr.flags;
476		− if (flags & RXRPC_JUMBO_PACKET) {
	476	+ if (sp->nr_subpackets > 1) {
477	477	if (call->nr_jumbo_bad > 3) {
478	478	ack = RXRPC_ACK_NOSPACE;
479	479	ack_serial = serial;
480	480	goto ack;
481	481	}
482		− annotation = 1;
483	482	}
484	483
485		−next_subpacket:
486		− queued = false;
487		− ix = seq & RXRPC_RXTX_BUFF_MASK;
488		− len = skb->len;
489		− if (flags & RXRPC_JUMBO_PACKET)
490		− len = RXRPC_JUMBO_DATALEN;
491		−
492		− if (flags & RXRPC_LAST_PACKET) {
493		− if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) &&
494		− seq != call->rx_top) {
495		− rxrpc_proto_abort("LSN", call, seq);
496		− goto unlock;
497		− }
498		− } else {
499		− if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) &&
500		− after_eq(seq, call->rx_top)) {
501		− rxrpc_proto_abort("LSA", call, seq);
502		− goto unlock;
	484	+ for (j = 0; j < sp->nr_subpackets; j++) {
	485	+ rxrpc_serial_t serial = sp->hdr.serial + j;
	486	+ rxrpc_seq_t seq = seq0 + j;
	487	+ unsigned int ix = seq & RXRPC_RXTX_BUFF_MASK;
	488	+ bool terminal = (j == sp->nr_subpackets - 1);
	489	+ bool last = terminal && (sp->rx_flags & RXRPC_SKB_INCL_LAST);
	490	+ u8 flags, annotation = j;
	491	+
	492	+ _proto("Rx DATA+%u %%%u { #%x t=%u l=%u }",
	493	+ j, serial, seq, terminal, last);
	494	+
	495	+ if (last) {
	496	+ if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) &&
	497	+ seq != call->rx_top) {
	498	+ rxrpc_proto_abort("LSN", call, seq);
	499	+ goto unlock;
	500	+ }
	501	+ } else {
	502	+ if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) &&
	503	+ after_eq(seq, call->rx_top)) {
	504	+ rxrpc_proto_abort("LSA", call, seq);
	505	+ goto unlock;
	506	+ }
503	507	}
504		− }
505	508
506		− trace_rxrpc_rx_data(call->debug_id, seq, serial, flags, annotation);
507		− if (before_eq(seq, hard_ack)) {
508		− ack = RXRPC_ACK_DUPLICATE;
509		− ack_serial = serial;
510		− goto skip;
511		− }
	509	+ flags = 0;
	510	+ if (last)
	511	+ flags \|= RXRPC_LAST_PACKET;
	512	+ if (!terminal)
	513	+ flags \|= RXRPC_JUMBO_PACKET;
	514	+ if (test_bit(j, sp->rx_req_ack))
	515	+ flags \|= RXRPC_REQUEST_ACK;
	516	+ trace_rxrpc_rx_data(call->debug_id, seq, serial, flags, annotation);
512	517
513		− if (flags & RXRPC_REQUEST_ACK && !ack) {
514		− ack = RXRPC_ACK_REQUESTED;
515		− ack_serial = serial;
516		− }
517		−
518		− if (call->rxtx_buffer[ix]) {
519		− rxrpc_input_dup_data(call, seq, annotation, &jumbo_bad);
520		− if (ack != RXRPC_ACK_DUPLICATE) {
	518	+ if (before_eq(seq, hard_ack)) {
521	519	ack = RXRPC_ACK_DUPLICATE;
522	520	ack_serial = serial;
	521	+ continue;
523	522	}
524		− immediate_ack = true;
525		− goto skip;
526		− }
527	523
528		− /* Queue the packet. We use a couple of memory barriers here as need
529		− * to make sure that rx_top is perceived to be set after the buffer
530		− * pointer and that the buffer pointer is set after the annotation and
531		− * the skb data.
532		− *
533		− * Barriers against rxrpc_recvmsg_data() and rxrpc_rotate_rx_window()
534		− * and also rxrpc_fill_out_ack().
535		− */
536		− rxrpc_get_skb(skb, rxrpc_skb_rx_got);
537		− call->rxtx_annotations[ix] = annotation;
538		− smp_wmb();
539		− call->rxtx_buffer[ix] = skb;
540		− if (after(seq, call->rx_top)) {
541		− smp_store_release(&call->rx_top, seq);
542		− } else if (before(seq, call->rx_top)) {
543		− /* Send an immediate ACK if we fill in a hole */
544		− if (!ack) {
545		− ack = RXRPC_ACK_DELAY;
546		− ack_serial = serial;
	524	+ if (call->rxtx_buffer[ix]) {
	525	+ rxrpc_input_dup_data(call, seq, sp->nr_subpackets > 1,
	526	+ &jumbo_bad);
	527	+ if (ack != RXRPC_ACK_DUPLICATE) {
	528	+ ack = RXRPC_ACK_DUPLICATE;
	529	+ ack_serial = serial;
	530	+ }
	531	+ immediate_ack = true;
	532	+ continue;
547	533	}
548		− immediate_ack = true;
549		− }
550		− if (flags & RXRPC_LAST_PACKET) {
551		− set_bit(RXRPC_CALL_RX_LAST, &call->flags);
552		− trace_rxrpc_receive(call, rxrpc_receive_queue_last, serial, seq);
553		− } else {
554		− trace_rxrpc_receive(call, rxrpc_receive_queue, serial, seq);
555		− }
556		− queued = true;
557	534
558		− if (after_eq(seq, call->rx_expect_next)) {
559		− if (after(seq, call->rx_expect_next)) {
560		− _net("OOS %u > %u", seq, call->rx_expect_next);
561		− ack = RXRPC_ACK_OUT_OF_SEQUENCE;
562		− ack_serial = serial;
563		− }
564		− call->rx_expect_next = seq + 1;
565		− }
566		−
567		−skip:
568		− offset += len;
569		− if (flags & RXRPC_JUMBO_PACKET) {
570		− if (skb_copy_bits(skb, offset, &flags, 1) < 0) {
571		− rxrpc_proto_abort("XJF", call, seq);
572		− goto unlock;
573		− }
574		− offset += sizeof(struct rxrpc_jumbo_header);
575		− seq++;
576		− serial++;
577		− annotation++;
578		− if (flags & RXRPC_JUMBO_PACKET)
579		− annotation \|= RXRPC_RX_ANNO_JLAST;
580	535	if (after(seq, hard_ack + call->rx_winsize)) {
581	536	ack = RXRPC_ACK_EXCEEDS_WINDOW;
582	537	ack_serial = serial;
583		− if (!jumbo_bad) {
584		− call->nr_jumbo_bad++;
585		− jumbo_bad = true;
	538	+ if (flags & RXRPC_JUMBO_PACKET) {
	539	+ if (!jumbo_bad) {
	540	+ call->nr_jumbo_bad++;
	541	+ jumbo_bad = true;
	542	+ }
586	543	}
	544	+
587	545	goto ack;
588	546	}
589	547
590		− _proto("Rx DATA Jumbo %%%u", serial);
591		− goto next_subpacket;
592		− }
	548	+ if (flags & RXRPC_REQUEST_ACK && !ack) {
	549	+ ack = RXRPC_ACK_REQUESTED;
	550	+ ack_serial = serial;
	551	+ }
	552	+
	553	+ /* Queue the packet. We use a couple of memory barriers here as need
	554	+ * to make sure that rx_top is perceived to be set after the buffer
	555	+ * pointer and that the buffer pointer is set after the annotation and
	556	+ * the skb data.
	557	+ *
	558	+ * Barriers against rxrpc_recvmsg_data() and rxrpc_rotate_rx_window()
	559	+ * and also rxrpc_fill_out_ack().
	560	+ */
	561	+ if (!terminal)
	562	+ rxrpc_get_skb(skb, rxrpc_skb_got);
	563	+ call->rxtx_annotations[ix] = annotation;
	564	+ smp_wmb();
	565	+ call->rxtx_buffer[ix] = skb;
	566	+ if (after(seq, call->rx_top)) {
	567	+ smp_store_release(&call->rx_top, seq);
	568	+ } else if (before(seq, call->rx_top)) {
	569	+ /* Send an immediate ACK if we fill in a hole */
	570	+ if (!ack) {
	571	+ ack = RXRPC_ACK_DELAY;
	572	+ ack_serial = serial;
	573	+ }
	574	+ immediate_ack = true;
	575	+ }
	576	+
	577	+ if (terminal) {
	578	+ /* From this point on, we're not allowed to touch the
	579	+ * packet any longer as its ref now belongs to the Rx
	580	+ * ring.
	581	+ */
	582	+ skb = NULL;
	583	+ }
593	584
594		− if (queued && flags & RXRPC_LAST_PACKET && !ack) {
595		− ack = RXRPC_ACK_DELAY;
596		− ack_serial = serial;
	585	+ if (last) {
	586	+ set_bit(RXRPC_CALL_RX_LAST, &call->flags);
	587	+ if (!ack) {
	588	+ ack = RXRPC_ACK_DELAY;
	589	+ ack_serial = serial;
	590	+ }
	591	+ trace_rxrpc_receive(call, rxrpc_receive_queue_last, serial, seq);
	592	+ } else {
	593	+ trace_rxrpc_receive(call, rxrpc_receive_queue, serial, seq);
	594	+ }
	595	+
	596	+ if (after_eq(seq, call->rx_expect_next)) {
	597	+ if (after(seq, call->rx_expect_next)) {
	598	+ _net("OOS %u > %u", seq, call->rx_expect_next);
	599	+ ack = RXRPC_ACK_OUT_OF_SEQUENCE;
	600	+ ack_serial = serial;
	601	+ }
	602	+ call->rx_expect_next = seq + 1;
	603	+ }
597	604	}
598	605
599	606	ack:
@@ -606,13 +613,14 @@ ack: · 6 unmodified lines
606	613	false, true,
607	614	rxrpc_propose_ack_input_data);
608	615
609		− if (sp->hdr.seq == READ_ONCE(call->rx_hard_ack) + 1) {
	616	+ if (seq0 == READ_ONCE(call->rx_hard_ack) + 1) {
610	617	trace_rxrpc_notify_socket(call->debug_id, serial);
611	618	rxrpc_notify_socket(call);
612	619	}
613	620
614	621	unlock:
615	622	spin_unlock(&call->input_lock);
	623	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
616	624	_leave(" [queued]");
617	625	}
618	626
@@ -1021,7 +1029,7 @@ static void rxrpc_input_call_packet(struct rxrpc_call *call, · 402 unmodified lines
1021	1029	switch (sp->hdr.type) {
1022	1030	case RXRPC_PACKET_TYPE_DATA:
1023	1031	rxrpc_input_data(call, skb);
1024		− break;
	1032	+ goto no_free;
1025	1033
1026	1034	case RXRPC_PACKET_TYPE_ACK:
1027	1035	rxrpc_input_ack(call, skb);
@@ -1048,6 +1056,8 @@ static void rxrpc_input_call_packet(struct rxrpc_call *call, · 20 unmodified lines
1048	1056	break;
1049	1057	}
1050	1058
	1059	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
	1060	+no_free:
1051	1061	_leave("");
1052	1062	}
1053	1063
@@ -1109,7 +1119,7 @@ static void rxrpc_post_packet_to_local(struct rxrpc_local *local, · 55 unmodified lines
1109	1119	skb_queue_tail(&local->event_queue, skb);
1110	1120	rxrpc_queue_local(local);
1111	1121	} else {
1112		− rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
	1122	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
1113	1123	}
1114	1124	}
1115	1125
@@ -1124,7 +1134,7 @@ static void rxrpc_reject_packet(struct rxrpc_local local, struct sk_buff skb) · 8 unmodified lines
1124	1134	skb_queue_tail(&local->reject_queue, skb);
1125	1135	rxrpc_queue_local(local);
1126	1136	} else {
1127		− rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
	1137	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
1128	1138	}
1129	1139	}
1130	1140
@@ -1188,7 +1198,7 @@ int rxrpc_input_packet(struct sock udp_sk, struct sk_buff skb) · 57 unmodified lines
1188	1198	if (skb->tstamp == 0)
1189	1199	skb->tstamp = ktime_get_real();
1190	1200
1191		− rxrpc_new_skb(skb, rxrpc_skb_rx_received);
	1201	+ rxrpc_new_skb(skb, rxrpc_skb_received);
1192	1202
1193	1203	skb_pull(skb, sizeof(struct udphdr));
1194	1204
@@ -1205,7 +1215,7 @@ int rxrpc_input_packet(struct sock udp_sk, struct sk_buff skb) · 10 unmodified lines
1205	1215	static int lose;
1206	1216	if ((lose++ & 7) == 7) {
1207	1217	trace_rxrpc_rx_lose(sp);
1208		− rxrpc_free_skb(skb, rxrpc_skb_rx_lost);
	1218	+ rxrpc_free_skb(skb, rxrpc_skb_lost);
1209	1219	return 0;
1210	1220	}
1211	1221	}
@@ -1237,9 +1247,26 @@ int rxrpc_input_packet(struct sock udp_sk, struct sk_buff skb) · 25 unmodified lines
1237	1247	if (sp->hdr.callNumber == 0 \|\|
1238	1248	sp->hdr.seq == 0)
1239	1249	goto bad_message;
1240		− if (sp->hdr.flags & RXRPC_JUMBO_PACKET &&
1241		− !rxrpc_validate_jumbo(skb))
	1250	+ if (!rxrpc_validate_data(skb))
1242	1251	goto bad_message;
	1252	+
	1253	+ /* Unshare the packet so that it can be modified for in-place
	1254	+ * decryption.
	1255	+ */
	1256	+ if (sp->hdr.securityIndex != 0) {
	1257	+ struct sk_buff *nskb = skb_unshare(skb, GFP_ATOMIC);
	1258	+ if (!nskb) {
	1259	+ rxrpc_eaten_skb(skb, rxrpc_skb_unshared_nomem);
	1260	+ goto out;
	1261	+ }
	1262	+
	1263	+ if (nskb != skb) {
	1264	+ rxrpc_eaten_skb(skb, rxrpc_skb_received);
	1265	+ rxrpc_new_skb(skb, rxrpc_skb_unshared);
	1266	+ skb = nskb;
	1267	+ sp = rxrpc_skb(skb);
	1268	+ }
	1269	+ }
1243	1270	break;
1244	1271
1245	1272	case RXRPC_PACKET_TYPE_CHALLENGE:
@@ -1373,11 +1400,14 @@ int rxrpc_input_packet(struct sock udp_sk, struct sk_buff skb) · 127 unmodified lines
1373	1400	mutex_unlock(&call->user_mutex);
1374	1401	}
1375	1402
	1403	+ /* Process a call packet; this either discards or passes on the ref
	1404	+ * elsewhere.
	1405	+ */
1376	1406	rxrpc_input_call_packet(call, skb);
1377		− goto discard;
	1407	+ goto out;
1378	1408
1379	1409	discard:
1380		− rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
	1410	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
1381	1411	out:
1382	1412	trace_rxrpc_rx_done(0, 0);
1383	1413	return 0;

net/rxrpc/local_event.c+2−2

@@ -90,7 +90,7 @@ void rxrpc_process_local_events(struct rxrpc_local *local) · 89 unmodified lines
90	90	if (skb) {
91	91	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
92	92
93		− rxrpc_see_skb(skb, rxrpc_skb_rx_seen);
	93	+ rxrpc_see_skb(skb, rxrpc_skb_seen);
94	94	_debug("{%d},{%u}", local->debug_id, sp->hdr.type);
95	95
96	96	switch (sp->hdr.type) {
@@ -108,7 +108,7 @@ void rxrpc_process_local_events(struct rxrpc_local *local) · 11 unmodified lines
108	108	break;
109	109	}
110	110
111		− rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
	111	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
112	112	}
113	113
114	114	_leave("");

net/rxrpc/output.c+3−3

@@ -565,7 +565,7 @@ void rxrpc_reject_packets(struct rxrpc_local *local) · 564 unmodified lines
565	565	memset(&whdr, 0, sizeof(whdr));
566	566
567	567	while ((skb = skb_dequeue(&local->reject_queue))) {
568		− rxrpc_see_skb(skb, rxrpc_skb_rx_seen);
	568	+ rxrpc_see_skb(skb, rxrpc_skb_seen);
569	569	sp = rxrpc_skb(skb);
570	570
571	571	switch (skb->mark) {
@@ -581,7 +581,7 @@ void rxrpc_reject_packets(struct rxrpc_local *local) · 9 unmodified lines
581	581	ioc = 2;
582	582	break;
583	583	default:
584		− rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
	584	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
585	585	continue;
586	586	}
587	587
@@ -606,7 +606,7 @@ void rxrpc_reject_packets(struct rxrpc_local *local) · 18 unmodified lines
606	606	rxrpc_tx_point_reject);
607	607	}
608	608
609		− rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
	609	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
610	610	}
611	611
612	612	_leave("");

net/rxrpc/peer_event.c+5−5

@@ -163,11 +163,11 @@ void rxrpc_error_report(struct sock *sk) · 162 unmodified lines
163	163	_leave("UDP socket errqueue empty");
164	164	return;
165	165	}
166		− rxrpc_new_skb(skb, rxrpc_skb_rx_received);
	166	+ rxrpc_new_skb(skb, rxrpc_skb_received);
167	167	serr = SKB_EXT_ERR(skb);
168	168	if (!skb->len && serr->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING) {
169	169	_leave("UDP empty message");
170		− rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
	170	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
171	171	return;
172	172	}
173	173
@@ -177,7 +177,7 @@ void rxrpc_error_report(struct sock *sk) · 3 unmodified lines
177	177	peer = NULL;
178	178	if (!peer) {
179	179	rcu_read_unlock();
180		− rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
	180	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
181	181	_leave(" [no peer]");
182	182	return;
183	183	}
@@ -189,7 +189,7 @@ void rxrpc_error_report(struct sock *sk) · 5 unmodified lines
189	189	serr->ee.ee_code == ICMP_FRAG_NEEDED)) {
190	190	rxrpc_adjust_mtu(peer, serr);
191	191	rcu_read_unlock();
192		− rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
	192	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
193	193	rxrpc_put_peer(peer);
194	194	_leave(" [MTU update]");
195	195	return;
@@ -197,7 +197,7 @@ void rxrpc_error_report(struct sock *sk) · 1 unmodified line
197	197
198	198	rxrpc_store_error(peer, serr);
199	199	rcu_read_unlock();
200		− rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
	200	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
201	201	rxrpc_put_peer(peer);
202	202
203	203	_leave("");

net/rxrpc/protocol.h+9−0

@@ -89,6 +89,15 @@ struct rxrpc_jumbo_header { · 88 unmodified lines
89	89	#define RXRPC_JUMBO_DATALEN 1412 /* non-terminal jumbo packet data length */
90	90	#define RXRPC_JUMBO_SUBPKTLEN (RXRPC_JUMBO_DATALEN + sizeof(struct rxrpc_jumbo_header))
91	91
	92	+/*
	93	+ * The maximum number of subpackets that can possibly fit in a UDP packet is:
	94	+ *
	95	+ * ((max_IP - IP_hdr - UDP_hdr) / RXRPC_JUMBO_SUBPKTLEN) + 1
	96	+ * = ((65535 - 28 - 28) / 1416) + 1
	97	+ * = 46 non-terminal packets and 1 terminal packet.
	98	+ */
	99	+#define RXRPC_MAX_NR_JUMBO 47
	100	+
92	101	/*****************************************************************************/
93	102	/*
94	103	* on-the-wire Rx ACK packet data payload

net/rxrpc/recvmsg.c+27−20

@@ -177,7 +177,8 @@ static void rxrpc_rotate_rx_window(struct rxrpc_call *call) · 176 unmodified lines
177	177	struct sk_buff *skb;
178	178	rxrpc_serial_t serial;
179	179	rxrpc_seq_t hard_ack, top;
180		− u8 flags;
	180	+ bool last = false;
	181	+ u8 subpacket;
181	182	int ix;
182	183
183	184	_enter("%d", call->debug_id);
@@ -189,23 +190,25 @@ static void rxrpc_rotate_rx_window(struct rxrpc_call *call) · 5 unmodified lines
189	190	hard_ack++;
190	191	ix = hard_ack & RXRPC_RXTX_BUFF_MASK;
191	192	skb = call->rxtx_buffer[ix];
192		− rxrpc_see_skb(skb, rxrpc_skb_rx_rotated);
	193	+ rxrpc_see_skb(skb, rxrpc_skb_rotated);
193	194	sp = rxrpc_skb(skb);
194		− flags = sp->hdr.flags;
195		− serial = sp->hdr.serial;
196		− if (call->rxtx_annotations[ix] & RXRPC_RX_ANNO_JUMBO)
197		− serial += (call->rxtx_annotations[ix] & RXRPC_RX_ANNO_JUMBO) - 1;
	195	+
	196	+ subpacket = call->rxtx_annotations[ix] & RXRPC_RX_ANNO_SUBPACKET;
	197	+ serial = sp->hdr.serial + subpacket;
	198	+
	199	+ if (subpacket == sp->nr_subpackets - 1 &&
	200	+ sp->rx_flags & RXRPC_SKB_INCL_LAST)
	201	+ last = true;
198	202
199	203	call->rxtx_buffer[ix] = NULL;
200	204	call->rxtx_annotations[ix] = 0;
201	205	/* Barrier against rxrpc_input_data(). */
202	206	smp_store_release(&call->rx_hard_ack, hard_ack);
203	207
204		− rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
	208	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
205	209
206		− _debug("%u,%u,%02x", hard_ack, top, flags);
207	210	trace_rxrpc_receive(call, rxrpc_receive_rotate, serial, hard_ack);
208		− if (flags & RXRPC_LAST_PACKET) {
	211	+ if (last) {
209	212	rxrpc_end_rx_phase(call, serial);
210	213	} else {
211	214	/* Check to see if there's an ACK that needs sending. */
@@ -233,18 +236,19 @@ static int rxrpc_verify_packet(struct rxrpc_call call, struct sk_buff skb, · 21 unmodified lines
233	236	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
234	237	rxrpc_seq_t seq = sp->hdr.seq;
235	238	u16 cksum = sp->hdr.cksum;
	239	+ u8 subpacket = annotation & RXRPC_RX_ANNO_SUBPACKET;
236	240
237	241	_enter("");
238	242
239	243	/* For all but the head jumbo subpacket, the security checksum is in a
240	244	* jumbo header immediately prior to the data.
241	245	*/
242		− if ((annotation & RXRPC_RX_ANNO_JUMBO) > 1) {
	246	+ if (subpacket > 0) {
243	247	__be16 tmp;
244	248	if (skb_copy_bits(skb, offset - 2, &tmp, 2) < 0)
245	249	BUG();
246	250	cksum = ntohs(tmp);
247		− seq += (annotation & RXRPC_RX_ANNO_JUMBO) - 1;
	251	+ seq += subpacket;
248	252	}
249	253
250	254	return call->conn->security->verify_packet(call, skb, offset, len,
@@ -265,19 +269,18 @@ static int rxrpc_locate_data(struct rxrpc_call call, struct sk_buff skb, · 14 unmodified lines
265	269	u8 *_annotation,
266	270	unsigned int _offset, unsigned int _len)
267	271	{
	272	+ struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
268	273	unsigned int offset = sizeof(struct rxrpc_wire_header);
269	274	unsigned int len;
270	275	int ret;
271	276	u8 annotation = *_annotation;
	277	+ u8 subpacket = annotation & RXRPC_RX_ANNO_SUBPACKET;
272	278
273	279	/* Locate the subpacket */
	280	+ offset += subpacket * RXRPC_JUMBO_SUBPKTLEN;
274	281	len = skb->len - offset;
275		− if ((annotation & RXRPC_RX_ANNO_JUMBO) > 0) {
276		− offset += (((annotation & RXRPC_RX_ANNO_JUMBO) - 1) *
277		− RXRPC_JUMBO_SUBPKTLEN);
278		− len = (annotation & RXRPC_RX_ANNO_JLAST) ?
279		− skb->len - offset : RXRPC_JUMBO_SUBPKTLEN;
280		− }
	282	+ if (subpacket < sp->nr_subpackets - 1)
	283	+ len = RXRPC_JUMBO_DATALEN;
281	284
282	285	if (!(annotation & RXRPC_RX_ANNO_VERIFIED)) {
283	286	ret = rxrpc_verify_packet(call, skb, annotation, offset, len);
@@ -303,6 +306,7 @@ static int rxrpc_recvmsg_data(struct socket sock, struct rxrpc_call call, · 19 unmodified lines
303	306	{
304	307	struct rxrpc_skb_priv *sp;
305	308	struct sk_buff *skb;
	309	+ rxrpc_serial_t serial;
306	310	rxrpc_seq_t hard_ack, top, seq;
307	311	size_t remain;
308	312	bool last;
@@ -336,12 +340,15 @@ static int rxrpc_recvmsg_data(struct socket sock, struct rxrpc_call call, · 27 unmodified lines
336	340	break;
337	341	}
338	342	smp_rmb();
339		− rxrpc_see_skb(skb, rxrpc_skb_rx_seen);
	343	+ rxrpc_see_skb(skb, rxrpc_skb_seen);
340	344	sp = rxrpc_skb(skb);
341	345
342		− if (!(flags & MSG_PEEK))
	346	+ if (!(flags & MSG_PEEK)) {
	347	+ serial = sp->hdr.serial;
	348	+ serial += call->rxtx_annotations[ix] & RXRPC_RX_ANNO_SUBPACKET;
343	349	trace_rxrpc_receive(call, rxrpc_receive_front,
344		− sp->hdr.serial, seq);
	350	+ serial, seq);
	351	+ }
345	352
346	353	if (msg)
347	354	sock_recv_timestamp(msg, sock->sk, skb);

net/rxrpc/rxkad.c+9−23

@@ -187,10 +187,8 @@ static int rxkad_secure_packet_encrypt(const struct rxrpc_call *call, · 186 unmodified lines
187	187	struct rxrpc_skb_priv *sp;
188	188	struct rxrpc_crypt iv;
189	189	struct scatterlist sg[16];
190		− struct sk_buff *trailer;
191	190	unsigned int len;
192	191	u16 check;
193		− int nsg;
194	192	int err;
195	193
196	194	sp = rxrpc_skb(skb);
@@ -214,15 +212,14 @@ static int rxkad_secure_packet_encrypt(const struct rxrpc_call *call, · 17 unmodified lines
214	212	crypto_skcipher_encrypt(req);
215	213
216	214	/* we want to encrypt the skbuff in-place */
217		− nsg = skb_cow_data(skb, 0, &trailer);
218		− err = -ENOMEM;
219		− if (nsg < 0 \|\| nsg > 16)
	215	+ err = -EMSGSIZE;
	216	+ if (skb_shinfo(skb)->nr_frags > 16)
220	217	goto out;
221	218
222	219	len = data_size + call->conn->size_align - 1;
223	220	len &= ~(call->conn->size_align - 1);
224	221
225		− sg_init_table(sg, nsg);
	222	+ sg_init_table(sg, ARRAY_SIZE(sg));
226	223	err = skb_to_sgvec(skb, sg, 0, len);
227	224	if (unlikely(err < 0))
228	225	goto out;
@@ -319,11 +316,10 @@ static int rxkad_verify_packet_1(struct rxrpc_call call, struct sk_buff skb, · 90 unmodified lines
319	316	struct rxkad_level1_hdr sechdr;
320	317	struct rxrpc_crypt iv;
321	318	struct scatterlist sg[16];
322		− struct sk_buff *trailer;
323	319	bool aborted;
324	320	u32 data_size, buf;
325	321	u16 check;
326		− int nsg, ret;
	322	+ int ret;
327	323
328	324	_enter("");
329	325
@@ -336,11 +332,7 @@ static int rxkad_verify_packet_1(struct rxrpc_call call, struct sk_buff skb, · 6 unmodified lines
336	332	/* Decrypt the skbuff in-place. TODO: We really want to decrypt
337	333	* directly into the target buffer.
338	334	*/
339		− nsg = skb_cow_data(skb, 0, &trailer);
340		− if (nsg < 0 \|\| nsg > 16)
341		− goto nomem;
342		−
343		− sg_init_table(sg, nsg);
	335	+ sg_init_table(sg, ARRAY_SIZE(sg));
344	336	ret = skb_to_sgvec(skb, sg, offset, 8);
345	337	if (unlikely(ret < 0))
346	338	return ret;
@@ -388,10 +380,6 @@ protocol_error: · 41 unmodified lines
388	380	if (aborted)
389	381	rxrpc_send_abort_packet(call);
390	382	return -EPROTO;
391		−
392		−nomem:
393		− _leave(" = -ENOMEM");
394		− return -ENOMEM;
395	383	}
396	384
397	385	/*
@@ -406,7 +394,6 @@ static int rxkad_verify_packet_2(struct rxrpc_call call, struct sk_buff skb, · 8 unmodified lines
406	394	struct rxkad_level2_hdr sechdr;
407	395	struct rxrpc_crypt iv;
408	396	struct scatterlist _sg[4], *sg;
409		− struct sk_buff *trailer;
410	397	bool aborted;
411	398	u32 data_size, buf;
412	399	u16 check;
@@ -423,12 +410,11 @@ static int rxkad_verify_packet_2(struct rxrpc_call call, struct sk_buff skb, · 10 unmodified lines
423	410	/* Decrypt the skbuff in-place. TODO: We really want to decrypt
424	411	* directly into the target buffer.
425	412	*/
426		− nsg = skb_cow_data(skb, 0, &trailer);
427		− if (nsg < 0)
428		− goto nomem;
429		−
430	413	sg = _sg;
431		− if (unlikely(nsg > 4)) {
	414	+ nsg = skb_shinfo(skb)->nr_frags;
	415	+ if (nsg <= 4) {
	416	+ nsg = 4;
	417	+ } else {
432	418	sg = kmalloc_array(nsg, sizeof(*sg), GFP_NOIO);
433	419	if (!sg)
434	420	goto nomem;

net/rxrpc/sendmsg.c+7−6

@@ -176,7 +176,7 @@ static int rxrpc_queue_packet(struct rxrpc_sock rx, struct rxrpc_call call, · 175 unmodified lines
176	176	skb->tstamp = ktime_get_real();
177	177
178	178	ix = seq & RXRPC_RXTX_BUFF_MASK;
179		− rxrpc_get_skb(skb, rxrpc_skb_tx_got);
	179	+ rxrpc_get_skb(skb, rxrpc_skb_got);
180	180	call->rxtx_annotations[ix] = annotation;
181	181	smp_wmb();
182	182	call->rxtx_buffer[ix] = skb;
@@ -248,7 +248,7 @@ static int rxrpc_queue_packet(struct rxrpc_sock rx, struct rxrpc_call call, · 65 unmodified lines
248	248	}
249	249
250	250	out:
251		− rxrpc_free_skb(skb, rxrpc_skb_tx_freed);
	251	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
252	252	_leave(" = %d", ret);
253	253	return ret;
254	254	}
@@ -289,7 +289,7 @@ static int rxrpc_send_data(struct rxrpc_sock *rx, · 34 unmodified lines
289	289
290	290	skb = call->tx_pending;
291	291	call->tx_pending = NULL;
292		− rxrpc_see_skb(skb, rxrpc_skb_tx_seen);
	292	+ rxrpc_see_skb(skb, rxrpc_skb_seen);
293	293
294	294	copied = 0;
295	295	do {
@@ -336,7 +336,9 @@ static int rxrpc_send_data(struct rxrpc_sock *rx, · 40 unmodified lines
336	336	if (!skb)
337	337	goto maybe_error;
338	338
339		− rxrpc_new_skb(skb, rxrpc_skb_tx_new);
	339	+ sp = rxrpc_skb(skb);
	340	+ sp->rx_flags \|= RXRPC_SKB_TX_BUFFER;
	341	+ rxrpc_new_skb(skb, rxrpc_skb_new);
340	342
341	343	_debug("ALLOC SEND %p", skb);
342	344
@@ -346,7 +348,6 @@ static int rxrpc_send_data(struct rxrpc_sock *rx, · 3 unmodified lines
346	348	skb_reserve(skb, call->conn->security_size);
347	349	skb->len += call->conn->security_size;
348	350
349		− sp = rxrpc_skb(skb);
350	351	sp->remain = chunk;
351	352	if (sp->remain > skb_tailroom(skb))
352	353	sp->remain = skb_tailroom(skb);
@@ -439,7 +440,7 @@ out: · 86 unmodified lines
439	440	return ret;
440	441
441	442	call_terminated:
442		− rxrpc_free_skb(skb, rxrpc_skb_tx_freed);
	443	+ rxrpc_free_skb(skb, rxrpc_skb_freed);
443	444	_leave(" = %d", call->error);
444	445	return call->error;
445	446

net/rxrpc/skbuff.c+28−12

@@ -14,7 +14,8 @@ · 13 unmodified lines
14	14	#include <net/af_rxrpc.h>
15	15	#include "ar-internal.h"
16	16
17		−#define select_skb_count(op) (op >= rxrpc_skb_tx_cleaned ? &rxrpc_n_tx_skbs : &rxrpc_n_rx_skbs)
	17	+#define is_tx_skb(skb) (rxrpc_skb(skb)->rx_flags & RXRPC_SKB_TX_BUFFER)
	18	+#define select_skb_count(skb) (is_tx_skb(skb) ? &rxrpc_n_tx_skbs : &rxrpc_n_rx_skbs)
18	19
19	20	/*
20	21	* Note the allocation or reception of a socket buffer.
@@ -22,8 +23,9 @@ · 1 unmodified line
22	23	void rxrpc_new_skb(struct sk_buff *skb, enum rxrpc_skb_trace op)
23	24	{
24	25	const void *here = __builtin_return_address(0);
25		− int n = atomic_inc_return(select_skb_count(op));
26		− trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n, here);
	26	+ int n = atomic_inc_return(select_skb_count(skb));
	27	+ trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n,
	28	+ rxrpc_skb(skb)->rx_flags, here);
27	29	}
28	30
29	31	/*
@@ -33,8 +35,9 @@ void rxrpc_see_skb(struct sk_buff *skb, enum rxrpc_skb_trace op) · 3 unmodified lines
33	35	{
34	36	const void *here = __builtin_return_address(0);
35	37	if (skb) {
36		− int n = atomic_read(select_skb_count(op));
37		− trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n, here);
	38	+ int n = atomic_read(select_skb_count(skb));
	39	+ trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n,
	40	+ rxrpc_skb(skb)->rx_flags, here);
38	41	}
39	42	}
40	43
@@ -44,11 +47,22 @@ void rxrpc_see_skb(struct sk_buff *skb, enum rxrpc_skb_trace op) · 3 unmodified lines
44	47	void rxrpc_get_skb(struct sk_buff *skb, enum rxrpc_skb_trace op)
45	48	{
46	49	const void *here = __builtin_return_address(0);
47		− int n = atomic_inc_return(select_skb_count(op));
48		− trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n, here);
	50	+ int n = atomic_inc_return(select_skb_count(skb));
	51	+ trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n,
	52	+ rxrpc_skb(skb)->rx_flags, here);
49	53	skb_get(skb);
50	54	}
51	55
	56	+/*
	57	+ * Note the dropping of a ref on a socket buffer by the core.
	58	+ */
	59	+void rxrpc_eaten_skb(struct sk_buff *skb, enum rxrpc_skb_trace op)
	60	+{
	61	+ const void *here = __builtin_return_address(0);
	62	+ int n = atomic_inc_return(&rxrpc_n_rx_skbs);
	63	+ trace_rxrpc_skb(skb, op, 0, n, 0, here);
	64	+}
	65	+
52	66	/*
53	67	* Note the destruction of a socket buffer.
54	68	*/
@@ -58,8 +72,9 @@ void rxrpc_free_skb(struct sk_buff *skb, enum rxrpc_skb_trace op) · 3 unmodified lines
58	72	if (skb) {
59	73	int n;
60	74	CHECK_SLAB_OKAY(&skb->users);
61		− n = atomic_dec_return(select_skb_count(op));
62		− trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n, here);
	75	+ n = atomic_dec_return(select_skb_count(skb));
	76	+ trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n,
	77	+ rxrpc_skb(skb)->rx_flags, here);
63	78	kfree_skb(skb);
64	79	}
65	80	}
@@ -72,9 +87,10 @@ void rxrpc_purge_queue(struct sk_buff_head *list) · 6 unmodified lines
72	87	const void *here = __builtin_return_address(0);
73	88	struct sk_buff *skb;
74	89	while ((skb = skb_dequeue((list))) != NULL) {
75		− int n = atomic_dec_return(select_skb_count(rxrpc_skb_rx_purged));
76		− trace_rxrpc_skb(skb, rxrpc_skb_rx_purged,
77		− refcount_read(&skb->users), n, here);
	90	+ int n = atomic_dec_return(select_skb_count(skb));
	91	+ trace_rxrpc_skb(skb, rxrpc_skb_purged,
	92	+ refcount_read(&skb->users), n,
	93	+ rxrpc_skb(skb)->rx_flags, here);
78	94	kfree_skb(skb);
79	95	}
80	96	}