History

Merge tag 'rxrpc-fixes-20190827' of git://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs

David Howells says: ==================== rxrpc: Fix use of skb_cow_data() Here's a series of patches that replaces the use of skb_cow_data() in rxrpc with skb_unshare() early on in the input process. The problem that is being seen is that skb_cow_data() indirectly requires that the maximum usage count on an sk_buff be 1, and it may generate an assertion failure in pskb_expand_head() if not. This can occur because rxrpc_input_data() may be still holding a ref when it has just attached the sk_buff to the rx ring and given that attachment its own ref. If recvmsg happens fast enough, skb_cow_data() can see the ref still held by the softirq handler. Further, a packet may contain multiple subpackets, each of which gets its own attachment to the ring and its own ref - also making skb_cow_data() go bang. Fix this by: (1) The DATA packet is currently parsed for subpackets twice by the input routines. Parse it just once instead and make notes in the sk_buff private data. (2) Use the notes from (1) when attaching the packet to the ring multiple times. Once the packet is attached to the ring, recvmsg can see it and start modifying it, so the softirq handler is not permitted to look inside it from that point. (3) Pass the ref from the input code to the ring rather than getting an extra ref. rxrpc_input_data() uses a ref on the second refcount to prevent the packet from evaporating under it. (4) Call skb_unshare() on secured DATA packets in rxrpc_input_packet() before we take call->input_lock. Other sorts of packets don't get modified and so can be left. A trace is emitted if skb_unshare() eats the skb. Note that skb_share() for our accounting in this regard as we can't see the parameters in the packet to log in a trace line if it releases it. (5) Remove the calls to skb_cow_data(). These are then no longer necessary. There are also patches to improve the rxrpc_skb tracepoint to make sure that Tx-derived buffers are identified separately from Rx-derived buffers in the trace. ====================

David S. Miller committed 7 years ago

Commit a285c1f2 parents 3b25528 d0d5c0c14 files · +319 −268Browse files

Unified · Split

14 files
  1. include/trace/events/rxrpc.h +28 −31
  2. net/rxrpc/ar-internal.h +11 −5
  3. net/rxrpc/call_event.c +4 −4
  4. net/rxrpc/call_object.c +16 −17
  5. net/rxrpc/conn_event.c +3 −3
  6. net/rxrpc/input.c +167 −137
  7. net/rxrpc/local_event.c +2 −2
  8. net/rxrpc/output.c +3 −3
  9. net/rxrpc/peer_event.c +5 −5
  10. net/rxrpc/protocol.h +9 −0
  11. net/rxrpc/recvmsg.c +27 −20
  12. net/rxrpc/rxkad.c +9 −23
  13. net/rxrpc/sendmsg.c +7 −6
  14. net/rxrpc/skbuff.c +28 −12
include/trace/events/rxrpc.h+28−31
@@ -23,20 +23,17 @@ · 22 unmodified lines
23#define __RXRPC_DECLARE_TRACE_ENUMS_ONCE_ONLY23#define __RXRPC_DECLARE_TRACE_ENUMS_ONCE_ONLY
24 24
25enum rxrpc_skb_trace {25enum rxrpc_skb_trace {
26 rxrpc_skb_rx_cleaned,26 rxrpc_skb_cleaned,
27 rxrpc_skb_rx_freed,27 rxrpc_skb_freed,
28 rxrpc_skb_rx_got,28 rxrpc_skb_got,
29 rxrpc_skb_rx_lost,29 rxrpc_skb_lost,
30 rxrpc_skb_rx_purged,30 rxrpc_skb_new,
31 rxrpc_skb_rx_received,31 rxrpc_skb_purged,
32 rxrpc_skb_rx_rotated,32 rxrpc_skb_received,
33 rxrpc_skb_rx_seen,33 rxrpc_skb_rotated,
34 rxrpc_skb_tx_cleaned,34 rxrpc_skb_seen,
35 rxrpc_skb_tx_freed,35 rxrpc_skb_unshared,
36 rxrpc_skb_tx_got,36 rxrpc_skb_unshared_nomem,
37 rxrpc_skb_tx_new,
38 rxrpc_skb_tx_rotated,
39 rxrpc_skb_tx_seen,
40};37};
41 38
42enum rxrpc_local_trace {39enum rxrpc_local_trace {
@@ -228,20 +225,17 @@ enum rxrpc_tx_point { · 185 unmodified lines
228 * Declare tracing information enums and their string mappings for display.225 * Declare tracing information enums and their string mappings for display.
229 */226 */
230#define rxrpc_skb_traces \227#define rxrpc_skb_traces \
231 EM(rxrpc_skb_rx_cleaned, "Rx CLN") \228 EM(rxrpc_skb_cleaned, "CLN") \
232 EM(rxrpc_skb_rx_freed, "Rx FRE") \229 EM(rxrpc_skb_freed, "FRE") \
233 EM(rxrpc_skb_rx_got, "Rx GOT") \230 EM(rxrpc_skb_got, "GOT") \
234 EM(rxrpc_skb_rx_lost, "Rx *L*") \231 EM(rxrpc_skb_lost, "*L*") \
235 EM(rxrpc_skb_rx_purged, "Rx PUR") \232 EM(rxrpc_skb_new, "NEW") \
236 EM(rxrpc_skb_rx_received, "Rx RCV") \233 EM(rxrpc_skb_purged, "PUR") \
237 EM(rxrpc_skb_rx_rotated, "Rx ROT") \234 EM(rxrpc_skb_received, "RCV") \
238 EM(rxrpc_skb_rx_seen, "Rx SEE") \235 EM(rxrpc_skb_rotated, "ROT") \
239 EM(rxrpc_skb_tx_cleaned, "Tx CLN") \236 EM(rxrpc_skb_seen, "SEE") \
240 EM(rxrpc_skb_tx_freed, "Tx FRE") \237 EM(rxrpc_skb_unshared, "UNS") \
241 EM(rxrpc_skb_tx_got, "Tx GOT") \238 E_(rxrpc_skb_unshared_nomem, "US0")
242 EM(rxrpc_skb_tx_new, "Tx NEW") \
243 EM(rxrpc_skb_tx_rotated, "Tx ROT") \
244 E_(rxrpc_skb_tx_seen, "Tx SEE")
245 239
246#define rxrpc_local_traces \240#define rxrpc_local_traces \
247 EM(rxrpc_local_got, "GOT") \241 EM(rxrpc_local_got, "GOT") \
@@ -643,13 +637,14 @@ TRACE_EVENT(rxrpc_call, · 395 unmodified lines
643 637
644TRACE_EVENT(rxrpc_skb,638TRACE_EVENT(rxrpc_skb,
645 TP_PROTO(struct sk_buff *skb, enum rxrpc_skb_trace op,639 TP_PROTO(struct sk_buff *skb, enum rxrpc_skb_trace op,
646 int usage, int mod_count, const void *where),640 int usage, int mod_count, u8 flags, const void *where),
647 641
648 TP_ARGS(skb, op, usage, mod_count, where),642 TP_ARGS(skb, op, usage, mod_count, flags, where),
649 643
650 TP_STRUCT__entry(644 TP_STRUCT__entry(
651 __field(struct sk_buff *, skb )645 __field(struct sk_buff *, skb )
652 __field(enum rxrpc_skb_trace, op )646 __field(enum rxrpc_skb_trace, op )
647 __field(u8, flags )
653 __field(int, usage )648 __field(int, usage )
654 __field(int, mod_count )649 __field(int, mod_count )
655 __field(const void *, where )650 __field(const void *, where )
@@ -657,14 +652,16 @@ TRACE_EVENT(rxrpc_skb, · 1 unmodified line
657 652
658 TP_fast_assign(653 TP_fast_assign(
659 __entry->skb = skb;654 __entry->skb = skb;
655 __entry->flags = flags;
660 __entry->op = op;656 __entry->op = op;
661 __entry->usage = usage;657 __entry->usage = usage;
662 __entry->mod_count = mod_count;658 __entry->mod_count = mod_count;
663 __entry->where = where;659 __entry->where = where;
664 ),660 ),
665 661
666 TP_printk("s=%p %s u=%d m=%d p=%pSR",662 TP_printk("s=%p %cx %s u=%d m=%d p=%pSR",
667 __entry->skb,663 __entry->skb,
664 __entry->flags & RXRPC_SKB_TX_BUFFER ? 'T' : 'R',
668 __print_symbolic(__entry->op, rxrpc_skb_traces),665 __print_symbolic(__entry->op, rxrpc_skb_traces),
669 __entry->usage,666 __entry->usage,
670 __entry->mod_count,667 __entry->mod_count,
net/rxrpc/ar-internal.h+11−5
@@ -185,11 +185,17 @@ struct rxrpc_host_header { · 184 unmodified lines
185 * - max 48 bytes (struct sk_buff::cb)185 * - max 48 bytes (struct sk_buff::cb)
186 */186 */
187struct rxrpc_skb_priv {187struct rxrpc_skb_priv {
188 union {188 atomic_t nr_ring_pins; /* Number of rxtx ring pins */
189 u8 nr_jumbo; /* Number of jumbo subpackets */189 u8 nr_subpackets; /* Number of subpackets */
190 };190 u8 rx_flags; /* Received packet flags */
191#define RXRPC_SKB_INCL_LAST 0x01 /* - Includes last packet */
192#define RXRPC_SKB_TX_BUFFER 0x02 /* - Is transmit buffer */
191 union {193 union {
192 int remain; /* amount of space remaining for next write */194 int remain; /* amount of space remaining for next write */
195
196 /* List of requested ACKs on subpackets */
197 unsigned long rx_req_ack[(RXRPC_MAX_NR_JUMBO + BITS_PER_LONG - 1) /
198 BITS_PER_LONG];
193 };199 };
194 200
195 struct rxrpc_host_header hdr; /* RxRPC packet header from this packet */201 struct rxrpc_host_header hdr; /* RxRPC packet header from this packet */
@@ -613,8 +619,7 @@ struct rxrpc_call { · 417 unmodified lines
613#define RXRPC_TX_ANNO_LAST 0x04619#define RXRPC_TX_ANNO_LAST 0x04
614#define RXRPC_TX_ANNO_RESENT 0x08620#define RXRPC_TX_ANNO_RESENT 0x08
615 621
616#define RXRPC_RX_ANNO_JUMBO 0x3f /* Jumbo subpacket number + 1 if not zero */622#define RXRPC_RX_ANNO_SUBPACKET 0x3f /* Subpacket number in jumbogram */
617#define RXRPC_RX_ANNO_JLAST 0x40 /* Set if last element of a jumbo packet */
618#define RXRPC_RX_ANNO_VERIFIED 0x80 /* Set if verified and decrypted */623#define RXRPC_RX_ANNO_VERIFIED 0x80 /* Set if verified and decrypted */
619 rxrpc_seq_t tx_hard_ack; /* Dead slot in buffer; the first transmitted but624 rxrpc_seq_t tx_hard_ack; /* Dead slot in buffer; the first transmitted but
620 * not hard-ACK'd packet follows this.625 * not hard-ACK'd packet follows this.
@@ -1105,6 +1110,7 @@ void rxrpc_kernel_data_consumed(struct rxrpc_call *, struct sk_buff *); · 484 unmodified lines
1105void rxrpc_packet_destructor(struct sk_buff *);1110void rxrpc_packet_destructor(struct sk_buff *);
1106void rxrpc_new_skb(struct sk_buff *, enum rxrpc_skb_trace);1111void rxrpc_new_skb(struct sk_buff *, enum rxrpc_skb_trace);
1107void rxrpc_see_skb(struct sk_buff *, enum rxrpc_skb_trace);1112void rxrpc_see_skb(struct sk_buff *, enum rxrpc_skb_trace);
1113void rxrpc_eaten_skb(struct sk_buff *, enum rxrpc_skb_trace);
1108void rxrpc_get_skb(struct sk_buff *, enum rxrpc_skb_trace);1114void rxrpc_get_skb(struct sk_buff *, enum rxrpc_skb_trace);
1109void rxrpc_free_skb(struct sk_buff *, enum rxrpc_skb_trace);1115void rxrpc_free_skb(struct sk_buff *, enum rxrpc_skb_trace);
1110void rxrpc_purge_queue(struct sk_buff_head *);1116void rxrpc_purge_queue(struct sk_buff_head *);
net/rxrpc/call_event.c+4−4
@@ -199,7 +199,7 @@ static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j) · 198 unmodified lines
199 continue;199 continue;
200 200
201 skb = call->rxtx_buffer[ix];201 skb = call->rxtx_buffer[ix];
202 rxrpc_see_skb(skb, rxrpc_skb_tx_seen);202 rxrpc_see_skb(skb, rxrpc_skb_seen);
203 203
204 if (anno_type == RXRPC_TX_ANNO_UNACK) {204 if (anno_type == RXRPC_TX_ANNO_UNACK) {
205 if (ktime_after(skb->tstamp, max_age)) {205 if (ktime_after(skb->tstamp, max_age)) {
@@ -255,18 +255,18 @@ static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j) · 49 unmodified lines
255 continue;255 continue;
256 256
257 skb = call->rxtx_buffer[ix];257 skb = call->rxtx_buffer[ix];
258 rxrpc_get_skb(skb, rxrpc_skb_tx_got);258 rxrpc_get_skb(skb, rxrpc_skb_got);
259 spin_unlock_bh(&call->lock);259 spin_unlock_bh(&call->lock);
260 260
261 if (rxrpc_send_data_packet(call, skb, true) < 0) {261 if (rxrpc_send_data_packet(call, skb, true) < 0) {
262 rxrpc_free_skb(skb, rxrpc_skb_tx_freed);262 rxrpc_free_skb(skb, rxrpc_skb_freed);
263 return;263 return;
264 }264 }
265 265
266 if (rxrpc_is_client_call(call))266 if (rxrpc_is_client_call(call))
267 rxrpc_expose_client_call(call);267 rxrpc_expose_client_call(call);
268 268
269 rxrpc_free_skb(skb, rxrpc_skb_tx_freed);269 rxrpc_free_skb(skb, rxrpc_skb_freed);
270 spin_lock_bh(&call->lock);270 spin_lock_bh(&call->lock);
271 271
272 /* We need to clear the retransmit state, but there are two272 /* We need to clear the retransmit state, but there are two
net/rxrpc/call_object.c+16−17
@@ -421,6 +421,19 @@ void rxrpc_get_call(struct rxrpc_call *call, enum rxrpc_call_trace op) · 420 unmodified lines
421 trace_rxrpc_call(call, op, n, here, NULL);421 trace_rxrpc_call(call, op, n, here, NULL);
422}422}
423 423
424/*
425 * Clean up the RxTx skb ring.
426 */
427static void rxrpc_cleanup_ring(struct rxrpc_call *call)
428{
429 int i;
430
431 for (i = 0; i < RXRPC_RXTX_BUFF_SIZE; i++) {
432 rxrpc_free_skb(call->rxtx_buffer[i], rxrpc_skb_cleaned);
433 call->rxtx_buffer[i] = NULL;
434 }
435}
436
424/*437/*
425 * Detach a call from its owning socket.438 * Detach a call from its owning socket.
426 */439 */
@@ -429,7 +442,6 @@ void rxrpc_release_call(struct rxrpc_sock *rx, struct rxrpc_call *call) · 2 unmodified lines
429 const void *here = __builtin_return_address(0);442 const void *here = __builtin_return_address(0);
430 struct rxrpc_connection *conn = call->conn;443 struct rxrpc_connection *conn = call->conn;
431 bool put = false;444 bool put = false;
432 int i;
433 445
434 _enter("{%d,%d}", call->debug_id, atomic_read(&call->usage));446 _enter("{%d,%d}", call->debug_id, atomic_read(&call->usage));
435 447
@@ -479,13 +491,7 @@ void rxrpc_release_call(struct rxrpc_sock *rx, struct rxrpc_call *call) · 43 unmodified lines
479 if (conn)491 if (conn)
480 rxrpc_disconnect_call(call);492 rxrpc_disconnect_call(call);
481 493
482 for (i = 0; i < RXRPC_RXTX_BUFF_SIZE; i++) {494 rxrpc_cleanup_ring(call);
483 rxrpc_free_skb(call->rxtx_buffer[i],
484 (call->tx_phase ? rxrpc_skb_tx_cleaned :
485 rxrpc_skb_rx_cleaned));
486 call->rxtx_buffer[i] = NULL;
487 }
488
489 _leave("");495 _leave("");
490}496}
491 497
@@ -568,8 +574,6 @@ static void rxrpc_rcu_destroy_call(struct rcu_head *rcu) · 76 unmodified lines
568 */574 */
569void rxrpc_cleanup_call(struct rxrpc_call *call)575void rxrpc_cleanup_call(struct rxrpc_call *call)
570{576{
571 int i;
572
573 _net("DESTROY CALL %d", call->debug_id);577 _net("DESTROY CALL %d", call->debug_id);
574 578
575 memset(&call->sock_node, 0xcd, sizeof(call->sock_node));579 memset(&call->sock_node, 0xcd, sizeof(call->sock_node));
@@ -580,13 +584,8 @@ void rxrpc_cleanup_call(struct rxrpc_call *call) · 4 unmodified lines
580 ASSERT(test_bit(RXRPC_CALL_RELEASED, &call->flags));584 ASSERT(test_bit(RXRPC_CALL_RELEASED, &call->flags));
581 ASSERTCMP(call->conn, ==, NULL);585 ASSERTCMP(call->conn, ==, NULL);
582 586
583 /* Clean up the Rx/Tx buffer */587 rxrpc_cleanup_ring(call);
584 for (i = 0; i < RXRPC_RXTX_BUFF_SIZE; i++)588 rxrpc_free_skb(call->tx_pending, rxrpc_skb_cleaned);
585 rxrpc_free_skb(call->rxtx_buffer[i],
586 (call->tx_phase ? rxrpc_skb_tx_cleaned :
587 rxrpc_skb_rx_cleaned));
588
589 rxrpc_free_skb(call->tx_pending, rxrpc_skb_tx_cleaned);
590 589
591 call_rcu(&call->rcu, rxrpc_rcu_destroy_call);590 call_rcu(&call->rcu, rxrpc_rcu_destroy_call);
592}591}
net/rxrpc/conn_event.c+3−3
@@ -472,7 +472,7 @@ void rxrpc_process_connection(struct work_struct *work) · 471 unmodified lines
472 /* go through the conn-level event packets, releasing the ref on this472 /* go through the conn-level event packets, releasing the ref on this
473 * connection that each one has when we've finished with it */473 * connection that each one has when we've finished with it */
474 while ((skb = skb_dequeue(&conn->rx_queue))) {474 while ((skb = skb_dequeue(&conn->rx_queue))) {
475 rxrpc_see_skb(skb, rxrpc_skb_rx_seen);475 rxrpc_see_skb(skb, rxrpc_skb_seen);
476 ret = rxrpc_process_event(conn, skb, &abort_code);476 ret = rxrpc_process_event(conn, skb, &abort_code);
477 switch (ret) {477 switch (ret) {
478 case -EPROTO:478 case -EPROTO:
@@ -484,7 +484,7 @@ void rxrpc_process_connection(struct work_struct *work) · 5 unmodified lines
484 goto requeue_and_leave;484 goto requeue_and_leave;
485 case -ECONNABORTED:485 case -ECONNABORTED:
486 default:486 default:
487 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);487 rxrpc_free_skb(skb, rxrpc_skb_freed);
488 break;488 break;
489 }489 }
490 }490 }
@@ -501,6 +501,6 @@ requeue_and_leave: · 10 unmodified lines
501protocol_error:501protocol_error:
502 if (rxrpc_abort_connection(conn, ret, abort_code) < 0)502 if (rxrpc_abort_connection(conn, ret, abort_code) < 0)
503 goto requeue_and_leave;503 goto requeue_and_leave;
504 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);504 rxrpc_free_skb(skb, rxrpc_skb_freed);
505 goto out;505 goto out;
506}506}
net/rxrpc/input.c+167−137
@@ -233,7 +233,7 @@ static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to, · 232 unmodified lines
233 ix = call->tx_hard_ack & RXRPC_RXTX_BUFF_MASK;233 ix = call->tx_hard_ack & RXRPC_RXTX_BUFF_MASK;
234 skb = call->rxtx_buffer[ix];234 skb = call->rxtx_buffer[ix];
235 annotation = call->rxtx_annotations[ix];235 annotation = call->rxtx_annotations[ix];
236 rxrpc_see_skb(skb, rxrpc_skb_tx_rotated);236 rxrpc_see_skb(skb, rxrpc_skb_rotated);
237 call->rxtx_buffer[ix] = NULL;237 call->rxtx_buffer[ix] = NULL;
238 call->rxtx_annotations[ix] = 0;238 call->rxtx_annotations[ix] = 0;
239 skb->next = list;239 skb->next = list;
@@ -258,7 +258,7 @@ static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to, · 18 unmodified lines
258 skb = list;258 skb = list;
259 list = skb->next;259 list = skb->next;
260 skb_mark_not_on_list(skb);260 skb_mark_not_on_list(skb);
261 rxrpc_free_skb(skb, rxrpc_skb_tx_freed);261 rxrpc_free_skb(skb, rxrpc_skb_freed);
262 }262 }
263 263
264 return rot_last;264 return rot_last;
@@ -347,7 +347,7 @@ static bool rxrpc_receiving_reply(struct rxrpc_call *call) · 82 unmodified lines
347}347}
348 348
349/*349/*
350 * Scan a jumbo packet to validate its structure and to work out how many350 * Scan a data packet to validate its structure and to work out how many
351 * subpackets it contains.351 * subpackets it contains.
352 *352 *
353 * A jumbo packet is a collection of consecutive packets glued together with353 * A jumbo packet is a collection of consecutive packets glued together with
@@ -358,16 +358,21 @@ static bool rxrpc_receiving_reply(struct rxrpc_call *call) · 4 unmodified lines
358 * the last are RXRPC_JUMBO_DATALEN in size. The last subpacket may be of any358 * the last are RXRPC_JUMBO_DATALEN in size. The last subpacket may be of any
359 * size.359 * size.
360 */360 */
361static bool rxrpc_validate_jumbo(struct sk_buff *skb)361static bool rxrpc_validate_data(struct sk_buff *skb)
362{362{
363 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);363 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
364 unsigned int offset = sizeof(struct rxrpc_wire_header);364 unsigned int offset = sizeof(struct rxrpc_wire_header);
365 unsigned int len = skb->len;365 unsigned int len = skb->len;
366 int nr_jumbo = 1;
367 u8 flags = sp->hdr.flags;366 u8 flags = sp->hdr.flags;
368 367
369 do {368 for (;;) {
370 nr_jumbo++;369 if (flags & RXRPC_REQUEST_ACK)
370 __set_bit(sp->nr_subpackets, sp->rx_req_ack);
371 sp->nr_subpackets++;
372
373 if (!(flags & RXRPC_JUMBO_PACKET))
374 break;
375
371 if (len - offset < RXRPC_JUMBO_SUBPKTLEN)376 if (len - offset < RXRPC_JUMBO_SUBPKTLEN)
372 goto protocol_error;377 goto protocol_error;
373 if (flags & RXRPC_LAST_PACKET)378 if (flags & RXRPC_LAST_PACKET)
@@ -376,9 +381,10 @@ static bool rxrpc_validate_jumbo(struct sk_buff *skb) · 2 unmodified lines
376 if (skb_copy_bits(skb, offset, &flags, 1) < 0)381 if (skb_copy_bits(skb, offset, &flags, 1) < 0)
377 goto protocol_error;382 goto protocol_error;
378 offset += sizeof(struct rxrpc_jumbo_header);383 offset += sizeof(struct rxrpc_jumbo_header);
379 } while (flags & RXRPC_JUMBO_PACKET);384 }
380 385
381 sp->nr_jumbo = nr_jumbo;386 if (flags & RXRPC_LAST_PACKET)
387 sp->rx_flags |= RXRPC_SKB_INCL_LAST;
382 return true;388 return true;
383 389
384protocol_error:390protocol_error:
@@ -399,10 +405,10 @@ protocol_error: · 14 unmodified lines
399 * (that information is encoded in the ACK packet).405 * (that information is encoded in the ACK packet).
400 */406 */
401static void rxrpc_input_dup_data(struct rxrpc_call *call, rxrpc_seq_t seq,407static void rxrpc_input_dup_data(struct rxrpc_call *call, rxrpc_seq_t seq,
402 u8 annotation, bool *_jumbo_bad)408 bool is_jumbo, bool *_jumbo_bad)
403{409{
404 /* Discard normal packets that are duplicates. */410 /* Discard normal packets that are duplicates. */
405 if (annotation == 0)411 if (is_jumbo)
406 return;412 return;
407 413
408 /* Skip jumbo subpackets that are duplicates. When we've had three or414 /* Skip jumbo subpackets that are duplicates. When we've had three or
@@ -416,29 +422,30 @@ static void rxrpc_input_dup_data(struct rxrpc_call *call, rxrpc_seq_t seq, · 7 unmodified lines
416}422}
417 423
418/*424/*
419 * Process a DATA packet, adding the packet to the Rx ring.425 * Process a DATA packet, adding the packet to the Rx ring. The caller's
426 * packet ref must be passed on or discarded.
420 */427 */
421static void rxrpc_input_data(struct rxrpc_call *call, struct sk_buff *skb)428static void rxrpc_input_data(struct rxrpc_call *call, struct sk_buff *skb)
422{429{
423 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);430 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
424 enum rxrpc_call_state state;431 enum rxrpc_call_state state;
425 unsigned int offset = sizeof(struct rxrpc_wire_header);432 unsigned int j;
426 unsigned int ix;
427 rxrpc_serial_t serial = sp->hdr.serial, ack_serial = 0;433 rxrpc_serial_t serial = sp->hdr.serial, ack_serial = 0;
428 rxrpc_seq_t seq = sp->hdr.seq, hard_ack;434 rxrpc_seq_t seq0 = sp->hdr.seq, hard_ack;
429 bool immediate_ack = false, jumbo_bad = false, queued;435 bool immediate_ack = false, jumbo_bad = false;
430 u16 len;436 u8 ack = 0;
431 u8 ack = 0, flags, annotation = 0;
432 437
433 _enter("{%u,%u},{%u,%u}",438 _enter("{%u,%u},{%u,%u}",
434 call->rx_hard_ack, call->rx_top, skb->len, seq);439 call->rx_hard_ack, call->rx_top, skb->len, seq0);
435 440
436 _proto("Rx DATA %%%u { #%u f=%02x }",441 _proto("Rx DATA %%%u { #%u f=%02x n=%u }",
437 sp->hdr.serial, seq, sp->hdr.flags);442 sp->hdr.serial, seq0, sp->hdr.flags, sp->nr_subpackets);
438 443
439 state = READ_ONCE(call->state);444 state = READ_ONCE(call->state);
440 if (state >= RXRPC_CALL_COMPLETE)445 if (state >= RXRPC_CALL_COMPLETE) {
446 rxrpc_free_skb(skb, rxrpc_skb_freed);
441 return;447 return;
448 }
442 449
443 if (call->state == RXRPC_CALL_SERVER_RECV_REQUEST) {450 if (call->state == RXRPC_CALL_SERVER_RECV_REQUEST) {
444 unsigned long timo = READ_ONCE(call->next_req_timo);451 unsigned long timo = READ_ONCE(call->next_req_timo);
@@ -463,137 +470,137 @@ static void rxrpc_input_data(struct rxrpc_call *call, struct sk_buff *skb) · 18 unmodified lines
463 !rxrpc_receiving_reply(call))470 !rxrpc_receiving_reply(call))
464 goto unlock;471 goto unlock;
465 472
466 call->ackr_prev_seq = seq;473 call->ackr_prev_seq = seq0;
467
468 hard_ack = READ_ONCE(call->rx_hard_ack);474 hard_ack = READ_ONCE(call->rx_hard_ack);
469 if (after(seq, hard_ack + call->rx_winsize)) {
470 ack = RXRPC_ACK_EXCEEDS_WINDOW;
471 ack_serial = serial;
472 goto ack;
473 }
474 475
475 flags = sp->hdr.flags;476 if (sp->nr_subpackets > 1) {
476 if (flags & RXRPC_JUMBO_PACKET) {
477 if (call->nr_jumbo_bad > 3) {477 if (call->nr_jumbo_bad > 3) {
478 ack = RXRPC_ACK_NOSPACE;478 ack = RXRPC_ACK_NOSPACE;
479 ack_serial = serial;479 ack_serial = serial;
480 goto ack;480 goto ack;
481 }481 }
482 annotation = 1;
483 }482 }
484 483
485next_subpacket:484 for (j = 0; j < sp->nr_subpackets; j++) {
486 queued = false;485 rxrpc_serial_t serial = sp->hdr.serial + j;
487 ix = seq & RXRPC_RXTX_BUFF_MASK;486 rxrpc_seq_t seq = seq0 + j;
488 len = skb->len;487 unsigned int ix = seq & RXRPC_RXTX_BUFF_MASK;
489 if (flags & RXRPC_JUMBO_PACKET)488 bool terminal = (j == sp->nr_subpackets - 1);
490 len = RXRPC_JUMBO_DATALEN;489 bool last = terminal && (sp->rx_flags & RXRPC_SKB_INCL_LAST);
491 490 u8 flags, annotation = j;
492 if (flags & RXRPC_LAST_PACKET) {491
493 if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) &&492 _proto("Rx DATA+%u %%%u { #%x t=%u l=%u }",
494 seq != call->rx_top) {493 j, serial, seq, terminal, last);
495 rxrpc_proto_abort("LSN", call, seq);494
496 goto unlock;495 if (last) {
497 }496 if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) &&
498 } else {497 seq != call->rx_top) {
499 if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) &&498 rxrpc_proto_abort("LSN", call, seq);
500 after_eq(seq, call->rx_top)) {499 goto unlock;
501 rxrpc_proto_abort("LSA", call, seq);500 }
502 goto unlock;501 } else {
502 if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) &&
503 after_eq(seq, call->rx_top)) {
504 rxrpc_proto_abort("LSA", call, seq);
505 goto unlock;
506 }
503 }507 }
504 }
505 508
506 trace_rxrpc_rx_data(call->debug_id, seq, serial, flags, annotation);509 flags = 0;
507 if (before_eq(seq, hard_ack)) {510 if (last)
508 ack = RXRPC_ACK_DUPLICATE;511 flags |= RXRPC_LAST_PACKET;
509 ack_serial = serial;512 if (!terminal)
510 goto skip;513 flags |= RXRPC_JUMBO_PACKET;
511 }514 if (test_bit(j, sp->rx_req_ack))
515 flags |= RXRPC_REQUEST_ACK;
516 trace_rxrpc_rx_data(call->debug_id, seq, serial, flags, annotation);
512 517
513 if (flags & RXRPC_REQUEST_ACK && !ack) {518 if (before_eq(seq, hard_ack)) {
514 ack = RXRPC_ACK_REQUESTED;
515 ack_serial = serial;
516 }
517
518 if (call->rxtx_buffer[ix]) {
519 rxrpc_input_dup_data(call, seq, annotation, &jumbo_bad);
520 if (ack != RXRPC_ACK_DUPLICATE) {
521 ack = RXRPC_ACK_DUPLICATE;519 ack = RXRPC_ACK_DUPLICATE;
522 ack_serial = serial;520 ack_serial = serial;
521 continue;
523 }522 }
524 immediate_ack = true;
525 goto skip;
526 }
527 523
528 /* Queue the packet. We use a couple of memory barriers here as need524 if (call->rxtx_buffer[ix]) {
529 * to make sure that rx_top is perceived to be set after the buffer525 rxrpc_input_dup_data(call, seq, sp->nr_subpackets > 1,
530 * pointer and that the buffer pointer is set after the annotation and526 &jumbo_bad);
531 * the skb data.527 if (ack != RXRPC_ACK_DUPLICATE) {
532 *528 ack = RXRPC_ACK_DUPLICATE;
533 * Barriers against rxrpc_recvmsg_data() and rxrpc_rotate_rx_window()529 ack_serial = serial;
534 * and also rxrpc_fill_out_ack().530 }
535 */531 immediate_ack = true;
536 rxrpc_get_skb(skb, rxrpc_skb_rx_got);532 continue;
537 call->rxtx_annotations[ix] = annotation;
538 smp_wmb();
539 call->rxtx_buffer[ix] = skb;
540 if (after(seq, call->rx_top)) {
541 smp_store_release(&call->rx_top, seq);
542 } else if (before(seq, call->rx_top)) {
543 /* Send an immediate ACK if we fill in a hole */
544 if (!ack) {
545 ack = RXRPC_ACK_DELAY;
546 ack_serial = serial;
547 }533 }
548 immediate_ack = true;
549 }
550 if (flags & RXRPC_LAST_PACKET) {
551 set_bit(RXRPC_CALL_RX_LAST, &call->flags);
552 trace_rxrpc_receive(call, rxrpc_receive_queue_last, serial, seq);
553 } else {
554 trace_rxrpc_receive(call, rxrpc_receive_queue, serial, seq);
555 }
556 queued = true;
557 534
558 if (after_eq(seq, call->rx_expect_next)) {
559 if (after(seq, call->rx_expect_next)) {
560 _net("OOS %u > %u", seq, call->rx_expect_next);
561 ack = RXRPC_ACK_OUT_OF_SEQUENCE;
562 ack_serial = serial;
563 }
564 call->rx_expect_next = seq + 1;
565 }
566
567skip:
568 offset += len;
569 if (flags & RXRPC_JUMBO_PACKET) {
570 if (skb_copy_bits(skb, offset, &flags, 1) < 0) {
571 rxrpc_proto_abort("XJF", call, seq);
572 goto unlock;
573 }
574 offset += sizeof(struct rxrpc_jumbo_header);
575 seq++;
576 serial++;
577 annotation++;
578 if (flags & RXRPC_JUMBO_PACKET)
579 annotation |= RXRPC_RX_ANNO_JLAST;
580 if (after(seq, hard_ack + call->rx_winsize)) {535 if (after(seq, hard_ack + call->rx_winsize)) {
581 ack = RXRPC_ACK_EXCEEDS_WINDOW;536 ack = RXRPC_ACK_EXCEEDS_WINDOW;
582 ack_serial = serial;537 ack_serial = serial;
583 if (!jumbo_bad) {538 if (flags & RXRPC_JUMBO_PACKET) {
584 call->nr_jumbo_bad++;539 if (!jumbo_bad) {
585 jumbo_bad = true;540 call->nr_jumbo_bad++;
541 jumbo_bad = true;
542 }
586 }543 }
544
587 goto ack;545 goto ack;
588 }546 }
589 547
590 _proto("Rx DATA Jumbo %%%u", serial);548 if (flags & RXRPC_REQUEST_ACK && !ack) {
591 goto next_subpacket;549 ack = RXRPC_ACK_REQUESTED;
592 }550 ack_serial = serial;
551 }
552
553 /* Queue the packet. We use a couple of memory barriers here as need
554 * to make sure that rx_top is perceived to be set after the buffer
555 * pointer and that the buffer pointer is set after the annotation and
556 * the skb data.
557 *
558 * Barriers against rxrpc_recvmsg_data() and rxrpc_rotate_rx_window()
559 * and also rxrpc_fill_out_ack().
560 */
561 if (!terminal)
562 rxrpc_get_skb(skb, rxrpc_skb_got);
563 call->rxtx_annotations[ix] = annotation;
564 smp_wmb();
565 call->rxtx_buffer[ix] = skb;
566 if (after(seq, call->rx_top)) {
567 smp_store_release(&call->rx_top, seq);
568 } else if (before(seq, call->rx_top)) {
569 /* Send an immediate ACK if we fill in a hole */
570 if (!ack) {
571 ack = RXRPC_ACK_DELAY;
572 ack_serial = serial;
573 }
574 immediate_ack = true;
575 }
576
577 if (terminal) {
578 /* From this point on, we're not allowed to touch the
579 * packet any longer as its ref now belongs to the Rx
580 * ring.
581 */
582 skb = NULL;
583 }
593 584
594 if (queued && flags & RXRPC_LAST_PACKET && !ack) {585 if (last) {
595 ack = RXRPC_ACK_DELAY;586 set_bit(RXRPC_CALL_RX_LAST, &call->flags);
596 ack_serial = serial;587 if (!ack) {
588 ack = RXRPC_ACK_DELAY;
589 ack_serial = serial;
590 }
591 trace_rxrpc_receive(call, rxrpc_receive_queue_last, serial, seq);
592 } else {
593 trace_rxrpc_receive(call, rxrpc_receive_queue, serial, seq);
594 }
595
596 if (after_eq(seq, call->rx_expect_next)) {
597 if (after(seq, call->rx_expect_next)) {
598 _net("OOS %u > %u", seq, call->rx_expect_next);
599 ack = RXRPC_ACK_OUT_OF_SEQUENCE;
600 ack_serial = serial;
601 }
602 call->rx_expect_next = seq + 1;
603 }
597 }604 }
598 605
599ack:606ack:
@@ -606,13 +613,14 @@ ack: · 6 unmodified lines
606 false, true,613 false, true,
607 rxrpc_propose_ack_input_data);614 rxrpc_propose_ack_input_data);
608 615
609 if (sp->hdr.seq == READ_ONCE(call->rx_hard_ack) + 1) {616 if (seq0 == READ_ONCE(call->rx_hard_ack) + 1) {
610 trace_rxrpc_notify_socket(call->debug_id, serial);617 trace_rxrpc_notify_socket(call->debug_id, serial);
611 rxrpc_notify_socket(call);618 rxrpc_notify_socket(call);
612 }619 }
613 620
614unlock:621unlock:
615 spin_unlock(&call->input_lock);622 spin_unlock(&call->input_lock);
623 rxrpc_free_skb(skb, rxrpc_skb_freed);
616 _leave(" [queued]");624 _leave(" [queued]");
617}625}
618 626
@@ -1021,7 +1029,7 @@ static void rxrpc_input_call_packet(struct rxrpc_call *call, · 402 unmodified lines
1021 switch (sp->hdr.type) {1029 switch (sp->hdr.type) {
1022 case RXRPC_PACKET_TYPE_DATA:1030 case RXRPC_PACKET_TYPE_DATA:
1023 rxrpc_input_data(call, skb);1031 rxrpc_input_data(call, skb);
1024 break;1032 goto no_free;
1025 1033
1026 case RXRPC_PACKET_TYPE_ACK:1034 case RXRPC_PACKET_TYPE_ACK:
1027 rxrpc_input_ack(call, skb);1035 rxrpc_input_ack(call, skb);
@@ -1048,6 +1056,8 @@ static void rxrpc_input_call_packet(struct rxrpc_call *call, · 20 unmodified lines
1048 break;1056 break;
1049 }1057 }
1050 1058
1059 rxrpc_free_skb(skb, rxrpc_skb_freed);
1060no_free:
1051 _leave("");1061 _leave("");
1052}1062}
1053 1063
@@ -1109,7 +1119,7 @@ static void rxrpc_post_packet_to_local(struct rxrpc_local *local, · 55 unmodified lines
1109 skb_queue_tail(&local->event_queue, skb);1119 skb_queue_tail(&local->event_queue, skb);
1110 rxrpc_queue_local(local);1120 rxrpc_queue_local(local);
1111 } else {1121 } else {
1112 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);1122 rxrpc_free_skb(skb, rxrpc_skb_freed);
1113 }1123 }
1114}1124}
1115 1125
@@ -1124,7 +1134,7 @@ static void rxrpc_reject_packet(struct rxrpc_local *local, struct sk_buff *skb) · 8 unmodified lines
1124 skb_queue_tail(&local->reject_queue, skb);1134 skb_queue_tail(&local->reject_queue, skb);
1125 rxrpc_queue_local(local);1135 rxrpc_queue_local(local);
1126 } else {1136 } else {
1127 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);1137 rxrpc_free_skb(skb, rxrpc_skb_freed);
1128 }1138 }
1129}1139}
1130 1140
@@ -1188,7 +1198,7 @@ int rxrpc_input_packet(struct sock *udp_sk, struct sk_buff *skb) · 57 unmodified lines
1188 if (skb->tstamp == 0)1198 if (skb->tstamp == 0)
1189 skb->tstamp = ktime_get_real();1199 skb->tstamp = ktime_get_real();
1190 1200
1191 rxrpc_new_skb(skb, rxrpc_skb_rx_received);1201 rxrpc_new_skb(skb, rxrpc_skb_received);
1192 1202
1193 skb_pull(skb, sizeof(struct udphdr));1203 skb_pull(skb, sizeof(struct udphdr));
1194 1204
@@ -1205,7 +1215,7 @@ int rxrpc_input_packet(struct sock *udp_sk, struct sk_buff *skb) · 10 unmodified lines
1205 static int lose;1215 static int lose;
1206 if ((lose++ & 7) == 7) {1216 if ((lose++ & 7) == 7) {
1207 trace_rxrpc_rx_lose(sp);1217 trace_rxrpc_rx_lose(sp);
1208 rxrpc_free_skb(skb, rxrpc_skb_rx_lost);1218 rxrpc_free_skb(skb, rxrpc_skb_lost);
1209 return 0;1219 return 0;
1210 }1220 }
1211 }1221 }
@@ -1237,9 +1247,26 @@ int rxrpc_input_packet(struct sock *udp_sk, struct sk_buff *skb) · 25 unmodified lines
1237 if (sp->hdr.callNumber == 0 ||1247 if (sp->hdr.callNumber == 0 ||
1238 sp->hdr.seq == 0)1248 sp->hdr.seq == 0)
1239 goto bad_message;1249 goto bad_message;
1240 if (sp->hdr.flags & RXRPC_JUMBO_PACKET &&1250 if (!rxrpc_validate_data(skb))
1241 !rxrpc_validate_jumbo(skb))
1242 goto bad_message;1251 goto bad_message;
1252
1253 /* Unshare the packet so that it can be modified for in-place
1254 * decryption.
1255 */
1256 if (sp->hdr.securityIndex != 0) {
1257 struct sk_buff *nskb = skb_unshare(skb, GFP_ATOMIC);
1258 if (!nskb) {
1259 rxrpc_eaten_skb(skb, rxrpc_skb_unshared_nomem);
1260 goto out;
1261 }
1262
1263 if (nskb != skb) {
1264 rxrpc_eaten_skb(skb, rxrpc_skb_received);
1265 rxrpc_new_skb(skb, rxrpc_skb_unshared);
1266 skb = nskb;
1267 sp = rxrpc_skb(skb);
1268 }
1269 }
1243 break;1270 break;
1244 1271
1245 case RXRPC_PACKET_TYPE_CHALLENGE:1272 case RXRPC_PACKET_TYPE_CHALLENGE:
@@ -1373,11 +1400,14 @@ int rxrpc_input_packet(struct sock *udp_sk, struct sk_buff *skb) · 127 unmodified lines
1373 mutex_unlock(&call->user_mutex);1400 mutex_unlock(&call->user_mutex);
1374 }1401 }
1375 1402
1403 /* Process a call packet; this either discards or passes on the ref
1404 * elsewhere.
1405 */
1376 rxrpc_input_call_packet(call, skb);1406 rxrpc_input_call_packet(call, skb);
1377 goto discard;1407 goto out;
1378 1408
1379discard:1409discard:
1380 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);1410 rxrpc_free_skb(skb, rxrpc_skb_freed);
1381out:1411out:
1382 trace_rxrpc_rx_done(0, 0);1412 trace_rxrpc_rx_done(0, 0);
1383 return 0;1413 return 0;
net/rxrpc/local_event.c+2−2
@@ -90,7 +90,7 @@ void rxrpc_process_local_events(struct rxrpc_local *local) · 89 unmodified lines
90 if (skb) {90 if (skb) {
91 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);91 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
92 92
93 rxrpc_see_skb(skb, rxrpc_skb_rx_seen);93 rxrpc_see_skb(skb, rxrpc_skb_seen);
94 _debug("{%d},{%u}", local->debug_id, sp->hdr.type);94 _debug("{%d},{%u}", local->debug_id, sp->hdr.type);
95 95
96 switch (sp->hdr.type) {96 switch (sp->hdr.type) {
@@ -108,7 +108,7 @@ void rxrpc_process_local_events(struct rxrpc_local *local) · 11 unmodified lines
108 break;108 break;
109 }109 }
110 110
111 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);111 rxrpc_free_skb(skb, rxrpc_skb_freed);
112 }112 }
113 113
114 _leave("");114 _leave("");
net/rxrpc/output.c+3−3
@@ -565,7 +565,7 @@ void rxrpc_reject_packets(struct rxrpc_local *local) · 564 unmodified lines
565 memset(&whdr, 0, sizeof(whdr));565 memset(&whdr, 0, sizeof(whdr));
566 566
567 while ((skb = skb_dequeue(&local->reject_queue))) {567 while ((skb = skb_dequeue(&local->reject_queue))) {
568 rxrpc_see_skb(skb, rxrpc_skb_rx_seen);568 rxrpc_see_skb(skb, rxrpc_skb_seen);
569 sp = rxrpc_skb(skb);569 sp = rxrpc_skb(skb);
570 570
571 switch (skb->mark) {571 switch (skb->mark) {
@@ -581,7 +581,7 @@ void rxrpc_reject_packets(struct rxrpc_local *local) · 9 unmodified lines
581 ioc = 2;581 ioc = 2;
582 break;582 break;
583 default:583 default:
584 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);584 rxrpc_free_skb(skb, rxrpc_skb_freed);
585 continue;585 continue;
586 }586 }
587 587
@@ -606,7 +606,7 @@ void rxrpc_reject_packets(struct rxrpc_local *local) · 18 unmodified lines
606 rxrpc_tx_point_reject);606 rxrpc_tx_point_reject);
607 }607 }
608 608
609 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);609 rxrpc_free_skb(skb, rxrpc_skb_freed);
610 }610 }
611 611
612 _leave("");612 _leave("");
net/rxrpc/peer_event.c+5−5
@@ -163,11 +163,11 @@ void rxrpc_error_report(struct sock *sk) · 162 unmodified lines
163 _leave("UDP socket errqueue empty");163 _leave("UDP socket errqueue empty");
164 return;164 return;
165 }165 }
166 rxrpc_new_skb(skb, rxrpc_skb_rx_received);166 rxrpc_new_skb(skb, rxrpc_skb_received);
167 serr = SKB_EXT_ERR(skb);167 serr = SKB_EXT_ERR(skb);
168 if (!skb->len && serr->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING) {168 if (!skb->len && serr->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING) {
169 _leave("UDP empty message");169 _leave("UDP empty message");
170 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);170 rxrpc_free_skb(skb, rxrpc_skb_freed);
171 return;171 return;
172 }172 }
173 173
@@ -177,7 +177,7 @@ void rxrpc_error_report(struct sock *sk) · 3 unmodified lines
177 peer = NULL;177 peer = NULL;
178 if (!peer) {178 if (!peer) {
179 rcu_read_unlock();179 rcu_read_unlock();
180 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);180 rxrpc_free_skb(skb, rxrpc_skb_freed);
181 _leave(" [no peer]");181 _leave(" [no peer]");
182 return;182 return;
183 }183 }
@@ -189,7 +189,7 @@ void rxrpc_error_report(struct sock *sk) · 5 unmodified lines
189 serr->ee.ee_code == ICMP_FRAG_NEEDED)) {189 serr->ee.ee_code == ICMP_FRAG_NEEDED)) {
190 rxrpc_adjust_mtu(peer, serr);190 rxrpc_adjust_mtu(peer, serr);
191 rcu_read_unlock();191 rcu_read_unlock();
192 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);192 rxrpc_free_skb(skb, rxrpc_skb_freed);
193 rxrpc_put_peer(peer);193 rxrpc_put_peer(peer);
194 _leave(" [MTU update]");194 _leave(" [MTU update]");
195 return;195 return;
@@ -197,7 +197,7 @@ void rxrpc_error_report(struct sock *sk) · 1 unmodified line
197 197
198 rxrpc_store_error(peer, serr);198 rxrpc_store_error(peer, serr);
199 rcu_read_unlock();199 rcu_read_unlock();
200 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);200 rxrpc_free_skb(skb, rxrpc_skb_freed);
201 rxrpc_put_peer(peer);201 rxrpc_put_peer(peer);
202 202
203 _leave("");203 _leave("");
net/rxrpc/protocol.h+9−0
@@ -89,6 +89,15 @@ struct rxrpc_jumbo_header { · 88 unmodified lines
89#define RXRPC_JUMBO_DATALEN 1412 /* non-terminal jumbo packet data length */89#define RXRPC_JUMBO_DATALEN 1412 /* non-terminal jumbo packet data length */
90#define RXRPC_JUMBO_SUBPKTLEN (RXRPC_JUMBO_DATALEN + sizeof(struct rxrpc_jumbo_header))90#define RXRPC_JUMBO_SUBPKTLEN (RXRPC_JUMBO_DATALEN + sizeof(struct rxrpc_jumbo_header))
91 91
92/*
93 * The maximum number of subpackets that can possibly fit in a UDP packet is:
94 *
95 * ((max_IP - IP_hdr - UDP_hdr) / RXRPC_JUMBO_SUBPKTLEN) + 1
96 * = ((65535 - 28 - 28) / 1416) + 1
97 * = 46 non-terminal packets and 1 terminal packet.
98 */
99#define RXRPC_MAX_NR_JUMBO 47
100
92/*****************************************************************************/101/*****************************************************************************/
93/*102/*
94 * on-the-wire Rx ACK packet data payload103 * on-the-wire Rx ACK packet data payload
net/rxrpc/recvmsg.c+27−20
@@ -177,7 +177,8 @@ static void rxrpc_rotate_rx_window(struct rxrpc_call *call) · 176 unmodified lines
177 struct sk_buff *skb;177 struct sk_buff *skb;
178 rxrpc_serial_t serial;178 rxrpc_serial_t serial;
179 rxrpc_seq_t hard_ack, top;179 rxrpc_seq_t hard_ack, top;
180 u8 flags;180 bool last = false;
181 u8 subpacket;
181 int ix;182 int ix;
182 183
183 _enter("%d", call->debug_id);184 _enter("%d", call->debug_id);
@@ -189,23 +190,25 @@ static void rxrpc_rotate_rx_window(struct rxrpc_call *call) · 5 unmodified lines
189 hard_ack++;190 hard_ack++;
190 ix = hard_ack & RXRPC_RXTX_BUFF_MASK;191 ix = hard_ack & RXRPC_RXTX_BUFF_MASK;
191 skb = call->rxtx_buffer[ix];192 skb = call->rxtx_buffer[ix];
192 rxrpc_see_skb(skb, rxrpc_skb_rx_rotated);193 rxrpc_see_skb(skb, rxrpc_skb_rotated);
193 sp = rxrpc_skb(skb);194 sp = rxrpc_skb(skb);
194 flags = sp->hdr.flags;195
195 serial = sp->hdr.serial;196 subpacket = call->rxtx_annotations[ix] & RXRPC_RX_ANNO_SUBPACKET;
196 if (call->rxtx_annotations[ix] & RXRPC_RX_ANNO_JUMBO)197 serial = sp->hdr.serial + subpacket;
197 serial += (call->rxtx_annotations[ix] & RXRPC_RX_ANNO_JUMBO) - 1;198
199 if (subpacket == sp->nr_subpackets - 1 &&
200 sp->rx_flags & RXRPC_SKB_INCL_LAST)
201 last = true;
198 202
199 call->rxtx_buffer[ix] = NULL;203 call->rxtx_buffer[ix] = NULL;
200 call->rxtx_annotations[ix] = 0;204 call->rxtx_annotations[ix] = 0;
201 /* Barrier against rxrpc_input_data(). */205 /* Barrier against rxrpc_input_data(). */
202 smp_store_release(&call->rx_hard_ack, hard_ack);206 smp_store_release(&call->rx_hard_ack, hard_ack);
203 207
204 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);208 rxrpc_free_skb(skb, rxrpc_skb_freed);
205 209
206 _debug("%u,%u,%02x", hard_ack, top, flags);
207 trace_rxrpc_receive(call, rxrpc_receive_rotate, serial, hard_ack);210 trace_rxrpc_receive(call, rxrpc_receive_rotate, serial, hard_ack);
208 if (flags & RXRPC_LAST_PACKET) {211 if (last) {
209 rxrpc_end_rx_phase(call, serial);212 rxrpc_end_rx_phase(call, serial);
210 } else {213 } else {
211 /* Check to see if there's an ACK that needs sending. */214 /* Check to see if there's an ACK that needs sending. */
@@ -233,18 +236,19 @@ static int rxrpc_verify_packet(struct rxrpc_call *call, struct sk_buff *skb, · 21 unmodified lines
233 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);236 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
234 rxrpc_seq_t seq = sp->hdr.seq;237 rxrpc_seq_t seq = sp->hdr.seq;
235 u16 cksum = sp->hdr.cksum;238 u16 cksum = sp->hdr.cksum;
239 u8 subpacket = annotation & RXRPC_RX_ANNO_SUBPACKET;
236 240
237 _enter("");241 _enter("");
238 242
239 /* For all but the head jumbo subpacket, the security checksum is in a243 /* For all but the head jumbo subpacket, the security checksum is in a
240 * jumbo header immediately prior to the data.244 * jumbo header immediately prior to the data.
241 */245 */
242 if ((annotation & RXRPC_RX_ANNO_JUMBO) > 1) {246 if (subpacket > 0) {
243 __be16 tmp;247 __be16 tmp;
244 if (skb_copy_bits(skb, offset - 2, &tmp, 2) < 0)248 if (skb_copy_bits(skb, offset - 2, &tmp, 2) < 0)
245 BUG();249 BUG();
246 cksum = ntohs(tmp);250 cksum = ntohs(tmp);
247 seq += (annotation & RXRPC_RX_ANNO_JUMBO) - 1;251 seq += subpacket;
248 }252 }
249 253
250 return call->conn->security->verify_packet(call, skb, offset, len,254 return call->conn->security->verify_packet(call, skb, offset, len,
@@ -265,19 +269,18 @@ static int rxrpc_locate_data(struct rxrpc_call *call, struct sk_buff *skb, · 14 unmodified lines
265 u8 *_annotation,269 u8 *_annotation,
266 unsigned int *_offset, unsigned int *_len)270 unsigned int *_offset, unsigned int *_len)
267{271{
272 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
268 unsigned int offset = sizeof(struct rxrpc_wire_header);273 unsigned int offset = sizeof(struct rxrpc_wire_header);
269 unsigned int len;274 unsigned int len;
270 int ret;275 int ret;
271 u8 annotation = *_annotation;276 u8 annotation = *_annotation;
277 u8 subpacket = annotation & RXRPC_RX_ANNO_SUBPACKET;
272 278
273 /* Locate the subpacket */279 /* Locate the subpacket */
280 offset += subpacket * RXRPC_JUMBO_SUBPKTLEN;
274 len = skb->len - offset;281 len = skb->len - offset;
275 if ((annotation & RXRPC_RX_ANNO_JUMBO) > 0) {282 if (subpacket < sp->nr_subpackets - 1)
276 offset += (((annotation & RXRPC_RX_ANNO_JUMBO) - 1) *283 len = RXRPC_JUMBO_DATALEN;
277 RXRPC_JUMBO_SUBPKTLEN);
278 len = (annotation & RXRPC_RX_ANNO_JLAST) ?
279 skb->len - offset : RXRPC_JUMBO_SUBPKTLEN;
280 }
281 284
282 if (!(annotation & RXRPC_RX_ANNO_VERIFIED)) {285 if (!(annotation & RXRPC_RX_ANNO_VERIFIED)) {
283 ret = rxrpc_verify_packet(call, skb, annotation, offset, len);286 ret = rxrpc_verify_packet(call, skb, annotation, offset, len);
@@ -303,6 +306,7 @@ static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call, · 19 unmodified lines
303{306{
304 struct rxrpc_skb_priv *sp;307 struct rxrpc_skb_priv *sp;
305 struct sk_buff *skb;308 struct sk_buff *skb;
309 rxrpc_serial_t serial;
306 rxrpc_seq_t hard_ack, top, seq;310 rxrpc_seq_t hard_ack, top, seq;
307 size_t remain;311 size_t remain;
308 bool last;312 bool last;
@@ -336,12 +340,15 @@ static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call, · 27 unmodified lines
336 break;340 break;
337 }341 }
338 smp_rmb();342 smp_rmb();
339 rxrpc_see_skb(skb, rxrpc_skb_rx_seen);343 rxrpc_see_skb(skb, rxrpc_skb_seen);
340 sp = rxrpc_skb(skb);344 sp = rxrpc_skb(skb);
341 345
342 if (!(flags & MSG_PEEK))346 if (!(flags & MSG_PEEK)) {
347 serial = sp->hdr.serial;
348 serial += call->rxtx_annotations[ix] & RXRPC_RX_ANNO_SUBPACKET;
343 trace_rxrpc_receive(call, rxrpc_receive_front,349 trace_rxrpc_receive(call, rxrpc_receive_front,
344 sp->hdr.serial, seq);350 serial, seq);
351 }
345 352
346 if (msg)353 if (msg)
347 sock_recv_timestamp(msg, sock->sk, skb);354 sock_recv_timestamp(msg, sock->sk, skb);
net/rxrpc/rxkad.c+9−23
@@ -187,10 +187,8 @@ static int rxkad_secure_packet_encrypt(const struct rxrpc_call *call, · 186 unmodified lines
187 struct rxrpc_skb_priv *sp;187 struct rxrpc_skb_priv *sp;
188 struct rxrpc_crypt iv;188 struct rxrpc_crypt iv;
189 struct scatterlist sg[16];189 struct scatterlist sg[16];
190 struct sk_buff *trailer;
191 unsigned int len;190 unsigned int len;
192 u16 check;191 u16 check;
193 int nsg;
194 int err;192 int err;
195 193
196 sp = rxrpc_skb(skb);194 sp = rxrpc_skb(skb);
@@ -214,15 +212,14 @@ static int rxkad_secure_packet_encrypt(const struct rxrpc_call *call, · 17 unmodified lines
214 crypto_skcipher_encrypt(req);212 crypto_skcipher_encrypt(req);
215 213
216 /* we want to encrypt the skbuff in-place */214 /* we want to encrypt the skbuff in-place */
217 nsg = skb_cow_data(skb, 0, &trailer);215 err = -EMSGSIZE;
218 err = -ENOMEM;216 if (skb_shinfo(skb)->nr_frags > 16)
219 if (nsg < 0 || nsg > 16)
220 goto out;217 goto out;
221 218
222 len = data_size + call->conn->size_align - 1;219 len = data_size + call->conn->size_align - 1;
223 len &= ~(call->conn->size_align - 1);220 len &= ~(call->conn->size_align - 1);
224 221
225 sg_init_table(sg, nsg);222 sg_init_table(sg, ARRAY_SIZE(sg));
226 err = skb_to_sgvec(skb, sg, 0, len);223 err = skb_to_sgvec(skb, sg, 0, len);
227 if (unlikely(err < 0))224 if (unlikely(err < 0))
228 goto out;225 goto out;
@@ -319,11 +316,10 @@ static int rxkad_verify_packet_1(struct rxrpc_call *call, struct sk_buff *skb, · 90 unmodified lines
319 struct rxkad_level1_hdr sechdr;316 struct rxkad_level1_hdr sechdr;
320 struct rxrpc_crypt iv;317 struct rxrpc_crypt iv;
321 struct scatterlist sg[16];318 struct scatterlist sg[16];
322 struct sk_buff *trailer;
323 bool aborted;319 bool aborted;
324 u32 data_size, buf;320 u32 data_size, buf;
325 u16 check;321 u16 check;
326 int nsg, ret;322 int ret;
327 323
328 _enter("");324 _enter("");
329 325
@@ -336,11 +332,7 @@ static int rxkad_verify_packet_1(struct rxrpc_call *call, struct sk_buff *skb, · 6 unmodified lines
336 /* Decrypt the skbuff in-place. TODO: We really want to decrypt332 /* Decrypt the skbuff in-place. TODO: We really want to decrypt
337 * directly into the target buffer.333 * directly into the target buffer.
338 */334 */
339 nsg = skb_cow_data(skb, 0, &trailer);335 sg_init_table(sg, ARRAY_SIZE(sg));
340 if (nsg < 0 || nsg > 16)
341 goto nomem;
342
343 sg_init_table(sg, nsg);
344 ret = skb_to_sgvec(skb, sg, offset, 8);336 ret = skb_to_sgvec(skb, sg, offset, 8);
345 if (unlikely(ret < 0))337 if (unlikely(ret < 0))
346 return ret;338 return ret;
@@ -388,10 +380,6 @@ protocol_error: · 41 unmodified lines
388 if (aborted)380 if (aborted)
389 rxrpc_send_abort_packet(call);381 rxrpc_send_abort_packet(call);
390 return -EPROTO;382 return -EPROTO;
391
392nomem:
393 _leave(" = -ENOMEM");
394 return -ENOMEM;
395}383}
396 384
397/*385/*
@@ -406,7 +394,6 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb, · 8 unmodified lines
406 struct rxkad_level2_hdr sechdr;394 struct rxkad_level2_hdr sechdr;
407 struct rxrpc_crypt iv;395 struct rxrpc_crypt iv;
408 struct scatterlist _sg[4], *sg;396 struct scatterlist _sg[4], *sg;
409 struct sk_buff *trailer;
410 bool aborted;397 bool aborted;
411 u32 data_size, buf;398 u32 data_size, buf;
412 u16 check;399 u16 check;
@@ -423,12 +410,11 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb, · 10 unmodified lines
423 /* Decrypt the skbuff in-place. TODO: We really want to decrypt410 /* Decrypt the skbuff in-place. TODO: We really want to decrypt
424 * directly into the target buffer.411 * directly into the target buffer.
425 */412 */
426 nsg = skb_cow_data(skb, 0, &trailer);
427 if (nsg < 0)
428 goto nomem;
429
430 sg = _sg;413 sg = _sg;
431 if (unlikely(nsg > 4)) {414 nsg = skb_shinfo(skb)->nr_frags;
415 if (nsg <= 4) {
416 nsg = 4;
417 } else {
432 sg = kmalloc_array(nsg, sizeof(*sg), GFP_NOIO);418 sg = kmalloc_array(nsg, sizeof(*sg), GFP_NOIO);
433 if (!sg)419 if (!sg)
434 goto nomem;420 goto nomem;
net/rxrpc/sendmsg.c+7−6
@@ -176,7 +176,7 @@ static int rxrpc_queue_packet(struct rxrpc_sock *rx, struct rxrpc_call *call, · 175 unmodified lines
176 skb->tstamp = ktime_get_real();176 skb->tstamp = ktime_get_real();
177 177
178 ix = seq & RXRPC_RXTX_BUFF_MASK;178 ix = seq & RXRPC_RXTX_BUFF_MASK;
179 rxrpc_get_skb(skb, rxrpc_skb_tx_got);179 rxrpc_get_skb(skb, rxrpc_skb_got);
180 call->rxtx_annotations[ix] = annotation;180 call->rxtx_annotations[ix] = annotation;
181 smp_wmb();181 smp_wmb();
182 call->rxtx_buffer[ix] = skb;182 call->rxtx_buffer[ix] = skb;
@@ -248,7 +248,7 @@ static int rxrpc_queue_packet(struct rxrpc_sock *rx, struct rxrpc_call *call, · 65 unmodified lines
248 }248 }
249 249
250out:250out:
251 rxrpc_free_skb(skb, rxrpc_skb_tx_freed);251 rxrpc_free_skb(skb, rxrpc_skb_freed);
252 _leave(" = %d", ret);252 _leave(" = %d", ret);
253 return ret;253 return ret;
254}254}
@@ -289,7 +289,7 @@ static int rxrpc_send_data(struct rxrpc_sock *rx, · 34 unmodified lines
289 289
290 skb = call->tx_pending;290 skb = call->tx_pending;
291 call->tx_pending = NULL;291 call->tx_pending = NULL;
292 rxrpc_see_skb(skb, rxrpc_skb_tx_seen);292 rxrpc_see_skb(skb, rxrpc_skb_seen);
293 293
294 copied = 0;294 copied = 0;
295 do {295 do {
@@ -336,7 +336,9 @@ static int rxrpc_send_data(struct rxrpc_sock *rx, · 40 unmodified lines
336 if (!skb)336 if (!skb)
337 goto maybe_error;337 goto maybe_error;
338 338
339 rxrpc_new_skb(skb, rxrpc_skb_tx_new);339 sp = rxrpc_skb(skb);
340 sp->rx_flags |= RXRPC_SKB_TX_BUFFER;
341 rxrpc_new_skb(skb, rxrpc_skb_new);
340 342
341 _debug("ALLOC SEND %p", skb);343 _debug("ALLOC SEND %p", skb);
342 344
@@ -346,7 +348,6 @@ static int rxrpc_send_data(struct rxrpc_sock *rx, · 3 unmodified lines
346 skb_reserve(skb, call->conn->security_size);348 skb_reserve(skb, call->conn->security_size);
347 skb->len += call->conn->security_size;349 skb->len += call->conn->security_size;
348 350
349 sp = rxrpc_skb(skb);
350 sp->remain = chunk;351 sp->remain = chunk;
351 if (sp->remain > skb_tailroom(skb))352 if (sp->remain > skb_tailroom(skb))
352 sp->remain = skb_tailroom(skb);353 sp->remain = skb_tailroom(skb);
@@ -439,7 +440,7 @@ out: · 86 unmodified lines
439 return ret;440 return ret;
440 441
441call_terminated:442call_terminated:
442 rxrpc_free_skb(skb, rxrpc_skb_tx_freed);443 rxrpc_free_skb(skb, rxrpc_skb_freed);
443 _leave(" = %d", call->error);444 _leave(" = %d", call->error);
444 return call->error;445 return call->error;
445 446
net/rxrpc/skbuff.c+28−12
@@ -14,7 +14,8 @@ · 13 unmodified lines
14#include <net/af_rxrpc.h>14#include <net/af_rxrpc.h>
15#include "ar-internal.h"15#include "ar-internal.h"
16 16
17#define select_skb_count(op) (op >= rxrpc_skb_tx_cleaned ? &rxrpc_n_tx_skbs : &rxrpc_n_rx_skbs)17#define is_tx_skb(skb) (rxrpc_skb(skb)->rx_flags & RXRPC_SKB_TX_BUFFER)
18#define select_skb_count(skb) (is_tx_skb(skb) ? &rxrpc_n_tx_skbs : &rxrpc_n_rx_skbs)
18 19
19/*20/*
20 * Note the allocation or reception of a socket buffer.21 * Note the allocation or reception of a socket buffer.
@@ -22,8 +23,9 @@ · 1 unmodified line
22void rxrpc_new_skb(struct sk_buff *skb, enum rxrpc_skb_trace op)23void rxrpc_new_skb(struct sk_buff *skb, enum rxrpc_skb_trace op)
23{24{
24 const void *here = __builtin_return_address(0);25 const void *here = __builtin_return_address(0);
25 int n = atomic_inc_return(select_skb_count(op));26 int n = atomic_inc_return(select_skb_count(skb));
26 trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n, here);27 trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n,
28 rxrpc_skb(skb)->rx_flags, here);
27}29}
28 30
29/*31/*
@@ -33,8 +35,9 @@ void rxrpc_see_skb(struct sk_buff *skb, enum rxrpc_skb_trace op) · 3 unmodified lines
33{35{
34 const void *here = __builtin_return_address(0);36 const void *here = __builtin_return_address(0);
35 if (skb) {37 if (skb) {
36 int n = atomic_read(select_skb_count(op));38 int n = atomic_read(select_skb_count(skb));
37 trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n, here);39 trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n,
40 rxrpc_skb(skb)->rx_flags, here);
38 }41 }
39}42}
40 43
@@ -44,11 +47,22 @@ void rxrpc_see_skb(struct sk_buff *skb, enum rxrpc_skb_trace op) · 3 unmodified lines
44void rxrpc_get_skb(struct sk_buff *skb, enum rxrpc_skb_trace op)47void rxrpc_get_skb(struct sk_buff *skb, enum rxrpc_skb_trace op)
45{48{
46 const void *here = __builtin_return_address(0);49 const void *here = __builtin_return_address(0);
47 int n = atomic_inc_return(select_skb_count(op));50 int n = atomic_inc_return(select_skb_count(skb));
48 trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n, here);51 trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n,
52 rxrpc_skb(skb)->rx_flags, here);
49 skb_get(skb);53 skb_get(skb);
50}54}
51 55
56/*
57 * Note the dropping of a ref on a socket buffer by the core.
58 */
59void rxrpc_eaten_skb(struct sk_buff *skb, enum rxrpc_skb_trace op)
60{
61 const void *here = __builtin_return_address(0);
62 int n = atomic_inc_return(&rxrpc_n_rx_skbs);
63 trace_rxrpc_skb(skb, op, 0, n, 0, here);
64}
65
52/*66/*
53 * Note the destruction of a socket buffer.67 * Note the destruction of a socket buffer.
54 */68 */
@@ -58,8 +72,9 @@ void rxrpc_free_skb(struct sk_buff *skb, enum rxrpc_skb_trace op) · 3 unmodified lines
58 if (skb) {72 if (skb) {
59 int n;73 int n;
60 CHECK_SLAB_OKAY(&skb->users);74 CHECK_SLAB_OKAY(&skb->users);
61 n = atomic_dec_return(select_skb_count(op));75 n = atomic_dec_return(select_skb_count(skb));
62 trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n, here);76 trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n,
77 rxrpc_skb(skb)->rx_flags, here);
63 kfree_skb(skb);78 kfree_skb(skb);
64 }79 }
65}80}
@@ -72,9 +87,10 @@ void rxrpc_purge_queue(struct sk_buff_head *list) · 6 unmodified lines
72 const void *here = __builtin_return_address(0);87 const void *here = __builtin_return_address(0);
73 struct sk_buff *skb;88 struct sk_buff *skb;
74 while ((skb = skb_dequeue((list))) != NULL) {89 while ((skb = skb_dequeue((list))) != NULL) {
75 int n = atomic_dec_return(select_skb_count(rxrpc_skb_rx_purged));90 int n = atomic_dec_return(select_skb_count(skb));
76 trace_rxrpc_skb(skb, rxrpc_skb_rx_purged,91 trace_rxrpc_skb(skb, rxrpc_skb_purged,
77 refcount_read(&skb->users), n, here);92 refcount_read(&skb->users), n,
93 rxrpc_skb(skb)->rx_flags, here);
78 kfree_skb(skb);94 kfree_skb(skb);
79 }95 }
80}96}