bpf_skb_store_bytes() invocations above L2 header need BPF_F_RECOMPUTE_CSUM flag for updates, so that CHECKSUM_COMPLETE will be fixed up along the way. Where we ran into an issue with bpf_skb_store_bytes() is when we did a single-byte update on the IPv6 hoplimit despite using BPF_F_RECOMPUTE_CSUM flag; simple ping via ICMPv6 triggered a hw csum failure as a result. The underlying issue has been tracked down to a buffer alignment issue. Meaning, that csum_partial() computations via skb_postpull_rcsum() and skb_postpush_rcsum() pair invoked had a wrong result since they operated on an odd address for the hoplimit, while other computations were done on an even address. This mix doesn't work as-is with skb_postpull_rcsum(), skb_postpush_rcsum() pair as it always expects at least half-word alignment of input buffers, which is normally the case. Thus, instead of these helpers using csum_sub() and (implicitly) csum_add(), we need to use csum_block_sub(), csum_block_add(), respectively. For unaligned offsets, they rotate the sum to align it to a half-word boundary again, otherwise they work the same as csum_sub() and csum_add(). Adding __skb_postpull_rcsum(), __skb_postpush_rcsum() variants that take the offset as an input and adapting bpf_skb_store_bytes() to them fixes the hw csum failures again. The skb_postpull_rcsum(), skb_postpush_rcsum() helpers use a 0 constant for offset so that the compiler optimizes the offset & 1 test away and generates the same code as with csum_sub()/_add().
Unified · Split
| @@ -2847,6 +2847,18 @@ static inline int skb_linearize_cow(struct sk_buff *skb) · 2846 unmodified lines | |||
| 2847 | __skb_linearize(skb) : 0; | 2847 | __skb_linearize(skb) : 0; |
| 2848 | } | 2848 | } |
| 2849 | 2849 | ||
| 2850 | static __always_inline void | ||
| 2851 | __skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len, | ||
| 2852 | unsigned int off) | ||
| 2853 | { | ||
| 2854 | if (skb->ip_summed == CHECKSUM_COMPLETE) | ||
| 2855 | skb->csum = csum_block_sub(skb->csum, | ||
| 2856 | csum_partial(start, len, 0), off); | ||
| 2857 | else if (skb->ip_summed == CHECKSUM_PARTIAL && | ||
| 2858 | skb_checksum_start_offset(skb) < 0) | ||
| 2859 | skb->ip_summed = CHECKSUM_NONE; | ||
| 2860 | } | ||
| 2861 | |||
| 2850 | /** | 2862 | /** |
| 2851 | * skb_postpull_rcsum - update checksum for received skb after pull | 2863 | * skb_postpull_rcsum - update checksum for received skb after pull |
| 2852 | * @skb: buffer to update | 2864 | * @skb: buffer to update |
| @@ -2857,36 +2869,38 @@ static inline int skb_linearize_cow(struct sk_buff *skb) · 4 unmodified lines | |||
| 2857 | * update the CHECKSUM_COMPLETE checksum, or set ip_summed to | 2869 | * update the CHECKSUM_COMPLETE checksum, or set ip_summed to |
| 2858 | * CHECKSUM_NONE so that it can be recomputed from scratch. | 2870 | * CHECKSUM_NONE so that it can be recomputed from scratch. |
| 2859 | */ | 2871 | */ |
| 2860 | |||
| 2861 | static inline void skb_postpull_rcsum(struct sk_buff *skb, | 2872 | static inline void skb_postpull_rcsum(struct sk_buff *skb, |
| 2862 | const void *start, unsigned int len) | 2873 | const void *start, unsigned int len) |
| 2863 | { | 2874 | { |
| 2864 | if (skb->ip_summed == CHECKSUM_COMPLETE) | 2875 | __skb_postpull_rcsum(skb, start, len, 0); |
| 2865 | skb->csum = csum_sub(skb->csum, csum_partial(start, len, 0)); | ||
| 2866 | else if (skb->ip_summed == CHECKSUM_PARTIAL && | ||
| 2867 | skb_checksum_start_offset(skb) < 0) | ||
| 2868 | skb->ip_summed = CHECKSUM_NONE; | ||
| 2869 | } | 2876 | } |
| 2870 | 2877 | ||
| 2871 | unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len); | 2878 | static __always_inline void |
| 2879 | __skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len, | ||
| 2880 | unsigned int off) | ||
| 2881 | { | ||
| 2882 | if (skb->ip_summed == CHECKSUM_COMPLETE) | ||
| 2883 | skb->csum = csum_block_add(skb->csum, | ||
| 2884 | csum_partial(start, len, 0), off); | ||
| 2885 | } | ||
| 2872 | 2886 | ||
| 2887 | /** | ||
| 2888 | * skb_postpush_rcsum - update checksum for received skb after push | ||
| 2889 | * @skb: buffer to update | ||
| 2890 | * @start: start of data after push | ||
| 2891 | * @len: length of data pushed | ||
| 2892 | * | ||
| 2893 | * After doing a push on a received packet, you need to call this to | ||
| 2894 | * update the CHECKSUM_COMPLETE checksum. | ||
| 2895 | */ | ||
| 2873 | static inline void skb_postpush_rcsum(struct sk_buff *skb, | 2896 | static inline void skb_postpush_rcsum(struct sk_buff *skb, |
| 2874 | const void *start, unsigned int len) | 2897 | const void *start, unsigned int len) |
| 2875 | { | 2898 | { |
| 2876 | /* For performing the reverse operation to skb_postpull_rcsum(), | 2899 | __skb_postpush_rcsum(skb, start, len, 0); |
| 2877 | * we can instead of ... | ||
| 2878 | * | ||
| 2879 | * skb->csum = csum_add(skb->csum, csum_partial(start, len, 0)); | ||
| 2880 | * | ||
| 2881 | * ... just use this equivalent version here to save a few | ||
| 2882 | * instructions. Feeding csum of 0 in csum_partial() and later | ||
| 2883 | * on adding skb->csum is equivalent to feed skb->csum in the | ||
| 2884 | * first place. | ||
| 2885 | */ | ||
| 2886 | if (skb->ip_summed == CHECKSUM_COMPLETE) | ||
| 2887 | skb->csum = csum_partial(start, len, skb->csum); | ||
| 2888 | } | 2900 | } |
| 2889 | 2901 | ||
| 2902 | unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len); | ||
| 2903 | |||
| 2890 | /** | 2904 | /** |
| 2891 | * skb_push_rcsum - push skb and update receive checksum | 2905 | * skb_push_rcsum - push skb and update receive checksum |
| 2892 | * @skb: buffer to update | 2906 | * @skb: buffer to update |
| @@ -1401,7 +1401,7 @@ static u64 bpf_skb_store_bytes(u64 r1, u64 r2, u64 r3, u64 r4, u64 flags) · 1400 unmodified lines | |||
| 1401 | return -EFAULT; | 1401 | return -EFAULT; |
| 1402 | 1402 | ||
| 1403 | if (flags & BPF_F_RECOMPUTE_CSUM) | 1403 | if (flags & BPF_F_RECOMPUTE_CSUM) |
| 1404 | skb_postpull_rcsum(skb, ptr, len); | 1404 | __skb_postpull_rcsum(skb, ptr, len, offset); |
| 1405 | 1405 | ||
| 1406 | memcpy(ptr, from, len); | 1406 | memcpy(ptr, from, len); |
| 1407 | 1407 | ||
| @@ -1410,7 +1410,7 @@ static u64 bpf_skb_store_bytes(u64 r1, u64 r2, u64 r3, u64 r4, u64 flags) · 2 unmodified lines | |||
| 1410 | skb_store_bits(skb, offset, ptr, len); | 1410 | skb_store_bits(skb, offset, ptr, len); |
| 1411 | 1411 | ||
| 1412 | if (flags & BPF_F_RECOMPUTE_CSUM) | 1412 | if (flags & BPF_F_RECOMPUTE_CSUM) |
| 1413 | skb_postpush_rcsum(skb, ptr, len); | 1413 | __skb_postpush_rcsum(skb, ptr, len, offset); |
| 1414 | if (flags & BPF_F_INVALIDATE_HASH) | 1414 | if (flags & BPF_F_INVALIDATE_HASH) |
| 1415 | skb_clear_hash(skb); | 1415 | skb_clear_hash(skb); |
| 1416 | 1416 | ||