bpf_skb_store_bytes() invocations above L2 header need BPF_F_RECOMPUTE_CSUM flag for updates, so that CHECKSUM_COMPLETE will be fixed up along the way. Where we ran into an issue with bpf_skb_store_bytes() is when we did a single-byte update on the IPv6 hoplimit despite using BPF_F_RECOMPUTE_CSUM flag; simple ping via ICMPv6 triggered a hw csum failure as a result. The underlying issue has been tracked down to a buffer alignment issue. Meaning, that csum_partial() computations via skb_postpull_rcsum() and skb_postpush_rcsum() pair invoked had a wrong result since they operated on an odd address for the hoplimit, while other computations were done on an even address. This mix doesn't work as-is with skb_postpull_rcsum(), skb_postpush_rcsum() pair as it always expects at least half-word alignment of input buffers, which is normally the case. Thus, instead of these helpers using csum_sub() and (implicitly) csum_add(), we need to use csum_block_sub(), csum_block_add(), respectively. For unaligned offsets, they rotate the sum to align it to a half-word boundary again, otherwise they work the same as csum_sub() and csum_add(). Adding __skb_postpull_rcsum(), __skb_postpush_rcsum() variants that take the offset as an input and adapting bpf_skb_store_bytes() to them fixes the hw csum failures again. The skb_postpull_rcsum(), skb_postpush_rcsum() helpers use a 0 constant for offset so that the compiler optimizes the offset & 1 test away and generates the same code as with csum_sub()/_add().
Showing the changes to include/linux (1 of 2 files). Show all
Unified · Split
| @@ -2847,6 +2847,18 @@ static inline int skb_linearize_cow(struct sk_buff *skb) · 2846 unmodified lines | ||
| 2847 | 2847 | __skb_linearize(skb) : 0; |
| 2848 | 2848 | } |
| 2849 | 2849 | |
| 2850 | +static __always_inline void | |
| 2851 | +__skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len, | |
| 2852 | + unsigned int off) | |
| 2853 | +{ | |
| 2854 | + if (skb->ip_summed == CHECKSUM_COMPLETE) | |
| 2855 | + skb->csum = csum_block_sub(skb->csum, | |
| 2856 | + csum_partial(start, len, 0), off); | |
| 2857 | + else if (skb->ip_summed == CHECKSUM_PARTIAL && | |
| 2858 | + skb_checksum_start_offset(skb) < 0) | |
| 2859 | + skb->ip_summed = CHECKSUM_NONE; | |
| 2860 | +} | |
| 2861 | + | |
| 2850 | 2862 | /** |
| 2851 | 2863 | * skb_postpull_rcsum - update checksum for received skb after pull |
| 2852 | 2864 | * @skb: buffer to update |
| @@ -2857,36 +2869,38 @@ static inline int skb_linearize_cow(struct sk_buff *skb) · 4 unmodified lines | ||
| 2857 | 2869 | * update the CHECKSUM_COMPLETE checksum, or set ip_summed to |
| 2858 | 2870 | * CHECKSUM_NONE so that it can be recomputed from scratch. |
| 2859 | 2871 | */ |
| 2860 | − | |
| 2861 | 2872 | static inline void skb_postpull_rcsum(struct sk_buff *skb, |
| 2862 | 2873 | const void *start, unsigned int len) |
| 2863 | 2874 | { |
| 2864 | − if (skb->ip_summed == CHECKSUM_COMPLETE) | |
| 2865 | − skb->csum = csum_sub(skb->csum, csum_partial(start, len, 0)); | |
| 2866 | − else if (skb->ip_summed == CHECKSUM_PARTIAL && | |
| 2867 | − skb_checksum_start_offset(skb) < 0) | |
| 2868 | − skb->ip_summed = CHECKSUM_NONE; | |
| 2875 | + __skb_postpull_rcsum(skb, start, len, 0); | |
| 2869 | 2876 | } |
| 2870 | 2877 | |
| 2871 | −unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len); | |
| 2878 | +static __always_inline void | |
| 2879 | +__skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len, | |
| 2880 | + unsigned int off) | |
| 2881 | +{ | |
| 2882 | + if (skb->ip_summed == CHECKSUM_COMPLETE) | |
| 2883 | + skb->csum = csum_block_add(skb->csum, | |
| 2884 | + csum_partial(start, len, 0), off); | |
| 2885 | +} | |
| 2872 | 2886 | |
| 2887 | +/** | |
| 2888 | + * skb_postpush_rcsum - update checksum for received skb after push | |
| 2889 | + * @skb: buffer to update | |
| 2890 | + * @start: start of data after push | |
| 2891 | + * @len: length of data pushed | |
| 2892 | + * | |
| 2893 | + * After doing a push on a received packet, you need to call this to | |
| 2894 | + * update the CHECKSUM_COMPLETE checksum. | |
| 2895 | + */ | |
| 2873 | 2896 | static inline void skb_postpush_rcsum(struct sk_buff *skb, |
| 2874 | 2897 | const void *start, unsigned int len) |
| 2875 | 2898 | { |
| 2876 | − /* For performing the reverse operation to skb_postpull_rcsum(), | |
| 2877 | − * we can instead of ... | |
| 2878 | − * | |
| 2879 | − * skb->csum = csum_add(skb->csum, csum_partial(start, len, 0)); | |
| 2880 | − * | |
| 2881 | − * ... just use this equivalent version here to save a few | |
| 2882 | − * instructions. Feeding csum of 0 in csum_partial() and later | |
| 2883 | − * on adding skb->csum is equivalent to feed skb->csum in the | |
| 2884 | − * first place. | |
| 2885 | − */ | |
| 2886 | − if (skb->ip_summed == CHECKSUM_COMPLETE) | |
| 2887 | − skb->csum = csum_partial(start, len, skb->csum); | |
| 2899 | + __skb_postpush_rcsum(skb, start, len, 0); | |
| 2888 | 2900 | } |
| 2889 | 2901 | |
| 2902 | +unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len); | |
| 2903 | + | |
| 2890 | 2904 | /** |
| 2891 | 2905 | * skb_push_rcsum - push skb and update receive checksum |
| 2892 | 2906 | * @skb: buffer to update |