This change actually defines the (initial) metadata layout that should be used by AF_XDP userspace (xsk_tx_metadata). The first field is flags which requests appropriate offloads, followed by the offload-specific fields. The supported per-device offloads are exported via netlink (new xsk-flags). The offloads themselves are still implemented in a bit of a framework-y fashion that's left from my initial kfunc attempt. I'm introducing new xsk_tx_metadata_ops which drivers are supposed to implement. The drivers are also supposed to call xsk_tx_metadata_request/xsk_tx_metadata_complete in the right places. Since xsk_tx_metadata_{request,_complete} are static inline, we don't incur any extra overhead doing indirect calls. The benefit of this scheme is as follows: - keeps all metadata layout parsing away from driver code - makes it easy to grep and see which drivers implement what - don't need any extra flags to maintain to keep track of what offloads are implemented; if the callback is implemented - the offload is supported (used by netlink reporting code) Two offloads are defined right now: 1. XDP_TXMD_FLAGS_CHECKSUM: skb-style csum_start+csum_offset 2. XDP_TXMD_FLAGS_TIMESTAMP: writes TX timestamp back into metadata area upon completion (tx_timestamp field) XDP_TXMD_FLAGS_TIMESTAMP is also implemented for XDP_COPY mode: it writes SW timestamp from the skb destructor (note I'm reusing hwtstamps to pass metadata pointer). The struct is forward-compatible and can be extended in the future by appending more fields.
Showing the changes to tools/include/uapi (2 of 15 files). Show all
Unified · Split
| @@ -26,11 +26,11 @@ · 25 unmodified lines | ||
| 26 | 26 | */ |
| 27 | 27 | #define XDP_USE_NEED_WAKEUP (1 << 3) |
| 28 | 28 | /* By setting this option, userspace application indicates that it can |
| 29 | − * handle multiple descriptors per packet thus enabling xsk core to split | |
| 29 | + * handle multiple descriptors per packet thus enabling AF_XDP to split | |
| 30 | 30 | * multi-buffer XDP frames into multiple Rx descriptors. Without this set |
| 31 | − * such frames will be dropped by xsk. | |
| 31 | + * such frames will be dropped. | |
| 32 | 32 | */ |
| 33 | −#define XDP_USE_SG (1 << 4) | |
| 33 | +#define XDP_USE_SG (1 << 4) | |
| 34 | 34 | |
| 35 | 35 | /* Flags for xsk_umem_config flags */ |
| 36 | 36 | #define XDP_UMEM_UNALIGNED_CHUNK_FLAG (1 << 0) |
| @@ -106,6 +106,41 @@ struct xdp_options { · 69 unmodified lines | ||
| 106 | 106 | #define XSK_UNALIGNED_BUF_ADDR_MASK \ |
| 107 | 107 | ((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1) |
| 108 | 108 | |
| 109 | +/* Request transmit timestamp. Upon completion, put it into tx_timestamp | |
| 110 | + * field of union xsk_tx_metadata. | |
| 111 | + */ | |
| 112 | +#define XDP_TXMD_FLAGS_TIMESTAMP (1 << 0) | |
| 113 | + | |
| 114 | +/* Request transmit checksum offload. Checksum start position and offset | |
| 115 | + * are communicated via csum_start and csum_offset fields of union | |
| 116 | + * xsk_tx_metadata. | |
| 117 | + */ | |
| 118 | +#define XDP_TXMD_FLAGS_CHECKSUM (1 << 1) | |
| 119 | + | |
| 120 | +/* AF_XDP offloads request. 'request' union member is consumed by the driver | |
| 121 | + * when the packet is being transmitted. 'completion' union member is | |
| 122 | + * filled by the driver when the transmit completion arrives. | |
| 123 | + */ | |
| 124 | +struct xsk_tx_metadata { | |
| 125 | + __u64 flags; | |
| 126 | + | |
| 127 | + union { | |
| 128 | + struct { | |
| 129 | + /* XDP_TXMD_FLAGS_CHECKSUM */ | |
| 130 | + | |
| 131 | + /* Offset from desc->addr where checksumming should start. */ | |
| 132 | + __u16 csum_start; | |
| 133 | + /* Offset from csum_start where checksum should be stored. */ | |
| 134 | + __u16 csum_offset; | |
| 135 | + } request; | |
| 136 | + | |
| 137 | + struct { | |
| 138 | + /* XDP_TXMD_FLAGS_TIMESTAMP */ | |
| 139 | + __u64 tx_timestamp; | |
| 140 | + } completion; | |
| 141 | + }; | |
| 142 | +}; | |
| 143 | + | |
| 109 | 144 | /* Rx/Tx descriptor */ |
| 110 | 145 | struct xdp_desc { |
| 111 | 146 | __u64 addr; |
| @@ -113,9 +148,16 @@ struct xdp_desc { · 1 unmodified line | ||
| 113 | 148 | __u32 options; |
| 114 | 149 | }; |
| 115 | 150 | |
| 116 | −/* Flag indicating packet constitutes of multiple buffers*/ | |
| 151 | +/* UMEM descriptor is __u64 */ | |
| 152 | + | |
| 153 | +/* Flag indicating that the packet continues with the buffer pointed out by the | |
| 154 | + * next frame in the ring. The end of the packet is signalled by setting this | |
| 155 | + * bit to zero. For single buffer packets, every descriptor has 'options' set | |
| 156 | + * to 0 and this maintains backward compatibility. | |
| 157 | + */ | |
| 117 | 158 | #define XDP_PKT_CONTD (1 << 0) |
| 118 | 159 | |
| 119 | −/* UMEM descriptor is __u64 */ | |
| 160 | +/* TX packet carries valid metadata. */ | |
| 161 | +#define XDP_TX_METADATA (1 << 1) | |
| 120 | 162 | |
| 121 | 163 | #endif /* _LINUX_IF_XDP_H */ |
| @@ -53,12 +53,28 @@ enum netdev_xdp_rx_metadata { · 52 unmodified lines | ||
| 53 | 53 | NETDEV_XDP_RX_METADATA_MASK = 3, |
| 54 | 54 | }; |
| 55 | 55 | |
| 56 | +/** | |
| 57 | + * enum netdev_xsk_flags | |
| 58 | + * @NETDEV_XSK_FLAGS_TX_TIMESTAMP: HW timestamping egress packets is supported | |
| 59 | + * by the driver. | |
| 60 | + * @NETDEV_XSK_FLAGS_TX_CHECKSUM: L3 checksum HW offload is supported by the | |
| 61 | + * driver. | |
| 62 | + */ | |
| 63 | +enum netdev_xsk_flags { | |
| 64 | + NETDEV_XSK_FLAGS_TX_TIMESTAMP = 1, | |
| 65 | + NETDEV_XSK_FLAGS_TX_CHECKSUM = 2, | |
| 66 | + | |
| 67 | + /* private: */ | |
| 68 | + NETDEV_XSK_FLAGS_MASK = 3, | |
| 69 | +}; | |
| 70 | + | |
| 56 | 71 | enum { |
| 57 | 72 | NETDEV_A_DEV_IFINDEX = 1, |
| 58 | 73 | NETDEV_A_DEV_PAD, |
| 59 | 74 | NETDEV_A_DEV_XDP_FEATURES, |
| 60 | 75 | NETDEV_A_DEV_XDP_ZC_MAX_SEGS, |
| 61 | 76 | NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, |
| 77 | + NETDEV_A_DEV_XSK_FEATURES, | |
| 62 | 78 | |
| 63 | 79 | __NETDEV_A_DEV_MAX, |
| 64 | 80 | NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1) |