This change actually defines the (initial) metadata layout that should be used by AF_XDP userspace (xsk_tx_metadata). The first field is flags which requests appropriate offloads, followed by the offload-specific fields. The supported per-device offloads are exported via netlink (new xsk-flags). The offloads themselves are still implemented in a bit of a framework-y fashion that's left from my initial kfunc attempt. I'm introducing new xsk_tx_metadata_ops which drivers are supposed to implement. The drivers are also supposed to call xsk_tx_metadata_request/xsk_tx_metadata_complete in the right places. Since xsk_tx_metadata_{request,_complete} are static inline, we don't incur any extra overhead doing indirect calls. The benefit of this scheme is as follows: - keeps all metadata layout parsing away from driver code - makes it easy to grep and see which drivers implement what - don't need any extra flags to maintain to keep track of what offloads are implemented; if the callback is implemented - the offload is supported (used by netlink reporting code) Two offloads are defined right now: 1. XDP_TXMD_FLAGS_CHECKSUM: skb-style csum_start+csum_offset 2. XDP_TXMD_FLAGS_TIMESTAMP: writes TX timestamp back into metadata area upon completion (tx_timestamp field) XDP_TXMD_FLAGS_TIMESTAMP is also implemented for XDP_COPY mode: it writes SW timestamp from the skb destructor (note I'm reusing hwtstamps to pass metadata pointer). The struct is forward-compatible and can be extended in the future by appending more fields.
Showing the changes to tools/include/uapi (2 of 15 files). Show all
Unified · Split
| @@ -26,11 +26,11 @@ · 25 unmodified lines | |||
| 26 | */ | 26 | */ |
| 27 | #define XDP_USE_NEED_WAKEUP (1 << 3) | 27 | #define XDP_USE_NEED_WAKEUP (1 << 3) |
| 28 | /* By setting this option, userspace application indicates that it can | 28 | /* By setting this option, userspace application indicates that it can |
| 29 | * handle multiple descriptors per packet thus enabling xsk core to split | 29 | * handle multiple descriptors per packet thus enabling AF_XDP to split |
| 30 | * multi-buffer XDP frames into multiple Rx descriptors. Without this set | 30 | * multi-buffer XDP frames into multiple Rx descriptors. Without this set |
| 31 | * such frames will be dropped by xsk. | 31 | * such frames will be dropped. |
| 32 | */ | 32 | */ |
| 33 | #define XDP_USE_SG (1 << 4) | 33 | #define XDP_USE_SG (1 << 4) |
| 34 | 34 | ||
| 35 | /* Flags for xsk_umem_config flags */ | 35 | /* Flags for xsk_umem_config flags */ |
| 36 | #define XDP_UMEM_UNALIGNED_CHUNK_FLAG (1 << 0) | 36 | #define XDP_UMEM_UNALIGNED_CHUNK_FLAG (1 << 0) |
| @@ -106,6 +106,41 @@ struct xdp_options { · 69 unmodified lines | |||
| 106 | #define XSK_UNALIGNED_BUF_ADDR_MASK \ | 106 | #define XSK_UNALIGNED_BUF_ADDR_MASK \ |
| 107 | ((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1) | 107 | ((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1) |
| 108 | 108 | ||
| 109 | /* Request transmit timestamp. Upon completion, put it into tx_timestamp | ||
| 110 | * field of union xsk_tx_metadata. | ||
| 111 | */ | ||
| 112 | #define XDP_TXMD_FLAGS_TIMESTAMP (1 << 0) | ||
| 113 | |||
| 114 | /* Request transmit checksum offload. Checksum start position and offset | ||
| 115 | * are communicated via csum_start and csum_offset fields of union | ||
| 116 | * xsk_tx_metadata. | ||
| 117 | */ | ||
| 118 | #define XDP_TXMD_FLAGS_CHECKSUM (1 << 1) | ||
| 119 | |||
| 120 | /* AF_XDP offloads request. 'request' union member is consumed by the driver | ||
| 121 | * when the packet is being transmitted. 'completion' union member is | ||
| 122 | * filled by the driver when the transmit completion arrives. | ||
| 123 | */ | ||
| 124 | struct xsk_tx_metadata { | ||
| 125 | __u64 flags; | ||
| 126 | |||
| 127 | union { | ||
| 128 | struct { | ||
| 129 | /* XDP_TXMD_FLAGS_CHECKSUM */ | ||
| 130 | |||
| 131 | /* Offset from desc->addr where checksumming should start. */ | ||
| 132 | __u16 csum_start; | ||
| 133 | /* Offset from csum_start where checksum should be stored. */ | ||
| 134 | __u16 csum_offset; | ||
| 135 | } request; | ||
| 136 | |||
| 137 | struct { | ||
| 138 | /* XDP_TXMD_FLAGS_TIMESTAMP */ | ||
| 139 | __u64 tx_timestamp; | ||
| 140 | } completion; | ||
| 141 | }; | ||
| 142 | }; | ||
| 143 | |||
| 109 | /* Rx/Tx descriptor */ | 144 | /* Rx/Tx descriptor */ |
| 110 | struct xdp_desc { | 145 | struct xdp_desc { |
| 111 | __u64 addr; | 146 | __u64 addr; |
| @@ -113,9 +148,16 @@ struct xdp_desc { · 1 unmodified line | |||
| 113 | __u32 options; | 148 | __u32 options; |
| 114 | }; | 149 | }; |
| 115 | 150 | ||
| 116 | /* Flag indicating packet constitutes of multiple buffers*/ | 151 | /* UMEM descriptor is __u64 */ |
| 152 | |||
| 153 | /* Flag indicating that the packet continues with the buffer pointed out by the | ||
| 154 | * next frame in the ring. The end of the packet is signalled by setting this | ||
| 155 | * bit to zero. For single buffer packets, every descriptor has 'options' set | ||
| 156 | * to 0 and this maintains backward compatibility. | ||
| 157 | */ | ||
| 117 | #define XDP_PKT_CONTD (1 << 0) | 158 | #define XDP_PKT_CONTD (1 << 0) |
| 118 | 159 | ||
| 119 | /* UMEM descriptor is __u64 */ | 160 | /* TX packet carries valid metadata. */ |
| 161 | #define XDP_TX_METADATA (1 << 1) | ||
| 120 | 162 | ||
| 121 | #endif /* _LINUX_IF_XDP_H */ | 163 | #endif /* _LINUX_IF_XDP_H */ |
| @@ -53,12 +53,28 @@ enum netdev_xdp_rx_metadata { · 52 unmodified lines | |||
| 53 | NETDEV_XDP_RX_METADATA_MASK = 3, | 53 | NETDEV_XDP_RX_METADATA_MASK = 3, |
| 54 | }; | 54 | }; |
| 55 | 55 | ||
| 56 | /** | ||
| 57 | * enum netdev_xsk_flags | ||
| 58 | * @NETDEV_XSK_FLAGS_TX_TIMESTAMP: HW timestamping egress packets is supported | ||
| 59 | * by the driver. | ||
| 60 | * @NETDEV_XSK_FLAGS_TX_CHECKSUM: L3 checksum HW offload is supported by the | ||
| 61 | * driver. | ||
| 62 | */ | ||
| 63 | enum netdev_xsk_flags { | ||
| 64 | NETDEV_XSK_FLAGS_TX_TIMESTAMP = 1, | ||
| 65 | NETDEV_XSK_FLAGS_TX_CHECKSUM = 2, | ||
| 66 | |||
| 67 | /* private: */ | ||
| 68 | NETDEV_XSK_FLAGS_MASK = 3, | ||
| 69 | }; | ||
| 70 | |||
| 56 | enum { | 71 | enum { |
| 57 | NETDEV_A_DEV_IFINDEX = 1, | 72 | NETDEV_A_DEV_IFINDEX = 1, |
| 58 | NETDEV_A_DEV_PAD, | 73 | NETDEV_A_DEV_PAD, |
| 59 | NETDEV_A_DEV_XDP_FEATURES, | 74 | NETDEV_A_DEV_XDP_FEATURES, |
| 60 | NETDEV_A_DEV_XDP_ZC_MAX_SEGS, | 75 | NETDEV_A_DEV_XDP_ZC_MAX_SEGS, |
| 61 | NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, | 76 | NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, |
| 77 | NETDEV_A_DEV_XSK_FEATURES, | ||
| 62 | 78 | ||
| 63 | __NETDEV_A_DEV_MAX, | 79 | __NETDEV_A_DEV_MAX, |
| 64 | NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1) | 80 | NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1) |